Forex Training (legacy v7) #686

Workflow file for this run

	name: Forex Training

	on:
	schedule:
	# Hourly, offset 30 min from the stock workflow so the two pipelines
	# never hit the shared HF repo at the same time. The 429 rate-limit
	# failures are handled by the atomic single-commit push (v1.1.0) and the
	# retrying warm-start download (scripts/hf_download.py).
	- cron: '30 * * * *'
	workflow_dispatch:

	concurrency:
	group: forex-training
	cancel-in-progress: false

	permissions:
	contents: write
	issues: write

	env:
	PYTHON_VERSION: '3.11'
	CACHE_VERSION: v3
	FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

	jobs:
	pipeline:
	name: Train & Push Forex Model
	runs-on: ubuntu-latest
	timeout-minutes: 360

	steps:
	- name: Checkout
	uses: actions/checkout@v6
	with:
	fetch-depth: 1

	- name: Run Information
	run: \|
	echo "=================================================="
	echo " MERIDIAN.AI * FOREX TRAINING PIPELINE"
	echo "=================================================="
	echo " Run : #${{ github.run_number }} (attempt ${{ github.run_attempt }})"
	echo " Triggered : ${{ github.event_name }}"
	echo " Commit : ${{ github.sha }}"
	echo " Branch : ${{ github.ref_name }}"
	echo " Started : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	echo " Runner : ${{ runner.os }} / ${{ runner.arch }}"
	echo "=================================================="
	echo ""
	echo "--- System Resources ---"
	echo " CPUs : $(nproc)"
	free -h \| awk '/^Mem:/ {printf " RAM : %s total, %s available\n", $2, $7}'
	df -h / \| awk 'NR==2 {printf " Disk : %s free of %s\n", $4, $2}'
	echo ""
	echo "--- CPU detail ---"
	cat /proc/cpuinfo \| grep "model name" \| head -1 \| sed 's/model name.*: / /'

	- name: Setup Python
	uses: actions/setup-python@v6
	with:
	python-version: ${{ env.PYTHON_VERSION }}

	- name: Cache Dependencies
	uses: actions/cache@v5
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ env.CACHE_VERSION }}-${{ hashFiles('requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-${{ env.CACHE_VERSION }}-

	- name: Install Dependencies
	timeout-minutes: 10
	run: \|
	echo "--- Installing dependencies ---"
	python -m pip install --upgrade pip
	pip install torch --index-url https://download.pytorch.org/whl/cpu
	pip install -r requirements.txt
	pip install comet-ml huggingface_hub
	echo ""
	echo "--- Installed versions ---"
	python --version
	pip show torch comet-ml huggingface-hub pandas numpy \| grep -E "^(Name\|Version):"

	- name: Free Disk Space
	run: \|
	echo "--- Disk before cleanup ---"
	df -h /
	sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost
	echo "--- Disk after cleanup ---"
	df -h /

	- name: Fetch Forex Data
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	run: \|
	echo "--- Fetching forex data (up to 30 pairs) ---"
	START=$(date +%s)
	python scripts/fetch_and_store_data.py \
	--db-file training.db \
	--asset-type forex \
	--limit 30
	ELAPSED=$(( $(date +%s) - START ))
	echo "--- Fetch completed in ${ELAPSED}s ---"

	- name: Database Stats
	run: \|
	if [ ! -f training.db ]; then
	echo "FATAL: training.db not produced — aborting"
	exit 1
	fi
	echo "=== Training Database ==="
	TOTAL=$(sqlite3 training.db "SELECT COUNT(*) FROM market_data WHERE asset_type='forex';")
	PAIRS=$(sqlite3 training.db "SELECT COUNT(DISTINCT symbol) FROM market_data WHERE asset_type='forex';")
	echo " Total rows : $TOTAL"
	echo " Pairs : $PAIRS"
	[ "$PAIRS" -gt "0" ] && echo " Avg rows/pair: $(( TOTAL / PAIRS ))"
	echo ""
	echo "--- Per-Pair Detail ---"
	sqlite3 -column -header training.db \
	"SELECT symbol,
	COUNT(*) AS rows,
	MIN(date) AS earliest,
	MAX(date) AS latest,
	CAST(ROUND(JULIANDAY(MAX(date)) - JULIANDAY(MIN(date))) AS INT) AS days
	FROM market_data
	WHERE asset_type = 'forex'
	GROUP BY symbol
	ORDER BY symbol;"

	- name: Download Existing Model
	timeout-minutes: 15
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	run: \|
	mkdir -p models
	# 429-aware retry wrapper — a transient HF rate limit must not
	# silently discard the warm start (always exits 0).
	python scripts/hf_download.py \
	--repo-id meridianal/ARA.AI \
	--filename models/Meridian.AI_Forex.pt \
	--local-dir .

	- name: Existing Model Info
	run: \|
	if [ -f models/Meridian.AI_Forex.pt ]; then
	SIZE=$(ls -lh models/Meridian.AI_Forex.pt \| awk '{print $5}')
	echo "Existing model: models/Meridian.AI_Forex.pt ($SIZE)"
	python -c "
	import torch
	try:
	m = torch.load('models/Meridian.AI_Forex.pt', map_location='cpu', weights_only=False)
	md = m.get('metadata', {})
	print(f' Architecture : {m.get(\"architecture\", \"N/A\")} v{m.get(\"version\", \"N/A\")}')
	print(f' Training date: {md.get(\"training_date\", m.get(\"training_date\", \"N/A\"))}')
	print(f' Best val loss: {md.get(\"best_val_loss\", \"N/A\")}')
	print(f' Symbols : {len(md.get(\"trained_symbols\", []))} trained')
	sd = m.get('model_state_dict', {})
	if sd:
	print(f' Parameters : {sum(v.numel() for v in sd.values()):,}')
	except Exception as e:
	print(f' Could not inspect model: {e}')
	" \|\| true
	else
	echo " No existing model on HF — will train from scratch"
	fi

	- name: Train Model
	env:
	COMET_API_KEY: ${{ secrets.COMET_API_KEY }}
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	PYTHONUNBUFFERED: "1"
	run: \|
	echo "=================================================="
	echo " STARTING FOREX MODEL TRAINING"
	echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	echo " Max steps: 2000 \| No time limit (~2.5h budget)"
	echo "=================================================="
	python -u scripts/train_forex.py \
	--db-file training.db \
	--output models/Meridian.AI_Forex.pt \
	--use-all-data \
	--epochs 999 \
	--max-steps 2000 \
	--comet-api-key "$COMET_API_KEY"
	EXIT_CODE=$?
	echo ""
	echo "=================================================="
	echo " TRAINING SCRIPT EXITED — code=$EXIT_CODE"
	echo " $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	echo "=================================================="
	# Don't propagate exit code — let Verify + Push step handle whatever
	# state the .pt is in (safety-save writes a valid checkpoint at
	# step-limit even if post-training validation gets SIGTERMed).
	exit 0

	- name: Verify Saved Model
	if: always()
	run: \|
	echo "=== Post-Training Verification ==="
	if [ -f models/Meridian.AI_Forex.pt ]; then
	SIZE=$(ls -lh models/Meridian.AI_Forex.pt \| awk '{print $5}')
	MTIME=$(stat -c '%y' models/Meridian.AI_Forex.pt)
	echo "OK Model file present"
	echo " Path : models/Meridian.AI_Forex.pt"
	echo " Size : $SIZE"
	echo " Modified : $MTIME"
	echo ""
	python -c "
	import torch
	try:
	m = torch.load('models/Meridian.AI_Forex.pt', map_location='cpu', weights_only=False)
	md = m.get('metadata', {})
	print(f' Architecture : {m.get(\"architecture\", \"N/A\")} v{m.get(\"version\", \"N/A\")}')
	print(f' Training date: {md.get(\"training_date\", \"N/A\")}')
	print(f' Global step : {md.get(\"global_step\", \"N/A\")}')
	print(f' Best val loss: {md.get(\"best_val_loss\", \"N/A\")}')
	print(f' Direction acc: {md.get(\"direction_accuracy\", \"N/A\")}')
	except Exception as e:
	print(f' Inspection error: {e}')
	" \|\| true
	else
	echo "FAIL Model file NOT present — HF push will be skipped"
	fi
	echo ""
	echo "--- models/ directory ---"
	ls -lh models/ 2>/dev/null \|\| true
	echo ""
	echo "--- disk ---"
	df -h /

	- name: Sanity Gate
	if: always() && hashFiles('models/Meridian.AI_Forex.pt') != ''
	run: \|
	echo "--- Running post-training sanity gate ---"
	# If the model is degenerate (constant output, directional collapse,
	# or blown-up target scale), delete the .pt so the push step below
	# skips it, then fail the run so a tracking issue is opened.
	if ! python scripts/sanity_check_model.py \
	--model-path models/Meridian.AI_Forex.pt \
	--model-type forex \
	--db-file training.db; then
	echo "Sanity gate FAILED — removing degenerate model so it is NOT pushed."
	rm -f models/Meridian.AI_Forex.pt
	exit 1
	fi

	- name: Push to Hugging Face
	if: always() && hashFiles('models/Meridian.AI_Forex.pt') != ''
	env:
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	run: \|
	echo "--- Pushing Forex Model to Hugging Face ---"
	SIZE=$(ls -lh models/Meridian.AI_Forex.pt \| awk '{print $5}')
	echo " File : models/Meridian.AI_Forex.pt ($SIZE)"
	echo " Repo : meridianal/ARA.AI"
	echo " Start : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	START=$(date +%s)
	python scripts/push_to_hf.py \
	--model-path models/Meridian.AI_Forex.pt \
	--model-type forex
	ELAPSED=$(( $(date +%s) - START ))
	echo "--- HF push completed in ${ELAPSED}s ---"

	- name: Pipeline Summary
	if: always()
	run: \|
	echo "=================================================="
	echo " MERIDIAN.AI * FOREX PIPELINE COMPLETE"
	echo "=================================================="
	echo " Run : #${{ github.run_number }}"
	echo " Finished : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	if [ -f models/Meridian.AI_Forex.pt ]; then
	echo " Status : OK — .pt saved + push attempted"
	else
	echo " Status : FAIL — no .pt produced, nothing pushed"
	fi
	echo "=================================================="

	- name: Close Open Failure Issues on Success
	if: success() && hashFiles('models/Meridian.AI_Forex.pt') != ''
	uses: actions/github-script@v8
	with:
	script: \|
	const issues = await github.rest.issues.listForRepo({
	owner: context.repo.owner,
	repo: context.repo.repo,
	labels: 'forex-training,automated',
	state: 'open',
	per_page: 10,
	});
	for (const issue of issues.data) {
	await github.rest.issues.update({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: issue.number,
	state: 'closed',
	state_reason: 'completed',
	});
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: issue.number,
	body: `Resolved by run [#${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId}).`,
	});
	}

	- name: Open Failure Issue on Failure
	if: failure()
	uses: actions/github-script@v8
	with:
	script: \|
	const existing = await github.rest.issues.listForRepo({
	owner: context.repo.owner,
	repo: context.repo.repo,
	labels: 'forex-training,automated',
	state: 'open',
	per_page: 5,
	});
	if (existing.data.length > 0) {
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: existing.data[0].number,
	body: `Still failing — run [#${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId})`,
	});
	return;
	}
	await github.rest.issues.create({
	owner: context.repo.owner,
	repo: context.repo.repo,
	title: `Forex Training Failed — Run ${context.runNumber}`,
	body: `Workflow: [Run #${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId})\nCommit: ${context.sha.substring(0, 7)}\n\nAuto-closes on next successful run.`,
	labels: ['bug', 'forex-training', 'automated']
	});

Navigation Menu

Forex Training (legacy v7) #686

Workflow file

Forex Training (legacy v7) #686

Uh oh!

Workflow file for this run