forex.yml

name: Forex Training (legacy v7)

# FROZEN — dispatch-only. Honest holdout accuracy was 48.68% against a 52.02%
# always-up baseline, and the source FX bars leak next-day information through
# day-t high/low (see LOCAL_BENCHMARK_REPORT.md). Not superseded by v8: v8 is
# stocks-only, because a 22-pair FX cross-section is too thin to rank.
on:
  workflow_dispatch:

concurrency:
  group: forex-training
  cancel-in-progress: false

permissions:
  contents: write
  issues: write

env:
  PYTHON_VERSION: '3.11'
  CACHE_VERSION: v3
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  pipeline:
    name: Train & Push Forex Model
    runs-on: ubuntu-latest
    timeout-minutes: 360

    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 1

      - name: Run Information
        run: |
          echo "=================================================="
          echo "   MERIDIAN.AI  *  FOREX TRAINING PIPELINE"
          echo "=================================================="
          echo "  Run       : #${{ github.run_number }} (attempt ${{ github.run_attempt }})"
          echo "  Triggered : ${{ github.event_name }}"
          echo "  Commit    : ${{ github.sha }}"
          echo "  Branch    : ${{ github.ref_name }}"
          echo "  Started   : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          echo "  Runner    : ${{ runner.os }} / ${{ runner.arch }}"
          echo "=================================================="
          echo ""
          echo "--- System Resources ---"
          echo "  CPUs  : $(nproc)"
          free -h | awk '/^Mem:/ {printf "  RAM   : %s total, %s available\n", $2, $7}'
          df -h / | awk 'NR==2    {printf "  Disk  : %s free of %s\n", $4, $2}'
          echo ""
          echo "--- CPU detail ---"
          cat /proc/cpuinfo | grep "model name" | head -1 | sed 's/model name.*: /  /'

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Cache Dependencies
        uses: actions/cache@v5
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ env.CACHE_VERSION }}-${{ hashFiles('legacy/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-${{ env.CACHE_VERSION }}-

      - name: Install Dependencies
        timeout-minutes: 10
        run: |
          echo "--- Installing dependencies ---"
          python -m pip install --upgrade pip
          pip install torch --index-url https://download.pytorch.org/whl/cpu
          pip install -r legacy/requirements.txt
          pip install comet-ml huggingface_hub
          echo ""
          echo "--- Installed versions ---"
          python --version
          pip show torch comet-ml huggingface-hub pandas numpy | grep -E "^(Name|Version):"

      - name: Free Disk Space
        run: |
          echo "--- Disk before cleanup ---"
          df -h /
          sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost
          echo "--- Disk after cleanup ---"
          df -h /

      - name: Fetch Forex Data
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          echo "--- Fetching forex data (up to 30 pairs) ---"
          START=$(date +%s)
          python scripts/fetch_and_store_data.py \
            --db-file training.db \
            --asset-type forex \
            --limit 30
          ELAPSED=$(( $(date +%s) - START ))
          echo "--- Fetch completed in ${ELAPSED}s ---"

      - name: Database Stats
        run: |
          if [ ! -f training.db ]; then
            echo "FATAL: training.db not produced — aborting"
            exit 1
          fi
          echo "=== Training Database ==="
          TOTAL=$(sqlite3 training.db "SELECT COUNT(*) FROM market_data WHERE asset_type='forex';")
          PAIRS=$(sqlite3 training.db "SELECT COUNT(DISTINCT symbol) FROM market_data WHERE asset_type='forex';")
          echo "  Total rows  : $TOTAL"
          echo "  Pairs       : $PAIRS"
          [ "$PAIRS" -gt "0" ] && echo "  Avg rows/pair: $(( TOTAL / PAIRS ))"
          echo ""
          echo "--- Per-Pair Detail ---"
          sqlite3 -column -header training.db \
            "SELECT symbol,
                    COUNT(*)   AS rows,
                    MIN(date)  AS earliest,
                    MAX(date)  AS latest,
                    CAST(ROUND(JULIANDAY(MAX(date)) - JULIANDAY(MIN(date))) AS INT) AS days
             FROM market_data
             WHERE asset_type = 'forex'
             GROUP BY symbol
             ORDER BY symbol;"

      - name: Download Existing Model
        timeout-minutes: 15
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          mkdir -p models
          # 429-aware retry wrapper — a transient HF rate limit must not
          # silently discard the warm start (always exits 0).
          python scripts/hf_download.py \
            --repo-id meridianal/ARA.AI \
            --filename models/Meridian.AI_Forex.pt \
            --local-dir .

      - name: Existing Model Info
        run: |
          if [ -f models/Meridian.AI_Forex.pt ]; then
            SIZE=$(ls -lh models/Meridian.AI_Forex.pt | awk '{print $5}')
            echo "Existing model: models/Meridian.AI_Forex.pt  ($SIZE)"
            python -c "
          import torch
          try:
              m = torch.load('models/Meridian.AI_Forex.pt', map_location='cpu', weights_only=False)
              md = m.get('metadata', {})
              print(f'  Architecture : {m.get(\"architecture\", \"N/A\")} v{m.get(\"version\", \"N/A\")}')
              print(f'  Training date: {md.get(\"training_date\", m.get(\"training_date\", \"N/A\"))}')
              print(f'  Best val loss: {md.get(\"best_val_loss\", \"N/A\")}')
              print(f'  Symbols      : {len(md.get(\"trained_symbols\", []))} trained')
              sd = m.get('model_state_dict', {})
              if sd:
                  print(f'  Parameters   : {sum(v.numel() for v in sd.values()):,}')
          except Exception as e:
              print(f'  Could not inspect model: {e}')
          " || true
          else
            echo "  No existing model on HF — will train from scratch"
          fi

      - name: Train Model
        env:
          COMET_API_KEY: ${{ secrets.COMET_API_KEY }}
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
          PYTHONUNBUFFERED: "1"
        run: |
          echo "=================================================="
          echo "   STARTING FOREX MODEL TRAINING"
          echo "   $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          echo "   Max steps: 2000 | No time limit (~2.5h budget)"
          echo "=================================================="
          python -u legacy/scripts/train_forex.py \
            --db-file training.db \
            --output models/Meridian.AI_Forex.pt \
            --use-all-data \
            --epochs 999 \
            --max-steps 2000 \
            --comet-api-key "$COMET_API_KEY"
          EXIT_CODE=$?
          echo ""
          echo "=================================================="
          echo "   TRAINING SCRIPT EXITED — code=$EXIT_CODE"
          echo "   $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          echo "=================================================="
          # Don't propagate exit code — let Verify + Push step handle whatever
          # state the .pt is in (safety-save writes a valid checkpoint at
          # step-limit even if post-training validation gets SIGTERMed).
          exit 0

      - name: Verify Saved Model
        if: always()
        run: |
          echo "=== Post-Training Verification ==="
          if [ -f models/Meridian.AI_Forex.pt ]; then
            SIZE=$(ls -lh models/Meridian.AI_Forex.pt | awk '{print $5}')
            MTIME=$(stat -c '%y' models/Meridian.AI_Forex.pt)
            echo "OK Model file present"
            echo "  Path     : models/Meridian.AI_Forex.pt"
            echo "  Size     : $SIZE"
            echo "  Modified : $MTIME"
            echo ""
            python -c "
          import torch
          try:
              m = torch.load('models/Meridian.AI_Forex.pt', map_location='cpu', weights_only=False)
              md = m.get('metadata', {})
              print(f'  Architecture : {m.get(\"architecture\", \"N/A\")} v{m.get(\"version\", \"N/A\")}')
              print(f'  Training date: {md.get(\"training_date\", \"N/A\")}')
              print(f'  Global step  : {md.get(\"global_step\", \"N/A\")}')
              print(f'  Best val loss: {md.get(\"best_val_loss\", \"N/A\")}')
              print(f'  Direction acc: {md.get(\"direction_accuracy\", \"N/A\")}')
          except Exception as e:
              print(f'  Inspection error: {e}')
          " || true
          else
            echo "FAIL Model file NOT present — HF push will be skipped"
          fi
          echo ""
          echo "--- models/ directory ---"
          ls -lh models/ 2>/dev/null || true
          echo ""
          echo "--- disk ---"
          df -h /

      - name: Sanity Gate
        if: always() && hashFiles('models/Meridian.AI_Forex.pt') != ''
        run: |
          echo "--- Running post-training sanity gate ---"
          # If the model is degenerate (constant output, directional collapse,
          # or blown-up target scale), delete the .pt so the push step below
          # skips it, then fail the run so a tracking issue is opened.
          if ! python legacy/scripts/sanity_check_model.py \
              --model-path models/Meridian.AI_Forex.pt \
              --model-type forex \
              --db-file training.db; then
            echo "Sanity gate FAILED — removing degenerate model so it is NOT pushed."
            rm -f models/Meridian.AI_Forex.pt
            exit 1
          fi

      - name: Push to Hugging Face
        if: always() && hashFiles('models/Meridian.AI_Forex.pt') != ''
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          echo "--- Pushing Forex Model to Hugging Face ---"
          SIZE=$(ls -lh models/Meridian.AI_Forex.pt | awk '{print $5}')
          echo "  File  : models/Meridian.AI_Forex.pt ($SIZE)"
          echo "  Repo  : meridianal/ARA.AI"
          echo "  Start : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          START=$(date +%s)
          python scripts/push_to_hf.py \
            --model-path models/Meridian.AI_Forex.pt \
            --model-type forex
          ELAPSED=$(( $(date +%s) - START ))
          echo "--- HF push completed in ${ELAPSED}s ---"

      - name: Pipeline Summary
        if: always()
        run: |
          echo "=================================================="
          echo "   MERIDIAN.AI  *  FOREX PIPELINE COMPLETE"
          echo "=================================================="
          echo "  Run      : #${{ github.run_number }}"
          echo "  Finished : $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
          if [ -f models/Meridian.AI_Forex.pt ]; then
            echo "  Status   : OK — .pt saved + push attempted"
          else
            echo "  Status   : FAIL — no .pt produced, nothing pushed"
          fi
          echo "=================================================="

      - name: Close Open Failure Issues on Success
        if: success() && hashFiles('models/Meridian.AI_Forex.pt') != ''
        uses: actions/github-script@v8
        with:
          script: |
            const issues = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: 'forex-training,automated',
              state: 'open',
              per_page: 10,
            });
            for (const issue of issues.data) {
              await github.rest.issues.update({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: issue.number,
                state: 'closed',
                state_reason: 'completed',
              });
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: issue.number,
                body: `Resolved by run [#${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId}).`,
              });
            }

      - name: Open Failure Issue on Failure
        if: failure()
        uses: actions/github-script@v8
        with:
          script: |
            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: 'forex-training,automated',
              state: 'open',
              per_page: 5,
            });
            if (existing.data.length > 0) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: existing.data[0].number,
                body: `Still failing — run [#${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId})`,
              });
              return;
            }
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: `Forex Training Failed — Run ${context.runNumber}`,
              body: `Workflow: [Run #${context.runNumber}](${context.payload.repository.html_url}/actions/runs/${context.runId})\nCommit: ${context.sha.substring(0, 7)}\n\nAuto-closes on next successful run.`,
              labels: ['bug', 'forex-training', 'automated']
            });