api-regression-tests.yml

name: API Regression Tests

on:
  workflow_dispatch:
    inputs:
      network:
        description: "Network to compare: live (baseline) against dead (candidate)"
        default: "mainnet"
        required: false
        type: choice
        options:
          - mainnet
          - testnet
      tests:
        description: "Subset to run: a suite name (stargate, b3tr, transfers, …) or a path regex. Empty runs every operation."
        default: ""
        required: false
        type: string
      max_block_gap:
        description: "Blocks an indexer on the candidate may trail the chain head by"
        default: 100
        required: false
        type: number
      sweep_windows:
        description: "Block windows to compare row by row (0 skips the sweep)"
        default: 6
        required: false
        type: number
  workflow_call:
    inputs:
      network:
        description: "Network to compare: mainnet or testnet"
        required: false
        type: string
        default: "mainnet"
      tests:
        description: "Subset to run: a suite name or a path regex; empty runs every operation"
        required: false
        type: string
        default: ""
      max_block_gap:
        description: "Blocks an indexer on the candidate may trail the chain head by"
        required: false
        type: number
        default: 100
      sweep_windows:
        description: "Block windows to compare row by row (0 skips the sweep)"
        required: false
        type: number
        default: 6

permissions:
  contents: read

jobs:
  regression:
    name: Regression comparison
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.14'

      - name: Resolve the network's endpoints
        id: net
        env:
          NETWORK: ${{ inputs.network || 'mainnet' }}
        run: |
          for field in baseline candidate; do
            echo "$field=$(python3 packages/api/scripts/networks.py "$NETWORK" "$field")" >> "$GITHUB_OUTPUT"
          done

      - name: Check the candidate has caught up with the chain
        id: sync
        continue-on-error: true
        env:
          NETWORK: ${{ inputs.network || 'mainnet' }}
          MAX_BLOCK_GAP: ${{ inputs.max_block_gap || 100 }}
        run: |
          packages/api/scripts/check_sync_gate.py \
            --network "$NETWORK" \
            --max-gap "$MAX_BLOCK_GAP"

      - name: Run regression comparison (candidate vs baseline)
        env:
          NETWORK: ${{ inputs.network || 'mainnet' }}
          TESTS: ${{ inputs.tests }}
          REGRESSION_SEED_METADATA_FILE: regression-output/seed-metadata.json
          RATE_LIMIT_BYPASS_TOKEN: ${{ secrets.RATE_LIMIT_BYPASS_TOKEN }}
        run: |
          mkdir -p regression-output
          filter=()
          if [[ -n "$TESTS" ]]; then
            filter=(--path-filter "$TESTS")
          fi
          packages/api/scripts/run_regression_tests.sh \
            "${filter[@]}" \
            --output regression-output/report.json

      - name: Generate summary
        if: always()
        env:
          NETWORK: ${{ inputs.network || 'mainnet' }}
          TESTS: ${{ inputs.tests }}
          CANDIDATE: ${{ steps.net.outputs.candidate }}
          BASELINE: ${{ steps.net.outputs.baseline }}
          SYNC_OUTCOME: ${{ steps.sync.outcome }}
        shell: bash
        run: |
          python3 packages/api/scripts/regression_summary.py \
            --report regression-output/report.json \
            --seed-metadata regression-output/seed-metadata.json \
            --candidate "$CANDIDATE" \
            --baseline "$BASELINE" \
            --output "$GITHUB_STEP_SUMMARY"

          echo "> Network \`$NETWORK\`, subset \`${TESTS:-all operations}\`." >> "$GITHUB_STEP_SUMMARY"

          if [[ "$SYNC_OUTCOME" != "success" ]]; then
            echo "> **The candidate had not caught up when this ran**, so differences below may" \
              "be lag rather than regressions. See the sync gate step." >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload regression report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: regression-report
          path: ./regression-output
          if-no-files-found: ignore

  sweep:
    name: Block window sweep
    runs-on: ubuntu-latest
    timeout-minutes: 45
    if: ${{ (inputs.sweep_windows == null && 6 || inputs.sweep_windows) != 0 }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.14'

      - name: Compare whole block windows
        env:
          NETWORK: ${{ inputs.network || 'mainnet' }}
          SWEEP_WINDOWS: ${{ inputs.sweep_windows || 6 }}
        run: |
          mkdir -p sweep-output
          printf '{"endpoints":{"baseline":"%s","candidate":"%s"}}' \
            "$(python3 packages/api/scripts/networks.py "$NETWORK" baseline)" \
            "$(python3 packages/api/scripts/networks.py "$NETWORK" candidate)" \
            > sweep-output/endpoints.json
          packages/api/scripts/sweep_blocks.py \
            --config-file sweep-output/endpoints.json \
            --windows "$SWEEP_WINDOWS" \
            --output sweep-output/sweep.json

      - name: Summarise the sweep
        if: always()
        run: |
          python3 - <<'PYEOF' >> "$GITHUB_STEP_SUMMARY"
          import json, pathlib
          path = pathlib.Path("sweep-output/sweep.json")
          if not path.exists():
              print("### Block window sweep\n\n> No report generated.")
              raise SystemExit(0)
          report = json.loads(path.read_text())
          s = report["summary"]
          print("### Block window sweep\n")
          print("| | |\n|---|---|")
          print(f"| **Candidate head** | {s['candidate_head']} |")
          print(f"| **Windows** | {s['windows']} |")
          print(f"| **Blocks compared** | {s['blocks_compared']} |")
          print(f"| **Transactions compared** | {s['transactions_compared']} |")
          print(f"| **Findings** | {s['findings']} |\n")
          for window in report["windows"]:
              if not window["findings"]:
                  continue
              start = window["window"]["start"]
              print(f"<details><summary>Blocks {start}-{start + window['window']['count'] - 1}</summary>\n")
              print("```")
              for finding in window["findings"][:20]:
                  subject = finding.get("block") or finding.get("transaction")
                  for path_, description in finding["diffs"][:5]:
                      print(f"  {subject} {path_}: {description}")
              print("```\n</details>\n")
          PYEOF

      - name: Upload sweep report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: sweep-report
          path: ./sweep-output
          if-no-files-found: ignore