Evaluate frozen V121 embeddings only #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Trace Ace V121 Eval Only | |
| on: | |
| pull_request: | |
| branches: [agent/v121-cache-batch2] | |
| paths: | |
| - '.github/workflows/trace-ace-v121-eval-only.yml' | |
| workflow_dispatch: | |
| concurrency: | |
| group: trace-ace-v121-eval-only | |
| cancel-in-progress: false | |
| env: | |
| PREPARED_RUN_ID: '32400309220' | |
| EMBEDDING_RUN_ID: '32402681183' | |
| jobs: | |
| evaluate: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| cache: pip | |
| - name: Install evaluation dependencies | |
| run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 | |
| - name: Download exact frozen V121 prepared artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: v121-prepared | |
| path: v121_prepared | |
| repository: heathsanchez/mathgraph | |
| run-id: ${{ env.PREPARED_RUN_ID }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Verify frozen manifest | |
| run: | | |
| test -f v121_prepared/manifest.json | |
| grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json | |
| - name: Download all 128 frozen embedding shards from original run | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| mkdir -p v121_embedding_shards | |
| for i in $(seq 0 127); do | |
| gh run download "$EMBEDDING_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/$i" | |
| done | |
| test "$(find v121_embedding_shards -name 'v121_embeddings_shard_*.npz' | wc -l)" -eq 128 | |
| - name: Evaluate unchanged frozen V121 precommit | |
| run: | | |
| cd competitions/trace_the_ace | |
| python v121_staged_transport.py evaluate --dir ../../v121_prepared \ | |
| --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json | |
| - name: Show decision | |
| run: cat v121_pretrained_semantic_residual.json | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: trace-ace-v121-pretrained-semantic-residual | |
| path: v121_pretrained_semantic_residual.json | |
| retention-days: 14 |