infra: recover frozen V121 tail shards only

Author: heathsanchez

.github/workflows/trace-ace-v121-tail-recovery.yml

@@ -0,0 +1,111 @@
+name: Trace Ace V121 Tail Recovery
+
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events, agent/v111-runner]
+    paths:
+      - '.github/workflows/trace-ace-v121-tail-recovery.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: trace-ace-v121-tail-recovery
+  cancel-in-progress: false
+
+env:
+  PREPARED_RUN_ID: '32400309220'
+  ORIGINAL_RUN_ID: '32402681183'
+
+jobs:
+  embed_tail:
+    strategy:
+      fail-fast: false
+      max-parallel: 16
+      matrix:
+        shard: [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install exact embedding dependencies
+        run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Verify frozen manifest
+        run: |
+          test -f v121_prepared/manifest.json
+          grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json
+      - name: Embed exact frozen shard ${{ matrix.shard }}
+        run: |
+          cd competitions/trace_the_ace
+          python v121_embed_batch1_transport.py --dir ../../v121_prepared \
+            --shard ${{ matrix.shard }} --shards 128 \
+            --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-embeddings-${{ matrix.shard }}
+          path: v121_embeddings_shard_${{ matrix.shard }}.npz
+          retention-days: 2
+          compression-level: 0
+
+  evaluate_union:
+    needs: embed_tail
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install evaluation dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download successful original shards 0 through 64
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p v121_embedding_shards/original
+          for i in $(seq 0 64); do
+            gh run download "$ORIGINAL_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/original/$i"
+          done
+      - name: Download recovered tail shards 65 through 127
+        uses: actions/download-artifact@v4
+        with:
+          pattern: v121-embeddings-*
+          path: v121_embedding_shards/recovered
+          merge-multiple: false
+      - name: Evaluate unchanged frozen V121 precommit
+        run: |
+          cd competitions/trace_the_ace
+          python v121_staged_transport.py evaluate --dir ../../v121_prepared \
+            --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json
+      - name: Show decision
+        run: cat v121_pretrained_semantic_residual.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v121-pretrained-semantic-residual
+          path: v121_pretrained_semantic_residual.json
+          retention-days: 14