Run memory-safe frozen V121 semantic residual

.github/workflows/trace-ace-v121-128shard.yml

@@ -0,0 +1,102 @@
+name: Trace Ace V121 Frozen 128 Shard
+
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events, agent/v111-runner]
+    paths:
+      - '.github/workflows/trace-ace-v121-128shard.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: trace-ace-v121-frozen-128shard
+  cancel-in-progress: false
+
+env:
+  PREPARED_RUN_ID: '32400309220'
+
+jobs:
+  embed:
+    strategy:
+      fail-fast: false
+      max-parallel: 16
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install exact embedding dependencies
+        run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Verify frozen manifest
+        run: |
+          test -f v121_prepared/manifest.json
+          grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json
+      - name: Embed exact frozen shard ${{ matrix.shard }}
+        run: |
+          cd competitions/trace_the_ace
+          python v121_embed_batch1_transport.py --dir ../../v121_prepared \
+            --shard ${{ matrix.shard }} --shards 128 \
+            --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-embeddings-${{ matrix.shard }}
+          path: v121_embeddings_shard_${{ matrix.shard }}.npz
+          retention-days: 2
+          compression-level: 0
+
+  evaluate:
+    needs: embed
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install evaluation dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download all exact embedding shards
+        uses: actions/download-artifact@v4
+        with:
+          pattern: v121-embeddings-*
+          path: v121_embedding_shards
+          merge-multiple: false
+      - name: Evaluate unchanged frozen V121 precommit
+        run: |
+          cd competitions/trace_the_ace
+          python v121_staged_transport.py evaluate --dir ../../v121_prepared \
+            --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json
+      - name: Show decision
+        run: cat v121_pretrained_semantic_residual.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v121-pretrained-semantic-residual
+          path: v121_pretrained_semantic_residual.json
+          retention-days: 14

.github/workflows/trace-ace-v121-embed-only.yml

@@ -0,0 +1,104 @@
+name: Trace Ace V121 Frozen Embed Only
+
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events, agent/v111-runner]
+    paths:
+      - '.github/workflows/trace-ace-v121-embed-only.yml'
+  workflow_dispatch:
+
+# Reuses the already-successful exact frozen V121 prepared artifact from run 32400309220.
+# Scientific model/text/sample/folds/gates are unchanged.
+concurrency:
+  group: trace-ace-v121-embed-only-16-p4
+  cancel-in-progress: false
+
+env:
+  PREPARED_RUN_ID: '32400309220'
+
+jobs:
+  embed:
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Verify frozen preparation manifest exists
+        run: |
+          test -f v121_prepared/manifest.json
+          cat v121_prepared/manifest.json
+      - name: Embed exact frozen V121 texts shard ${{ matrix.shard }}
+        run: |
+          cd competitions/trace_the_ace
+          python v121_staged_transport.py embed --dir ../../v121_prepared \
+            --shard ${{ matrix.shard }} --shards 16 \
+            --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-embeddings-${{ matrix.shard }}
+          path: v121_embeddings_shard_${{ matrix.shard }}.npz
+          retention-days: 2
+          compression-level: 0
+
+  evaluate:
+    needs: embed
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0
+      - name: Download exact frozen V121 prepared artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: v121-prepared
+          path: v121_prepared
+          repository: heathsanchez/mathgraph
+          run-id: ${{ env.PREPARED_RUN_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download all exact embedding shards
+        uses: actions/download-artifact@v4
+        with:
+          pattern: v121-embeddings-*
+          path: v121_embedding_shards
+          merge-multiple: false
+      - name: Evaluate frozen V121 precommit
+        run: |
+          cd competitions/trace_the_ace
+          python v121_staged_transport.py evaluate --dir ../../v121_prepared \
+            --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json
+      - name: Show decision
+        run: cat v121_pretrained_semantic_residual.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v121-pretrained-semantic-residual
+          path: v121_pretrained_semantic_residual.json
+          retention-days: 14

.github/workflows/trace-ace-v121-local-bootstrap.yml

@@ -0,0 +1,49 @@
+name: Trace Ace V121 Local Bootstrap
+
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events, agent/v111-runner]
+    paths:
+      - '.github/workflows/trace-ace-v121-local-bootstrap.yml'
+  workflow_dispatch:
+
+jobs:
+  bootstrap:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Download Python 3.13 wheelhouse
+        run: |
+          mkdir -p wheelhouse
+          python -m pip download --only-binary=:all: --dest wheelhouse fastembed==0.8.0
+      - name: Install FastEmbed and fetch exact Jina model
+        run: |
+          python -m pip install --no-index --find-links wheelhouse fastembed==0.8.0
+          python - <<'PY'
+          from fastembed import TextEmbedding
+          m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en')
+          print('model_ready', type(m).__name__)
+          PY
+          mkdir -p model_cache
+          for p in "$HOME/.cache/fastembed" "$HOME/.cache/huggingface"; do
+            if [ -d "$p" ]; then cp -a "$p" model_cache/; fi
+          done
+          find model_cache -maxdepth 4 -type f -printf '%p %s\n' | head -100
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-python313-wheelhouse
+          path: wheelhouse/
+          retention-days: 2
+          compression-level: 0
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-jina-model-cache
+          path: model_cache/
+          retention-days: 2
+          compression-level: 0

.github/workflows/trace-ace-v121-model-export.yml

@@ -0,0 +1,41 @@
+name: Trace Ace V121 Model Export
+
+on:
+  pull_request:
+    branches: [agent/trace-ace-mastery-events, agent/v111-runner]
+    paths:
+      - '.github/workflows/trace-ace-v121-model-export.yml'
+  workflow_dispatch:
+
+jobs:
+  export:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Install exact FastEmbed
+        run: python -m pip install --disable-pip-version-check fastembed==0.8.0
+      - name: Fetch and locate exact Jina model
+        run: |
+          python - <<'PY'
+          from fastembed import TextEmbedding
+          import tempfile, pathlib, json
+          cache = pathlib.Path('/tmp/v121_fastembed_cache')
+          m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en', cache_dir=str(cache))
+          print('MODEL_DICT', json.dumps({k:str(v) for k,v in m.__dict__.items()}, default=str, indent=2))
+          print('CACHE', cache)
+          PY
+          echo 'MODEL FILES:'
+          find /tmp/v121_fastembed_cache -type f -printf '%p %s\n' | sort
+          du -sh /tmp/v121_fastembed_cache
+      - uses: actions/upload-artifact@v4
+        with:
+          name: v121-jina-fastembed-cache
+          path: /tmp/v121_fastembed_cache/
+          retention-days: 2
+          compression-level: 0