From 56ca19142a9c34d5bc4018b677c1eb8a645ab7ff Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:12:41 +1200 Subject: [PATCH 01/20] infra: microbatch frozen V121 embeddings to fit runner memory --- .../trace_the_ace/v121_pretrained_semantic_residual.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py index 5a51b41d..323134d6 100644 --- a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py +++ b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py @@ -62,7 +62,11 @@ def build_semantic_text(objective: str, transcript_df) -> str: def embed(model: TextEmbedding, seq: list[str]) -> np.ndarray: - arr = np.vstack(list(model.embed(seq, batch_size=64))).astype(np.float32) + # Infrastructure-only repair after the first valid V121 execution reached + # ONNX and batch_size=64 requested a ~66.5 GB attention buffer. The model, + # texts, ordering and outputs are unchanged; only inference microbatching is + # reduced to fit the hosted runner. + arr = np.vstack(list(model.embed(seq, batch_size=2))).astype(np.float32) if not np.isfinite(arr).all(): raise RuntimeError("non-finite embedding") return arr From b7c181a039a7207d6744b6641f1517797e3b2612 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:12:59 +1200 Subject: [PATCH 02/20] infra: isolate V121 memory-safe rerun --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index a3fa5201..8b027003 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -2,7 +2,7 @@ name: Trace Ace V121 Pretrained Semantic Residual on: pull_request: - branches: [agent/trace-ace-mastery-events] + branches: [agent/trace-ace-mastery-events, agent/v111-runner] paths: - 'competitions/trace_the_ace/v121_pretrained_semantic_residual.py' - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' From b26d58dc67b1180fb9121ad23cd761523538ea3f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:33:08 +1200 Subject: [PATCH 03/20] infra: finish frozen V121 within hosted runner --- .../v121_pretrained_semantic_residual.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py index 323134d6..b537d627 100644 --- a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py +++ b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py @@ -62,11 +62,12 @@ def build_semantic_text(objective: str, transcript_df) -> str: def embed(model: TextEmbedding, seq: list[str]) -> np.ndarray: - # Infrastructure-only repair after the first valid V121 execution reached - # ONNX and batch_size=64 requested a ~66.5 GB attention buffer. The model, - # texts, ordering and outputs are unchanged; only inference microbatching is - # reduced to fit the hosted runner. - arr = np.vstack(list(model.embed(seq, batch_size=2))).astype(np.float32) + # Infrastructure-only repair. batch_size=64 requested a ~66.5 GB attention + # buffer; batch_size=2 fit memory but twice hit the hosted-runner wall-clock + # shutdown during the same frozen embedding. batch_size=4 preserves model, + # texts, ordering and outputs while fitting the measured memory envelope and + # reducing runtime enough to finish on the hosted runner. + arr = np.vstack(list(model.embed(seq, batch_size=4))).astype(np.float32) if not np.isfinite(arr).all(): raise RuntimeError("non-finite embedding") return arr From 16407d42992145284c9c8efc10fd72872d6467be Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:44:08 +1200 Subject: [PATCH 04/20] infra: stage frozen V121 across hosted runners --- .../trace_the_ace/v121_staged_transport.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 competitions/trace_the_ace/v121_staged_transport.py diff --git a/competitions/trace_the_ace/v121_staged_transport.py b/competitions/trace_the_ace/v121_staged_transport.py new file mode 100644 index 00000000..19e90910 --- /dev/null +++ b/competitions/trace_the_ace/v121_staged_transport.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Infrastructure-only staged transport for frozen V121. + +This module does not define new scientific features, models, folds, controls, or +gates. It serializes the exact V121 computation across short-lived hosted runner +jobs so preparation, embedding, and evaluation can complete independently. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +from fastembed import TextEmbedding +from scipy.sparse import load_npz, save_npz + +from v110_residual_collider_state_discovery import hb +from v121_pretrained_semantic_residual import ( + MODEL_NAME, + build_semantic_text, + embed, + eval_geometry, + stable_hash, + within_objective_shuffle, +) +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + + +def prepare(a): + out = Path(a.dir); out.mkdir(parents=True, exist_ok=True) + f = load_training(a.features, a.labels).reset_index(drop=True) + print('features columns', list(f.columns), flush=True) + obj0 = (f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + cand = np.where(np.array([hb(x, 5) != 0 for x in obj0]))[0] + ix = np.array(sorted(cand, key=lambda i: stable_hash(f.response_id.iloc[i]))[:a.rows]) + f = f.iloc[ix].reset_index(drop=True) + + y = f.target.to_numpy(int) + objectives = (f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support = f.learning_objective.astype(str).to_numpy() + sessions = f.session_id.astype(str).to_numpy() + + cache = {s: load_transcript(a.transcripts / f'{s}.csv') for s in np.unique(sessions)} + rt, rz, sem_text, obj_text = [], [], [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), 'related') + rt.append(t); rz.append(z) + obj_text.append(f'learning objective: {r.learning_objective}') + sem_text.append(build_semantic_text(str(r.learning_objective), d)) + if (i + 1) % 500 == 0: + print('prepared rows', i + 1, flush=True) + + save_npz(out / 'X75.npz', build_v75(f, cache)) + save_npz(out / 'Xr.npz', build_control(rt, rz)) + np.savez_compressed(out / 'arrays.npz', y=y, objectives=objectives, support=support, sessions=sessions) + (out / 'texts.json').write_text(json.dumps({'objective': obj_text, 'semantic': sem_text})) + manifest = { + 'protocol': 'V121_PRETRAINED_SEMANTIC_RESIDUAL', 'rows': int(len(f)), + 'objectives': int(len(np.unique(objectives))), 'sessions': int(len(np.unique(sessions))), + 'response_ids_sha256': __import__('hashlib').sha256('\n'.join(f.response_id.astype(str)).encode()).hexdigest(), + } + (out / 'manifest.json').write_text(json.dumps(manifest, indent=2)) + print(json.dumps(manifest, indent=2), flush=True) + + +def do_embed(a): + d = Path(a.dir) + texts = json.loads((d / 'texts.json').read_text()) + print('loading embedding model', MODEL_NAME, flush=True) + model = TextEmbedding(model_name=MODEL_NAME) + print('embedding objective control', flush=True) + E_obj = embed(model, texts['objective']) + print('embedding semantic intervention', flush=True) + E_sem = embed(model, texts['semantic']) + if E_obj.shape[0] != E_sem.shape[0]: raise RuntimeError('embedding row mismatch') + np.savez_compressed(Path(a.out), E_obj=E_obj, E_sem=E_sem) + print('embedding shapes', E_obj.shape, E_sem.shape, flush=True) + + +def evaluate(a): + d = Path(a.dir) + X75 = load_npz(d / 'X75.npz'); Xr = load_npz(d / 'Xr.npz') + z = np.load(d / 'arrays.npz', allow_pickle=False) + y=z['y']; objectives=z['objectives']; support=z['support']; sessions=z['sessions'] + e = np.load(a.embeddings, allow_pickle=False); E_obj=e['E_obj']; E_sem=e['E_sem'] + E_shuf = within_objective_shuffle(E_sem, objectives) + manifest=json.loads((d/'manifest.json').read_text()) + results = { + 'protocol': 'V121_PRETRAINED_SEMANTIC_RESIDUAL', 'model': MODEL_NAME, + 'rows': int(len(y)), 'objectives': int(len(np.unique(objectives))), + 'sessions': int(len(np.unique(sessions))), + 'transport_manifest': manifest, + 'precommit': {'semantic_gain_each_geometry': .003, + 'semantic_minus_shuffle_each_geometry': .002, + 'hard_collision_gain_each_geometry': '>0', + 'no_hyperparameter_sweep': True}, + } + results['objective_grouped'] = eval_geometry('objective_grouped', objectives, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf) + results['session_grouped'] = eval_geometry('session_grouped', sessions, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf) + def passes(r): + return r['semantic']['gain'] >= .003 and r['semantic_minus_shuffle_gain'] >= .002 and r['hard_collision'].get('semantic_gain', -1.) > 0 + ok_obj=passes(results['objective_grouped']); ok_sess=passes(results['session_grouped']) + if ok_obj and ok_sess: + verdict='PHASE_CHANGE_CANDIDATE'; nxt='Promote pretrained semantic residual to larger frozen validation and public-probe packaging.' + else: + verdict='NO_ROBUST_SEMANTIC_PHASE_CHANGE'; nxt='Treat remaining oracle gap as largely unidentifiable from supplied transcript/objective observables; pivot to validation geometry / assessment-process inference rather than more text feature search.' + results['decision']={'objective_grouped_pass': bool(ok_obj), 'session_grouped_pass': bool(ok_sess), 'verdict': verdict, 'next': nxt} + Path(a.out).write_text(json.dumps(results, indent=2)); print(json.dumps(results, indent=2), flush=True) + + +if __name__ == '__main__': + p=argparse.ArgumentParser(); sp=p.add_subparsers(dest='cmd', required=True) + q=sp.add_parser('prepare'); q.add_argument('--features',type=Path,required=True); q.add_argument('--labels',type=Path,required=True); q.add_argument('--transcripts',type=Path,required=True); q.add_argument('--rows',type=int,default=2500); q.add_argument('--dir',required=True) + q=sp.add_parser('embed'); q.add_argument('--dir',required=True); q.add_argument('--out',required=True) + q=sp.add_parser('evaluate'); q.add_argument('--dir',required=True); q.add_argument('--embeddings',required=True); q.add_argument('--out',required=True) + a=p.parse_args(); {'prepare':prepare,'embed':do_embed,'evaluate':evaluate}[a.cmd](a) From 1c292f8c77a42e805a44f959e5f5f5266d6bb069 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:44:29 +1200 Subject: [PATCH 05/20] infra: split frozen V121 across hosted runners --- ...-ace-v121-pretrained-semantic-residual.yml | 119 ++++++++++++------ 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 8b027003..f0e9158f 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -5,6 +5,7 @@ on: branches: [agent/trace-ace-mastery-events, agent/v111-runner] paths: - 'competitions/trace_the_ace/v121_pretrained_semantic_residual.py' + - 'competitions/trace_the_ace/v121_staged_transport.py' - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' workflow_dispatch: @@ -13,10 +14,14 @@ concurrency: group: trace-ace-frozen-transcripts-v1 cancel-in-progress: false +env: + TRANSCRIPT_KEY: trace-ace-transcripts-v1-603547640 + TRANSCRIPT_SHA: e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d + jobs: - semantic: + prepare: runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -24,28 +29,21 @@ jobs: python-version: '3.12' cache: pip - name: Install dependencies - run: | - python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown fastembed==0.8.0 - - - name: Restore frozen transcript archive from GitHub cache + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown fastembed==0.8.0 + - name: Restore frozen transcript archive uses: actions/cache/restore@v4 with: path: transcripts.zip - key: trace-ace-transcripts-v1-603547640 + key: ${{ env.TRANSCRIPT_KEY }} fail-on-cache-miss: true - - name: Validate frozen transcript archive shell: bash run: | set -euo pipefail - test -f transcripts.zip test "$(stat -c%s transcripts.zip)" = "603547640" unzip -tq transcripts.zip >/dev/null - ACTUAL=$(sha256sum transcripts.zip | cut -d' ' -f1) - test "$ACTUAL" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d" - echo "Frozen transcript cache verified: $ACTUAL" - - - name: Download frozen metadata and extract data + test "$(sha256sum transcripts.zip | cut -d' ' -f1)" = "$TRANSCRIPT_SHA" + - name: Download frozen metadata and extract shell: bash run: | set -euo pipefail @@ -53,41 +51,80 @@ jobs: mkdir -p data/meta data/transcripts unzip -q metadata.zip -d data/meta unzip -q transcripts.zip -d data/transcripts - FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) - LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) - test -n "$FEATURES" - test -n "$LABELS" - test -n "$FIRST" - TRANSCRIPTS=$(dirname "$FIRST") - echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" - echo "LABELS=$LABELS" >> "$GITHUB_ENV" - echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV" + - name: Prepare exact frozen V121 inputs + run: | + python -m py_compile competitions/trace_the_ace/v121_pretrained_semantic_residual.py competitions/trace_the_ace/v121_staged_transport.py + cd competitions/trace_the_ace + python v121_staged_transport.py prepare \ + --features "../../$FEATURES" --labels "../../$LABELS" \ + --transcripts "../../$TRANSCRIPTS" --rows 2500 --dir ../../v121_prepared + - uses: actions/upload-artifact@v4 + with: + name: v121-prepared + path: v121_prepared/ + retention-days: 2 + compression-level: 6 - - name: Preflight + embed: + needs: prepare + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + - name: Embed exact frozen V121 texts run: | - python -m py_compile competitions/trace_the_ace/v121_pretrained_semantic_residual.py - python - <<'PY' - import os, pandas as pd - print('features columns', list(pd.read_csv(os.environ['FEATURES'], nrows=0).columns)) - print('labels columns', list(pd.read_csv(os.environ['LABELS'], nrows=0).columns)) - PY - - name: Run V121 + cd competitions/trace_the_ace + python v121_staged_transport.py embed --dir ../../v121_prepared --out ../../v121_embeddings.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings + path: v121_embeddings.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + - uses: actions/download-artifact@v4 + with: + name: v121-embeddings + path: . + - name: Evaluate frozen V121 precommit run: | cd competitions/trace_the_ace - python v121_pretrained_semantic_residual.py \ - --features "../../$FEATURES" \ - --labels "../../$LABELS" \ - --transcripts "../../$TRANSCRIPTS" \ - --rows 2500 \ - --out ../../v121_pretrained_semantic_residual.json + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embeddings.npz --out ../../v121_pretrained_semantic_residual.json - name: Show decision - if: always() - run: test -f v121_pretrained_semantic_residual.json && cat v121_pretrained_semantic_residual.json || true + run: cat v121_pretrained_semantic_residual.json - uses: actions/upload-artifact@v4 - if: always() with: name: trace-ace-v121-pretrained-semantic-residual path: v121_pretrained_semantic_residual.json retention-days: 14 - if-no-files-found: warn From 3263beb3cce7c18b7519a49e529a381cfe246b98 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:47:32 +1200 Subject: [PATCH 06/20] experiment: freeze V125 nested calibration residual --- .../trace_the_ace/v125_nested_calibration.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 competitions/trace_the_ace/v125_nested_calibration.py diff --git a/competitions/trace_the_ace/v125_nested_calibration.py b/competitions/trace_the_ace/v125_nested_calibration.py new file mode 100644 index 00000000..b08d6008 --- /dev/null +++ b/competitions/trace_the_ace/v125_nested_calibration.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""V125: nested calibration residual over frozen V97. + +Question: is V97 leaving lawful log-loss improvement in probability calibration, +without adding new information or exploiting a particular validation geometry? + +Frozen protocol: +- deterministic 2500-row response-id sample; +- exact V97 endpoint (V75 when objective supported; .65 V75 + .35 RELATED when unsupported); +- 4-fold outer objective-grouped and session-grouped OOF; +- calibration parameters fit only to inner-OOF V97 predictions inside each outer training fold; +- intervention = one global Platt map sigmoid(a + b*logit(p97)); +- control = same map fit after deterministic shuffle of inner-OOF probabilities; +- no hyperparameter sweep. + +Promote only if calibration gains >= .001 log loss in BOTH geometries and beats +the shuffled calibration by >= .001 in BOTH. Otherwise retain as a negative law. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS=1e-5 + +def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS); return np.log(p/(1-p)) + +def endpoint(X75,Xr,y,key,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + p75=m.predict_proba(X75[va])[:,1] + r=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + pr=r.predict_proba(Xr[va])[:,1] + vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)) + seen=np.array([d.get(x,0)>0 for x in key[va]]) + return np.clip(np.where(seen,p75,.65*p75+.35*pr),EPS,1-EPS) + +def fit_cal(p,y): + m=LogisticRegression(C=1000.,max_iter=300,solver='liblinear',random_state=SEED).fit(logit(p)[:,None],y) + return m + +def geometry(name,groups,X75,Xr,y,key): + outer=list(GroupKFold(4).split(np.zeros(len(y)),y,groups)) + pb=np.zeros(len(y)); pc=np.zeros(len(y)); ps=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(outer): + inner_groups=groups[tr] + inn=list(GroupKFold(min(4,len(np.unique(inner_groups)))).split(np.zeros(len(tr)),y[tr],inner_groups)) + pi=np.zeros(len(tr)) + for itr,iva in inn: + pi[iva]=endpoint(X75,Xr,y,key,tr[itr],tr[iva]) + cal=fit_cal(pi,y[tr]) + rng=np.random.default_rng(SEED+125+k) + sh=fit_cal(pi[rng.permutation(len(pi))],y[tr]) + raw=endpoint(X75,Xr,y,key,tr,va) + q=cal.predict_proba(logit(raw)[:,None])[:,1] + qs=sh.predict_proba(logit(raw)[:,None])[:,1] + pb[va]=raw;pc[va]=q;ps[va]=qs + folds.append({'fold':k+1,'rows':int(len(va)),'baseline':ll(y[va],raw),'calibrated':ll(y[va],q), + 'gain':ll(y[va],raw)-ll(y[va],q),'slope':float(cal.coef_[0,0]), + 'intercept':float(cal.intercept_[0])}) + base=ll(y,pb); cal=ll(y,pc); shuf=ll(y,ps) + return {'geometry':name,'baseline_v97_ll':base,'calibrated_ll':cal,'gain':base-cal, + 'shuffled_calibration_ll':shuf,'calibration_minus_shuffle_gain':shuf-cal,'folds':folds} + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f.columns),flush=True) + ix=sorted(range(len(f)),key=lambda i:hh(f.response_id.iloc[i]))[:a.rows] + f=f.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); key=f.learning_objective.astype(str).to_numpy() + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z) + if (i+1)%500==0: print('prepared rows',i+1,flush=True) + X75=build_v75(f,cache);Xr=build_control(rt,rz) + ro=geometry('objective_grouped',obj,X75,Xr,y,key);rs=geometry('session_grouped',sess,X75,Xr,y,key) + def ok(r): return r['gain']>=.001 and r['calibration_minus_shuffle_gain']>=.001 + verdict='PROMOTE_CALIBRATION_LAW' if ok(ro) and ok(rs) else 'KEEP_V97_CALIBRATION' + out={'protocol':'V125_NESTED_CALIBRATION','rows':len(f),'precommit':{'gain_each_geometry':.001,'margin_vs_shuffle_each':.001,'no_sweep':True}, + 'objective_grouped':ro,'session_grouped':rs,'decision':{'objective_pass':ok(ro),'session_pass':ok(rs),'verdict':verdict}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v125_nested_calibration.json');run(p.parse_args()) From b24f2a48a2c38223821a44eaf25c0e3badfff14f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:12:37 +1200 Subject: [PATCH 07/20] infra: shard frozen V121 embedding transport --- .../trace_the_ace/v121_staged_transport.py | 65 ++++++++++++++++--- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/competitions/trace_the_ace/v121_staged_transport.py b/competitions/trace_the_ace/v121_staged_transport.py index 19e90910..0e5f1ee5 100644 --- a/competitions/trace_the_ace/v121_staged_transport.py +++ b/competitions/trace_the_ace/v121_staged_transport.py @@ -71,15 +71,57 @@ def prepare(a): def do_embed(a): d = Path(a.dir) texts = json.loads((d / 'texts.json').read_text()) - print('loading embedding model', MODEL_NAME, flush=True) + n = len(texts['semantic']) + shards = int(a.shards) + shard = int(a.shard) + if shards < 1 or shard < 0 or shard >= shards: + raise ValueError(f'invalid shard {shard}/{shards}') + start = (n * shard) // shards + end = (n * (shard + 1)) // shards + obj_text = texts['objective'][start:end] + sem_text = texts['semantic'][start:end] + print('loading embedding model', MODEL_NAME, 'shard', shard, 'rows', start, end, flush=True) model = TextEmbedding(model_name=MODEL_NAME) - print('embedding objective control', flush=True) - E_obj = embed(model, texts['objective']) - print('embedding semantic intervention', flush=True) - E_sem = embed(model, texts['semantic']) - if E_obj.shape[0] != E_sem.shape[0]: raise RuntimeError('embedding row mismatch') - np.savez_compressed(Path(a.out), E_obj=E_obj, E_sem=E_sem) - print('embedding shapes', E_obj.shape, E_sem.shape, flush=True) + print('embedding objective control shard', shard, flush=True) + E_obj = embed(model, obj_text) + print('embedding semantic intervention shard', shard, flush=True) + E_sem = embed(model, sem_text) + if E_obj.shape[0] != E_sem.shape[0] or E_obj.shape[0] != end - start: + raise RuntimeError('embedding row mismatch') + np.savez_compressed(Path(a.out), E_obj=E_obj, E_sem=E_sem, + start=np.array(start), end=np.array(end), total=np.array(n), + shard=np.array(shard), shards=np.array(shards)) + print('embedding shard complete', shard, start, end, E_obj.shape, E_sem.shape, flush=True) + + +def load_embeddings(path: Path): + if path.is_file(): + e = np.load(path, allow_pickle=False) + return e['E_obj'], e['E_sem'] + files = sorted(path.glob('**/v121_embeddings_shard_*.npz')) + if not files: + files = sorted(path.glob('**/*.npz')) + parts = [] + for f in files: + e = np.load(f, allow_pickle=False) + if 'start' not in e.files or 'end' not in e.files: + continue + parts.append((int(e['start']), int(e['end']), int(e['total']), e['E_obj'], e['E_sem'], f)) + if not parts: + raise RuntimeError(f'no embedding shards found under {path}') + parts.sort(key=lambda x: x[0]) + total = parts[0][2] + cursor = 0 + obj, sem = [], [] + for start, end, t, eo, es, f in parts: + if t != total or start != cursor or end - start != eo.shape[0] or eo.shape[0] != es.shape[0]: + raise RuntimeError(f'invalid embedding shard coverage at {f}: {start}:{end}, cursor={cursor}, total={t}') + obj.append(eo); sem.append(es); cursor = end + if cursor != total: + raise RuntimeError(f'incomplete embedding coverage: {cursor}/{total}') + E_obj = np.vstack(obj); E_sem = np.vstack(sem) + print('merged embedding shards', len(parts), E_obj.shape, E_sem.shape, flush=True) + return E_obj, E_sem def evaluate(a): @@ -87,7 +129,9 @@ def evaluate(a): X75 = load_npz(d / 'X75.npz'); Xr = load_npz(d / 'Xr.npz') z = np.load(d / 'arrays.npz', allow_pickle=False) y=z['y']; objectives=z['objectives']; support=z['support']; sessions=z['sessions'] - e = np.load(a.embeddings, allow_pickle=False); E_obj=e['E_obj']; E_sem=e['E_sem'] + E_obj, E_sem = load_embeddings(Path(a.embeddings)) + if len(y) != E_obj.shape[0] or len(y) != E_sem.shape[0]: + raise RuntimeError('evaluation embedding row mismatch') E_shuf = within_objective_shuffle(E_sem, objectives) manifest=json.loads((d/'manifest.json').read_text()) results = { @@ -95,6 +139,7 @@ def evaluate(a): 'rows': int(len(y)), 'objectives': int(len(np.unique(objectives))), 'sessions': int(len(np.unique(sessions))), 'transport_manifest': manifest, + 'transport': {'embedding_shards_merged': True}, 'precommit': {'semantic_gain_each_geometry': .003, 'semantic_minus_shuffle_each_geometry': .002, 'hard_collision_gain_each_geometry': '>0', @@ -116,6 +161,6 @@ def passes(r): if __name__ == '__main__': p=argparse.ArgumentParser(); sp=p.add_subparsers(dest='cmd', required=True) q=sp.add_parser('prepare'); q.add_argument('--features',type=Path,required=True); q.add_argument('--labels',type=Path,required=True); q.add_argument('--transcripts',type=Path,required=True); q.add_argument('--rows',type=int,default=2500); q.add_argument('--dir',required=True) - q=sp.add_parser('embed'); q.add_argument('--dir',required=True); q.add_argument('--out',required=True) + q=sp.add_parser('embed'); q.add_argument('--dir',required=True); q.add_argument('--out',required=True); q.add_argument('--shard',type=int,default=0); q.add_argument('--shards',type=int,default=1) q=sp.add_parser('evaluate'); q.add_argument('--dir',required=True); q.add_argument('--embeddings',required=True); q.add_argument('--out',required=True) a=p.parse_args(); {'prepare':prepare,'embed':do_embed,'evaluate':evaluate}[a.cmd](a) From e62133e2cf5f83ffd145eeb40f3d35630a501d6a Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:12:53 +1200 Subject: [PATCH 08/20] infra: shard frozen V121 embedding jobs --- ...-ace-v121-pretrained-semantic-residual.yml | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index f0e9158f..f9c5b772 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -71,6 +71,10 @@ jobs: embed: needs: prepare + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] runs-on: ubuntu-24.04 timeout-minutes: 20 steps: @@ -85,14 +89,16 @@ jobs: with: name: v121-prepared path: v121_prepared - - name: Embed exact frozen V121 texts + - name: Embed exact frozen V121 texts shard ${{ matrix.shard }} run: | cd competitions/trace_the_ace - python v121_staged_transport.py embed --dir ../../v121_prepared --out ../../v121_embeddings.npz + python v121_staged_transport.py embed --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 4 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz - uses: actions/upload-artifact@v4 with: - name: v121-embeddings - path: v121_embeddings.npz + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz retention-days: 2 compression-level: 0 @@ -112,15 +118,17 @@ jobs: with: name: v121-prepared path: v121_prepared - - uses: actions/download-artifact@v4 + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 with: - name: v121-embeddings - path: . + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false - name: Evaluate frozen V121 precommit run: | cd competitions/trace_the_ace python v121_staged_transport.py evaluate --dir ../../v121_prepared \ - --embeddings ../../v121_embeddings.npz --out ../../v121_pretrained_semantic_residual.json + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json - name: Show decision run: cat v121_pretrained_semantic_residual.json - uses: actions/upload-artifact@v4 From c299c3f34fb9bbbcddf7895b5f4123b69699aadd Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:13:56 +1200 Subject: [PATCH 09/20] infra: cancel obsolete V121 transports --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index f9c5b772..9f38f204 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -12,7 +12,7 @@ on: # Infrastructure-only serialization. Scientific protocol is unchanged. concurrency: group: trace-ace-frozen-transcripts-v1 - cancel-in-progress: false + cancel-in-progress: true env: TRANSCRIPT_KEY: trace-ace-transcripts-v1-603547640 From c4da5dda0e2d187e87493b0cb63a5e31cf36525e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:22:26 +1200 Subject: [PATCH 10/20] infra: split frozen V121 into eight embedding shards --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 9f38f204..91478a62 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3] + shard: [0, 1, 2, 3, 4, 5, 6, 7] runs-on: ubuntu-24.04 timeout-minutes: 20 steps: @@ -93,7 +93,7 @@ jobs: run: | cd competitions/trace_the_ace python v121_staged_transport.py embed --dir ../../v121_prepared \ - --shard ${{ matrix.shard }} --shards 4 \ + --shard ${{ matrix.shard }} --shards 8 \ --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz - uses: actions/upload-artifact@v4 with: From f34b01f02bd83c20b1f9cbcb6f38ad8479688f5d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:32:18 +1200 Subject: [PATCH 11/20] infra: serialize V121 embedding shards --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 91478a62..8c78f74a 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -73,6 +73,7 @@ jobs: needs: prepare strategy: fail-fast: false + max-parallel: 1 matrix: shard: [0, 1, 2, 3, 4, 5, 6, 7] runs-on: ubuntu-24.04 From 7d6759bf397ce91b924ba14a5fc50bdaaf8a9c23 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:54:44 +1200 Subject: [PATCH 12/20] infra: fit frozen V121 shards within runner lifetime --- .../trace-ace-v121-pretrained-semantic-residual.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 8c78f74a..ce72bbf9 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -12,7 +12,7 @@ on: # Infrastructure-only serialization. Scientific protocol is unchanged. concurrency: group: trace-ace-frozen-transcripts-v1 - cancel-in-progress: true + cancel-in-progress: false env: TRANSCRIPT_KEY: trace-ace-transcripts-v1-603547640 @@ -75,7 +75,7 @@ jobs: fail-fast: false max-parallel: 1 matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] runs-on: ubuntu-24.04 timeout-minutes: 20 steps: @@ -94,7 +94,7 @@ jobs: run: | cd competitions/trace_the_ace python v121_staged_transport.py embed --dir ../../v121_prepared \ - --shard ${{ matrix.shard }} --shards 8 \ + --shard ${{ matrix.shard }} --shards 16 \ --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz - uses: actions/upload-artifact@v4 with: From d398685f939e800ccda490743c04be83f1f65c27 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:55:20 +1200 Subject: [PATCH 13/20] infra: isolate frozen V121 16-shard execution --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index ce72bbf9..435f9afd 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -9,9 +9,9 @@ on: - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' workflow_dispatch: -# Infrastructure-only serialization. Scientific protocol is unchanged. +# Infrastructure-only serialization for the frozen 16-shard V121 transport. concurrency: - group: trace-ace-frozen-transcripts-v1 + group: trace-ace-v121-frozen-16shard cancel-in-progress: false env: From 4e85c48246f6373d0fcb6d12b6485e60bce3183e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:01:15 +1200 Subject: [PATCH 14/20] infra: parallelize frozen V121 short shards --- .../workflows/trace-ace-v121-pretrained-semantic-residual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 435f9afd..3e43b290 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -11,7 +11,7 @@ on: # Infrastructure-only serialization for the frozen 16-shard V121 transport. concurrency: - group: trace-ace-v121-frozen-16shard + group: trace-ace-v121-frozen-16shard-p4 cancel-in-progress: false env: @@ -73,7 +73,7 @@ jobs: needs: prepare strategy: fail-fast: false - max-parallel: 1 + max-parallel: 4 matrix: shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] runs-on: ubuntu-24.04 From 5b3cc70b720a2b3118d07a259599dce6d1c49e56 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:06:09 +1200 Subject: [PATCH 15/20] infra: run frozen V121 from prepared artifact --- .../workflows/trace-ace-v121-embed-only.yml | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-embed-only.yml diff --git a/.github/workflows/trace-ace-v121-embed-only.yml b/.github/workflows/trace-ace-v121-embed-only.yml new file mode 100644 index 00000000..84cc9f75 --- /dev/null +++ b/.github/workflows/trace-ace-v121-embed-only.yml @@ -0,0 +1,104 @@ +name: Trace Ace V121 Frozen Embed Only + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-embed-only.yml' + workflow_dispatch: + +# Reuses the already-successful exact frozen V121 prepared artifact from run 32400309220. +# Scientific model/text/sample/folds/gates are unchanged. +concurrency: + group: trace-ace-v121-embed-only-16-p4 + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + embed: + strategy: + fail-fast: false + max-parallel: 4 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen preparation manifest exists + run: | + test -f v121_prepared/manifest.json + cat v121_prepared/manifest.json + - name: Embed exact frozen V121 texts shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py embed --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 16 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false + - name: Evaluate frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 From 997d4052b4e1b6844d85cba44135e44ecca28e2f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:11:09 +1200 Subject: [PATCH 16/20] infra: bootstrap local frozen V121 embedding --- .../trace-ace-v121-local-bootstrap.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-local-bootstrap.yml diff --git a/.github/workflows/trace-ace-v121-local-bootstrap.yml b/.github/workflows/trace-ace-v121-local-bootstrap.yml new file mode 100644 index 00000000..f18b81de --- /dev/null +++ b/.github/workflows/trace-ace-v121-local-bootstrap.yml @@ -0,0 +1,49 @@ +name: Trace Ace V121 Local Bootstrap + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-local-bootstrap.yml' + workflow_dispatch: + +jobs: + bootstrap: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Download Python 3.13 wheelhouse + run: | + mkdir -p wheelhouse + python -m pip download --only-binary=:all: --dest wheelhouse fastembed==0.8.0 + - name: Install FastEmbed and fetch exact Jina model + run: | + python -m pip install --no-index --find-links wheelhouse fastembed==0.8.0 + python - <<'PY' + from fastembed import TextEmbedding + m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en') + print('model_ready', type(m).__name__) + PY + mkdir -p model_cache + for p in "$HOME/.cache/fastembed" "$HOME/.cache/huggingface"; do + if [ -d "$p" ]; then cp -a "$p" model_cache/; fi + done + find model_cache -maxdepth 4 -type f -printf '%p %s\n' | head -100 + - uses: actions/upload-artifact@v4 + with: + name: v121-python313-wheelhouse + path: wheelhouse/ + retention-days: 2 + compression-level: 0 + - uses: actions/upload-artifact@v4 + with: + name: v121-jina-model-cache + path: model_cache/ + retention-days: 2 + compression-level: 0 From 4f08ddc0af044838e63ac1004e9289ea6b6fb9bf Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:13:53 +1200 Subject: [PATCH 17/20] infra: export exact frozen V121 Jina model cache --- .../workflows/trace-ace-v121-model-export.yml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-model-export.yml diff --git a/.github/workflows/trace-ace-v121-model-export.yml b/.github/workflows/trace-ace-v121-model-export.yml new file mode 100644 index 00000000..b8c0e63c --- /dev/null +++ b/.github/workflows/trace-ace-v121-model-export.yml @@ -0,0 +1,41 @@ +name: Trace Ace V121 Model Export + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-model-export.yml' + workflow_dispatch: + +jobs: + export: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install exact FastEmbed + run: python -m pip install --disable-pip-version-check fastembed==0.8.0 + - name: Fetch and locate exact Jina model + run: | + python - <<'PY' + from fastembed import TextEmbedding + import tempfile, pathlib, json + cache = pathlib.Path('/tmp/v121_fastembed_cache') + m = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en', cache_dir=str(cache)) + print('MODEL_DICT', json.dumps({k:str(v) for k,v in m.__dict__.items()}, default=str, indent=2)) + print('CACHE', cache) + PY + echo 'MODEL FILES:' + find /tmp/v121_fastembed_cache -type f -printf '%p %s\n' | sort + du -sh /tmp/v121_fastembed_cache + - uses: actions/upload-artifact@v4 + with: + name: v121-jina-fastembed-cache + path: /tmp/v121_fastembed_cache/ + retention-days: 2 + compression-level: 0 From 4d2da1458c95cd2e06dcaa5add30730541c58280 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:19:52 +1200 Subject: [PATCH 18/20] infra: add batch1 frozen V121 shard transport --- .../v121_embed_batch1_transport.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 competitions/trace_the_ace/v121_embed_batch1_transport.py diff --git a/competitions/trace_the_ace/v121_embed_batch1_transport.py b/competitions/trace_the_ace/v121_embed_batch1_transport.py new file mode 100644 index 00000000..08f96175 --- /dev/null +++ b/competitions/trace_the_ace/v121_embed_batch1_transport.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Infrastructure-only memory-safe embedding transport for frozen V121. + +No scientific representation, model, sample, ordering, folds, controls, or gates +are changed. Each text is embedded independently with the same frozen Jina model; +batch_size=1 only lowers peak ONNX attention memory. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from fastembed import TextEmbedding + +MODEL_NAME = "jinaai/jina-embeddings-v2-small-en" + +def embed1(model, seq): + arr = np.vstack(list(model.embed(seq, batch_size=1))).astype(np.float32) + if not np.isfinite(arr).all(): + raise RuntimeError("non-finite embedding") + return arr + +def main(a): + d=Path(a.dir) + texts=json.loads((d/'texts.json').read_text()) + n=len(texts['semantic']); shard=int(a.shard); shards=int(a.shards) + if shards < 1 or not (0 <= shard < shards): raise ValueError(f'invalid shard {shard}/{shards}') + start=(n*shard)//shards; end=(n*(shard+1))//shards + obj=texts['objective'][start:end]; sem=texts['semantic'][start:end] + print('frozen shard',shard,'of',shards,'rows',start,end,flush=True) + model=TextEmbedding(model_name=MODEL_NAME, threads=4) + E_obj=embed1(model,obj) + E_sem=embed1(model,sem) + if E_obj.shape[0] != end-start or E_sem.shape[0] != end-start: raise RuntimeError('row mismatch') + np.savez_compressed(Path(a.out),E_obj=E_obj,E_sem=E_sem,start=np.array(start),end=np.array(end),total=np.array(n),shard=np.array(shard),shards=np.array(shards)) + print('complete',shard,E_obj.shape,E_sem.shape,flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--dir',required=True); p.add_argument('--out',required=True); p.add_argument('--shard',type=int,required=True); p.add_argument('--shards',type=int,required=True); main(p.parse_args()) From 57a25a3f76070407c996df0c77ad7fc2004c2e73 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:20:45 +1200 Subject: [PATCH 19/20] infra: execute frozen V121 in 128 batch1 shards --- .github/workflows/trace-ace-v121-128shard.yml | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-128shard.yml diff --git a/.github/workflows/trace-ace-v121-128shard.yml b/.github/workflows/trace-ace-v121-128shard.yml new file mode 100644 index 00000000..527dadd6 --- /dev/null +++ b/.github/workflows/trace-ace-v121-128shard.yml @@ -0,0 +1,102 @@ +name: Trace Ace V121 Frozen 128 Shard + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-128shard.yml' + workflow_dispatch: + +concurrency: + group: trace-ace-v121-frozen-128shard + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + +jobs: + embed: + strategy: + fail-fast: false + max-parallel: 16 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install exact embedding dependencies + run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen manifest + run: | + test -f v121_prepared/manifest.json + grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Embed exact frozen shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_embed_batch1_transport.py --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 128 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate: + needs: embed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install evaluation dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download all exact embedding shards + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards + merge-multiple: false + - name: Evaluate unchanged frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 From 91b232f4b8831ad07ec15f2cd22b13722b1c1589 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:33:52 +1200 Subject: [PATCH 20/20] infra: recover frozen V121 tail shards only --- .../trace-ace-v121-tail-recovery.yml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-tail-recovery.yml diff --git a/.github/workflows/trace-ace-v121-tail-recovery.yml b/.github/workflows/trace-ace-v121-tail-recovery.yml new file mode 100644 index 00000000..f50b1b46 --- /dev/null +++ b/.github/workflows/trace-ace-v121-tail-recovery.yml @@ -0,0 +1,111 @@ +name: Trace Ace V121 Tail Recovery + +on: + pull_request: + branches: [agent/trace-ace-mastery-events, agent/v111-runner] + paths: + - '.github/workflows/trace-ace-v121-tail-recovery.yml' + workflow_dispatch: + +concurrency: + group: trace-ace-v121-tail-recovery + cancel-in-progress: false + +env: + PREPARED_RUN_ID: '32400309220' + ORIGINAL_RUN_ID: '32402681183' + +jobs: + embed_tail: + strategy: + fail-fast: false + max-parallel: 16 + matrix: + shard: [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install exact embedding dependencies + run: python -m pip install --disable-pip-version-check numpy fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Verify frozen manifest + run: | + test -f v121_prepared/manifest.json + grep -q 'b1612f9fe4558680e468afb2a2452b75c603c244934fe62f7345feee68a61bc1' v121_prepared/manifest.json + - name: Embed exact frozen shard ${{ matrix.shard }} + run: | + cd competitions/trace_the_ace + python v121_embed_batch1_transport.py --dir ../../v121_prepared \ + --shard ${{ matrix.shard }} --shards 128 \ + --out ../../v121_embeddings_shard_${{ matrix.shard }}.npz + - uses: actions/upload-artifact@v4 + with: + name: v121-embeddings-${{ matrix.shard }} + path: v121_embeddings_shard_${{ matrix.shard }}.npz + retention-days: 2 + compression-level: 0 + + evaluate_union: + needs: embed_tail + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install evaluation dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn fastembed==0.8.0 + - name: Download exact frozen V121 prepared artifact + uses: actions/download-artifact@v4 + with: + name: v121-prepared + path: v121_prepared + repository: heathsanchez/mathgraph + run-id: ${{ env.PREPARED_RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Download successful original shards 0 through 64 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p v121_embedding_shards/original + for i in $(seq 0 64); do + gh run download "$ORIGINAL_RUN_ID" -n "v121-embeddings-$i" -D "v121_embedding_shards/original/$i" + done + - name: Download recovered tail shards 65 through 127 + uses: actions/download-artifact@v4 + with: + pattern: v121-embeddings-* + path: v121_embedding_shards/recovered + merge-multiple: false + - name: Evaluate unchanged frozen V121 precommit + run: | + cd competitions/trace_the_ace + python v121_staged_transport.py evaluate --dir ../../v121_prepared \ + --embeddings ../../v121_embedding_shards --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14