Skip to content

Commit b32344c

Browse files
heathsanchezheathsanchez
committed
Wire V76/V77 and private metadata transport into Actions
1 parent aedf0a7 commit b32344c

1 file changed

Lines changed: 61 additions & 81 deletions

File tree

‎.github/workflows/trace-ace-mastery.yml‎

Lines changed: 61 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
workflow_dispatch:
55
inputs:
66
run_full:
7-
description: "Run full experiment using public Google Drive dataset"
7+
description: "Run full experiment using public transcripts + private metadata bundle"
88
required: false
99
default: false
1010
type: boolean
@@ -42,14 +42,19 @@ jobs:
4242
run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test
4343
- name: Run canonical trajectory self-test
4444
run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test
45+
- name: Run unseen validation self-test
46+
run: python competitions/trace_the_ace/v76_unseen_validation.py --self-test
47+
- name: Run incremental mastery stack self-test
48+
run: python competitions/trace_the_ace/v77_incremental_mastery_stack.py --self-test
4549

4650
full-experiment:
4751
if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }}
4852
needs: self-test
4953
runs-on: ubuntu-latest
5054
timeout-minutes: 360
5155
env:
52-
TRACE_ACE_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI
56+
TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI
57+
TRACE_ACE_METADATA_URL: ${{ secrets.TRACE_ACE_METADATA_URL }}
5358
steps:
5459
- uses: actions/checkout@v4
5560
- uses: actions/setup-python@v5
@@ -58,115 +63,88 @@ jobs:
5863
cache: pip
5964
- name: Install experiment dependencies
6065
run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
61-
- name: Download public Drive dataset
66+
- name: Require private metadata transport
6267
shell: bash
6368
run: |
6469
set -euo pipefail
65-
mkdir -p /tmp/trace_ace
70+
if [ -z "${TRACE_ACE_METADATA_URL:-}" ]; then
71+
echo "TRACE_ACE_METADATA_URL repository secret is required for the competition feature/label bundle." >&2
72+
exit 2
73+
fi
74+
- name: Download public transcript archive
75+
shell: bash
76+
run: |
77+
set -euo pipefail
78+
mkdir -p /tmp/trace_ace/transcripts
6679
python - <<'PY'
6780
import os, gdown
68-
file_id = os.environ['TRACE_ACE_DRIVE_FILE_ID']
69-
out = '/tmp/trace_ace/dataset_download'
70-
url = f'https://drive.google.com/uc?id={file_id}'
71-
path = gdown.download(url, out, quiet=False)
81+
file_id = os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID']
82+
out = '/tmp/trace_ace/transcripts_download'
83+
path = gdown.download(f'https://drive.google.com/uc?id={file_id}', out, quiet=False)
7284
if not path:
73-
raise SystemExit('Google Drive download failed')
74-
print(f'downloaded to {path}')
85+
raise SystemExit('Google Drive transcript download failed')
7586
PY
76-
- name: Extract dataset archive
87+
unzip -q /tmp/trace_ace/transcripts_download -d /tmp/trace_ace/transcripts
88+
rm -f /tmp/trace_ace/transcripts_download
89+
- name: Download private feature/label bundle
7790
shell: bash
7891
run: |
7992
set -euo pipefail
80-
FILE=/tmp/trace_ace/dataset_download
81-
MIME=$(file -b --mime-type "$FILE")
82-
echo "download mime: $MIME"
83-
case "$MIME" in
84-
application/zip)
85-
mkdir -p /tmp/trace_ace/data && unzip -q "$FILE" -d /tmp/trace_ace/data ;;
86-
application/gzip|application/x-gzip)
87-
mkdir -p /tmp/trace_ace/data && tar -xzf "$FILE" -C /tmp/trace_ace/data ;;
88-
application/x-tar)
89-
mkdir -p /tmp/trace_ace/data && tar -xf "$FILE" -C /tmp/trace_ace/data ;;
90-
*)
91-
echo "Unsupported dataset archive MIME: $MIME" >&2
92-
exit 1 ;;
93-
esac
94-
rm -f "$FILE"
95-
- name: Locate inputs and run experiments
93+
mkdir -p /tmp/trace_ace/meta
94+
curl --fail --location --silent --show-error "$TRACE_ACE_METADATA_URL" -o /tmp/trace_ace/meta.zip
95+
unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta
96+
rm -f /tmp/trace_ace/meta.zip
97+
- name: Locate inputs by schema and run experiments
9698
shell: bash
9799
run: |
98100
set -euo pipefail
99101
python - <<'PY'
100102
import csv, shlex
101103
from pathlib import Path
102104
103-
root = Path('/tmp/trace_ace/data')
105+
roots = [Path('/tmp/trace_ace/meta'), Path('/tmp/trace_ace/transcripts')]
104106
features = labels = transcript_dir = None
105-
inspected = 0
106-
for path in root.rglob('*.csv'):
107-
try:
108-
with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f:
109-
header = next(csv.reader(f))
110-
except Exception:
111-
continue
112-
inspected += 1
113-
cols = set(header)
114-
if features is None and {'response_id', 'session_id', 'learning_objective'}.issubset(cols):
115-
features = path
116-
print('FEATURE HEADER', path, header)
117-
if labels is None and 'response_id' in cols and ({'is_correct'} <= cols or {'correct'} <= cols):
118-
labels = path
119-
print('LABEL HEADER', path, header)
120-
if transcript_dir is None and {'session_id', 'utterance_id', 'role', 'content', 'timestamp'}.issubset(cols):
121-
transcript_dir = path.parent
122-
print('TRANSCRIPT HEADER', path, header)
107+
for root in roots:
108+
for path in root.rglob('*.csv'):
109+
try:
110+
with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f:
111+
header = next(csv.reader(f))
112+
except Exception:
113+
continue
114+
cols = set(header)
115+
if features is None and {'response_id','session_id','learning_objective'}.issubset(cols):
116+
features = path
117+
print('FEATURE HEADER', header)
118+
if labels is None and 'response_id' in cols and ('is_correct' in cols or 'correct' in cols):
119+
labels = path
120+
print('LABEL HEADER', header)
121+
if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(cols):
122+
transcript_dir = path.parent
123+
print('TRANSCRIPT HEADER', header)
124+
if features and labels and transcript_dir:
125+
break
123126
if features and labels and transcript_dir:
124127
break
125128
if not (features and labels and transcript_dir):
126-
sample = [str(p) for p in list(root.rglob('*'))[:80]]
127-
raise SystemExit(f'Could not identify inputs by schema after {inspected} CSVs. Sample paths: {sample}')
128-
with open('/tmp/trace_ace/paths.env', 'w') as f:
129+
raise SystemExit('Could not identify all Trace the Ace inputs by schema')
130+
with open('/tmp/trace_ace/paths.env','w') as f:
129131
f.write('FEATURES=' + shlex.quote(str(features)) + '\n')
130132
f.write('LABELS=' + shlex.quote(str(labels)) + '\n')
131133
f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n')
132-
print('resolved features:', features)
133-
print('resolved labels:', labels)
134-
print('resolved transcripts:', transcript_dir)
135134
PY
136135
source /tmp/trace_ace/paths.env
137136
LIMIT="${{ inputs.limit }}"
138137
LIMIT="${LIMIT:-0}"
139138
EXTRA=()
140139
if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi
141-
python competitions/trace_the_ace/v71_mastery_events.py \
142-
--features "$FEATURES" \
143-
--labels "$LABELS" \
144-
--transcripts "$TRANSCRIPTS" \
145-
--out v71_mastery_results.json \
146-
"${EXTRA[@]}"
147-
python competitions/trace_the_ace/v72_supervision_audit.py \
148-
--features "$FEATURES" \
149-
--labels "$LABELS" \
150-
--transcripts "$TRANSCRIPTS" \
151-
--out v72_supervision_audit.json \
152-
"${EXTRA[@]}"
153-
python competitions/trace_the_ace/v73_contrastive_mastery.py \
154-
--features "$FEATURES" \
155-
--labels "$LABELS" \
156-
--transcripts "$TRANSCRIPTS" \
157-
--out v73_contrastive_mastery.json \
158-
"${EXTRA[@]}"
159-
python competitions/trace_the_ace/v74_semantic_objective_prior.py \
160-
--features "$FEATURES" \
161-
--labels "$LABELS" \
162-
--out v74_semantic_objective_prior.json \
163-
"${EXTRA[@]}"
164-
python competitions/trace_the_ace/v75_canonical_trajectory.py \
165-
--features "$FEATURES" \
166-
--labels "$LABELS" \
167-
--transcripts "$TRANSCRIPTS" \
168-
--out v75_canonical_trajectory.json \
169-
"${EXTRA[@]}"
140+
141+
python competitions/trace_the_ace/v71_mastery_events.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v71_mastery_results.json "${EXTRA[@]}"
142+
python competitions/trace_the_ace/v72_supervision_audit.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v72_supervision_audit.json "${EXTRA[@]}"
143+
python competitions/trace_the_ace/v73_contrastive_mastery.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v73_contrastive_mastery.json "${EXTRA[@]}"
144+
python competitions/trace_the_ace/v74_semantic_objective_prior.py --features "$FEATURES" --labels "$LABELS" --out v74_semantic_objective_prior.json "${EXTRA[@]}"
145+
python competitions/trace_the_ace/v75_canonical_trajectory.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v75_canonical_trajectory.json "${EXTRA[@]}"
146+
python competitions/trace_the_ace/v76_unseen_validation.py --features "$FEATURES" --labels "$LABELS" --out-protocol v76_validation_protocol.csv --out-summary v76_validation_summary.json
147+
python competitions/trace_the_ace/v77_incremental_mastery_stack.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v77_incremental_mastery_stack.json "${EXTRA[@]}"
170148
- name: Upload aggregate results only
171149
uses: actions/upload-artifact@v4
172150
with:
@@ -177,4 +155,6 @@ jobs:
177155
v73_contrastive_mastery.json
178156
v74_semantic_objective_prior.json
179157
v75_canonical_trajectory.json
158+
v76_validation_summary.json
159+
v77_incremental_mastery_stack.json
180160
retention-days: 14

0 commit comments

Comments
 (0)