Fix V75 short mathematical answer evidence [run-full] #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Trace the Ace mastery experiment | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| run_full: | |
| description: "Run full experiment using public Google Drive dataset" | |
| required: false | |
| default: false | |
| type: boolean | |
| limit: | |
| description: "Optional row limit (0 = all rows)" | |
| required: false | |
| default: "0" | |
| type: string | |
| push: | |
| branches: | |
| - agent/trace-ace-mastery-events | |
| paths: | |
| - "competitions/trace_the_ace/**" | |
| - ".github/workflows/trace-ace-mastery.yml" | |
| jobs: | |
| self-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| - name: Install experiment dependencies | |
| run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown | |
| - name: Run mastery extractor self-test | |
| run: python competitions/trace_the_ace/v71_mastery_events.py --self-test | |
| - name: Run supervision audit self-test | |
| run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test | |
| - name: Run contrastive mastery self-test | |
| run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test | |
| - name: Run semantic objective prior self-test | |
| run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test | |
| - name: Run canonical trajectory self-test | |
| run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test | |
| full-experiment: | |
| if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }} | |
| needs: self-test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| env: | |
| TRACE_ACE_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: pip | |
| - name: Install experiment dependencies | |
| run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown | |
| - name: Download public Drive dataset | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p /tmp/trace_ace | |
| python - <<'PY' | |
| import os, gdown | |
| file_id = os.environ['TRACE_ACE_DRIVE_FILE_ID'] | |
| out = '/tmp/trace_ace/dataset_download' | |
| url = f'https://drive.google.com/uc?id={file_id}' | |
| path = gdown.download(url, out, quiet=False, fuzzy=True) | |
| if not path: | |
| raise SystemExit('Google Drive download failed') | |
| print(f'downloaded to {path}') | |
| PY | |
| - name: Extract dataset archive | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| FILE=/tmp/trace_ace/dataset_download | |
| MIME=$(file -b --mime-type "$FILE") | |
| echo "download mime: $MIME" | |
| case "$MIME" in | |
| application/zip) | |
| mkdir -p /tmp/trace_ace/data && unzip -q "$FILE" -d /tmp/trace_ace/data ;; | |
| application/gzip|application/x-gzip) | |
| mkdir -p /tmp/trace_ace/data && tar -xzf "$FILE" -C /tmp/trace_ace/data ;; | |
| application/x-tar) | |
| mkdir -p /tmp/trace_ace/data && tar -xf "$FILE" -C /tmp/trace_ace/data ;; | |
| *) | |
| echo "Unsupported dataset archive MIME: $MIME" >&2 | |
| exit 1 ;; | |
| esac | |
| rm -f "$FILE" | |
| - name: Locate inputs and run experiments | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| FEATURES=$(find /tmp/trace_ace -type f -name 'train_features*.csv' | head -1) | |
| LABELS=$(find /tmp/trace_ace -type f -name 'train_labels*.csv' | head -1) | |
| TRANSCRIPTS=$(find /tmp/trace_ace -type d -name 'train_transcripts*' | head -1) | |
| test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" | |
| LIMIT="${{ inputs.limit }}" | |
| LIMIT="${LIMIT:-0}" | |
| EXTRA=() | |
| if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi | |
| python competitions/trace_the_ace/v71_mastery_events.py \ | |
| --features "$FEATURES" \ | |
| --labels "$LABELS" \ | |
| --transcripts "$TRANSCRIPTS" \ | |
| --out v71_mastery_results.json \ | |
| "${EXTRA[@]}" | |
| python competitions/trace_the_ace/v72_supervision_audit.py \ | |
| --features "$FEATURES" \ | |
| --labels "$LABELS" \ | |
| --transcripts "$TRANSCRIPTS" \ | |
| --out v72_supervision_audit.json \ | |
| "${EXTRA[@]}" | |
| python competitions/trace_the_ace/v73_contrastive_mastery.py \ | |
| --features "$FEATURES" \ | |
| --labels "$LABELS" \ | |
| --transcripts "$TRANSCRIPTS" \ | |
| --out v73_contrastive_mastery.json \ | |
| "${EXTRA[@]}" | |
| python competitions/trace_the_ace/v74_semantic_objective_prior.py \ | |
| --features "$FEATURES" \ | |
| --labels "$LABELS" \ | |
| --out v74_semantic_objective_prior.json \ | |
| "${EXTRA[@]}" | |
| python competitions/trace_the_ace/v75_canonical_trajectory.py \ | |
| --features "$FEATURES" \ | |
| --labels "$LABELS" \ | |
| --transcripts "$TRANSCRIPTS" \ | |
| --out v75_canonical_trajectory.json \ | |
| "${EXTRA[@]}" | |
| - name: Upload aggregate results only | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: trace-ace-aggregate-results | |
| path: | | |
| v71_mastery_results.json | |
| v72_supervision_audit.json | |
| v73_contrastive_mastery.json | |
| v74_semantic_objective_prior.json | |
| v75_canonical_trajectory.json | |
| retention-days: 14 |