Bump beta release to 0.3.1 #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AgentDojo benchmark | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| paths: | |
| - 'benchmarks/agentdojo/**' | |
| - 'core/**' | |
| - 'sdk/python/**' | |
| - '.github/workflows/agentdojo-benchmark.yml' | |
| workflow_dispatch: | |
| inputs: | |
| live: | |
| description: 'Run the opt-in live OpenAI/Azure path instead of only scripted deterministic mode' | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: read | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| scripted: | |
| name: Scripted AgentDojo smoke | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install benchmark dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r benchmarks/agentdojo/requirements.txt | |
| python -m pip install ./sdk/python | |
| - name: Smoke test harness import and gate | |
| run: PYTHONPATH=benchmarks/agentdojo pytest -q benchmarks/agentdojo/test_smoke.py | |
| - name: Run deterministic scripted benchmark | |
| env: | |
| ACS_AGENTDOJO_ENABLE: '1' | |
| PYTHONPATH: benchmarks/agentdojo | |
| run: | | |
| python -m acs_agentdojo_bench.runner \ | |
| --mode both \ | |
| --llm scripted \ | |
| --user-task user_task_0 \ | |
| --injection-task injection_task_0 \ | |
| --no-write | |
| live: | |
| name: Live AgentDojo benchmark | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.live }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install benchmark dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r benchmarks/agentdojo/requirements.txt | |
| python -m pip install ./sdk/python | |
| - name: Run live benchmark when credentials are present | |
| env: | |
| ACS_AGENTDOJO_ENABLE: '1' | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }} | |
| PYTHONPATH: benchmarks/agentdojo | |
| run: | | |
| python -m acs_agentdojo_bench.runner \ | |
| --mode both \ | |
| --user-task user_task_0 \ | |
| --injection-task injection_task_0 \ | |
| --no-write |