diff --git a/.github/workflows/tutorial-evaluation-scheduled.yml b/.github/workflows/tutorial-evaluation-scheduled.yml index de4b148..f3bef60 100644 --- a/.github/workflows/tutorial-evaluation-scheduled.yml +++ b/.github/workflows/tutorial-evaluation-scheduled.yml @@ -34,7 +34,7 @@ jobs: learning_path: tutorial/absence-of-change - tutorial: risky-containers devcontainer: .devcontainer/risky-containers/devcontainer.json - docs_path: docs/content/tutorials/risky-containers + docs_path: docs/content/drasi-kubernetes/tutorials/risky-containers learning_path: tutorial/risky-containers steps: @@ -83,8 +83,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 @@ -126,8 +125,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 @@ -168,8 +166,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 diff --git a/.github/workflows/tutorial-evaluation.yml b/.github/workflows/tutorial-evaluation.yml index 4584848..63f8dd8 100644 --- a/.github/workflows/tutorial-evaluation.yml +++ b/.github/workflows/tutorial-evaluation.yml @@ -18,8 +18,10 @@ on: description: 'Branch from docs repo to use' default: 'main' model: - description: 'AI model to use for evaluation' - default: 'claude-opus-4.7' + description: 'Optional model override (leave blank for Copilot CLI default)' + type: string + required: false + default: '' jobs: setup: @@ -65,7 +67,7 @@ jobs: ;; risky-containers) echo "devcontainer=.devcontainer/risky-containers/devcontainer.json" >> $GITHUB_OUTPUT - echo "docs_path=docs/content/tutorials/risky-containers" >> $GITHUB_OUTPUT + echo "docs_path=docs/content/drasi-kubernetes/tutorials/risky-containers" >> $GITHUB_OUTPUT echo "learning_path=tutorial/risky-containers" >> $GITHUB_OUTPUT ;; *) @@ -102,10 +104,17 @@ jobs: env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_PAT_TOKEN }} DRASI_TUTORIAL_EVALUATION: "true" + EVALUATION_MODEL: ${{ inputs.model }} with: configFile: ${{ needs.setup.outputs.devcontainer }} push: never + env: | + EVALUATION_MODEL runCmd: | + set -- + if [ -n "$EVALUATION_MODEL" ]; then + set -- --model "$EVALUATION_MODEL" + fi timeout 1200 copilot -p "$(cat prompt.md)" \ --allow-all-tools \ --allow-all-paths \ @@ -123,7 +132,7 @@ jobs: --deny-tool 'shell(dd *)' \ --allow-url localhost \ --allow-url 127.0.0.1 \ - --model ${{ inputs.model }} + "$@" - name: Check evaluation status if: always() diff --git a/README.md b/README.md index b56066f..3738283 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,14 @@ This repo contains code for a tutorial and a few sample applications that use Dr [The step-by-step tutorial linked here](https://drasi.io/getting-started/) will help you get Drasi up and running quickly and show you how easy it is to create Sources, Continuous Queries, and Reactions. +## Automated Tutorial Evaluation + +The [manual evaluation workflow](.github/workflows/tutorial-evaluation.yml) and +[scheduled evaluation workflow](.github/workflows/tutorial-evaluation-scheduled.yml) +use Copilot CLI's default model rather than a pinned model ID. The manual workflow +accepts an optional `model` input to override that default; leave it blank to use +the CLI default. Scheduled evaluations always use the CLI default. + ## Sample Applications Following sample applications demonstrate usage of Drasi in different scenarios: 1. [Building comfort](apps/building-comfort)- Drasi for a building management scenario