From f91d17e983165d3bc02eaa8f6def11648d3bb392 Mon Sep 17 00:00:00 2001 From: Ruokun Niu Date: Wed, 14 Oct 2026 09:57:13 -0700 Subject: [PATCH 1/2] switch to default model Signed-off-by: Ruokun Niu --- .../workflows/tutorial-evaluation-scheduled.yml | 9 +++------ .github/workflows/tutorial-evaluation.yml | 15 ++++++++++++--- README.md | 8 ++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tutorial-evaluation-scheduled.yml b/.github/workflows/tutorial-evaluation-scheduled.yml index de4b148..da35b8f 100644 --- a/.github/workflows/tutorial-evaluation-scheduled.yml +++ b/.github/workflows/tutorial-evaluation-scheduled.yml @@ -83,8 +83,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 @@ -126,8 +125,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 @@ -168,8 +166,7 @@ jobs: --deny-tool 'shell(rm -rf /*)' \ --deny-tool 'shell(dd *)' \ --allow-url localhost \ - --allow-url 127.0.0.1 \ - --model claude-opus-4.7 || true + --allow-url 127.0.0.1 || true REPORT=$(find . -name "report.md" -path "*/evaluation-*/*" | sort -r | head -1) if [ -z "$REPORT" ] || ! grep -qiE "^##\s*STATUS:\s*SUCCESS" "$REPORT"; then exit 1 diff --git a/.github/workflows/tutorial-evaluation.yml b/.github/workflows/tutorial-evaluation.yml index 4584848..40870e1 100644 --- a/.github/workflows/tutorial-evaluation.yml +++ b/.github/workflows/tutorial-evaluation.yml @@ -18,8 +18,10 @@ on: description: 'Branch from docs repo to use' default: 'main' model: - description: 'AI model to use for evaluation' - default: 'claude-opus-4.7' + description: 'Optional model override (leave blank for Copilot CLI default)' + type: string + required: false + default: '' jobs: setup: @@ -102,10 +104,17 @@ jobs: env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_PAT_TOKEN }} DRASI_TUTORIAL_EVALUATION: "true" + EVALUATION_MODEL: ${{ inputs.model }} with: configFile: ${{ needs.setup.outputs.devcontainer }} push: never + env: | + EVALUATION_MODEL runCmd: | + set -- + if [ -n "$EVALUATION_MODEL" ]; then + set -- --model "$EVALUATION_MODEL" + fi timeout 1200 copilot -p "$(cat prompt.md)" \ --allow-all-tools \ --allow-all-paths \ @@ -123,7 +132,7 @@ jobs: --deny-tool 'shell(dd *)' \ --allow-url localhost \ --allow-url 127.0.0.1 \ - --model ${{ inputs.model }} + "$@" - name: Check evaluation status if: always() diff --git a/README.md b/README.md index b56066f..3738283 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,14 @@ This repo contains code for a tutorial and a few sample applications that use Dr [The step-by-step tutorial linked here](https://drasi.io/getting-started/) will help you get Drasi up and running quickly and show you how easy it is to create Sources, Continuous Queries, and Reactions. +## Automated Tutorial Evaluation + +The [manual evaluation workflow](.github/workflows/tutorial-evaluation.yml) and +[scheduled evaluation workflow](.github/workflows/tutorial-evaluation-scheduled.yml) +use Copilot CLI's default model rather than a pinned model ID. The manual workflow +accepts an optional `model` input to override that default; leave it blank to use +the CLI default. Scheduled evaluations always use the CLI default. + ## Sample Applications Following sample applications demonstrate usage of Drasi in different scenarios: 1. [Building comfort](apps/building-comfort)- Drasi for a building management scenario From 9095683690e3aba6c77cd478098251f57e738309 Mon Sep 17 00:00:00 2001 From: Ruokun Niu Date: Wed, 14 Oct 2026 11:45:23 -0700 Subject: [PATCH 2/2] updated links Signed-off-by: Ruokun Niu --- .github/workflows/tutorial-evaluation-scheduled.yml | 2 +- .github/workflows/tutorial-evaluation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tutorial-evaluation-scheduled.yml b/.github/workflows/tutorial-evaluation-scheduled.yml index da35b8f..f3bef60 100644 --- a/.github/workflows/tutorial-evaluation-scheduled.yml +++ b/.github/workflows/tutorial-evaluation-scheduled.yml @@ -34,7 +34,7 @@ jobs: learning_path: tutorial/absence-of-change - tutorial: risky-containers devcontainer: .devcontainer/risky-containers/devcontainer.json - docs_path: docs/content/tutorials/risky-containers + docs_path: docs/content/drasi-kubernetes/tutorials/risky-containers learning_path: tutorial/risky-containers steps: diff --git a/.github/workflows/tutorial-evaluation.yml b/.github/workflows/tutorial-evaluation.yml index 40870e1..63f8dd8 100644 --- a/.github/workflows/tutorial-evaluation.yml +++ b/.github/workflows/tutorial-evaluation.yml @@ -67,7 +67,7 @@ jobs: ;; risky-containers) echo "devcontainer=.devcontainer/risky-containers/devcontainer.json" >> $GITHUB_OUTPUT - echo "docs_path=docs/content/tutorials/risky-containers" >> $GITHUB_OUTPUT + echo "docs_path=docs/content/drasi-kubernetes/tutorials/risky-containers" >> $GITHUB_OUTPUT echo "learning_path=tutorial/risky-containers" >> $GITHUB_OUTPUT ;; *)