EvalOpsBot review request canary #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: EvalOpsBot review request canary | |
| on: | |
| schedule: | |
| - cron: "37 15 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| cleanup: | |
| description: "Close the canary PR after the deep review is observed" | |
| required: false | |
| default: "true" | |
| permissions: | |
| checks: read | |
| contents: write | |
| pull-requests: write | |
| statuses: read | |
| concurrency: | |
| group: evalopsbot-review-canary | |
| cancel-in-progress: false | |
| jobs: | |
| canary: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| CANARY_BRANCH: evalopsbot-review-canary | |
| CANARY_REPO: evalops/.github | |
| CLEANUP: ${{ github.event_name == 'workflow_dispatch' && inputs.cleanup || 'true' }} | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| persist-credentials: false | |
| - name: Create canary review request | |
| id: canary | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| git config user.name "EvalOpsBot canary" | |
| git config user.email "evalopsbot-canary@users.noreply.github.com" | |
| gh auth setup-git | |
| git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${CANARY_REPO}.git" | |
| git fetch origin main --depth=1 | |
| git checkout -B "${CANARY_BRANCH}" origin/main | |
| mkdir -p .github/evalopsbot-canary | |
| cat > .github/evalopsbot-canary/review-request.md <<MARKDOWN | |
| # EvalOpsBot Review Canary | |
| This disposable canary file proves that requesting a review from EvalOpsBot | |
| on a normal pull request triggers the central deep-review dispatcher. | |
| Run: ${GITHUB_RUN_ID} | |
| MARKDOWN | |
| git add .github/evalopsbot-canary/review-request.md | |
| git commit -m "test: refresh evalopsbot review canary fixture" | |
| git push --force-with-lease origin "${CANARY_BRANCH}" | |
| pr_number="$( | |
| gh pr list \ | |
| --repo "${CANARY_REPO}" \ | |
| --head "${CANARY_BRANCH}" \ | |
| --state open \ | |
| --json number \ | |
| --jq '.[0].number // empty' | |
| )" | |
| if [ -z "${pr_number}" ]; then | |
| pr_url="$( | |
| gh pr create \ | |
| --repo "${CANARY_REPO}" \ | |
| --head "${CANARY_BRANCH}" \ | |
| --base main \ | |
| --title "test: EvalOpsBot review request canary" \ | |
| --body $'Canary PR for the EvalOpsBot requested-review deep review path.\n\nTest Plan:\n- This workflow requests an EvalOpsBot review and waits for evalops-pr-lens/meta-review on the canary head SHA.' | |
| )" | |
| pr_number="${pr_url##*/}" | |
| fi | |
| gh api \ | |
| --method POST \ | |
| "repos/${CANARY_REPO}/pulls/${pr_number}/requested_reviewers" \ | |
| -f reviewers[]=EvalOpsBot >/dev/null | |
| head_sha="$(git rev-parse HEAD)" | |
| { | |
| echo "pr_number=${pr_number}" | |
| echo "head_sha=${head_sha}" | |
| } >> "${GITHUB_OUTPUT}" | |
| - name: Wait for deep review signal | |
| shell: bash | |
| env: | |
| PR_NUMBER: ${{ steps.canary.outputs.pr_number }} | |
| HEAD_SHA: ${{ steps.canary.outputs.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| ruby -rjson -ropen3 -e ' | |
| repo = ENV.fetch("CANARY_REPO") | |
| sha = ENV.fetch("HEAD_SHA") | |
| context = "evalops-pr-lens/meta-review" | |
| 12.times do |attempt| | |
| status_json, status_err, status = Open3.capture3("gh", "api", "repos/#{repo}/commits/#{sha}/status") | |
| check_json, check_err, check_status = Open3.capture3("gh", "api", "repos/#{repo}/commits/#{sha}/check-runs?check_name=#{context}&per_page=100") | |
| statuses = status.success? ? JSON.parse(status_json).fetch("statuses", []) : [] | |
| checks = check_status.success? ? JSON.parse(check_json).fetch("check_runs", []) : [] | |
| status_match = statuses.find { |row| row["context"] == context && row["state"] != "pending" } | |
| check_match = checks.find { |row| row["name"] == context && row["status"] == "completed" } | |
| if status_match || check_match | |
| puts "observed #{context} for #{repo}@#{sha}" | |
| exit 0 | |
| end | |
| warn "attempt #{attempt + 1}/12: no completed #{context} yet" | |
| sleep 45 | |
| end | |
| abort "timed out waiting for #{context} on #{repo}@#{sha}" | |
| ' | |
| - name: Close canary PR | |
| if: ${{ always() && env.CLEANUP == 'true' && steps.canary.outputs.pr_number != '' }} | |
| shell: bash | |
| env: | |
| PR_NUMBER: ${{ steps.canary.outputs.pr_number }} | |
| run: | | |
| set -euo pipefail | |
| gh pr close "${PR_NUMBER}" \ | |
| --repo "${CANARY_REPO}" \ | |
| --comment "EvalOpsBot requested-review canary complete for ${GITHUB_RUN_ID}." \ | |
| --delete-branch || true |