EvalOps PR lens review #378
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: EvalOps PR lens review | |
| on: | |
| schedule: | |
| - cron: "23 */2 * * *" | |
| repository_dispatch: | |
| types: [evalopsbot-review-requested] | |
| workflow_dispatch: | |
| inputs: | |
| target_repos: | |
| description: "Comma-separated repositories to sweep" | |
| required: false | |
| default: "evalops/platform,evalops/deploy,evalops/maestro-internal" | |
| target_prs: | |
| description: "Optional comma-separated repo#number filters, for example platform#2023,deploy#17" | |
| required: false | |
| default: "" | |
| min_confidence: | |
| description: "Deprecated alias for block_min_confidence (kept for back-compat)" | |
| required: false | |
| default: "" | |
| comment_min_confidence: | |
| description: "Minimum confidence for surfacing a finding (inline or summary)" | |
| required: false | |
| default: "0.55" | |
| block_min_confidence: | |
| description: "Minimum confidence for a P0/P1 finding to fail the meta-review status" | |
| required: false | |
| default: "0.80" | |
| model: | |
| description: "Model for lens reviewers" | |
| required: false | |
| default: "claude-opus-4-7" | |
| provider: | |
| description: "LLM provider for lens reviewers: anthropic or openai" | |
| required: false | |
| default: "anthropic" | |
| max_diff_bytes: | |
| description: "Maximum unified diff bytes sent to each lens reviewer" | |
| required: false | |
| default: "180000" | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: evalops-pr-lens-review-${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_prs || github.event.client_payload.target_pr) || github.event_name == 'workflow_dispatch' && inputs.target_prs || 'all-open' }} | |
| cancel-in-progress: false | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| matrix: ${{ steps.discover.outputs.matrix }} | |
| has_work: ${{ steps.discover.outputs.has_work }} | |
| pr_count: ${{ steps.discover.outputs.pr_count }} | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN || secrets.EVALOPS_ORG_READ_TOKEN }} | |
| TARGET_REPOS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_repos || github.event.client_payload.target_repo) || inputs.target_repos || 'evalops/platform,evalops/deploy,evalops/maestro-internal' }} | |
| TARGET_PRS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_prs || github.event.client_payload.target_pr) || inputs.target_prs || '' }} | |
| FORCE_EXPLICIT_REVIEW_LENSES: ${{ github.event_name == 'repository_dispatch' && 'true' || 'false' }} | |
| EXPLICIT_REVIEW_LENSES: migration-safety,nats-contract-drift,argo-manifest-skew,iam-blast-radius,generated-sdk-delta,eval-regression-risk | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby .github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Require cross-repo GitHub token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_ORG_READ_TOKEN or EVALOPS_PR_LENS_TOKEN with read access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Discover open PRs | |
| id: discover | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| args=( | |
| discover | |
| --repos "${TARGET_REPOS}" | |
| --target-prs "${TARGET_PRS}" | |
| --github-output "${GITHUB_OUTPUT}" | |
| --matrix-output pr-lens-matrix.json | |
| --targets-output pr-lens-targets.json | |
| ) | |
| if [ "${FORCE_EXPLICIT_REVIEW_LENSES}" = "true" ] && [ -n "${TARGET_PRS}" ]; then | |
| args+=(--force-lenses "${EXPLICIT_REVIEW_LENSES}") | |
| fi | |
| ruby .github/scripts/evalops-pr-lens-review.rb "${args[@]}" | |
| - name: Upload discovery ledger | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-discovery | |
| path: | | |
| pr-lens-matrix.json | |
| pr-lens-targets.json | |
| if-no-files-found: error | |
| retention-days: 30 | |
| lens-review: | |
| needs: discover | |
| if: ${{ needs.discover.outputs.has_work == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| REVIEW_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY || secrets.EVALOPS_ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || secrets.EVALOPS_OPENAI_API_KEY }} | |
| PR_LENS_PROVIDER_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.provider || github.event_name == 'workflow_dispatch' && inputs.provider || '' }} | |
| PR_LENS_MODEL_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.model || github.event_name == 'workflow_dispatch' && inputs.model || '' }} | |
| PR_LENS_MAX_DIFF_BYTES_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.max_diff_bytes || github.event_name == 'workflow_dispatch' && inputs.max_diff_bytes || '' }} | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: Checkout org review helpers | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| path: org-defaults | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| echo "REVIEW_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Require cross-repo write token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${REVIEW_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_PR_LENS_TOKEN with read/write access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Require model provider key | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${ANTHROPIC_API_KEY}" ] && [ -z "${OPENAI_API_KEY}" ]; then | |
| echo "::error::Set ANTHROPIC_API_KEY/EVALOPS_ANTHROPIC_API_KEY or OPENAI_API_KEY/EVALOPS_OPENAI_API_KEY for PR lens review." | |
| exit 2 | |
| fi | |
| - name: Mark lens pending | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state pending \ | |
| --description "Running ${{ matrix.lens }} lens review" \ | |
| --target-url "${RUN_URL}" | |
| - name: Prepare target pull request head | |
| id: refs | |
| shell: bash | |
| env: | |
| TARGET_REPO: ${{ matrix.repo }} | |
| PR_NUMBER: ${{ matrix.pr }} | |
| LENS: ${{ matrix.lens }} | |
| SNAPSHOT_HEAD_SHA: ${{ matrix.head_sha }} | |
| SNAPSHOT_BASE_SHA: ${{ matrix.base_sha }} | |
| run: | | |
| set -euo pipefail | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb prepare-workspace \ | |
| --repo "${TARGET_REPO}" \ | |
| --pr "${PR_NUMBER}" \ | |
| --lens "${LENS}" \ | |
| --workspace target \ | |
| --output lens-review.json \ | |
| --github-output "${GITHUB_OUTPUT}" \ | |
| --snapshot-head-sha "${SNAPSHOT_HEAD_SHA}" \ | |
| --snapshot-base-sha "${SNAPSHOT_BASE_SHA}" | |
| - name: Run lens reviewer | |
| if: ${{ steps.refs.outputs.skip != 'true' }} | |
| shell: bash | |
| env: | |
| TARGET_REPO: ${{ matrix.repo }} | |
| PR_NUMBER: ${{ matrix.pr }} | |
| LENS: ${{ matrix.lens }} | |
| BASE_SHA: ${{ steps.refs.outputs.base_sha }} | |
| HEAD_SHA: ${{ steps.refs.outputs.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| args=( | |
| run-lens | |
| --repo "${TARGET_REPO}" \ | |
| --pr "${PR_NUMBER}" \ | |
| --lens "${LENS}" \ | |
| --workspace target \ | |
| --base-sha "${BASE_SHA}" \ | |
| --head-sha "${HEAD_SHA}" \ | |
| --output lens-review.json | |
| --routing-config org-defaults/.github/pr-lens-routing.yml | |
| ) | |
| if [ -n "${PR_LENS_PROVIDER_OVERRIDE}" ]; then | |
| args+=(--provider "${PR_LENS_PROVIDER_OVERRIDE}") | |
| fi | |
| if [ -n "${PR_LENS_MODEL_OVERRIDE}" ]; then | |
| args+=(--model "${PR_LENS_MODEL_OVERRIDE}") | |
| fi | |
| if [ -n "${PR_LENS_MAX_DIFF_BYTES_OVERRIDE}" ]; then | |
| args+=(--max-diff-bytes "${PR_LENS_MAX_DIFF_BYTES_OVERRIDE}") | |
| fi | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb "${args[@]}" | |
| - name: Complete lens status | |
| if: ${{ success() && steps.refs.outputs.skip != 'true' }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| description="$( | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb \ | |
| lens-status-description --review-json lens-review.json | |
| )" | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state success \ | |
| --description "${description}" \ | |
| --target-url "${RUN_URL}" | |
| - name: Complete skipped lens status | |
| if: ${{ steps.refs.outputs.skip == 'true' }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| description="$( | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb \ | |
| lens-status-description --review-json lens-review.json | |
| )" | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha || matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state success \ | |
| --description "${description}" \ | |
| --target-url "${RUN_URL}" | |
| - name: Mark lens errored | |
| if: ${{ failure() }} | |
| shell: bash | |
| run: | | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha || matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state error \ | |
| --description "${{ matrix.lens }} lens review failed" \ | |
| --target-url "${RUN_URL}" | |
| - name: Upload lens review | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-${{ matrix.repo_slug }}-${{ matrix.pr }}-${{ matrix.lens }} | |
| path: lens-review.json | |
| if-no-files-found: ignore | |
| retention-days: 30 | |
| meta-review: | |
| needs: | |
| - discover | |
| - lens-review | |
| if: ${{ always() && needs.discover.outputs.has_work == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| # Back-compat alias: PR_LENS_MIN_CONFIDENCE maps to the block threshold. | |
| PR_LENS_MIN_CONFIDENCE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.min_confidence || inputs.min_confidence || '' }} | |
| PR_LENS_COMMENT_MIN_CONFIDENCE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.comment_min_confidence || inputs.comment_min_confidence || '0.55' }} | |
| PR_LENS_BLOCK_MIN_CONFIDENCE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.block_min_confidence || inputs.block_min_confidence || '0.80' }} | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: Checkout org review helpers | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby .github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Download lens artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Require cross-repo write token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_PR_LENS_TOKEN with read/write access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Publish findings as inline PR review | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| args=( | |
| meta-review | |
| --artifact-root artifacts | |
| --comment-min-confidence "${PR_LENS_COMMENT_MIN_CONFIDENCE}" | |
| --block-min-confidence "${PR_LENS_BLOCK_MIN_CONFIDENCE}" | |
| --output meta-review.json | |
| --markdown-output "${GITHUB_STEP_SUMMARY}" | |
| ) | |
| # Back-compat: a non-empty legacy min_confidence overrides the block threshold. | |
| if [ -n "${PR_LENS_MIN_CONFIDENCE}" ]; then | |
| args+=(--min-confidence "${PR_LENS_MIN_CONFIDENCE}") | |
| fi | |
| ruby .github/scripts/evalops-pr-lens-review.rb "${args[@]}" | |
| - name: Upload meta review ledger | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-meta-review | |
| path: meta-review.json | |
| if-no-files-found: error | |
| retention-days: 30 |