EvalOps PR lens review #348
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: EvalOps PR lens review | |
| on: | |
| schedule: | |
| - cron: "23 */2 * * *" | |
| repository_dispatch: | |
| types: [evalopsbot-review-requested] | |
| workflow_dispatch: | |
| inputs: | |
| target_repos: | |
| description: "Comma-separated repositories to sweep" | |
| required: false | |
| default: "evalops/platform,evalops/deploy,evalops/maestro-internal" | |
| target_prs: | |
| description: "Optional comma-separated repo#number filters, for example platform#2023,deploy#17" | |
| required: false | |
| default: "" | |
| min_confidence: | |
| description: "Minimum confidence for PR comment publication" | |
| required: false | |
| default: "0.82" | |
| model: | |
| description: "Model for lens reviewers" | |
| required: false | |
| default: "claude-opus-4-7" | |
| provider: | |
| description: "LLM provider for lens reviewers: anthropic or openai" | |
| required: false | |
| default: "anthropic" | |
| max_diff_bytes: | |
| description: "Maximum unified diff bytes sent to each lens reviewer" | |
| required: false | |
| default: "180000" | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: evalops-pr-lens-review-${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_prs || github.event.client_payload.target_pr) || github.event_name == 'workflow_dispatch' && inputs.target_prs || 'all-open' }} | |
| cancel-in-progress: false | |
| jobs: | |
| discover: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| matrix: ${{ steps.discover.outputs.matrix }} | |
| has_work: ${{ steps.discover.outputs.has_work }} | |
| pr_count: ${{ steps.discover.outputs.pr_count }} | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN || secrets.EVALOPS_ORG_READ_TOKEN }} | |
| TARGET_REPOS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_repos || github.event.client_payload.target_repo) || inputs.target_repos || 'evalops/platform,evalops/deploy,evalops/maestro-internal' }} | |
| TARGET_PRS: ${{ github.event_name == 'repository_dispatch' && (github.event.client_payload.target_prs || github.event.client_payload.target_pr) || inputs.target_prs || '' }} | |
| FORCE_EXPLICIT_REVIEW_LENSES: ${{ github.event_name == 'repository_dispatch' && 'true' || 'false' }} | |
| EXPLICIT_REVIEW_LENSES: migration-safety,nats-contract-drift,argo-manifest-skew,iam-blast-radius,generated-sdk-delta,eval-regression-risk | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby .github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Require cross-repo GitHub token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_ORG_READ_TOKEN or EVALOPS_PR_LENS_TOKEN with read access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Discover open PRs | |
| id: discover | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| args=( | |
| discover | |
| --repos "${TARGET_REPOS}" | |
| --target-prs "${TARGET_PRS}" | |
| --github-output "${GITHUB_OUTPUT}" | |
| --matrix-output pr-lens-matrix.json | |
| --targets-output pr-lens-targets.json | |
| ) | |
| if [ "${FORCE_EXPLICIT_REVIEW_LENSES}" = "true" ] && [ -n "${TARGET_PRS}" ]; then | |
| args+=(--force-lenses "${EXPLICIT_REVIEW_LENSES}") | |
| fi | |
| ruby .github/scripts/evalops-pr-lens-review.rb "${args[@]}" | |
| - name: Upload discovery ledger | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-discovery | |
| path: | | |
| pr-lens-matrix.json | |
| pr-lens-targets.json | |
| if-no-files-found: error | |
| retention-days: 30 | |
| lens-review: | |
| needs: discover | |
| if: ${{ needs.discover.outputs.has_work == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| REVIEW_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY || secrets.EVALOPS_ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || secrets.EVALOPS_OPENAI_API_KEY }} | |
| PR_LENS_PROVIDER_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.provider || github.event_name == 'workflow_dispatch' && inputs.provider || '' }} | |
| PR_LENS_MODEL_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.model || github.event_name == 'workflow_dispatch' && inputs.model || '' }} | |
| PR_LENS_MAX_DIFF_BYTES_OVERRIDE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.max_diff_bytes || github.event_name == 'workflow_dispatch' && inputs.max_diff_bytes || '' }} | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: Checkout org review helpers | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| with: | |
| path: org-defaults | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| echo "REVIEW_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Require cross-repo write token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${REVIEW_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_PR_LENS_TOKEN with read/write access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Require model provider key | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${ANTHROPIC_API_KEY}" ] && [ -z "${OPENAI_API_KEY}" ]; then | |
| echo "::error::Set ANTHROPIC_API_KEY/EVALOPS_ANTHROPIC_API_KEY or OPENAI_API_KEY/EVALOPS_OPENAI_API_KEY for PR lens review." | |
| exit 2 | |
| fi | |
| - name: Mark lens pending | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state pending \ | |
| --description "Running ${{ matrix.lens }} lens review" \ | |
| --target-url "${RUN_URL}" | |
| - name: Prepare target pull request head | |
| id: refs | |
| shell: bash | |
| env: | |
| TARGET_REPO: ${{ matrix.repo }} | |
| PR_NUMBER: ${{ matrix.pr }} | |
| LENS: ${{ matrix.lens }} | |
| SNAPSHOT_HEAD_SHA: ${{ matrix.head_sha }} | |
| SNAPSHOT_BASE_SHA: ${{ matrix.base_sha }} | |
| run: | | |
| set -euo pipefail | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb prepare-workspace \ | |
| --repo "${TARGET_REPO}" \ | |
| --pr "${PR_NUMBER}" \ | |
| --lens "${LENS}" \ | |
| --workspace target \ | |
| --output lens-review.json \ | |
| --github-output "${GITHUB_OUTPUT}" \ | |
| --snapshot-head-sha "${SNAPSHOT_HEAD_SHA}" \ | |
| --snapshot-base-sha "${SNAPSHOT_BASE_SHA}" | |
| - name: Run lens reviewer | |
| if: ${{ steps.refs.outputs.skip != 'true' }} | |
| shell: bash | |
| env: | |
| TARGET_REPO: ${{ matrix.repo }} | |
| PR_NUMBER: ${{ matrix.pr }} | |
| LENS: ${{ matrix.lens }} | |
| BASE_SHA: ${{ steps.refs.outputs.base_sha }} | |
| HEAD_SHA: ${{ steps.refs.outputs.head_sha }} | |
| run: | | |
| set -euo pipefail | |
| args=( | |
| run-lens | |
| --repo "${TARGET_REPO}" \ | |
| --pr "${PR_NUMBER}" \ | |
| --lens "${LENS}" \ | |
| --workspace target \ | |
| --base-sha "${BASE_SHA}" \ | |
| --head-sha "${HEAD_SHA}" \ | |
| --output lens-review.json | |
| --routing-config org-defaults/.github/pr-lens-routing.yml | |
| ) | |
| if [ -n "${PR_LENS_PROVIDER_OVERRIDE}" ]; then | |
| args+=(--provider "${PR_LENS_PROVIDER_OVERRIDE}") | |
| fi | |
| if [ -n "${PR_LENS_MODEL_OVERRIDE}" ]; then | |
| args+=(--model "${PR_LENS_MODEL_OVERRIDE}") | |
| fi | |
| if [ -n "${PR_LENS_MAX_DIFF_BYTES_OVERRIDE}" ]; then | |
| args+=(--max-diff-bytes "${PR_LENS_MAX_DIFF_BYTES_OVERRIDE}") | |
| fi | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb "${args[@]}" | |
| - name: Complete lens status | |
| if: ${{ success() && steps.refs.outputs.skip != 'true' }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| description="$( | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb \ | |
| lens-status-description --review-json lens-review.json | |
| )" | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state success \ | |
| --description "${description}" \ | |
| --target-url "${RUN_URL}" | |
| - name: Complete skipped lens status | |
| if: ${{ steps.refs.outputs.skip == 'true' }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| description="$( | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb \ | |
| lens-status-description --review-json lens-review.json | |
| )" | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha || matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state success \ | |
| --description "${description}" \ | |
| --target-url "${RUN_URL}" | |
| - name: Mark lens errored | |
| if: ${{ failure() }} | |
| shell: bash | |
| run: | | |
| ruby org-defaults/.github/scripts/evalops-pr-lens-review.rb post-status \ | |
| --repo "${{ matrix.repo }}" \ | |
| --sha "${{ steps.refs.outputs.head_sha || matrix.head_sha }}" \ | |
| --context "${{ matrix.check_context }}" \ | |
| --state error \ | |
| --description "${{ matrix.lens }} lens review failed" \ | |
| --target-url "${RUN_URL}" | |
| - name: Upload lens review | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-${{ matrix.repo_slug }}-${{ matrix.pr }}-${{ matrix.lens }} | |
| path: lens-review.json | |
| if-no-files-found: ignore | |
| retention-days: 30 | |
| meta-review: | |
| needs: | |
| - discover | |
| - lens-review | |
| if: ${{ always() && needs.discover.outputs.has_work == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| GH_TOKEN: ${{ secrets.EVALOPS_PR_LENS_TOKEN || secrets.EVALOPS_REVIEW_GUARD_TOKEN }} | |
| PR_LENS_MIN_CONFIDENCE: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.min_confidence || inputs.min_confidence || '0.82' }} | |
| PR_LENS_APP_REPOSITORIES: ".github,platform,deploy,maestro-internal,maestro,ensemble,diffscope,chat,cerebro" | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| steps: | |
| - name: Checkout org review helpers | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 | |
| - name: Configure GitHub App token | |
| shell: bash | |
| env: | |
| APP_ID: ${{ secrets.EVALOPS_PR_LENS_APP_ID }} | |
| APP_PRIVATE_KEY: ${{ secrets.EVALOPS_PR_LENS_APP_PRIVATE_KEY }} | |
| APP_INSTALLATION_ID: ${{ secrets.EVALOPS_PR_LENS_APP_INSTALLATION_ID }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "${APP_ID}" ] && [ -n "${APP_PRIVATE_KEY}" ]; then | |
| token="$( | |
| EVALOPS_PR_LENS_APP_ID="${APP_ID}" \ | |
| EVALOPS_PR_LENS_APP_PRIVATE_KEY="${APP_PRIVATE_KEY}" \ | |
| EVALOPS_PR_LENS_APP_INSTALLATION_ID="${APP_INSTALLATION_ID}" \ | |
| ruby .github/scripts/evalops-pr-lens-review.rb mint-app-token \ | |
| --owner evalops \ | |
| --repositories "${PR_LENS_APP_REPOSITORIES}" | |
| )" | |
| echo "::add-mask::${token}" | |
| echo "GH_TOKEN=${token}" >> "${GITHUB_ENV}" | |
| fi | |
| - name: Download lens artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Require cross-repo write token | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::error::Set EVALOPS_PR_LENS_TOKEN with read/write access to platform, deploy, and maestro-internal." | |
| exit 2 | |
| fi | |
| - name: Publish high-confidence findings | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ruby .github/scripts/evalops-pr-lens-review.rb meta-review \ | |
| --artifact-root artifacts \ | |
| --min-confidence "${PR_LENS_MIN_CONFIDENCE}" \ | |
| --output meta-review.json \ | |
| --markdown-output "${GITHUB_STEP_SUMMARY}" | |
| - name: Upload meta review ledger | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-lens-meta-review | |
| path: meta-review.json | |
| if-no-files-found: error | |
| retention-days: 30 |