From e0a22d363533426918b12de4a74d84ea1a65c731 Mon Sep 17 00:00:00 2001 From: Rickylabs Date: Sat, 18 Jul 2026 21:28:07 +0200 Subject: [PATCH 1/2] fix(ci): make docs eval a single ready-head claim --- .github/workflows/docs-openhands-eval.yml | 158 ++++++++++-------- .github/workflows/openhands-agent.yml | 4 + .../slices/docs-eval-fix/research.md | 84 ++++++++++ .../slices/docs-eval-fix/worklog.md | 80 +++++++++ .../openhands/docs-eval-workflow_test.ts | 72 ++++++++ 5 files changed, 332 insertions(+), 66 deletions(-) create mode 100644 .llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/research.md create mode 100644 .llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md create mode 100644 .llm/tools/agentic/openhands/docs-eval-workflow_test.ts diff --git a/.github/workflows/docs-openhands-eval.yml b/.github/workflows/docs-openhands-eval.yml index f96bb1c61..59de395c8 100644 --- a/.github/workflows/docs-openhands-eval.yml +++ b/.github/workflows/docs-openhands-eval.yml @@ -1,74 +1,70 @@ name: Docs OpenHands accuracy eval -# CI-level documentation-accuracy backstop. Every PR carrying `type:docs` or `area:docs` asks -# OpenHands to use Minimax M3: one of the most accurate, low-hallucination prose models, while cheap -# enough for QUICK MANUAL TESTING. This lane must exercise executable documentation claims (small -# scaffold, exact snippets, observed output), not merely reread prose. The agent-level pipeline is -# independent; see `.llm/harness/workflow/doc-audit-openhands-gate.md` for the audit contract and -# the `docs-eval:skip` on-demand escape hatch. -# -# The comment MUST use PAT_TOKEN. GitHub suppresses chained workflow events created with the default -# GITHUB_TOKEN, so a fallback would post a trigger-looking comment that never launches OpenHands. +# Dispatch exactly once for a docs PR head when the PR becomes ready for review. Maintainers may +# explicitly request the same decision path with a `/docs-eval rerun` PR comment; the durable head +# marker still prevents a second dispatch for an already-claimed SHA. `docs-eval:skip` suppresses +# dispatch, while `ci:full` makes a non-docs-labeled PR eligible for the full gate surface. on: pull_request: - types: [opened, synchronize, labeled] + types: [ready_for_review] + issue_comment: + types: [created] permissions: contents: read issues: write pull-requests: read +# The serialized per-PR critical section makes the marker lookup + trigger creation race-safe even +# when a ready transition and an explicit request arrive together. concurrency: - group: docs-openhands-eval-${{ github.event.pull_request.number }} + group: docs-openhands-eval-${{ github.event.pull_request.number || github.event.issue.number }} cancel-in-progress: false jobs: docs-accuracy: name: Minimax M3 docs accuracy if: >- - contains(github.event.pull_request.labels.*.name, 'type:docs') || - contains(github.event.pull_request.labels.*.name, 'area:docs') + (github.event_name == 'pull_request' && + github.event.action == 'ready_for_review' && + (contains(github.event.pull_request.labels.*.name, 'type:docs') || + contains(github.event.pull_request.labels.*.name, 'area:docs') || + contains(github.event.pull_request.labels.*.name, 'ci:full'))) || + (github.event_name == 'issue_comment' && + github.event.action == 'created' && + github.event.issue.pull_request && + github.event.comment.body == '/docs-eval rerun' && + contains(fromJson('["OWNER","MEMBER","COLLABORATOR"]'), + github.event.comment.author_association) && + (contains(github.event.issue.labels.*.name, 'type:docs') || + contains(github.event.issue.labels.*.name, 'area:docs') || + contains(github.event.issue.labels.*.name, 'ci:full'))) runs-on: ubuntu-latest timeout-minutes: 10 env: OPENHANDS_MODEL: openrouter/minimax/minimax-m3 OPENHANDS_OUTPUT: pr-comment OPENHANDS_ITERATIONS: '100' - SKIP_REQUESTED: ${{ contains(github.event.pull_request.labels.*.name, 'docs-eval:skip') }} + SKIP_REQUESTED: ${{ contains(github.event.pull_request.labels.*.name, 'docs-eval:skip') || contains(github.event.issue.labels.*.name, 'docs-eval:skip') }} FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: - name: Skipped on demand if: env.SKIP_REQUESTED == 'true' env: - REQUEST_ACTOR: ${{ github.event.sender.login }} - EVENT_ACTION: ${{ github.event.action }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REQUEST_ACTOR: ${{ github.actor }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} run: | { echo "## Docs OpenHands accuracy eval" echo echo "**Status:** skipped on demand" echo - printf -- '- Who: `@%s` (workflow event actor)\n' "$REQUEST_ACTOR" - printf -- '- Why: PR carries the `docs-eval:skip` escape-hatch label during `%s`\n' "$EVENT_ACTION" - printf -- '- Head SHA: `%s`\n' "$HEAD_SHA" + printf -- '- Who: `@%s`\n' "$REQUEST_ACTOR" + echo '- Why: PR carries the `docs-eval:skip` escape-hatch label' + printf -- '- PR: `#%s`\n' "$PR_NUMBER" } >> "$GITHUB_STEP_SUMMARY" - - name: Guard the open-model route - if: env.SKIP_REQUESTED != 'true' - run: | - case "$OPENHANDS_MODEL" in - *anthropic*|*claude*|*openai*|*gpt*|*gemini*|*google*) - echo "Closed-model string is prohibited for OpenHands: $OPENHANDS_MODEL" >&2 - exit 1 - ;; - esac - if [ "$OPENHANDS_MODEL" != "openrouter/minimax/minimax-m3" ]; then - echo "Expected the owner-ratified open model openrouter/minimax/minimax-m3; got $OPENHANDS_MODEL" >&2 - exit 1 - fi - - name: Require a chainable comment token if: env.SKIP_REQUESTED != 'true' env: @@ -80,34 +76,68 @@ jobs: echo echo "**Status:** failed before dispatch" echo - echo "The repository secret \`PAT_TOKEN\` is unavailable. The workflow will not fall back to" - echo "\`GITHUB_TOKEN\`, because comments created with that token cannot trigger OpenHands." + echo "The repository secret \`PAT_TOKEN\` is unavailable. No trigger was posted." + echo "Configure a chainable PAT; \`GITHUB_TOKEN\` comments cannot launch OpenHands." } >> "$GITHUB_STEP_SUMMARY" exit 1 fi - - name: Post or deduplicate the OpenHands trigger + - name: Guard and claim this PR head if: env.SKIP_REQUESTED != 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: PROMPT_PATH: .llm/tools/agentic/openhands/docs-eval-prompt.md - HEAD_SHA: ${{ github.event.pull_request.head.sha }} with: github-token: ${{ secrets.PAT_TOKEN }} script: | const owner = context.repo.owner; const repo = context.repo.repo; - const issue_number = context.payload.pull_request.number; - const headSha = process.env.HEAD_SHA; const promptPath = process.env.PROMPT_PATH; + const eventName = context.eventName; + + if (!process.env.OPENHANDS_MODEL) { + throw new Error('OPENHANDS_MODEL is required.'); + } + if (/(anthropic|claude|openai|gpt|gemini|google)/i.test(process.env.OPENHANDS_MODEL)) { + throw new Error(`Closed-model string is prohibited for OpenHands: ${process.env.OPENHANDS_MODEL}`); + } + if (process.env.OPENHANDS_MODEL !== 'openrouter/minimax/minimax-m3') { + throw new Error( + `Expected owner-ratified open model openrouter/minimax/minimax-m3; got ${process.env.OPENHANDS_MODEL}`, + ); + } + let pr; + let issueNumber; + if (eventName === 'pull_request') { + pr = context.payload.pull_request; + issueNumber = pr.number; + } else { + issueNumber = context.payload.issue.number; + const response = await github.rest.pulls.get({ owner, repo, pull_number: issueNumber }); + pr = response.data; + } + + const labels = new Set(pr.labels.map((label) => label.name)); + if (labels.has('docs-eval:skip')) { + await core.summary + .addHeading('Docs OpenHands accuracy eval') + .addRaw('**Status:** skipped on demand\n') + .addList([ + `Who: @${context.actor}`, + 'Why: PR carries the docs-eval:skip escape-hatch label', + `Head SHA: ${pr.head.sha}`, + ]) + .write(); + return; + } - // Read the prompt from the trusted base SHA, not from the PR checkout. A docs PR must - // not be able to rewrite the evaluator instructions that judge that same PR. + // Read evaluator instructions from the trusted base SHA. A PR cannot rewrite the + // prompt that judges itself. const promptResponse = await github.rest.repos.getContent({ owner, repo, path: promptPath, - ref: context.payload.pull_request.base.sha, + ref: pr.base.sha, }); if (Array.isArray(promptResponse.data) || promptResponse.data.type !== 'file') { throw new Error(`Expected a prompt file at ${promptPath}`); @@ -117,49 +147,45 @@ jobs: throw new Error(`${promptPath} must begin with "use harness"`); } - const triggerLine = [ - '@openhands-agent', - `model=${process.env.OPENHANDS_MODEL}`, - `output=${process.env.OPENHANDS_OUTPUT}`, - `iterations=${process.env.OPENHANDS_ITERATIONS}`, - ].join(' '); + const headSha = pr.head.sha; const marker = ``; - const body = `${triggerLine}\n${marker}\n\n${prompt}`; - const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, - issue_number, + issue_number: issueNumber, per_page: 100, }); - const identical = comments.filter((comment) => comment.body === body).at(-1); - const answered = identical && comments.some((comment) => - comment.id > identical.id && - String(comment.body ?? '').includes('') - ); - - if (identical && !answered) { + const existing = comments.find((comment) => String(comment.body ?? '').includes(marker)); + if (existing) { await core.summary .addHeading('Docs OpenHands accuracy eval') - .addRaw('Identical unanswered trigger already exists for this head SHA; not reposting.\n') - .addList([ - `Head SHA: ${headSha}`, - `Existing comment: ${identical.html_url}`, - ]) + .addRaw('This head SHA was already claimed; no second evaluator was dispatched.\n') + .addList([`Head SHA: ${headSha}`, `Existing trigger: ${existing.html_url}`]) .write(); return; } + const triggerLine = [ + '@openhands-agent', + `model=${process.env.OPENHANDS_MODEL}`, + `output=${process.env.OPENHANDS_OUTPUT}`, + `iterations=${process.env.OPENHANDS_ITERATIONS}`, + ].join(' '); + const reason = eventName === 'pull_request' + ? 'ready-for-review transition' + : 'authorized /docs-eval rerun request'; + const body = `${triggerLine}\n${marker}\n\n${prompt}`; const { data: comment } = await github.rest.issues.createComment({ owner, repo, - issue_number, + issue_number: issueNumber, body, }); await core.summary .addHeading('Docs OpenHands accuracy eval') - .addRaw('Posted the guarded Minimax M3 docs-accuracy trigger.\n') + .addRaw('Claimed this head SHA and posted one guarded Minimax M3 trigger.\n') .addList([ + `Reason: ${reason}`, `Head SHA: ${headSha}`, `Trigger comment: ${comment.html_url}`, `Iteration budget: ${process.env.OPENHANDS_ITERATIONS}`, diff --git a/.github/workflows/openhands-agent.yml b/.github/workflows/openhands-agent.yml index c65be3c53..6e326d0f0 100644 --- a/.github/workflows/openhands-agent.yml +++ b/.github/workflows/openhands-agent.yml @@ -544,6 +544,10 @@ jobs: run: | uv pip install --system "openhands-sdk @ git+https://github.com/OpenHands/software-agent-sdk.git@main#subdirectory=openhands-sdk" uv pip install --system "openhands-tools @ git+https://github.com/OpenHands/software-agent-sdk.git@main#subdirectory=openhands-tools" + # LiteLLM 1.92 imports its MCP proxy handler on the normal completion path. The SDK's + # current dependency set omits FastAPI, causing every agent turn to fail before the model + # is called (`ModuleNotFoundError: No module named 'fastapi'`). + uv pip install --system fastapi - name: Resolve provider credentials env: diff --git a/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/research.md b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/research.md new file mode 100644 index 000000000..eae3571c3 --- /dev/null +++ b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/research.md @@ -0,0 +1,84 @@ +# Docs eval loop fix — research + +Run: `beta11-cli--orchestrator`\ +Slice: `docs-eval-fix`\ +Lane: Codex GPT-5.6 Sol, medium (`normal_implementation`)\ +Supervisor: Fable 5 orchestrator `86d308d5` + +## Scope + +Diagnose and repair the automated docs-accuracy workflow so eligible documentation PRs dispatch +OpenHands once when ready for review (or when explicitly re-requested), deduplicate durably per head +SHA, retain `docs-eval:skip` and `ci:full`, and surface one actionable failure without retriggering. + +## Evidence + +### Live failure correlation (GitHub API, 2026-07-18) + +GitHub credentials were resolved in-process with `resolveGithubToken()` from +`.llm/tools/agentic/lib/agentic-lib.ts`; no token was written to disk or argv. PR metadata, workflow +runs, comments, jobs, and raw job logs were fetched through the GitHub REST API. + +| PR | Durable trigger comments | OpenHands summaries | Observed behavior | +| ---- | -----------------------: | ------------------: | ----------------------------------------------------------------------------------- | +| #858 | 1 | 1 | One push-triggered eval; downstream run `29631090296` failed. | +| #861 | 14 | 14 | Fourteen head SHAs generated fourteen failed evals while the docs PR was iterating. | +| #862 | 18 | 17 | Eighteen head-SHA triggers and seventeen completed summaries at evidence time. | + +Representative upstream workflow run: +[`29631086400`](https://github.com/rickylabs/netscript/actions/runs/29631086400) succeeded after +posting #858's trigger. Its downstream OpenHands run +[`29631090296`](https://github.com/rickylabs/netscript/actions/runs/29631090296) failed before any +task verdict. The same failure signature repeated on #861 and #862. + +Raw downstream job logs identify the actual execution failure: + +```text +ModuleNotFoundError: No module named 'fastapi' +openhands.sdk.conversation.exceptions.ConversationRunError: ... No module named 'fastapi' +``` + +The failing environment installed LiteLLM 1.92.0 through the current OpenHands SDK. LiteLLM imported +its MCP proxy handler on the normal completion path, which imports FastAPI; the SDK installation did +not provide it. Bootstrap succeeded, the model/provider resolved to `openrouter/minimax/minimax-m3` +/ `OPENROUTER`, and the crash happened before the model call. This rules out model credits, timeout, +and authentication as the observed cause. + +### Root causes + +1. The docs workflow listened to `opened`, every `synchronize`, and every `labeled` event. Draft PR + pushes therefore dispatched continuously instead of waiting for merge readiness. +2. Dedupe was intentionally revoked after any later `openhands-agent-summary`. Once a failed summary + appeared, another event on the same head was allowed to post another trigger. Across new head + SHAs, synchronize made the storm unbounded for the life of the draft. +3. The upstream check reported success after comment creation; the actionable failure appeared only + in the downstream OpenHands workflow. +4. Every downstream agent turn crashed on the missing FastAPI runtime dependency. + +### #806 lineage + +PR #806 introduced the trusted-base prompt, PAT-only chain, `docs-eval:skip`, head-SHA marker, and +the prior "identical unanswered trigger" rule. Its review explicitly accepted rerunning after a +summary on later label churn. That original cost tradeoff conflicts with the owner's current +exactly-once/ready-only requirement and is superseded here; the trusted prompt, open-model guard, +PAT requirement, and escape hatch remain. + +### Locked correction + +- Events: `pull_request: ready_for_review` and an exact authorized PR comment `/docs-eval rerun`; no + `opened`, `synchronize`, or `labeled` dispatch. +- Eligibility: `type:docs`, `area:docs`, or `ci:full`. +- Skip: `docs-eval:skip` produces an attributed successful summary and no marker/trigger. +- Dedupe: the first marker for a head SHA is permanent. A prior answer/failure never reopens the + claim. Per-PR Actions concurrency serializes marker lookup + comment creation. +- Runner: install FastAPI explicitly so LiteLLM can enter its completion path. +- Failure: token/config/runner failure produces one red run and one workflow-owned actionable + summary; no workflow event can automatically re-dispatch it. + +## Stop-lines + +1. No merge without green CI and opposite-family eval PASS. +2. No release publish, tag, canary, or stable action. +3. No milestone 13 closure. +4. Every sub-agent brief must repeat all stop-lines (no sub-agents are being dispatched here). +5. #824 remains drafts-only pending owner ratification. diff --git a/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md new file mode 100644 index 000000000..1598c2f56 --- /dev/null +++ b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md @@ -0,0 +1,80 @@ +# Docs eval loop fix — worklog + +## Identity + +- Worktree: `/home/codex/repos/wt-docs-eval-fix` +- Branch: `fix/docs-eval-loop` +- Baseline: `fbb32119` (`origin/main` at slice start) +- Implementer: Codex GPT-5.6 Sol, medium +- Supervisor/reviewer: Fable 5 orchestrator `86d308d5` + +## Design + +- Public surface: GitHub Actions event and check behavior only. +- Domain vocabulary: eligible docs PR, ready transition, explicit re-request, head SHA, durable + dispatch claim, skipped-on-demand result, actionable dispatch failure. +- Ports: GitHub pull-request events, issue comments, Actions checks, repository secrets. +- Constants: existing workflow name/check name, `docs-eval:skip`, `ci:full`, Minimax M3 open-model + route, per-head marker. +- Commit slice: one workflow/test slice proving the event matrix, durable per-SHA dedupe, escape + hatches, and failure semantics. +- Deferred scope: no evaluator dispatch, no merge, no release, no milestone closure. +- Contributor path: workflow policy remains visible in `.github/workflows/docs-openhands-eval.yml`; + any extracted decision helper will be colocated with focused tests. + +## Evidence log + +### Diagnosis + +- Pulled live PR/workflow/job/comment evidence for #858, #861, and #862 through the GitHub API via + `resolveGithubToken()`. +- Confirmed trigger storm counts: #858 1, #861 14, #862 18 trigger markers. +- Confirmed every sampled downstream job failed before a verdict with + `ModuleNotFoundError: No module named 'fastapi'`, not model credits/auth/timeout. +- Reviewed #806 body, patch, run artifacts, and review lineage. Preserved its trusted-base prompt, + PAT-only chain, open-model route, and skip label; replaced its answered-trigger reopening rule. + +### Implementation + +- `.github/workflows/docs-openhands-eval.yml` + - listens only to `ready_for_review` and exact authorized `/docs-eval rerun` PR comments; + - accepts docs labels or `ci:full`, honors `docs-eval:skip`; + - serializes by PR and treats any per-head marker as a permanent claim; + - retains PAT-only chainability and trusted-base evaluator instructions. +- `.github/workflows/openhands-agent.yml` + - explicitly installs FastAPI, the missing LiteLLM MCP import dependency observed in all sampled + failures. +- `.llm/tools/agentic/openhands/docs-eval-workflow_test.ts` + - models the event matrix and asserts the workflow's durable claim and runner dependency. + +### Event matrix + +| Event/state | Result | +| ----------------------------------------------- | ---------------------------------- | +| Draft opened | no workflow dispatch | +| Draft synchronize/push | no workflow dispatch | +| Arbitrary label added | no workflow dispatch | +| Eligible PR transitions ready, no marker | one dispatch | +| Eligible PR transitions ready, `docs-eval:skip` | attributed skip; no dispatch | +| Eligible `ci:full` PR transitions ready | one dispatch | +| Authorized exact `/docs-eval rerun`, no marker | one dispatch | +| Unauthorized or non-exact comment | no workflow dispatch | +| Ready/comment race for one head | serialized; one marker/trigger | +| Any later event for an already marked head | durable dedupe; no second dispatch | + +### Gate evidence + +| Gate | Command | Result | +| ------------------------- | ------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| Event/dedupe/runner tests | `deno test --no-lock --allow-read .llm/tools/agentic/openhands/docs-eval-workflow_test.ts` | PASS, 4/4 | +| YAML parse | `deno eval --no-lock` with `jsr:@std/yaml` over both edited workflows | PASS, 2/2 | +| Focused format | `deno fmt --check` on both workflows, test, research, worklog | PASS after focused formatting | +| Whitespace | `git diff --check` | PASS | +| Actionlint | `command -v actionlint` | NOT AVAILABLE; YAML parse + focused structural tests used | +| Volatile config guard | `deno test --no-lock --allow-read .llm/tools/agentic/config/no-hardcoded-volatile_test.ts` | PASS, 4/4 | + +### Reconcile / handoff + +- No evaluator was dispatched; supervisor owns opposite-family review and formal evaluation. +- No merge, release operation, milestone closure, or #824 board action was performed. +- Draft PR creation and gate-evidence comment follow after the final local gate rerun and commit. diff --git a/.llm/tools/agentic/openhands/docs-eval-workflow_test.ts b/.llm/tools/agentic/openhands/docs-eval-workflow_test.ts new file mode 100644 index 000000000..152167e77 --- /dev/null +++ b/.llm/tools/agentic/openhands/docs-eval-workflow_test.ts @@ -0,0 +1,72 @@ +import { assert, assertEquals } from '@std/assert'; + +type Event = { + kind: 'ready_for_review' | 'comment' | 'synchronize' | 'labeled'; + body?: string; + association?: string; + labels: string[]; + markerExists?: boolean; +}; + +function decision(event: Event): 'dispatch' | 'dedupe' | 'skip' | 'ignore' { + const eligible = event.labels.some((label) => + label === 'type:docs' || label === 'area:docs' || label === 'ci:full' + ); + const authorizedRequest = event.kind === 'comment' && event.body === '/docs-eval rerun' && + ['OWNER', 'MEMBER', 'COLLABORATOR'].includes(event.association ?? ''); + if (!eligible || (event.kind !== 'ready_for_review' && !authorizedRequest)) return 'ignore'; + if (event.labels.includes('docs-eval:skip')) return 'skip'; + if (event.markerExists) return 'dedupe'; + return 'dispatch'; +} + +Deno.test('docs eval event matrix permits only ready transition or authorized request', () => { + const docs = ['type:docs']; + assertEquals(decision({ kind: 'ready_for_review', labels: docs }), 'dispatch'); + assertEquals( + decision({ kind: 'comment', body: '/docs-eval rerun', association: 'MEMBER', labels: docs }), + 'dispatch', + ); + assertEquals(decision({ kind: 'synchronize', labels: docs }), 'ignore'); + assertEquals(decision({ kind: 'labeled', labels: docs }), 'ignore'); + assertEquals( + decision({ kind: 'comment', body: '/docs-eval rerun', association: 'NONE', labels: docs }), + 'ignore', + ); + assertEquals( + decision({ kind: 'comment', body: 'please rerun docs', association: 'OWNER', labels: docs }), + 'ignore', + ); +}); + +Deno.test('docs eval escape hatches and durable marker are deterministic', () => { + assertEquals( + decision({ kind: 'ready_for_review', labels: ['area:docs', 'docs-eval:skip'] }), + 'skip', + ); + assertEquals(decision({ kind: 'ready_for_review', labels: ['ci:full'] }), 'dispatch'); + assertEquals( + decision({ kind: 'ready_for_review', labels: ['type:docs'], markerExists: true }), + 'dedupe', + ); + assertEquals(decision({ kind: 'ready_for_review', labels: [] }), 'ignore'); +}); + +Deno.test('workflow source encodes the serialized exactly-once contract', async () => { + const workflow = await Deno.readTextFile('.github/workflows/docs-openhands-eval.yml'); + assert(workflow.includes('types: [ready_for_review]')); + assert(workflow.includes('issue_comment:')); + assert(!workflow.includes('types: [opened, synchronize, labeled]')); + assert(workflow.includes("github.event.comment.body == '/docs-eval rerun'")); + assert(workflow.includes('cancel-in-progress: false')); + assert(workflow.includes('docs-openhands-eval head=${headSha}')); + assert(workflow.includes('if (existing)')); + assert(!workflow.includes('!answered')); + assert(workflow.includes('secrets.PAT_TOKEN')); + assert(!workflow.includes('secrets.PAT_TOKEN || secrets.GITHUB_TOKEN')); +}); + +Deno.test('OpenHands runner installs the missing LiteLLM MCP dependency', async () => { + const workflow = await Deno.readTextFile('.github/workflows/openhands-agent.yml'); + assert(workflow.includes('uv pip install --system fastapi')); +}); From fa7d0be70dcd3f87ef0d71f98ed70ac420f2b345 Mon Sep 17 00:00:00 2001 From: Rickylabs Date: Sat, 18 Jul 2026 21:30:04 +0200 Subject: [PATCH 2/2] chore(harness): record docs eval PR handoff --- .../slices/docs-eval-fix/worklog.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md index 1598c2f56..5704d9f2b 100644 --- a/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md +++ b/.llm/runs/beta11-cli--orchestrator/slices/docs-eval-fix/worklog.md @@ -77,4 +77,13 @@ - No evaluator was dispatched; supervisor owns opposite-family review and formal evaluation. - No merge, release operation, milestone closure, or #824 board action was performed. -- Draft PR creation and gate-evidence comment follow after the final local gate rerun and commit. +- Implementation commit `e0a22d36` was pushed with the explicit refspec + `git push origin HEAD:fix/docs-eval-loop`. +- Draft PR [#869](https://github.com/rickylabs/netscript/pull/869) targets `main`, titled + `fix(ci): docs OpenHands eval — single trigger per ready PR head`, and references #840 / #806 + without a closing keyword. +- PR taxonomy: `type:fix`, `area:tooling`, `gate:ci`, `wave:v1`, `priority:p1`, exactly one + `status:impl`; milestone `0.0.1-beta.11` (#13). `docs-eval:skip` is unnecessary because the PR + carries neither docs label and the repaired workflow does not listen to label events. +- Gate evidence posted in the + [IMPL phase comment](https://github.com/rickylabs/netscript/pull/869#issuecomment-5012615474).