diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index c06d7fe2f73..13eeb203133 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -1439,9 +1439,20 @@ test("work-resolve-issue.yml: a WIP cap check that fails holds pick-slot green w assert.equal(wip["continue-on-error"], true, "a failed cap check holds green"); const hold = pick.steps.find((item) => item.name === "Hold on an unknown WIP cap"); assert.ok(pick.steps.indexOf(hold) > pick.steps.indexOf(wip), "the hold notice follows the check"); - for (const [capped, runs] of [["", true], ["true", false], ["false", false]]) { - assert.equal(evaluate(hold.if, { "steps.wip.outputs.capped": capped }), runs, `hold notice on capped ${JSON.stringify(capped)}`); + // A kill-switch check that could not run sets no proceed and holds as before; + // a switch that is on skips the check and every hold. + for (const proceed of ["true", ""]) { + for (const [capped, runs] of [["", true], ["true", false], ["false", false]]) { + const context = { "steps.kill.outputs.proceed": proceed, "steps.wip.outputs.capped": capped }; + assert.equal(evaluate(hold.if, context), runs, `hold notice on proceed ${JSON.stringify(proceed)}, capped ${JSON.stringify(capped)}`); + } + assert.equal(evaluate(wip.if, { "steps.kill.outputs.proceed": proceed }), true, `cap check on proceed ${JSON.stringify(proceed)}`); } + assert.equal(evaluate(wip.if, { "steps.kill.outputs.proceed": "false" }), false, "the switch on skips the cap check"); + assert.equal(evaluate(hold.if, { "steps.kill.outputs.proceed": "false", "steps.wip.outputs.capped": "" }), false, "the switch on names no hold"); + const marker = pick.steps.find((item) => item.id === "hold-marker"); + assert.equal(evaluate(marker.if, { "steps.kill.outputs.proceed": "false", "steps.wip.outputs.held": "", "steps.wip.outputs.capped": "" }), false, "the switch on marks no hold"); + assert.equal(evaluate(marker.if, { "steps.kill.outputs.proceed": "", "steps.wip.outputs.held": "", "steps.wip.outputs.capped": "" }), true, "an unknown cap marks the hold"); assert.match(hold.run, /^echo "::notice title=WIP cap hold::/); assert.equal(hold["continue-on-error"], undefined); // A held run reads no more config, so an unreadable base config cannot fail diff --git a/.github/scripts/work-unblock/label-unblocked.mjs b/.github/scripts/work-unblock/label-unblocked.mjs index cf0faad7005..b4b4c287080 100755 --- a/.github/scripts/work-unblock/label-unblocked.mjs +++ b/.github/scripts/work-unblock/label-unblocked.mjs @@ -192,15 +192,29 @@ // the issue lane's WIP cap check (check-wip-cap.mjs) does not pause it. It is no // limit stop, so it never counts toward failing the issue. // +// So is a run whose admit, pick-slot, or a call's broker-check or run job +// failed on a GitHub API rate-limit stop, in a run where no model step ran +// (the read runner's `Run the skill on` legs and its reused-branch summary +// pass): the failed step's last line is rate-guard's or api-budget.bash's +// stop, a call that hit a pool's rate limit or left it at or under its +// stop-floor share, naming the reset. It is re-fired as a failed mint, under +// the same bound and backoff, once both that reset and limits.reset-clamp-min +// hours after the run ended have passed. The same line where a model step ran, +// or a log that cannot be read, leaves the run as before. +// // A run the token broker refused for the lanes kill switch, at broker-check's // `Ask the broker whether the lane may run` or at a `Request the lane token` // step, is a breaker trip, not a fault in the issue or the lanes; only the // refused job's log names the reason (the jobs listing reads the same for a // workflow-modified refusal). So is a run whose broker-check `Check the kill -// switch` step saw a kill switch on and skipped the broker ask: that run ends -// green, and the jobs listing alone shows it. A trip at broker-check ran no -// model, and the sweep runs only with the kill switch off, so the first sweep -// after the switch clears re-fires it; a trip at a mint ran the model and is +// switch` step saw a kill switch on and skipped the broker ask, or whose admit +// or pick-slot job's own `Check the kill switch` did and the job ended green +// with its gated steps skipped (admit's `Install resolve-config's packages` +// and `Check the trigger`, pick-slot's `Check the WIP cap`; a failed install +// is a failed run, not the switch): that run ends green, and +// the jobs listing alone shows it; both count as trips at broker-check. A +// trip at broker-check ran no model, and the sweep runs only with the kill +// switch off, so the first sweep after the switch clears re-fires it; a trip at a mint ran the model and is // re-fired limits.reset-clamp-min hours after the run ended. A run whose // report-outcome mint alone stopped, after another mint took its token, is a // trip at a mint too. Each gets a breaker record of its own, and after @@ -215,9 +229,10 @@ // queue that each finished run's sweep tops up. A breaker // record never spends the once-per-version re-fire, never counts toward the // failed-mint bound and never raises another cause's backoff. Each run's log -// is read once, from its one refused job and at most LOG_CAP_BYTES, and only -// its failed step's last line counts: a refusal for another reason, or a log -// that cannot be read, is a failed mint. +// is read once, from its one refused job (or its one failed job before any +// model turn) and at most LOG_CAP_BYTES, and only its failed step's last line +// counts: a refusal for another reason, or a log that cannot be read, is a +// failed mint. // // A run admit's upstream-status stop skipped (check-upstream-status: Actions // or npm installs listed an outage) ran no model either. Once the status pages @@ -374,7 +389,15 @@ const UNTRUSTED_STEP = "Stop on untrusted provenance"; // broker-check's step, which asks the broker before any resolve-issue run job. const BROKER_STEP = "Ask the broker whether the lane may run"; // broker-check's step before BROKER_STEP, which skips it while a kill switch is on. +// admit and pick-slot run it first too, and while a switch is on skip their +// gated steps: INSTALL_STEP and TRIGGER_STEP in admit, WIP_STEP in pick-slot. const KILL_STEP = "Check the kill switch"; +const INSTALL_STEP = "Install resolve-config's packages"; +const TRIGGER_STEP = "Check the trigger"; +const WIP_STEP = "Check the WIP cap"; +// The steps that run a model: the read runner's legs and its reused-branch +// summary pass. A run where one ran is never a rate-limit stop before the model. +const MODEL_STEP = /^(?:Run the skill on|Summarize the reused branch on) /; // A raw job log line, its timestamp first, whose rest matches the pattern `rest`. const stamped = (rest) => new RegExp(String.raw`^\d{4}-\d\d-\d\dT[\d:.]+Z ` + rest); // request-lane-token's last line when the broker refuses it, in check mode @@ -383,6 +406,12 @@ const stamped = (rest) => new RegExp(String.raw`^\d{4}-\d\d-\d\dT[\d:.]+Z ` + re const denial = (mode) => stamped(String.raw`##\[error\]lane-${mode}-denied: the broker answered 403 with reason ([a-z]+(?:-[a-z]+)*)(?::|$)`); const DENIAL = { check: denial("check"), token: denial("token") }; const STEP_FAILED = stamped(String.raw`##\[error\]Process completed with exit code [0-9]+\.$`); +// rate-guard's (and api-budget.bash's) stop line, as the failed step's last +// line: a call that hit a pool's rate limit or left it at or under its +// stop-floor share, and the reset it names. Only the reset is read. +const RATE_LIMITED = stamped( + String.raw`##\[error\].* (?:hit GitHub's .+ rate limit|left GitHub's .+ requests, at or under its stop-floor share \S+)(?: \(status \d+\))?: no call to it before (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ) \(`, +); // The write runner's step after MINT_STEP that runs exactly when a token was // minted; a MINT_STEP success with it skipped is a mint that stopped green. const STAGE_STEP = "Stage the activity from the base"; @@ -1552,6 +1581,9 @@ const strandedCause = { }; const MINT_CAUSE = `The work-resolve-issue run that \`${LABEL}\` started could not get its lane token, or the token broker's leave to run.`; +// A rate-limit stop's cause, naming the latest reset (ms) its stop lines named. +const rateLimitCause = (reset) => + `The work-resolve-issue run that \`${LABEL}\` started stopped on GitHub's API rate limit before any model turn; the limit reset at ${new Date(reset).toISOString().replace(/\.\d{3}Z$/, "Z")}.`; // The record of a re-fire after a kill-switch refusal, posted before the // label moves, once per re-fire. It names the refused run and when it ended, @@ -1687,8 +1719,12 @@ export function sharedLaneRuns(actions, repo) { // MINT_STEP, so its broker ask is the one the broker can fail), had the broker // refuse it for the kill switch at broker-check or a mint (failed, or stopped // green with its STOPPED notice: at broker-check, BROKER_STEP succeeded and -// LEASE_STEP skipped), or had broker-check -// skip its broker ask for the kill switch (`breaker`), was skipped by the WIP +// LEASE_STEP skipped), or had broker-check skip its broker ask, a green admit +// its INSTALL_STEP and TRIGGER_STEP or a green pick-slot its WIP_STEP for the +// kill switch (`breaker`), or, +// with no MODEL_STEP run, had admit, pick-slot or a call's broker-check or run +// job fail on a RATE_LIMITED last line (`mint-failed`, with `reset` the latest +// reset named), was skipped by the WIP // cap (`capped`: pick-slot ran, no resolve-issue call did), or was skipped by the // admit job's blocker gate (`blocked`: its BLOCKER_STEP ran and pick-slot did // not, or a call's broker-check stopped at RECHECK_STOP_STEP), or by admit's @@ -1704,6 +1740,8 @@ async function didNoWork(jobsOf, refusalOf, runs) { let stalled = null; let tripped = null; let stage = null; + // The latest reset a rate-limit stop names, in ms; 0 for none. + let reset = 0; const atLeast = (kind, run, at) => { if (rank[kind] > (rank[stalled] ?? 0)) { stalled = kind; @@ -1729,7 +1767,19 @@ async function didNoWork(jobsOf, refusalOf, runs) { const named = (name) => namedJobs(jobs, name); // Any call's broker-check: a later leg asks the broker again for its slot. const calls = RESOLVE_CALLS.flatMap(named); - const killed = calls.some((job) => endedAt(job, KILL_STEP, "success") && endedAt(job, BROKER_STEP, "skipped")); + const killedBefore = (step) => (job) => endedAt(job, KILL_STEP, "success") && endedAt(job, step, "skipped"); + // admit's first gated step is INSTALL_STEP: a failed install also skips + // TRIGGER_STEP, and that job fails, so only a green job with both skipped + // is the switch. pick-slot's gated WIP_STEP follows only continue-on-error + // steps, so a green job with it skipped is the switch too. + const switchedOff = (...steps) => (job) => job.conclusion === "success" && steps.every((step) => killedBefore(step)(job)); + const killed = + calls.some(killedBefore(BROKER_STEP)) || named("admit").some(switchedOff(INSTALL_STEP, TRIGGER_STEP)) || named("pick-slot").some(switchedOff(WIP_STEP)); + // A failed job before any model turn: admit, pick-slot, or a call's + // broker-check or run job, in a run where no model step ran. + const modelRan = jobs.some((job) => (job.steps ?? []).some((step) => MODEL_STEP.test(String(step?.name)) && step.conclusion && step.conclusion !== "skipped")); + const preModel = [...named("admit"), ...named("pick-slot"), ...calls.filter((job) => / \/ (?:broker-check|run)$/.test(String(job.name)))]; + const limited = modelRan ? undefined : preModel.find((job) => job.conclusion === "failure"); const checked = calls.find((job) => failedAt(job, BROKER_STEP)); const checkStopped = calls.find((job) => job.conclusion === "success" && endedAt(job, BROKER_STEP, "success") && endedAt(job, LEASE_STEP, "skipped")); const minting = jobs.find((job) => failedAt(job, MINT_STEP)); @@ -1748,8 +1798,9 @@ async function didNoWork(jobsOf, refusalOf, runs) { named("pick-slot").length > 0 && named("pick-slot").every((job) => job.conclusion === "skipped"); // One log per run: broker-check's when it refused, failed or stopped - // green, else the first failed mint's, else the first stopped mint's. - const refused = killed + // green, else the first failed mint's, else the first stopped mint's, + // else the first failed job's before any model turn. + const refused = await (killed ? null : checked ? refusalOf(checked, "check") @@ -1759,24 +1810,30 @@ async function didNoWork(jobsOf, refusalOf, runs) { ? refusalOf(minting, "token") : stopped ? refusalOf(stopped, "stop") - : null; - if (killed || (await refused) === BREAKER) { + : limited + ? refusalOf(limited, "limit") + : null); + if (killed || refused === BREAKER) { atLeast("breaker", run, killed || checked || checkStopped ? "check" : "token"); } else if (minting || checked || checkStopped || stopped) { atLeast("mint-failed"); + } else if (limited && refused) { + atLeast("mint-failed"); + reset = Math.max(reset, Date.parse(refused)); } else if (capped) { atLeast("capped"); } else if (blocked) { atLeast(outage ? "outage" : "blocked"); } } - return stalled && { stalled, run: tripped, stage }; + return stalled && { stalled, run: tripped, stage, reset: stalled === "mint-failed" ? reset : 0 }; } // The reason the broker refused `job`, from its log: the failed step's last // line before the runner's first `Process completed`, in `mode` (check or -// token), or BREAKER for a `stop` mode log with a STOPPED line inside the -// mint step's MINT_START section. Null when there is no such line or the log +// token; in `limit` mode the reset a RATE_LIMITED line names), or BREAKER +// for a `stop` mode log with a STOPPED line inside the mint step's MINT_START +// section. Null when there is no such line or the log // cannot be read within LOG_CAP_BYTES, or no log reader was given; the log is // data: check and token modes read only that one line, and a STOPPED line // outside the section never counts. A job's log never changes once it ends, @@ -1816,7 +1873,14 @@ function refusalIn(lines, mode) { return stop > 0 && lines.slice(start, stop).some((line) => STOPPED.test(line)) ? BREAKER : null; } const end = lines.findIndex((line) => STEP_FAILED.test(line)); - return end > 0 ? (DENIAL[mode].exec(lines[end - 1])?.[1] ?? null) : null; + if (end <= 0) { + return null; + } + if (mode === "limit") { + const reset = RATE_LIMITED.exec(lines[end - 1])?.[1]; + return reset && Number.isFinite(Date.parse(reset)) ? reset : null; + } + return DENIAL[mode].exec(lines[end - 1])?.[1] ?? null; } // Each run's latest-attempt job listing, read once per run id. A completed @@ -2187,7 +2251,7 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss // A stale stop ran no model; a failed mint in its re-fire job is no failed // mint of the lane's, so it skips the did-no-work clock below. const stale = await stoppedStale(jobsOf, runs); - const { stalled, run: refused, stage } = (!stale && (await didNoWork(jobsOf, refusalReader(jobLog, repo, carry), runs))) || {}; + const { stalled, run: refused, stage, reset } = (!stale && (await didNoWork(jobsOf, refusalReader(jobLog, repo, carry), runs))) || {}; if (stalled) { // Breaker trips since a listed human last added agent-ready or a run last // reported; past limits.limit-stops-per-issue of them, a flat wait of the @@ -2209,14 +2273,17 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss // next sweep, which reads the page again. const outage = stalled === "outage"; const clamp = (stalled === "breaker" && stage === "check") || (outage && (await upstreamClear())) ? 0 : limits["reset-clamp-min"]; - const due = ended + (over >= 0 ? hoursFor["kill-switch"] : clamp) * HOUR_MS; + // A rate-limit stop waits past the reset it names too. + const due = Math.max(ended + (over >= 0 ? hoursFor["kill-switch"] : clamp) * HOUR_MS, reset); if (now < due) { const why = stalled === "breaker" ? `the broker refused its lane run for the lanes kill switch (trip ${trips + 1})` : outage ? "the upstream-status stop skipped its lane run, and the status page does not read clear" - : "its lane run did no work"; + : reset + ? "its lane run stopped on GitHub's API rate limit before any model turn" + : "its lane run did no work"; return { reason: `${why}; it is re-fired after ${new Date(due).toISOString()}`, ...(!outage && { due }) }; } if (await wipCapped()) { @@ -2232,12 +2299,12 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss // attempt, and the cap and restarts-per-run bound it. const fired = firedOn(key).length; if (stalled === "mint-failed" && !recorded && fired >= limits["limit-stops-per-issue"]) { - return backoff(key, ended, MINT_CAUSE, hoursFor["failed-mint"]); + return backoff(key, ended, reset ? rateLimitCause(reset) : MINT_CAUSE, hoursFor["failed-mint"]); } const trip = stalled === "breaker" ? { run: refused.id, ended: Date.parse(refused.updated_at) || ended } : undefined; // Trips sort by when their run ended, not when they fell due, so the sweeps // after the switch clears drain them oldest first whatever their waits. - return { reason: null, resets: trip ? ended : due, refire: key, recorded, stall: stalled, trip }; + return { reason: null, resets: trip ? ended : due, refire: key, recorded, stall: stalled, trip, ...(reset && { cause: rateLimitCause(reset) }) }; } if (!stale && !runs.some((run) => FAILED.has(run.conclusion)) && !(await stoppedMint(jobsOf, runs)) && !(await cancelledBeforeWork(jobsOf, runs))) { return stranded(ended, "its lane run did not fail", "no-report"); diff --git a/.github/scripts/work-unblock/label-unblocked.test.mjs b/.github/scripts/work-unblock/label-unblocked.test.mjs index 6e18944d7cb..3715013bc1c 100644 --- a/.github/scripts/work-unblock/label-unblocked.test.mjs +++ b/.github/scripts/work-unblock/label-unblocked.test.mjs @@ -3843,6 +3843,218 @@ test("a broker-check breaker trip is not re-fired while the issue lane is paused } }); +// A GitHub API rate-limit stop before any model turn. Each line is a stop +// line's shape as its writer prints it: rate-guard.mjs's hit-the-limit line +// (run 42976925201's run-checks line, its reset moved to this test's day), +// rate-guard.mjs's stop-floor line (rate-guard.test.mjs's stop-floor +// expectation), and api-budget.bash's api_budget_refuse line, each the failed +// step's last line. +const LIMIT_ID = 120990000001; +const RESET = "2026-10-16T12:30:00Z"; +const STOP_LINES = { + "hit the limit": `GET https://api.github.com/repos/${REPO}/issues/30 hit GitHub's core API rate limit (status 403): no call to it before ${RESET} (x-ratelimit-reset); this step fails`, + "stop floor": `GET https://api.github.com/repos/${REPO}/issues/30 left GitHub's core API 97 of 5000 requests, at or under its stop-floor share 0.02 (status 200): no call to it before ${RESET} (x-ratelimit-reset); this step fails`, + "api-budget.bash": `'gh api repos/${REPO}/issues/30' hit GitHub's core API rate limit: no call to it before ${RESET} (x-ratelimit-reset); the call returns 1`, +}; +const limitLog = (line) => + ["2026-10-16T10:02:00.0000000Z ##[group]Run node .base/.github/scripts/work-resolve-issue/check-blockers.mjs", `2026-10-16T10:02:01.0000000Z ##[error]${line}`, "2026-10-16T10:02:01.1000000Z ##[error]Process completed with exit code 1.", ""].join("\n"); +const failedAt = (name, steps) => ({ id: LIMIT_ID, name, conclusion: "failure", runner_id: 13, started_at: "2026-10-16T10:01:00Z", steps }); +const SKIPPED_REST = [ + { name: "resolve-issue", conclusion: "skipped", steps: [] }, + { name: "report-outcome", conclusion: "skipped", steps: [] }, +]; +const PRE_MODEL_LIMITED = { + admit: [failedAt("admit", [{ name: "Check the kill switch", conclusion: "success" }, { name: "Check the trigger", conclusion: "success" }, { name: "Wait for open blockers", conclusion: "failure" }]), { name: "pick-slot", conclusion: "skipped", steps: [] }, ...SKIPPED_REST], + "pick-slot": [ADMITTED, failedAt("pick-slot", [{ name: "Check the WIP cap", conclusion: "success" }, { name: "Resolve the issue dimensions' timeout", conclusion: "failure" }]), ...SKIPPED_REST], + "broker-check": [ + ADMITTED, + pickSlot, + failedAt("resolve-issue / broker-check", [ + { name: "Check the kill switch", conclusion: "success" }, + { name: "Ask the broker whether the lane may run", conclusion: "success" }, + { name: "Claim the issue with a lane lease", conclusion: "failure" }, + ]), + { name: "resolve-issue / run", conclusion: "skipped", steps: [] }, + { name: "report-outcome", conclusion: "skipped", steps: [] }, + ], + "a call's run job": [ + ADMITTED, + pickSlot, + BROKER_ALLOWED, + failedAt("resolve-issue / run", [{ name: "Resolve the config", conclusion: "failure" }, { name: "Run the skill on the API key", conclusion: "skipped" }]), + { name: "report-outcome", conclusion: "skipped", steps: [] }, + ], +}; +const AFTER_RESET = Date.parse("2026-10-16T12:31:00Z"); +// A rate-limit stop's records name the GitHub API rate limit and the reset, never the broker. +const limitCause = (reset) => `The work-resolve-issue run that \`agent-ready\` started stopped on GitHub's API rate limit before any model turn; the limit reset at ${reset}.`; +const limitRefired = (reset) => [`POST /repos/${REPO}/issues/30/comments ${JSON.stringify({ body: refireComment(KEY, limitCause(reset)) })}`, ...restarted(30)]; + +test("a run a GitHub rate-limit stop failed before any model turn is re-fired as a failed mint once the reset it names has passed, from one log", async () => { + for (const [job, jobs] of Object.entries(PRE_MODEL_LIMITED)) { + for (const [wording, line] of Object.entries(STOP_LINES)) { + const name = `${job}, ${wording}`; + const logs = { [LIMIT_ID]: limitLog(line) }; + // 12:00 is past the hour after the 10:05 end, but before the 12:30 reset. + const early = stalledRun({ jobs, logs }); + assert.equal(await sweep({ github: early.github, actions: early.actions }), 0, name); + assert.deepEqual(writes(early.calls), [], `${name}: re-fired before the reset`); + assert.deepEqual(logReads(early.calls), [`GET /repos/${REPO}/actions/jobs/${LIMIT_ID}/logs`], name); + const due = stalledRun({ jobs, logs }); + assert.equal(await sweep({ github: due.github, actions: due.actions, now: AFTER_RESET }), 0, name); + assert.deepEqual(writes(due.calls), limitRefired(RESET), `${name}: not re-fired after the reset`); + assert.ok(!writes(due.calls).some((call) => call.includes("needs-human")), name); + } + } +}); + +test("a rate-limit stop before any model turn is on the failed-mint bound and backoff, not the failed-run one", async () => { + const jobs = PRE_MODEL_LIMITED.admit; + const logs = { [LIMIT_ID]: limitLog(STOP_LINES["hit the limit"].replace(RESET, "2026-10-16T10:51:58Z")) }; + // Two re-fires on these workflow files: a failed run waits 3 hours, a failed mint goes again. + const again = stalledRun({ jobs, logs, ...earlierPumps(2) }); + assert.equal(await sweep({ github: again.github, actions: again.actions }), 0); + assert.deepEqual(writes(again.calls), limitRefired("2026-10-16T10:51:58Z")); + // Past the bound: the failed-mint backoff, 24 hours after the run ended at 10:05, its record naming the rate limit. + const over = stalledRun({ jobs, logs, ...earlierPumps(3) }); + assert.equal(await sweep({ github: over.github, actions: over.actions }), 0); + assert.deepEqual(writes(over.calls), []); + const due = stalledRun({ jobs, logs, ...earlierPumps(3) }); + assert.equal(await sweep({ github: due.github, actions: due.actions, now: Date.parse("2026-10-17T10:05:00Z") }), 0); + const backoffRecord = writes(due.calls); + assert.deepEqual(backoffRecord, [`POST /repos/${REPO}/issues/30/comments ${JSON.stringify({ body: infraComment(KEY, limitCause("2026-10-16T10:51:58Z"), 4) })}`, ...restarted(30)]); + assert.ok(!backoffRecord[0].includes("broker"), backoffRecord[0]); + // An early reset still waits limits.reset-clamp-min after the run ended. + const recent = stalledRun({ jobs, logs, ended: "2026-10-16T11:30:00Z" }); + assert.equal(await sweep({ github: recent.github, actions: recent.actions }), 0); + assert.deepEqual(writes(recent.calls), []); +}); + +// Run 42976995398: the subscription leg ran the model, then +// run-checks and apply-issue-patch's trigger check failed. The same stop line +// in a pre-model job of a run where a model step ran is no rate-limit stop. +test("a rate-limit stop line in a run where a model step ran is not did-no-work: no log is read and the failed-run re-fire is unchanged", async () => { + const modelRan = [ + ADMITTED, + pickSlot, + { ...BROKER_ALLOWED, name: "resolve-issue-subscription / broker-check" }, + { name: "resolve-issue-subscription / run", conclusion: "success", steps: [{ name: "Run the skill on the OAuth token", conclusion: "success" }] }, + { ...failedAt("resolve-issue-fallback / broker-check", [{ name: "Claim the issue with a lane lease", conclusion: "failure" }]), id: LIMIT_ID + 1 }, + { name: "apply-issue-patch / run", conclusion: "failure", steps: [{ name: "Check the trigger", conclusion: "failure" }, { name: "Request the lane token", conclusion: "skipped" }] }, + failedAt("run-checks", [{ name: "Resolve the config", conclusion: "failure" }]), + ]; + const summarized = [ + ...PRE_MODEL_LIMITED["broker-check"].slice(0, 3), + { name: "resolve-issue-subscription / summary", conclusion: "success", steps: [{ name: "Summarize the reused branch on the OAuth token", conclusion: "success" }] }, + ]; + for (const [name, jobs] of [ + ["the model ran", modelRan], + ["the summary pass ran", summarized], + ]) { + const line = limitLog(STOP_LINES["hit the limit"]); + const { github, actions, calls } = stalledRun({ jobs, logs: { [LIMIT_ID]: line, [LIMIT_ID + 1]: line } }); + assert.equal(await sweep({ github, actions }), 0, name); + assert.deepEqual(logReads(calls), [], name); + // The failed-run path re-fires it at once, inside the reset. + assert.deepEqual(writes(calls), refired(30), name); + } +}); + +test("a pre-model failure whose log cannot be read, or whose last line is no rate-limit stop, keeps the failed-run re-fire", async () => { + const notStop = limitLog("GET https://api.github.com/repos/o/r/issues/30 failed with status 502"); + const earlierLine = ["2026-10-16T10:02:00.0000000Z ##[error]" + STOP_LINES["hit the limit"], notStop].join("\n"); + for (const [name, logs] of [ + ["no log", {}], + ["another error", { [LIMIT_ID]: notStop }], + ["a stop line before the step's last line", { [LIMIT_ID]: earlierLine }], + ["over the size cap", { [LIMIT_ID]: `${"x".repeat(2 * 1024 * 1024)}\n${limitLog(STOP_LINES["hit the limit"])}` }], + ]) { + const { github, actions, calls } = stalledRun({ jobs: PRE_MODEL_LIMITED.admit, logs }); + assert.equal(await sweep({ github, actions }), 0, name); + assert.deepEqual(writes(calls), refired(30), name); + // Past the per-version re-fire it waits the failed-again hours, as a failed run does. + const again = stalledRun({ jobs: PRE_MODEL_LIMITED.admit, logs, ...earlierPumps(2) }); + assert.equal(await sweep({ github: again.github, actions: again.actions }), 0, name); + assert.deepEqual(writes(again.calls), [], name); + } +}); + +// admit and pick-slot check the kill switch first; a switch that is on skips +// their first API step and ends the run green with every later job skipped. +const ADMIT_KILLED = [ + { + name: "admit", + conclusion: "success", + steps: [ + { name: "Check the kill switch", conclusion: "success" }, + { name: "Install resolve-config's packages", conclusion: "skipped" }, + { name: "Check the trigger", conclusion: "skipped" }, + { name: "Wait for open blockers", conclusion: "skipped" }, + ], + }, + { name: "pick-slot", conclusion: "skipped", steps: [] }, + ...SKIPPED_REST, +]; +const PICK_SLOT_KILLED = [ + ADMITTED, + { + name: "pick-slot", + conclusion: "success", + steps: [ + { name: "Check the kill switch", conclusion: "success" }, + { name: "Check the WIP cap", conclusion: "skipped" }, + { name: "Hold on an unknown WIP cap", conclusion: "skipped" }, + ], + }, + { name: "report-skip", conclusion: "skipped", steps: [] }, + ...SKIPPED_REST, +]; + +test("an admit or pick-slot kill-switch skip is a breaker trip at broker-check, read from the job listing alone and re-fired at the first sweep", async () => { + for (const [name, jobs] of [ + ["admit", ADMIT_KILLED], + ["pick-slot", PICK_SLOT_KILLED], + ]) { + const { github, actions, calls } = stalledRun({ jobs, conclusion: "success" }); + assert.equal(await sweep({ github, actions }), 0, name); + tripped(calls); + assert.deepEqual(logReads(calls), [], name); + const recent = stalledRun({ jobs, conclusion: "success", ended: "2026-10-16T11:58:00Z" }); + assert.equal(await sweep({ github: recent.github, actions: recent.actions }), 0, name); + tripped(recent.calls); + } + // The same jobs with their API steps run are no trip: pick-slot's is a WIP + // cap skip, re-fired with a plain record, and admit's a blocker-gate skip, + // which waits while its blocker is open. + const ran = (jobs, job) => jobs.map((item) => (item.name === job ? { ...item, steps: item.steps.map((step) => ({ ...step, conclusion: "success" })) } : item)); + const capped = stalledRun({ jobs: ran(PICK_SLOT_KILLED, "pick-slot"), conclusion: "success" }); + assert.equal(await sweep({ github: capped.github, actions: capped.actions }), 0); + assert.deepEqual(writes(capped.calls), refired(30)); + const admitted = stalledRun({ jobs: ran(ADMIT_KILLED, "admit"), conclusion: "success", blockedBy: { 30: [issue(31)] } }); + assert.equal(await sweep({ github: admitted.github, actions: admitted.actions }), 0); + assert.deepEqual(writes(admitted.calls), []); +}); + +// With the switch off, a failed admit install also leaves `Check the trigger` +// skipped, and a failed pick-slot can leave `Check the WIP cap` skipped: each +// job ends red, so the run is a failed run, never a kill-switch trip. +test("an admit whose install failed, or a pick-slot that failed, with its gated step skipped is a failed run, not a breaker trip", async () => { + const failedJob = (jobs, name, edit) => jobs.map((item) => (item.name === name ? { ...item, conclusion: "failure", steps: item.steps.map(edit) } : item)); + for (const [name, jobs] of [ + ["admit install failed", failedJob(ADMIT_KILLED, "admit", (step) => (step.name === "Install resolve-config's packages" ? { ...step, conclusion: "failure" } : step))], + ["pick-slot failed", failedJob(PICK_SLOT_KILLED, "pick-slot", (step) => step)], + ]) { + // The failed-run path: re-fired at once with its plain record, no breaker record. + const fresh = stalledRun({ jobs, conclusion: "failure" }); + assert.equal(await sweep({ github: fresh.github, actions: fresh.actions }), 0, name); + assert.deepEqual(writes(fresh.calls), refired(30), name); + // Re-fired twice on these workflow files already: a failed run waits the failed-again hours, a breaker trip would not. + const again = stalledRun({ jobs, conclusion: "failure", ...earlierPumps(2) }); + assert.equal(await sweep({ github: again.github, actions: again.actions }), 0, name); + assert.deepEqual(writes(again.calls), [], name); + } +}); + test("a green run whose stale stop did not run, or a stop step outside resolve-issue, is not re-fired", async () => { for (const [name, jobs] of [ ["stop step skipped", [ADMITTED, pickSlot, BROKER_ALLOWED, staleRunJob("skipped")]], @@ -3876,6 +4088,15 @@ test("the job ids and step names the sweep matches exist in the lane workflows", assert.ok(steps(caller["pick-slot"]).includes("Check the WIP cap")); assert.ok(steps(caller.admit).includes("Wait for open blockers")); assert.ok(steps(caller.admit).includes("Stop on untrusted provenance")); + // A kill-switch skip in admit or pick-slot is read as its first API step skipped after the check. + for (const [id, step] of [ + ["admit", "Check the trigger"], + ["pick-slot", "Check the WIP cap"], + ]) { + const names = steps(caller[id]); + assert.ok(names.indexOf("Check the kill switch") >= 0 && names.indexOf("Check the kill switch") < names.indexOf(step), id); + assert.match(caller[id], new RegExp(String.raw`^ {6}- name: ${step}\n(?: {8}.*\n)*? {8}if: steps\.kill\.outputs\.proceed (?:== 'true'|!= 'false')\n`, "m"), id); + } // Every call of the read runner is a leg stoppedStale reads. const calls = Object.keys(caller).filter((id) => /^ {4}uses: \.\/\.github\/workflows\/pr-run-activity-read\.yml\b/m.test(caller[id])); assert.deepEqual(calls, ["resolve-issue", "resolve-issue-subscription", "resolve-issue-fallback"]); diff --git a/.github/scripts/work-unblock/verdict-carry.mjs b/.github/scripts/work-unblock/verdict-carry.mjs index 6fa82556202..8c6b8dc678a 100755 --- a/.github/scripts/work-unblock/verdict-carry.mjs +++ b/.github/scripts/work-unblock/verdict-carry.mjs @@ -72,10 +72,11 @@ export const DECIDED_MAX = 1000; export const WAIT_MAX_HOURS = 24; const WAIT_KEY = /^[0-9a-f]{40}-[0-9a-f]{40}$/; const RUN_KEY = /^[1-9][0-9]{0,14}\|[1-9][0-9]{0,3}$/; -const REFUSAL_KEY = /^[1-9][0-9]{0,14}\|(?:check|token|stop)$/; +const REFUSAL_KEY = /^[1-9][0-9]{0,14}\|(?:check|token|stop|limit)$/; const ISSUE_KEY = /^[1-9][0-9]{0,9}$/; -// A refusal reason as label-unblocked.mjs reads it (DENIAL, BREAKER), or null. -const REASON = /^[a-z]+(?:-[a-z]+)*$/; +// A refusal reason as label-unblocked.mjs reads it (DENIAL, BREAKER), or the +// reset a rate-limit stop names (RATE_LIMITED), or null. +const REASON = /^(?:[a-z]+(?:-[a-z]+)*|\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ)$/; const ZONED = /(Z|[+-]\d\d:\d\d)$/; const REPO = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const DAY_MS = 86_400_000; diff --git a/.github/scripts/work-unblock/verdict-carry.test.mjs b/.github/scripts/work-unblock/verdict-carry.test.mjs index 5915fc59e09..17af5c3f853 100644 --- a/.github/scripts/work-unblock/verdict-carry.test.mjs +++ b/.github/scripts/work-unblock/verdict-carry.test.mjs @@ -127,6 +127,19 @@ test("a carry kept by the same code is used, and one kept by other code is read assert.deepEqual(JSON.parse(readFileSync(join(dir, "next.json"), "utf8")), { code: "b".repeat(64), jobs: {}, refusals: {}, decided: {} }); }); +test("a rate-limit stop's reset is carried by its job and the limit mode, and a value of another shape is read fresh", () => { + const dir = tempDir("verdict-carry-limit-"); + const now = Date.parse("2026-11-10T03:00:00Z"); + const old = loadCarry(undefined, now - 60_000, "a".repeat(64)); + old.keepRefusal(71, "limit", "2026-11-10T03:30:00Z"); + old.keepRefusal(72, "limit", "2026-11-10 03:30"); + old.save(join(dir, "kept.json")); + const next = loadCarry(join(dir, "kept.json"), now, "a".repeat(64)); + assert.deepEqual(next.refusal(71, "limit"), { value: "2026-11-10T03:30:00Z" }); + assert.equal(next.refusal(71, "check"), undefined, "the reset is kept for the limit mode only"); + assert.equal(next.refusal(72, "limit"), undefined, "a reset not in the stop line's form is dropped"); +}); + test("the decided times carry each issue's last decision, and a file kept before them carries none", () => { const dir = tempDir("verdict-carry-decided-"); const now = Date.parse("2026-11-10T03:00:00Z"); diff --git a/.github/workflows/work-resolve-issue.yml b/.github/workflows/work-resolve-issue.yml index d7c31983d5d..22a989f8c24 100644 --- a/.github/workflows/work-resolve-issue.yml +++ b/.github/workflows/work-resolve-issue.yml @@ -145,15 +145,27 @@ jobs: node-version: 24 package-manager-cache: false + # A switch that is on skips every API step below and leaves the run + # green; work-unblock's sweep reads that skip as a kill-switch trip + # (label-unblocked.mjs didNoWork) and re-fires it once the switch clears. + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + # check-blockers.mjs reads the issue's Work status through # resolve-config.mjs, whose packages only the resolve-config action # installs; this job does not run it. - name: Install resolve-config's packages + if: steps.kill.outputs.proceed == 'true' timeout-minutes: 3 run: npm ci --ignore-scripts --omit=dev --no-audit --no-fund --prefix .base/.github/actions/resolve-config - name: Check the trigger id: gate + if: steps.kill.outputs.proceed == 'true' uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] with: trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json @@ -264,8 +276,22 @@ jobs: path: .base persist-credentials: false + # A switch that is on skips every API step below and leaves the job + # green with no `capped`, so resolve-issue skips; work-unblock's sweep + # reads that skip as a kill-switch trip (label-unblocked.mjs didNoWork). + # A check that cannot run (no checkout) sets no `proceed` and leaves the + # hold below as it was; broker-check asks again before any model turn. + - name: Check the kill switch + id: kill + continue-on-error: true + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + - name: Check the WIP cap id: wip + if: steps.kill.outputs.proceed != 'false' continue-on-error: true uses: ./.base/.github/actions/check-wip-cap # zizmor: ignore[self-repository] with: @@ -275,7 +301,7 @@ jobs: # The check names its own hold; this names one it could not, when it # failed or never ran. - name: Hold on an unknown WIP cap - if: steps.wip.outputs.capped != 'true' && steps.wip.outputs.capped != 'false' + if: steps.kill.outputs.proceed != 'false' && steps.wip.outputs.capped != 'true' && steps.wip.outputs.capped != 'false' run: echo "::notice title=WIP cap hold::the WIP cap check did not finish, so the pause is unknown and this run starts no work; work-unblock's next sweep re-fires it and the check reads again" # The key-credit gate, read before the slot queue: a credit stop inside @@ -353,7 +379,7 @@ jobs: # artifact for monitor signal 13 (lane-hold-marker.mjs). - name: Mark the lane hold id: hold-marker - if: steps.wip.outputs.held == 'true' || (steps.wip.outputs.capped != 'true' && steps.wip.outputs.capped != 'false') + if: steps.kill.outputs.proceed != 'false' && (steps.wip.outputs.held == 'true' || (steps.wip.outputs.capped != 'true' && steps.wip.outputs.capped != 'false')) continue-on-error: true run: mkdir -p "$RUNNER_TEMP/lane-hold" && echo "work-resolve-issue $GITHUB_RUN_ID" >"$RUNNER_TEMP/lane-hold/work-resolve-issue" - name: Keep the lane hold marker