fix(work-unblock): re-fire pre-model rate-limit stops and admit/pick-slot kill-switch skips

.github/actions/resolve-config/callers.test.mjs

@@ -1439,9 +1439,20 @@ test("work-resolve-issue.yml: a WIP cap check that fails holds pick-slot green w
   assert.equal(wip["continue-on-error"], true, "a failed cap check holds green");
   const hold = pick.steps.find((item) => item.name === "Hold on an unknown WIP cap");
   assert.ok(pick.steps.indexOf(hold) > pick.steps.indexOf(wip), "the hold notice follows the check");
-  for (const [capped, runs] of [["", true], ["true", false], ["false", false]]) {
-    assert.equal(evaluate(hold.if, { "steps.wip.outputs.capped": capped }), runs, `hold notice on capped ${JSON.stringify(capped)}`);
+  // A kill-switch check that could not run sets no proceed and holds as before;
+  // a switch that is on skips the check and every hold.
+  for (const proceed of ["true", ""]) {
+    for (const [capped, runs] of [["", true], ["true", false], ["false", false]]) {
+      const context = { "steps.kill.outputs.proceed": proceed, "steps.wip.outputs.capped": capped };
+      assert.equal(evaluate(hold.if, context), runs, `hold notice on proceed ${JSON.stringify(proceed)}, capped ${JSON.stringify(capped)}`);
+    }
+    assert.equal(evaluate(wip.if, { "steps.kill.outputs.proceed": proceed }), true, `cap check on proceed ${JSON.stringify(proceed)}`);
   }
+  assert.equal(evaluate(wip.if, { "steps.kill.outputs.proceed": "false" }), false, "the switch on skips the cap check");
+  assert.equal(evaluate(hold.if, { "steps.kill.outputs.proceed": "false", "steps.wip.outputs.capped": "" }), false, "the switch on names no hold");
+  const marker = pick.steps.find((item) => item.id === "hold-marker");
+  assert.equal(evaluate(marker.if, { "steps.kill.outputs.proceed": "false", "steps.wip.outputs.held": "", "steps.wip.outputs.capped": "" }), false, "the switch on marks no hold");
+  assert.equal(evaluate(marker.if, { "steps.kill.outputs.proceed": "", "steps.wip.outputs.held": "", "steps.wip.outputs.capped": "" }), true, "an unknown cap marks the hold");
   assert.match(hold.run, /^echo "::notice title=WIP cap hold::/);
   assert.equal(hold["continue-on-error"], undefined);
   // A held run reads no more config, so an unreadable base config cannot fail

.github/scripts/work-unblock/label-unblocked.mjs

@@ -192,15 +192,29 @@
 // the issue lane's WIP cap check (check-wip-cap.mjs) does not pause it. It is no
 // limit stop, so it never counts toward failing the issue.
 //
+// So is a run whose admit, pick-slot, or a call's broker-check or run job
+// failed on a GitHub API rate-limit stop, in a run where no model step ran
+// (the read runner's `Run the skill on` legs and its reused-branch summary
+// pass): the failed step's last line is rate-guard's or api-budget.bash's
+// stop, a call that hit a pool's rate limit or left it at or under its
+// stop-floor share, naming the reset. It is re-fired as a failed mint, under
+// the same bound and backoff, once both that reset and limits.reset-clamp-min
+// hours after the run ended have passed. The same line where a model step ran,
+// or a log that cannot be read, leaves the run as before.
+//
 // A run the token broker refused for the lanes kill switch, at broker-check's
 // `Ask the broker whether the lane may run` or at a `Request the lane token`
 // step, is a breaker trip, not a fault in the issue or the lanes; only the
 // refused job's log names the reason (the jobs listing reads the same for a
 // workflow-modified refusal). So is a run whose broker-check `Check the kill
-// switch` step saw a kill switch on and skipped the broker ask: that run ends
-// green, and the jobs listing alone shows it. A trip at broker-check ran no
-// model, and the sweep runs only with the kill switch off, so the first sweep
-// after the switch clears re-fires it; a trip at a mint ran the model and is
+// switch` step saw a kill switch on and skipped the broker ask, or whose admit
+// or pick-slot job's own `Check the kill switch` did and the job ended green
+// with its gated steps skipped (admit's `Install resolve-config's packages`
+// and `Check the trigger`, pick-slot's `Check the WIP cap`; a failed install
+// is a failed run, not the switch): that run ends green, and
+// the jobs listing alone shows it; both count as trips at broker-check. A
+// trip at broker-check ran no model, and the sweep runs only with the kill
+// switch off, so the first sweep after the switch clears re-fires it; a trip at a mint ran the model and is
 // re-fired limits.reset-clamp-min hours after the run ended. A run whose
 // report-outcome mint alone stopped, after another mint took its token, is a
 // trip at a mint too. Each gets a breaker record of its own, and after
@@ -215,9 +229,10 @@
 // queue that each finished run's sweep tops up. A breaker
 // record never spends the once-per-version re-fire, never counts toward the
 // failed-mint bound and never raises another cause's backoff. Each run's log
-// is read once, from its one refused job and at most LOG_CAP_BYTES, and only
-// its failed step's last line counts: a refusal for another reason, or a log
-// that cannot be read, is a failed mint.
+// is read once, from its one refused job (or its one failed job before any
+// model turn) and at most LOG_CAP_BYTES, and only its failed step's last line
+// counts: a refusal for another reason, or a log that cannot be read, is a
+// failed mint.
 //
 // A run admit's upstream-status stop skipped (check-upstream-status: Actions
 // or npm installs listed an outage) ran no model either. Once the status pages
@@ -374,7 +389,15 @@ const UNTRUSTED_STEP = "Stop on untrusted provenance";
 // broker-check's step, which asks the broker before any resolve-issue run job.
 const BROKER_STEP = "Ask the broker whether the lane may run";
 // broker-check's step before BROKER_STEP, which skips it while a kill switch is on.
+// admit and pick-slot run it first too, and while a switch is on skip their
+// gated steps: INSTALL_STEP and TRIGGER_STEP in admit, WIP_STEP in pick-slot.
 const KILL_STEP = "Check the kill switch";
+const INSTALL_STEP = "Install resolve-config's packages";
+const TRIGGER_STEP = "Check the trigger";
+const WIP_STEP = "Check the WIP cap";
+// The steps that run a model: the read runner's legs and its reused-branch
+// summary pass. A run where one ran is never a rate-limit stop before the model.
+const MODEL_STEP = /^(?:Run the skill on|Summarize the reused branch on) /;
 // A raw job log line, its timestamp first, whose rest matches the pattern `rest`.
 const stamped = (rest) => new RegExp(String.raw`^\d{4}-\d\d-\d\dT[\d:.]+Z ` + rest);
 // request-lane-token's last line when the broker refuses it, in check mode
@@ -383,6 +406,12 @@ const stamped = (rest) => new RegExp(String.raw`^\d{4}-\d\d-\d\dT[\d:.]+Z ` + re
 const denial = (mode) => stamped(String.raw`##\[error\]lane-${mode}-denied: the broker answered 403 with reason ([a-z]+(?:-[a-z]+)*)(?::|$)`);
 const DENIAL = { check: denial("check"), token: denial("token") };
 const STEP_FAILED = stamped(String.raw`##\[error\]Process completed with exit code [0-9]+\.$`);
+// rate-guard's (and api-budget.bash's) stop line, as the failed step's last
+// line: a call that hit a pool's rate limit or left it at or under its
+// stop-floor share, and the reset it names. Only the reset is read.
+const RATE_LIMITED = stamped(
+  String.raw`##\[error\].* (?:hit GitHub's .+ rate limit|left GitHub's .+ requests, at or under its stop-floor share \S+)(?: \(status \d+\))?: no call to it before (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ) \(`,
+);
 // The write runner's step after MINT_STEP that runs exactly when a token was
 // minted; a MINT_STEP success with it skipped is a mint that stopped green.
 const STAGE_STEP = "Stage the activity from the base";
@@ -1552,6 +1581,9 @@ const strandedCause = {
 };
 
 const MINT_CAUSE = `The work-resolve-issue run that \`${LABEL}\` started could not get its lane token, or the token broker's leave to run.`;
+// A rate-limit stop's cause, naming the latest reset (ms) its stop lines named.
+const rateLimitCause = (reset) =>
+  `The work-resolve-issue run that \`${LABEL}\` started stopped on GitHub's API rate limit before any model turn; the limit reset at ${new Date(reset).toISOString().replace(/\.\d{3}Z$/, "Z")}.`;
 
 // The record of a re-fire after a kill-switch refusal, posted before the
 // label moves, once per re-fire. It names the refused run and when it ended,
@@ -1687,8 +1719,12 @@ export function sharedLaneRuns(actions, repo) {
 // MINT_STEP, so its broker ask is the one the broker can fail), had the broker
 // refuse it for the kill switch at broker-check or a mint (failed, or stopped
 // green with its STOPPED notice: at broker-check, BROKER_STEP succeeded and
-// LEASE_STEP skipped), or had broker-check
-// skip its broker ask for the kill switch (`breaker`), was skipped by the WIP
+// LEASE_STEP skipped), or had broker-check skip its broker ask, a green admit
+// its INSTALL_STEP and TRIGGER_STEP or a green pick-slot its WIP_STEP for the
+// kill switch (`breaker`), or,
+// with no MODEL_STEP run, had admit, pick-slot or a call's broker-check or run
+// job fail on a RATE_LIMITED last line (`mint-failed`, with `reset` the latest
+// reset named), was skipped by the WIP
 // cap (`capped`: pick-slot ran, no resolve-issue call did), or was skipped by the
 // admit job's blocker gate (`blocked`: its BLOCKER_STEP ran and pick-slot did
 // not, or a call's broker-check stopped at RECHECK_STOP_STEP), or by admit's
@@ -1704,6 +1740,8 @@ async function didNoWork(jobsOf, refusalOf, runs) {
   let stalled = null;
   let tripped = null;
   let stage = null;
+  // The latest reset a rate-limit stop names, in ms; 0 for none.
+  let reset = 0;
   const atLeast = (kind, run, at) => {
     if (rank[kind] > (rank[stalled] ?? 0)) {
       stalled = kind;
@@ -1729,7 +1767,19 @@ async function didNoWork(jobsOf, refusalOf, runs) {
     const named = (name) => namedJobs(jobs, name);
     // Any call's broker-check: a later leg asks the broker again for its slot.
     const calls = RESOLVE_CALLS.flatMap(named);
-    const killed = calls.some((job) => endedAt(job, KILL_STEP, "success") && endedAt(job, BROKER_STEP, "skipped"));
+    const killedBefore = (step) => (job) => endedAt(job, KILL_STEP, "success") && endedAt(job, step, "skipped");
+    // admit's first gated step is INSTALL_STEP: a failed install also skips
+    // TRIGGER_STEP, and that job fails, so only a green job with both skipped
+    // is the switch. pick-slot's gated WIP_STEP follows only continue-on-error
+    // steps, so a green job with it skipped is the switch too.
+    const switchedOff = (...steps) => (job) => job.conclusion === "success" && steps.every((step) => killedBefore(step)(job));
+    const killed =
+      calls.some(killedBefore(BROKER_STEP)) || named("admit").some(switchedOff(INSTALL_STEP, TRIGGER_STEP)) || named("pick-slot").some(switchedOff(WIP_STEP));
+    // A failed job before any model turn: admit, pick-slot, or a call's
+    // broker-check or run job, in a run where no model step ran.
+    const modelRan = jobs.some((job) => (job.steps ?? []).some((step) => MODEL_STEP.test(String(step?.name)) && step.conclusion && step.conclusion !== "skipped"));
+    const preModel = [...named("admit"), ...named("pick-slot"), ...calls.filter((job) => / \/ (?:broker-check|run)$/.test(String(job.name)))];
+    const limited = modelRan ? undefined : preModel.find((job) => job.conclusion === "failure");
     const checked = calls.find((job) => failedAt(job, BROKER_STEP));
     const checkStopped = calls.find((job) => job.conclusion === "success" && endedAt(job, BROKER_STEP, "success") && endedAt(job, LEASE_STEP, "skipped"));
     const minting = jobs.find((job) => failedAt(job, MINT_STEP));
@@ -1748,8 +1798,9 @@ async function didNoWork(jobsOf, refusalOf, runs) {
       named("pick-slot").length > 0 &&
       named("pick-slot").every((job) => job.conclusion === "skipped");
     // One log per run: broker-check's when it refused, failed or stopped
-    // green, else the first failed mint's, else the first stopped mint's.
-    const refused = killed
+    // green, else the first failed mint's, else the first stopped mint's,
+    // else the first failed job's before any model turn.
+    const refused = await (killed
       ? null
       : checked
         ? refusalOf(checked, "check")
@@ -1759,24 +1810,30 @@ async function didNoWork(jobsOf, refusalOf, runs) {
             ? refusalOf(minting, "token")
             : stopped
               ? refusalOf(stopped, "stop")
-              : null;
-    if (killed || (await refused) === BREAKER) {
+              : limited
+                ? refusalOf(limited, "limit")
+                : null);
+    if (killed || refused === BREAKER) {
       atLeast("breaker", run, killed || checked || checkStopped ? "check" : "token");
     } else if (minting || checked || checkStopped || stopped) {
       atLeast("mint-failed");
+    } else if (limited && refused) {
+      atLeast("mint-failed");
+      reset = Math.max(reset, Date.parse(refused));
     } else if (capped) {
       atLeast("capped");
     } else if (blocked) {
       atLeast(outage ? "outage" : "blocked");
     }
   }
-  return stalled && { stalled, run: tripped, stage };
+  return stalled && { stalled, run: tripped, stage, reset: stalled === "mint-failed" ? reset : 0 };
 }
 
 // The reason the broker refused `job`, from its log: the failed step's last
 // line before the runner's first `Process completed`, in `mode` (check or
-// token), or BREAKER for a `stop` mode log with a STOPPED line inside the
-// mint step's MINT_START section. Null when there is no such line or the log
+// token; in `limit` mode the reset a RATE_LIMITED line names), or BREAKER
+// for a `stop` mode log with a STOPPED line inside the mint step's MINT_START
+// section. Null when there is no such line or the log
 // cannot be read within LOG_CAP_BYTES, or no log reader was given; the log is
 // data: check and token modes read only that one line, and a STOPPED line
 // outside the section never counts. A job's log never changes once it ends,
@@ -1816,7 +1873,14 @@ function refusalIn(lines, mode) {
     return stop > 0 && lines.slice(start, stop).some((line) => STOPPED.test(line)) ? BREAKER : null;
   }
   const end = lines.findIndex((line) => STEP_FAILED.test(line));
-  return end > 0 ? (DENIAL[mode].exec(lines[end - 1])?.[1] ?? null) : null;
+  if (end <= 0) {
+    return null;
+  }
+  if (mode === "limit") {
+    const reset = RATE_LIMITED.exec(lines[end - 1])?.[1];
+    return reset && Number.isFinite(Date.parse(reset)) ? reset : null;
+  }
+  return DENIAL[mode].exec(lines[end - 1])?.[1] ?? null;
 }
 
 // Each run's latest-attempt job listing, read once per run id. A completed
@@ -2187,7 +2251,7 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss
   // A stale stop ran no model; a failed mint in its re-fire job is no failed
   // mint of the lane's, so it skips the did-no-work clock below.
   const stale = await stoppedStale(jobsOf, runs);
-  const { stalled, run: refused, stage } = (!stale && (await didNoWork(jobsOf, refusalReader(jobLog, repo, carry), runs))) || {};
+  const { stalled, run: refused, stage, reset } = (!stale && (await didNoWork(jobsOf, refusalReader(jobLog, repo, carry), runs))) || {};
   if (stalled) {
     // Breaker trips since a listed human last added agent-ready or a run last
     // reported; past limits.limit-stops-per-issue of them, a flat wait of the
@@ -2209,14 +2273,17 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss
     // next sweep, which reads the page again.
     const outage = stalled === "outage";
     const clamp = (stalled === "breaker" && stage === "check") || (outage && (await upstreamClear())) ? 0 : limits["reset-clamp-min"];
-    const due = ended + (over >= 0 ? hoursFor["kill-switch"] : clamp) * HOUR_MS;
+    // A rate-limit stop waits past the reset it names too.
+    const due = Math.max(ended + (over >= 0 ? hoursFor["kill-switch"] : clamp) * HOUR_MS, reset);
     if (now < due) {
       const why =
         stalled === "breaker"
           ? `the broker refused its lane run for the lanes kill switch (trip ${trips + 1})`
           : outage
             ? "the upstream-status stop skipped its lane run, and the status page does not read clear"
-            : "its lane run did no work";
+            : reset
+              ? "its lane run stopped on GitHub's API rate limit before any model turn"
+              : "its lane run did no work";
       return { reason: `${why}; it is re-fired after ${new Date(due).toISOString()}`, ...(!outage && { due }) };
     }
     if (await wipCapped()) {
@@ -2232,12 +2299,12 @@ async function refireRefusal({ github, actions, jobLog, carry, repo, number, iss
     // attempt, and the cap and restarts-per-run bound it.
     const fired = firedOn(key).length;
     if (stalled === "mint-failed" && !recorded && fired >= limits["limit-stops-per-issue"]) {
-      return backoff(key, ended, MINT_CAUSE, hoursFor["failed-mint"]);
+      return backoff(key, ended, reset ? rateLimitCause(reset) : MINT_CAUSE, hoursFor["failed-mint"]);
     }
     const trip = stalled === "breaker" ? { run: refused.id, ended: Date.parse(refused.updated_at) || ended } : undefined;
     // Trips sort by when their run ended, not when they fell due, so the sweeps
     // after the switch clears drain them oldest first whatever their waits.
-    return { reason: null, resets: trip ? ended : due, refire: key, recorded, stall: stalled, trip };
+    return { reason: null, resets: trip ? ended : due, refire: key, recorded, stall: stalled, trip, ...(reset && { cause: rateLimitCause(reset) }) };
   }
   if (!stale && !runs.some((run) => FAILED.has(run.conclusion)) && !(await stoppedMint(jobsOf, runs)) && !(await cancelledBeforeWork(jobsOf, runs))) {
     return stranded(ended, "its lane run did not fail", "no-report");