From 561ec9b911c51071b4f7bc185f9c8ae8481bfd08 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 13 Oct 2026 07:25:25 -0400 Subject: [PATCH] bot-runs: Take --pool mistral, and let auto pick it for triage The operator asked for Mistral as a pool for toil (cgwalters-forge/tracker#524), and the tracker's pools.json now has it. A triage run is toil, so auto sends it there instead of spending from a paced pool, but only where the caller's pools.json names mistral: a caller without it would fail the run on an unknown pool, so it keeps today's choice by pace. The mistral pool has no reading or pace of its own yet, so it is never weighed against the other two. Generated-by: AI --- bin/bot-runs | 33 ++++++++++++++-------- docs/devspace-agent-runs.md | 14 ++++++---- lib/pool-choice.js | 55 +++++++++++++++++++++++++++++++++---- tests/bot-runs.sh | 29 +++++++++++++++++-- tests/pools.test.js | 19 +++++++++++++ 5 files changed, 125 insertions(+), 25 deletions(-) diff --git a/bin/bot-runs b/bin/bot-runs index 6bf0d9a..eb5f593 100755 --- a/bin/bot-runs +++ b/bin/bot-runs @@ -49,6 +49,9 @@ fi readonly RUNS_REPO RUNS_WORKFLOW RUNS_REF POOL_CHOICE=${BOT_RUNS_POOL_CHOICE:-$(dirname "$(readlink -f "$0")")/../lib/pool-choice.js} readonly POOL_CHOICE +# The pools a caller's dispatch.yml takes, in its repository. +readonly CALLER_POOLS_PATH=.github/agentic-job/pools.json +readonly POOL_USAGE='claude, openai, mistral, auto or none' readonly SUMMARY_ARTIFACT=agent-run readonly TRANSCRIPT_ARTIFACT=agent-transcript # What a run hands back, as gh-aw's safe outputs: outputs.jsonl, a patch and @@ -167,13 +170,15 @@ docs/devspace-agent-runs.md in homegit. Commands: dispatch --backend agentic-job --caller OWNER/REPO --repo OWNER/REPO --item NUMBER --kind implement|review|triage|research|fix - [--pool claude|openai|auto|none] [--dry-run] BRIEF + [--pool claude|openai|mistral|auto|none] [--dry-run] BRIEF Use the shipped dispatch.yml on main. NUMBER is the target issue (review: PR), not a board id. The caller owns model, limits and preamble; overrides are refused, not silently dropped. These runs apply themselves. --pool picks the inference pool: auto (the - default) the one furthest under its pace by bot-capacity's readings, - or none if there are none; the choice is printed. The pool input is + default) mistral for a triage run where the caller's pools.json has + it, else the one of claude and openai furthest under its pace by + bot-capacity's readings, or none if there are none; the choice is + printed. The pool input is sent only if the caller's dispatch.yml on its default branch declares it. Readers also take --backend agentic-job --caller. dispatch --item ITEM --repo OWNER/REPO [--base REF] [--agent AGENT] @@ -880,17 +885,19 @@ caller_declares_pool() { node "${POOL_CHOICE}" declares <<<"${yaml}" || { test $? -eq 1 && return 1; return 2; } } -# choose_pool POOL: sets chosen_pool ("" for none) from --pool POOL, saying -# on stderr which and why in one line. auto picks the pool furthest under its -# pace, and sends none to a caller that doesn't declare the input or when +# choose_pool POOL KIND: sets chosen_pool ("" for none) from --pool POOL, +# saying on stderr which and why in one line. auto picks the toil pool for a +# KIND that is toil, where the caller's pools.json has it (lib/pool-choice.js; +# a file that can't be read is one without it), else the pool furthest under +# its pace, and sends none to a caller that doesn't declare the input or when # that can't be told; only an explicit pool is refused for want of the input. choose_pool() { - local pool=$1 choice rc=0 + local pool=$1 kind=$2 choice rc=0 chosen_pool="" case "${pool}" in none) return ;; - claude|openai|auto) ;; - *) fatal "--pool must be claude, openai, auto or none" ;; + claude|openai|mistral|auto) ;; + *) fatal "--pool must be ${POOL_USAGE}" ;; esac require_cmd gh "install the GitHub CLI (https://cli.github.com) and authenticate it" require_cmd node "install Node.js 22 or newer" @@ -911,7 +918,9 @@ choose_pool() { echo "pool: ${pool} (--pool)" >&2 return fi - if ! choice=$(node "${POOL_CHOICE}" choose); then + ghapi -H 'Accept: application/vnd.github.raw' "repos/${RUNS_REPO}/contents/${CALLER_POOLS_PATH}" >"${scratch}/caller-pools.json" || rc=$? + pass_ratelimit "${rc}" + if ! choice=$(node "${POOL_CHOICE}" choose --kind "${kind}" --caller-pools "${scratch}/caller-pools.json"); then warn "cannot read the pools' pace; sending no pool input" return fi @@ -946,12 +955,12 @@ cmd_dispatch_agentic_job() { fatal "--item must be a positive safe issue/PR number" fi [[ "${kind}" =~ ^(implement|review|triage|research|fix)$ ]] || fatal "--kind must be implement, review, triage, research or fix" - [[ "${pool}" =~ ^(claude|openai|auto|none)$ ]] || fatal "--pool must be claude, openai, auto or none" + [[ "${pool}" =~ ^(claude|openai|mistral|auto|none)$ ]] || fatal "--pool must be ${POOL_USAGE}" test -n "${brief_file}" || fatal "missing BRIEF (a file, or - for stdin)" if test "${brief_file}" = -; then cat >"${scratch}/brief"; else test -r "${brief_file}" || fatal "cannot read ${brief_file}"; cp "${brief_file}" "${scratch}/brief"; fi grep -q '[^[:space:]]' "${scratch}/brief" || fatal "the brief is empty" - choose_pool "${pool}" + choose_pool "${pool}" "${kind}" body=$(jq -nc --arg ref "${RUNS_REF}" --arg repo "${repo}" --arg item "${item}" --arg kind "${kind}" --arg pool "${chosen_pool}" --rawfile task "${scratch}/brief" \ '{ref: $ref, return_run_details: true, inputs: ({repo: $repo, item: $item, kind: $kind, task: ($task | rtrimstr("\n"))} + (if $pool == "" then {} else {pool: $pool} end))}') test "$(jq '.inputs | tojson | length' <<<"${body}")" -le "${MAX_INPUTS_LEN}" || fatal "dispatch inputs exceed ${MAX_INPUTS_LEN}; shorten the brief" diff --git a/docs/devspace-agent-runs.md b/docs/devspace-agent-runs.md index 3643ebf..90012cd 100644 --- a/docs/devspace-agent-runs.md +++ b/docs/devspace-agent-runs.md @@ -648,17 +648,21 @@ caller configuration. Explicit overrides are refused rather than dropped or sent as unknown inputs. This is a remaining caller-interface gap before per-run model/limit selection. -`--pool claude|openai|auto|none` picks the inference pool, a closed set that +`--pool claude|openai|mistral|auto|none` picks the inference pool, a closed set that the caller maps to its own agent and model; an empty `pool` leaves the choice -to the caller's repository variables. `none` sends no input. `auto`, the -default, asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the -pool with the largest margin, what its pace allows now minus what it has used +to the caller's repository variables. `none` sends no input. `mistral` is the +pool for toil and has no pace: `auto`, the default, picks it for `--kind +triage` when the caller's `.github/agentic-job/pools.json` has a `mistral` +entry, printing `pool: mistral: triage is toil, and the caller's pools.json +has mistral`. For any other kind, or a caller without that entry, `auto` +asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the +pool of `claude` and `openai` with the largest margin, what its pace allows now minus what it has used (a pool that holds new work only when neither is free; one without a reading never over one with); with no reading of either, it warns and sends none. Whatever the value, the input is sent only when `dispatch.yml` on the caller's default branch declares a `pool` input, so a caller without it keeps working: `auto` then sends none (as it does when the chooser or the file cannot be -read, with a warning), and an explicit `claude` or `openai` is refused. The +read, with a warning), and an explicit `claude`, `openai` or `mistral` is refused. The choice is printed on stderr in one line, for example `pool: openai: furthest under its pace (claude 40% used of 48% allowed, openai 10% used of 48% allowed)`, or `pool: claude: openai is over its pace (...)` when the other diff --git a/lib/pool-choice.js b/lib/pool-choice.js index e76867d..71f11cd 100644 --- a/lib/pool-choice.js +++ b/lib/pool-choice.js @@ -1,15 +1,24 @@ // Which inference pool an agentic-job dispatch asks for (bot-runs dispatch -// --pool): the pool furthest under its pace, from the same readings -// bin/bot-capacity paces, and whether the caller's dispatch.yml takes a +// --pool): the toil pool for a kind of run that is toil, where the caller +// has one, else the pool furthest under its pace, from the same readings +// bin/bot-capacity paces; and whether the caller's dispatch.yml takes a // `pool` input at all. Node's standard library only. -// pool-choice.js choose prints {pool, line} (pool null: none to ask for) +// pool-choice.js choose [--kind KIND] [--caller-pools FILE] +// prints {pool, line} (pool null: none to ask +// for); FILE is the caller's pools.json // pool-choice.js declares reads a dispatch.yml on stdin; exit 0 if it // declares the pool input, 1 if not "use strict"; const path = require("node:path"); +// The paced pools: auto chooses between these by their readings. const POOLS = ["claude", "openai"]; +// The pool for toil, runs a small model does well enough, and the kinds +// that are toil. It has no reading and no pace: a run on it is one not +// spent from a paced pool. +const TOIL_POOL = "mistral"; +const TOIL_KINDS = ["triage"]; const INPUT = "pool"; // Exit status of a failure, apart from declares' 1 for "no". const EX_ERROR = 3; @@ -57,6 +66,37 @@ function choosePool(paces) { return { pool: best, line: `${best}: ${why} (${POOLS.map(detail).join(", ")})` }; } +// toilPool(kind, callerPools): {pool, line} when a run of KIND is toil and +// the caller's pools.json (parsed; null where it could not be read) has the +// toil pool, else null: the choice is then by pace, as for any other kind. +function toilPool(kind, callerPools) { + if (!TOIL_KINDS.includes(kind)) return null; + if (!callerPools || typeof callerPools !== "object" || !Object.hasOwn(callerPools, TOIL_POOL)) return null; + return { pool: TOIL_POOL, line: `${TOIL_POOL}: ${kind} is toil, and the caller's pools.json has ${TOIL_POOL}` }; +} + +// readCallerPools(file): the caller's pools.json, or null without a file or +// where it is not JSON: a caller without one has no toil pool. +function readCallerPools(file) { + if (!file) return null; + try { + return JSON.parse(require("node:fs").readFileSync(file, "utf8")); + } catch { + return null; + } +} + +// chooseArgs(argv): {kind, callerPools} of choose's options, each null +// when absent; throws on anything else. +function chooseArgs(argv) { + const opts = { "--kind": null, "--caller-pools": null }; + for (let i = 0; i < argv.length; i += 2) { + if (!Object.hasOwn(opts, argv[i]) || argv[i + 1] === undefined) throw new Error(`choose: unexpected argument '${argv[i]}'`); + opts[argv[i]] = argv[i + 1]; + } + return { kind: opts["--kind"], callerPools: opts["--caller-pools"] }; +} + async function readPaces() { const capacity = require(path.join(__dirname, "..", "bin", "bot-capacity")); const pools = require(path.join(__dirname, "pools.js")); @@ -73,10 +113,13 @@ async function main(argv) { return declaresPool(Buffer.concat(chunks).toString("utf8")) ? 0 : 1; } if (argv[0] === "choose") { - process.stdout.write(`${JSON.stringify(choosePool(await readPaces()))}\n`); + const { kind, callerPools } = chooseArgs(argv.slice(1)); + // Toil first: it needs no reading of the paced pools. + const choice = toilPool(kind, readCallerPools(callerPools)) ?? choosePool(await readPaces()); + process.stdout.write(`${JSON.stringify(choice)}\n`); return 0; } - process.stderr.write("usage: pool-choice.js choose | declares\n"); + process.stderr.write("usage: pool-choice.js choose [--kind KIND] [--caller-pools FILE] | declares\n"); return 2; } @@ -85,4 +128,4 @@ if (require.main === module) { process.stderr.write(`pool-choice: error: ${e.stack || e.message}\n`); process.exit(EX_ERROR); }); -} else module.exports = { POOLS, declaresPool, choosePool }; +} else module.exports = { POOLS, TOIL_POOL, TOIL_KINDS, declaresPool, choosePool, toilPool }; diff --git a/tests/bot-runs.sh b/tests/bot-runs.sh index c769624..8ea10c5 100755 --- a/tests/bot-runs.sh +++ b/tests/bot-runs.sh @@ -201,6 +201,11 @@ case "${method} ${path%%\?*}" in test -e "${store}/dispatch.yml" || fail_http 404 "Not Found" cat "${store}/dispatch.yml" ;; + # The caller's pools, if it has any: $FAKE_GH/pools.json. + "GET repos/${repo}/contents/.github/agentic-job/pools.json") + test -e "${store}/pools.json" || fail_http 404 "Not Found" + cat "${store}/pools.json" + ;; # A target repository: private-org's are private, gone's don't exist. "GET repos/"*/*) [[ "${path}" =~ ^repos/[^/]+/[^/]+$ ]] || { echo "fake gh: unexpected call: ${method} ${path}" 1>&2; exit 1; } @@ -1476,8 +1481,28 @@ test_agentic_pool() { auto|40|10|openai|^pool: openai: furthest claude|90|10|claude|^pool: claude \(--pool\)$ openai|10|90|openai|^pool: openai \(--pool\)$ +mistral|10|40|mistral|^pool: mistral \(--pool\)$ none|10|40||^$ CASES + # A triage run is toil: auto sends it to the mistral pool where the + # caller's pools.json has one, whatever the paced pools read, and + # chooses by pace as before where it has none or for another kind. + # pools.json | kind | pool sent | line on stderr + while IFS='|' read -r pools kind want line; do + pool_world 40 10 + rm -f "${FAKE_GH}/pools.json" + test -z "${pools}" || echo "${pools}" >"${FAKE_GH}/pools.json" + err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --kind "${kind}" --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out") || fail "dispatch failed: ${err}" + expect_lines "${err}" "${line}" + expect_eq "$(tail -n +2 "${WORK}/out" | jq -r '.inputs.pool // ""')" "${want}" "pool sent for ${kind} with pools '${pools}'" + done <<'CASES' +{"claude": {}, "mistral": {}}|triage|mistral|^pool: mistral: triage is toil, and the caller's pools.json has mistral$ +{"claude": {}, "mistral": {}}|fix|openai|^pool: openai: furthest under its pace +{"claude": {}, "openai": {}}|triage|openai|^pool: openai: furthest under its pace +|triage|openai|^pool: openai: furthest under its pace +not json|triage|openai|^pool: openai: furthest under its pace +CASES + rm -f "${FAKE_GH}/pools.json" # A caller that declares no pool input gets today's body, and says so. rm "${FAKE_GH}/dispatch.yml" err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out") @@ -1504,8 +1529,8 @@ CASES rm "${FAKE_GH}/dispatch.yml" # A bad value is refused before anything is read or dispatched. : >"${FAKE_GH}/calls" - out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool mistral "${WORK}/brief" 2>&1) && fail 'accepted a bad pool' - expect_lines "${out}" '--pool must be claude, openai, auto or none' + out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool gemini "${WORK}/brief" 2>&1) && fail 'accepted a bad pool' + expect_lines "${out}" '--pool must be claude, openai, mistral, auto or none' expect_eq "$(calls .)" 0 'API calls for a bad pool' # A real dispatch sends the chosen pool. pool_world 40 10 diff --git a/tests/pools.test.js b/tests/pools.test.js index 8496a73..0d08460 100644 --- a/tests/pools.test.js +++ b/tests/pools.test.js @@ -222,6 +222,25 @@ test("pool-choice: the pool with the largest margin under its pace", () => { assert.equal(line({ claude: p(40, 48), openai: p(10, 48) }), "openai: furthest under its pace"); }); +test("pool-choice: toil goes to the mistral pool where the caller has one", () => { + const both = { claude: {}, openai: {}, mistral: { agent: "vibe" } }; + // kind | the caller's pools.json | pool (null: choose by pace) + const cases = [ + ["triage", both, "mistral"], + ["triage", { claude: {}, openai: {} }, null], + ["triage", null, null], + ["triage", "mistral", null], + ["triage", ["mistral"], null], + ...["implement", "review", "research", "fix", "toString", null, undefined].map((kind) => [kind, both, null]), + ]; + for (const [kind, callerPools, want] of cases) { + assert.equal(choice.toilPool(kind, callerPools)?.pool ?? null, want, `${kind} ${JSON.stringify(callerPools)}`); + } + assert.equal(choice.toilPool("triage", both).line, "mistral: triage is toil, and the caller's pools.json has mistral"); + // The toil pool is not a paced one: choosePool never names it. + assert.ok(!choice.POOLS.includes(choice.TOIL_POOL)); +}); + test("pool-choice: whether a dispatch.yml declares the pool input", () => { const wf = (inputs, extra = "") => `name: Dispatch\non:\n workflow_dispatch:\n inputs:\n${inputs}${extra}`; const cases = [