diff --git a/bin/bot-runs b/bin/bot-runs index 6bf0d9a..eb5f593 100755 --- a/bin/bot-runs +++ b/bin/bot-runs @@ -49,6 +49,9 @@ fi readonly RUNS_REPO RUNS_WORKFLOW RUNS_REF POOL_CHOICE=${BOT_RUNS_POOL_CHOICE:-$(dirname "$(readlink -f "$0")")/../lib/pool-choice.js} readonly POOL_CHOICE +# The pools a caller's dispatch.yml takes, in its repository. +readonly CALLER_POOLS_PATH=.github/agentic-job/pools.json +readonly POOL_USAGE='claude, openai, mistral, auto or none' readonly SUMMARY_ARTIFACT=agent-run readonly TRANSCRIPT_ARTIFACT=agent-transcript # What a run hands back, as gh-aw's safe outputs: outputs.jsonl, a patch and @@ -167,13 +170,15 @@ docs/devspace-agent-runs.md in homegit. Commands: dispatch --backend agentic-job --caller OWNER/REPO --repo OWNER/REPO --item NUMBER --kind implement|review|triage|research|fix - [--pool claude|openai|auto|none] [--dry-run] BRIEF + [--pool claude|openai|mistral|auto|none] [--dry-run] BRIEF Use the shipped dispatch.yml on main. NUMBER is the target issue (review: PR), not a board id. The caller owns model, limits and preamble; overrides are refused, not silently dropped. These runs apply themselves. --pool picks the inference pool: auto (the - default) the one furthest under its pace by bot-capacity's readings, - or none if there are none; the choice is printed. The pool input is + default) mistral for a triage run where the caller's pools.json has + it, else the one of claude and openai furthest under its pace by + bot-capacity's readings, or none if there are none; the choice is + printed. The pool input is sent only if the caller's dispatch.yml on its default branch declares it. Readers also take --backend agentic-job --caller. dispatch --item ITEM --repo OWNER/REPO [--base REF] [--agent AGENT] @@ -880,17 +885,19 @@ caller_declares_pool() { node "${POOL_CHOICE}" declares <<<"${yaml}" || { test $? -eq 1 && return 1; return 2; } } -# choose_pool POOL: sets chosen_pool ("" for none) from --pool POOL, saying -# on stderr which and why in one line. auto picks the pool furthest under its -# pace, and sends none to a caller that doesn't declare the input or when +# choose_pool POOL KIND: sets chosen_pool ("" for none) from --pool POOL, +# saying on stderr which and why in one line. auto picks the toil pool for a +# KIND that is toil, where the caller's pools.json has it (lib/pool-choice.js; +# a file that can't be read is one without it), else the pool furthest under +# its pace, and sends none to a caller that doesn't declare the input or when # that can't be told; only an explicit pool is refused for want of the input. choose_pool() { - local pool=$1 choice rc=0 + local pool=$1 kind=$2 choice rc=0 chosen_pool="" case "${pool}" in none) return ;; - claude|openai|auto) ;; - *) fatal "--pool must be claude, openai, auto or none" ;; + claude|openai|mistral|auto) ;; + *) fatal "--pool must be ${POOL_USAGE}" ;; esac require_cmd gh "install the GitHub CLI (https://cli.github.com) and authenticate it" require_cmd node "install Node.js 22 or newer" @@ -911,7 +918,9 @@ choose_pool() { echo "pool: ${pool} (--pool)" >&2 return fi - if ! choice=$(node "${POOL_CHOICE}" choose); then + ghapi -H 'Accept: application/vnd.github.raw' "repos/${RUNS_REPO}/contents/${CALLER_POOLS_PATH}" >"${scratch}/caller-pools.json" || rc=$? + pass_ratelimit "${rc}" + if ! choice=$(node "${POOL_CHOICE}" choose --kind "${kind}" --caller-pools "${scratch}/caller-pools.json"); then warn "cannot read the pools' pace; sending no pool input" return fi @@ -946,12 +955,12 @@ cmd_dispatch_agentic_job() { fatal "--item must be a positive safe issue/PR number" fi [[ "${kind}" =~ ^(implement|review|triage|research|fix)$ ]] || fatal "--kind must be implement, review, triage, research or fix" - [[ "${pool}" =~ ^(claude|openai|auto|none)$ ]] || fatal "--pool must be claude, openai, auto or none" + [[ "${pool}" =~ ^(claude|openai|mistral|auto|none)$ ]] || fatal "--pool must be ${POOL_USAGE}" test -n "${brief_file}" || fatal "missing BRIEF (a file, or - for stdin)" if test "${brief_file}" = -; then cat >"${scratch}/brief"; else test -r "${brief_file}" || fatal "cannot read ${brief_file}"; cp "${brief_file}" "${scratch}/brief"; fi grep -q '[^[:space:]]' "${scratch}/brief" || fatal "the brief is empty" - choose_pool "${pool}" + choose_pool "${pool}" "${kind}" body=$(jq -nc --arg ref "${RUNS_REF}" --arg repo "${repo}" --arg item "${item}" --arg kind "${kind}" --arg pool "${chosen_pool}" --rawfile task "${scratch}/brief" \ '{ref: $ref, return_run_details: true, inputs: ({repo: $repo, item: $item, kind: $kind, task: ($task | rtrimstr("\n"))} + (if $pool == "" then {} else {pool: $pool} end))}') test "$(jq '.inputs | tojson | length' <<<"${body}")" -le "${MAX_INPUTS_LEN}" || fatal "dispatch inputs exceed ${MAX_INPUTS_LEN}; shorten the brief" diff --git a/docs/devspace-agent-runs.md b/docs/devspace-agent-runs.md index 3643ebf..90012cd 100644 --- a/docs/devspace-agent-runs.md +++ b/docs/devspace-agent-runs.md @@ -648,17 +648,21 @@ caller configuration. Explicit overrides are refused rather than dropped or sent as unknown inputs. This is a remaining caller-interface gap before per-run model/limit selection. -`--pool claude|openai|auto|none` picks the inference pool, a closed set that +`--pool claude|openai|mistral|auto|none` picks the inference pool, a closed set that the caller maps to its own agent and model; an empty `pool` leaves the choice -to the caller's repository variables. `none` sends no input. `auto`, the -default, asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the -pool with the largest margin, what its pace allows now minus what it has used +to the caller's repository variables. `none` sends no input. `mistral` is the +pool for toil and has no pace: `auto`, the default, picks it for `--kind +triage` when the caller's `.github/agentic-job/pools.json` has a `mistral` +entry, printing `pool: mistral: triage is toil, and the caller's pools.json +has mistral`. For any other kind, or a caller without that entry, `auto` +asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the +pool of `claude` and `openai` with the largest margin, what its pace allows now minus what it has used (a pool that holds new work only when neither is free; one without a reading never over one with); with no reading of either, it warns and sends none. Whatever the value, the input is sent only when `dispatch.yml` on the caller's default branch declares a `pool` input, so a caller without it keeps working: `auto` then sends none (as it does when the chooser or the file cannot be -read, with a warning), and an explicit `claude` or `openai` is refused. The +read, with a warning), and an explicit `claude`, `openai` or `mistral` is refused. The choice is printed on stderr in one line, for example `pool: openai: furthest under its pace (claude 40% used of 48% allowed, openai 10% used of 48% allowed)`, or `pool: claude: openai is over its pace (...)` when the other diff --git a/lib/pool-choice.js b/lib/pool-choice.js index e76867d..71f11cd 100644 --- a/lib/pool-choice.js +++ b/lib/pool-choice.js @@ -1,15 +1,24 @@ // Which inference pool an agentic-job dispatch asks for (bot-runs dispatch -// --pool): the pool furthest under its pace, from the same readings -// bin/bot-capacity paces, and whether the caller's dispatch.yml takes a +// --pool): the toil pool for a kind of run that is toil, where the caller +// has one, else the pool furthest under its pace, from the same readings +// bin/bot-capacity paces; and whether the caller's dispatch.yml takes a // `pool` input at all. Node's standard library only. -// pool-choice.js choose prints {pool, line} (pool null: none to ask for) +// pool-choice.js choose [--kind KIND] [--caller-pools FILE] +// prints {pool, line} (pool null: none to ask +// for); FILE is the caller's pools.json // pool-choice.js declares reads a dispatch.yml on stdin; exit 0 if it // declares the pool input, 1 if not "use strict"; const path = require("node:path"); +// The paced pools: auto chooses between these by their readings. const POOLS = ["claude", "openai"]; +// The pool for toil, runs a small model does well enough, and the kinds +// that are toil. It has no reading and no pace: a run on it is one not +// spent from a paced pool. +const TOIL_POOL = "mistral"; +const TOIL_KINDS = ["triage"]; const INPUT = "pool"; // Exit status of a failure, apart from declares' 1 for "no". const EX_ERROR = 3; @@ -57,6 +66,37 @@ function choosePool(paces) { return { pool: best, line: `${best}: ${why} (${POOLS.map(detail).join(", ")})` }; } +// toilPool(kind, callerPools): {pool, line} when a run of KIND is toil and +// the caller's pools.json (parsed; null where it could not be read) has the +// toil pool, else null: the choice is then by pace, as for any other kind. +function toilPool(kind, callerPools) { + if (!TOIL_KINDS.includes(kind)) return null; + if (!callerPools || typeof callerPools !== "object" || !Object.hasOwn(callerPools, TOIL_POOL)) return null; + return { pool: TOIL_POOL, line: `${TOIL_POOL}: ${kind} is toil, and the caller's pools.json has ${TOIL_POOL}` }; +} + +// readCallerPools(file): the caller's pools.json, or null without a file or +// where it is not JSON: a caller without one has no toil pool. +function readCallerPools(file) { + if (!file) return null; + try { + return JSON.parse(require("node:fs").readFileSync(file, "utf8")); + } catch { + return null; + } +} + +// chooseArgs(argv): {kind, callerPools} of choose's options, each null +// when absent; throws on anything else. +function chooseArgs(argv) { + const opts = { "--kind": null, "--caller-pools": null }; + for (let i = 0; i < argv.length; i += 2) { + if (!Object.hasOwn(opts, argv[i]) || argv[i + 1] === undefined) throw new Error(`choose: unexpected argument '${argv[i]}'`); + opts[argv[i]] = argv[i + 1]; + } + return { kind: opts["--kind"], callerPools: opts["--caller-pools"] }; +} + async function readPaces() { const capacity = require(path.join(__dirname, "..", "bin", "bot-capacity")); const pools = require(path.join(__dirname, "pools.js")); @@ -73,10 +113,13 @@ async function main(argv) { return declaresPool(Buffer.concat(chunks).toString("utf8")) ? 0 : 1; } if (argv[0] === "choose") { - process.stdout.write(`${JSON.stringify(choosePool(await readPaces()))}\n`); + const { kind, callerPools } = chooseArgs(argv.slice(1)); + // Toil first: it needs no reading of the paced pools. + const choice = toilPool(kind, readCallerPools(callerPools)) ?? choosePool(await readPaces()); + process.stdout.write(`${JSON.stringify(choice)}\n`); return 0; } - process.stderr.write("usage: pool-choice.js choose | declares\n"); + process.stderr.write("usage: pool-choice.js choose [--kind KIND] [--caller-pools FILE] | declares\n"); return 2; } @@ -85,4 +128,4 @@ if (require.main === module) { process.stderr.write(`pool-choice: error: ${e.stack || e.message}\n`); process.exit(EX_ERROR); }); -} else module.exports = { POOLS, declaresPool, choosePool }; +} else module.exports = { POOLS, TOIL_POOL, TOIL_KINDS, declaresPool, choosePool, toilPool }; diff --git a/tests/bot-runs.sh b/tests/bot-runs.sh index c769624..8ea10c5 100755 --- a/tests/bot-runs.sh +++ b/tests/bot-runs.sh @@ -201,6 +201,11 @@ case "${method} ${path%%\?*}" in test -e "${store}/dispatch.yml" || fail_http 404 "Not Found" cat "${store}/dispatch.yml" ;; + # The caller's pools, if it has any: $FAKE_GH/pools.json. + "GET repos/${repo}/contents/.github/agentic-job/pools.json") + test -e "${store}/pools.json" || fail_http 404 "Not Found" + cat "${store}/pools.json" + ;; # A target repository: private-org's are private, gone's don't exist. "GET repos/"*/*) [[ "${path}" =~ ^repos/[^/]+/[^/]+$ ]] || { echo "fake gh: unexpected call: ${method} ${path}" 1>&2; exit 1; } @@ -1476,8 +1481,28 @@ test_agentic_pool() { auto|40|10|openai|^pool: openai: furthest claude|90|10|claude|^pool: claude \(--pool\)$ openai|10|90|openai|^pool: openai \(--pool\)$ +mistral|10|40|mistral|^pool: mistral \(--pool\)$ none|10|40||^$ CASES + # A triage run is toil: auto sends it to the mistral pool where the + # caller's pools.json has one, whatever the paced pools read, and + # chooses by pace as before where it has none or for another kind. + # pools.json | kind | pool sent | line on stderr + while IFS='|' read -r pools kind want line; do + pool_world 40 10 + rm -f "${FAKE_GH}/pools.json" + test -z "${pools}" || echo "${pools}" >"${FAKE_GH}/pools.json" + err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --kind "${kind}" --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out") || fail "dispatch failed: ${err}" + expect_lines "${err}" "${line}" + expect_eq "$(tail -n +2 "${WORK}/out" | jq -r '.inputs.pool // ""')" "${want}" "pool sent for ${kind} with pools '${pools}'" + done <<'CASES' +{"claude": {}, "mistral": {}}|triage|mistral|^pool: mistral: triage is toil, and the caller's pools.json has mistral$ +{"claude": {}, "mistral": {}}|fix|openai|^pool: openai: furthest under its pace +{"claude": {}, "openai": {}}|triage|openai|^pool: openai: furthest under its pace +|triage|openai|^pool: openai: furthest under its pace +not json|triage|openai|^pool: openai: furthest under its pace +CASES + rm -f "${FAKE_GH}/pools.json" # A caller that declares no pool input gets today's body, and says so. rm "${FAKE_GH}/dispatch.yml" err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out") @@ -1504,8 +1529,8 @@ CASES rm "${FAKE_GH}/dispatch.yml" # A bad value is refused before anything is read or dispatched. : >"${FAKE_GH}/calls" - out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool mistral "${WORK}/brief" 2>&1) && fail 'accepted a bad pool' - expect_lines "${out}" '--pool must be claude, openai, auto or none' + out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool gemini "${WORK}/brief" 2>&1) && fail 'accepted a bad pool' + expect_lines "${out}" '--pool must be claude, openai, mistral, auto or none' expect_eq "$(calls .)" 0 'API calls for a bad pool' # A real dispatch sends the chosen pool. pool_world 40 10 diff --git a/tests/pools.test.js b/tests/pools.test.js index 8496a73..0d08460 100644 --- a/tests/pools.test.js +++ b/tests/pools.test.js @@ -222,6 +222,25 @@ test("pool-choice: the pool with the largest margin under its pace", () => { assert.equal(line({ claude: p(40, 48), openai: p(10, 48) }), "openai: furthest under its pace"); }); +test("pool-choice: toil goes to the mistral pool where the caller has one", () => { + const both = { claude: {}, openai: {}, mistral: { agent: "vibe" } }; + // kind | the caller's pools.json | pool (null: choose by pace) + const cases = [ + ["triage", both, "mistral"], + ["triage", { claude: {}, openai: {} }, null], + ["triage", null, null], + ["triage", "mistral", null], + ["triage", ["mistral"], null], + ...["implement", "review", "research", "fix", "toString", null, undefined].map((kind) => [kind, both, null]), + ]; + for (const [kind, callerPools, want] of cases) { + assert.equal(choice.toilPool(kind, callerPools)?.pool ?? null, want, `${kind} ${JSON.stringify(callerPools)}`); + } + assert.equal(choice.toilPool("triage", both).line, "mistral: triage is toil, and the caller's pools.json has mistral"); + // The toil pool is not a paced one: choosePool never names it. + assert.ok(!choice.POOLS.includes(choice.TOIL_POOL)); +}); + test("pool-choice: whether a dispatch.yml declares the pool input", () => { const wf = (inputs, extra = "") => `name: Dispatch\non:\n workflow_dispatch:\n inputs:\n${inputs}${extra}`; const cases = [