Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions bin/bot-runs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ fi
readonly RUNS_REPO RUNS_WORKFLOW RUNS_REF
POOL_CHOICE=${BOT_RUNS_POOL_CHOICE:-$(dirname "$(readlink -f "$0")")/../lib/pool-choice.js}
readonly POOL_CHOICE
# The pools a caller's dispatch.yml takes, in its repository.
readonly CALLER_POOLS_PATH=.github/agentic-job/pools.json
readonly POOL_USAGE='claude, openai, mistral, auto or none'
readonly SUMMARY_ARTIFACT=agent-run
readonly TRANSCRIPT_ARTIFACT=agent-transcript
# What a run hands back, as gh-aw's safe outputs: outputs.jsonl, a patch and
Expand Down Expand Up @@ -167,13 +170,15 @@ docs/devspace-agent-runs.md in homegit.
Commands:
dispatch --backend agentic-job --caller OWNER/REPO --repo OWNER/REPO
--item NUMBER --kind implement|review|triage|research|fix
[--pool claude|openai|auto|none] [--dry-run] BRIEF
[--pool claude|openai|mistral|auto|none] [--dry-run] BRIEF
Use the shipped dispatch.yml on main. NUMBER is the target issue
(review: PR), not a board id. The caller owns model, limits and
preamble; overrides are refused, not silently dropped. These runs
apply themselves. --pool picks the inference pool: auto (the
default) the one furthest under its pace by bot-capacity's readings,
or none if there are none; the choice is printed. The pool input is
default) mistral for a triage run where the caller's pools.json has
it, else the one of claude and openai furthest under its pace by
bot-capacity's readings, or none if there are none; the choice is
printed. The pool input is
sent only if the caller's dispatch.yml on its default branch declares
it. Readers also take --backend agentic-job --caller.
dispatch --item ITEM --repo OWNER/REPO [--base REF] [--agent AGENT]
Expand Down Expand Up @@ -880,17 +885,19 @@ caller_declares_pool() {
node "${POOL_CHOICE}" declares <<<"${yaml}" || { test $? -eq 1 && return 1; return 2; }
}

# choose_pool POOL: sets chosen_pool ("" for none) from --pool POOL, saying
# on stderr which and why in one line. auto picks the pool furthest under its
# pace, and sends none to a caller that doesn't declare the input or when
# choose_pool POOL KIND: sets chosen_pool ("" for none) from --pool POOL,
# saying on stderr which and why in one line. auto picks the toil pool for a
# KIND that is toil, where the caller's pools.json has it (lib/pool-choice.js;
# a file that can't be read is one without it), else the pool furthest under
# its pace, and sends none to a caller that doesn't declare the input or when
# that can't be told; only an explicit pool is refused for want of the input.
choose_pool() {
local pool=$1 choice rc=0
local pool=$1 kind=$2 choice rc=0
chosen_pool=""
case "${pool}" in
none) return ;;
claude|openai|auto) ;;
*) fatal "--pool must be claude, openai, auto or none" ;;
claude|openai|mistral|auto) ;;
*) fatal "--pool must be ${POOL_USAGE}" ;;
esac
require_cmd gh "install the GitHub CLI (https://cli.github.com) and authenticate it"
require_cmd node "install Node.js 22 or newer"
Expand All @@ -911,7 +918,9 @@ choose_pool() {
echo "pool: ${pool} (--pool)" >&2
return
fi
if ! choice=$(node "${POOL_CHOICE}" choose); then
ghapi -H 'Accept: application/vnd.github.raw' "repos/${RUNS_REPO}/contents/${CALLER_POOLS_PATH}" >"${scratch}/caller-pools.json" || rc=$?
pass_ratelimit "${rc}"
if ! choice=$(node "${POOL_CHOICE}" choose --kind "${kind}" --caller-pools "${scratch}/caller-pools.json"); then
warn "cannot read the pools' pace; sending no pool input"
return
fi
Expand Down Expand Up @@ -946,12 +955,12 @@ cmd_dispatch_agentic_job() {
fatal "--item must be a positive safe issue/PR number"
fi
[[ "${kind}" =~ ^(implement|review|triage|research|fix)$ ]] || fatal "--kind must be implement, review, triage, research or fix"
[[ "${pool}" =~ ^(claude|openai|auto|none)$ ]] || fatal "--pool must be claude, openai, auto or none"
[[ "${pool}" =~ ^(claude|openai|mistral|auto|none)$ ]] || fatal "--pool must be ${POOL_USAGE}"
test -n "${brief_file}" || fatal "missing BRIEF (a file, or - for stdin)"
if test "${brief_file}" = -; then cat >"${scratch}/brief";
else test -r "${brief_file}" || fatal "cannot read ${brief_file}"; cp "${brief_file}" "${scratch}/brief"; fi
grep -q '[^[:space:]]' "${scratch}/brief" || fatal "the brief is empty"
choose_pool "${pool}"
choose_pool "${pool}" "${kind}"
body=$(jq -nc --arg ref "${RUNS_REF}" --arg repo "${repo}" --arg item "${item}" --arg kind "${kind}" --arg pool "${chosen_pool}" --rawfile task "${scratch}/brief" \
'{ref: $ref, return_run_details: true, inputs: ({repo: $repo, item: $item, kind: $kind, task: ($task | rtrimstr("\n"))} + (if $pool == "" then {} else {pool: $pool} end))}')
test "$(jq '.inputs | tojson | length' <<<"${body}")" -le "${MAX_INPUTS_LEN}" || fatal "dispatch inputs exceed ${MAX_INPUTS_LEN}; shorten the brief"
Expand Down
14 changes: 9 additions & 5 deletions docs/devspace-agent-runs.md
Original file line number Diff line number Diff line change
Expand Up @@ -648,17 +648,21 @@ caller configuration. Explicit overrides are refused rather than dropped or
sent as unknown inputs. This is a remaining caller-interface gap before
per-run model/limit selection.

`--pool claude|openai|auto|none` picks the inference pool, a closed set that
`--pool claude|openai|mistral|auto|none` picks the inference pool, a closed set that
the caller maps to its own agent and model; an empty `pool` leaves the choice
to the caller's repository variables. `none` sends no input. `auto`, the
default, asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the
pool with the largest margin, what its pace allows now minus what it has used
to the caller's repository variables. `none` sends no input. `mistral` is the
pool for toil and has no pace: `auto`, the default, picks it for `--kind
triage` when the caller's `.github/agentic-job/pools.json` has a `mistral`
entry, printing `pool: mistral: triage is toil, and the caller's pools.json
has mistral`. For any other kind, or a caller without that entry, `auto`
asks the readings `bot-capacity` paces (`lib/pools.js`) and picks the
pool of `claude` and `openai` with the largest margin, what its pace allows now minus what it has used
(a pool that holds new work only when neither is free; one without a reading
never over one with); with no reading of either, it warns and sends none.
Whatever the value, the input is sent only when `dispatch.yml` on the caller's
default branch declares a `pool` input, so a caller without it keeps working:
`auto` then sends none (as it does when the chooser or the file cannot be
read, with a warning), and an explicit `claude` or `openai` is refused. The
read, with a warning), and an explicit `claude`, `openai` or `mistral` is refused. The
choice is printed on stderr in one line, for example `pool: openai: furthest
under its pace (claude 40% used of 48% allowed, openai 10% used of 48%
allowed)`, or `pool: claude: openai is over its pace (...)` when the other
Expand Down
55 changes: 49 additions & 6 deletions lib/pool-choice.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,24 @@
// Which inference pool an agentic-job dispatch asks for (bot-runs dispatch
// --pool): the pool furthest under its pace, from the same readings
// bin/bot-capacity paces, and whether the caller's dispatch.yml takes a
// --pool): the toil pool for a kind of run that is toil, where the caller
// has one, else the pool furthest under its pace, from the same readings
// bin/bot-capacity paces; and whether the caller's dispatch.yml takes a
// `pool` input at all. Node's standard library only.
// pool-choice.js choose prints {pool, line} (pool null: none to ask for)
// pool-choice.js choose [--kind KIND] [--caller-pools FILE]
// prints {pool, line} (pool null: none to ask
// for); FILE is the caller's pools.json
// pool-choice.js declares reads a dispatch.yml on stdin; exit 0 if it
// declares the pool input, 1 if not
"use strict";

const path = require("node:path");

// The paced pools: auto chooses between these by their readings.
const POOLS = ["claude", "openai"];
// The pool for toil, runs a small model does well enough, and the kinds
// that are toil. It has no reading and no pace: a run on it is one not
// spent from a paced pool.
const TOIL_POOL = "mistral";
const TOIL_KINDS = ["triage"];
const INPUT = "pool";
// Exit status of a failure, apart from declares' 1 for "no".
const EX_ERROR = 3;
Expand Down Expand Up @@ -57,6 +66,37 @@ function choosePool(paces) {
return { pool: best, line: `${best}: ${why} (${POOLS.map(detail).join(", ")})` };
}

// toilPool(kind, callerPools): {pool, line} when a run of KIND is toil and
// the caller's pools.json (parsed; null where it could not be read) has the
// toil pool, else null: the choice is then by pace, as for any other kind.
function toilPool(kind, callerPools) {
if (!TOIL_KINDS.includes(kind)) return null;
if (!callerPools || typeof callerPools !== "object" || !Object.hasOwn(callerPools, TOIL_POOL)) return null;
return { pool: TOIL_POOL, line: `${TOIL_POOL}: ${kind} is toil, and the caller's pools.json has ${TOIL_POOL}` };
}

// readCallerPools(file): the caller's pools.json, or null without a file or
// where it is not JSON: a caller without one has no toil pool.
function readCallerPools(file) {
if (!file) return null;
try {
return JSON.parse(require("node:fs").readFileSync(file, "utf8"));
} catch {
return null;
}
}

// chooseArgs(argv): {kind, callerPools} of choose's options, each null
// when absent; throws on anything else.
function chooseArgs(argv) {
const opts = { "--kind": null, "--caller-pools": null };
for (let i = 0; i < argv.length; i += 2) {
if (!Object.hasOwn(opts, argv[i]) || argv[i + 1] === undefined) throw new Error(`choose: unexpected argument '${argv[i]}'`);
opts[argv[i]] = argv[i + 1];
}
return { kind: opts["--kind"], callerPools: opts["--caller-pools"] };
}

async function readPaces() {
const capacity = require(path.join(__dirname, "..", "bin", "bot-capacity"));
const pools = require(path.join(__dirname, "pools.js"));
Expand All @@ -73,10 +113,13 @@ async function main(argv) {
return declaresPool(Buffer.concat(chunks).toString("utf8")) ? 0 : 1;
}
if (argv[0] === "choose") {
process.stdout.write(`${JSON.stringify(choosePool(await readPaces()))}\n`);
const { kind, callerPools } = chooseArgs(argv.slice(1));
// Toil first: it needs no reading of the paced pools.
const choice = toilPool(kind, readCallerPools(callerPools)) ?? choosePool(await readPaces());
process.stdout.write(`${JSON.stringify(choice)}\n`);
return 0;
}
process.stderr.write("usage: pool-choice.js choose | declares\n");
process.stderr.write("usage: pool-choice.js choose [--kind KIND] [--caller-pools FILE] | declares\n");
return 2;
}

Expand All @@ -85,4 +128,4 @@ if (require.main === module) {
process.stderr.write(`pool-choice: error: ${e.stack || e.message}\n`);
process.exit(EX_ERROR);
});
} else module.exports = { POOLS, declaresPool, choosePool };
} else module.exports = { POOLS, TOIL_POOL, TOIL_KINDS, declaresPool, choosePool, toilPool };
29 changes: 27 additions & 2 deletions tests/bot-runs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,11 @@ case "${method} ${path%%\?*}" in
test -e "${store}/dispatch.yml" || fail_http 404 "Not Found"
cat "${store}/dispatch.yml"
;;
# The caller's pools, if it has any: $FAKE_GH/pools.json.
"GET repos/${repo}/contents/.github/agentic-job/pools.json")
test -e "${store}/pools.json" || fail_http 404 "Not Found"
cat "${store}/pools.json"
;;
# A target repository: private-org's are private, gone's don't exist.
"GET repos/"*/*)
[[ "${path}" =~ ^repos/[^/]+/[^/]+$ ]] || { echo "fake gh: unexpected call: ${method} ${path}" 1>&2; exit 1; }
Expand Down Expand Up @@ -1476,8 +1481,28 @@ test_agentic_pool() {
auto|40|10|openai|^pool: openai: furthest
claude|90|10|claude|^pool: claude \(--pool\)$
openai|10|90|openai|^pool: openai \(--pool\)$
mistral|10|40|mistral|^pool: mistral \(--pool\)$
none|10|40||^$
CASES
# A triage run is toil: auto sends it to the mistral pool where the
# caller's pools.json has one, whatever the paced pools read, and
# chooses by pace as before where it has none or for another kind.
# pools.json | kind | pool sent | line on stderr
while IFS='|' read -r pools kind want line; do
pool_world 40 10
rm -f "${FAKE_GH}/pools.json"
test -z "${pools}" || echo "${pools}" >"${FAKE_GH}/pools.json"
err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --kind "${kind}" --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out") || fail "dispatch failed: ${err}"
expect_lines "${err}" "${line}"
expect_eq "$(tail -n +2 "${WORK}/out" | jq -r '.inputs.pool // ""')" "${want}" "pool sent for ${kind} with pools '${pools}'"
done <<'CASES'
{"claude": {}, "mistral": {}}|triage|mistral|^pool: mistral: triage is toil, and the caller's pools.json has mistral$
{"claude": {}, "mistral": {}}|fix|openai|^pool: openai: furthest under its pace
{"claude": {}, "openai": {}}|triage|openai|^pool: openai: furthest under its pace
|triage|openai|^pool: openai: furthest under its pace
not json|triage|openai|^pool: openai: furthest under its pace
CASES
rm -f "${FAKE_GH}/pools.json"
# A caller that declares no pool input gets today's body, and says so.
rm "${FAKE_GH}/dispatch.yml"
err=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --dry-run "${WORK}/brief" 2>&1 >"${WORK}/out")
Expand All @@ -1504,8 +1529,8 @@ CASES
rm "${FAKE_GH}/dispatch.yml"
# A bad value is refused before anything is read or dispatched.
: >"${FAKE_GH}/calls"
out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool mistral "${WORK}/brief" 2>&1) && fail 'accepted a bad pool'
expect_lines "${out}" '--pool must be claude, openai, auto or none'
out=$(agentic_runs dispatch --item 1 --repo composefs/composefs-rs --pool gemini "${WORK}/brief" 2>&1) && fail 'accepted a bad pool'
expect_lines "${out}" '--pool must be claude, openai, mistral, auto or none'
expect_eq "$(calls .)" 0 'API calls for a bad pool'
# A real dispatch sends the chosen pool.
pool_world 40 10
Expand Down
19 changes: 19 additions & 0 deletions tests/pools.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,25 @@ test("pool-choice: the pool with the largest margin under its pace", () => {
assert.equal(line({ claude: p(40, 48), openai: p(10, 48) }), "openai: furthest under its pace");
});

test("pool-choice: toil goes to the mistral pool where the caller has one", () => {
const both = { claude: {}, openai: {}, mistral: { agent: "vibe" } };
// kind | the caller's pools.json | pool (null: choose by pace)
const cases = [
["triage", both, "mistral"],
["triage", { claude: {}, openai: {} }, null],
["triage", null, null],
["triage", "mistral", null],
["triage", ["mistral"], null],
...["implement", "review", "research", "fix", "toString", null, undefined].map((kind) => [kind, both, null]),
];
for (const [kind, callerPools, want] of cases) {
assert.equal(choice.toilPool(kind, callerPools)?.pool ?? null, want, `${kind} ${JSON.stringify(callerPools)}`);
}
assert.equal(choice.toilPool("triage", both).line, "mistral: triage is toil, and the caller's pools.json has mistral");
// The toil pool is not a paced one: choosePool never names it.
assert.ok(!choice.POOLS.includes(choice.TOIL_POOL));
});

test("pool-choice: whether a dispatch.yml declares the pool input", () => {
const wf = (inputs, extra = "") => `name: Dispatch\non:\n workflow_dispatch:\n inputs:\n${inputs}${extra}`;
const cases = [
Expand Down
Loading