diff --git a/packages/loopover-engine/src/calibration/candidate-sandbox.ts b/packages/loopover-engine/src/calibration/candidate-sandbox.ts new file mode 100644 index 000000000..512ddaecb --- /dev/null +++ b/packages/loopover-engine/src/calibration/candidate-sandbox.ts @@ -0,0 +1,171 @@ +// Untrusted candidate-agent sandbox policy (#9264, harness #9216, epic #8534) — the pure half. +// +// Candidate agents are untrusted BY CONSTRUCTION: arbitrary third-party code, submitted to be scored. Two +// facts drive everything here. +// +// ATTESTATION IS NOT A SANDBOX. Attestation proves WHAT ran; it does nothing to constrain what that code +// can reach while running. So every property below must hold with or without SNP hardware, and none of +// them may be skipped because attestation is present. They are enforced by the container runtime, which +// exists today, rather than waiting on the TEE fleet (#8535/#8536). +// +// THE NETWORK IS THE WHOLE BALLGAME. A candidate agent that can reach the network can simply fetch the +// realized future — the very outcomes it is being asked to predict — and score perfectly while having +// learned nothing. #9259 leak-proofs the DATA; this module leak-proofs the RUNTIME. Both are required: +// a perfect snapshot handed to a networked process is not a benchmark. +// +// This module is pure policy + honest labeling. It does not spawn anything: it produces the exact argument +// list a runner must use, and it decides what a completed run is allowed to CLAIM. Keeping those two pure +// means both are testable without a container runtime, and means the CLI cannot quietly weaken a limit — it +// has no limits of its own to weaken. +// +// ── THE LIMITS, AND WHY EACH ONE ───────────────────────────────────────────────────────────────────── +// network: none — see above. The single most important control here. +// read-only rootfs — a candidate cannot persist anything between work units, so it cannot accumulate +// state across a benchmark it is supposed to answer one unit at a time. +// task input read-only — the frozen snapshot is mounted read-only, so an agent cannot edit the question. +// no host filesystem — no bind mounts beyond the read-only task input; nothing of ours is reachable. +// non-root — the runner image already creates uid 10001; running as it means a container +// escape lands as an unprivileged user rather than root on the host. +// memory / cpu / wall — bounded so one submission cannot deny service to the queue. Wall-clock matters +// most: an agent that never returns would otherwise hold a slot forever. +// no new privileges — blocks setuid escalation inside the container. +// all capabilities dropped — a scoring run needs none of them. + +/** Defaults, chosen to be generous for honest work and far too small to be useful for anything else. */ +export const DEFAULT_CANDIDATE_LIMITS = { + memoryMb: 2048, + cpus: 2, + wallClockSeconds: 600, + pidsLimit: 256, +} as const; + +export type CandidateResourceLimits = { + memoryMb: number; + cpus: number; + wallClockSeconds: number; + pidsLimit: number; +}; + +export type CandidateSandboxSpec = { + image: string; + /** Host path of the frozen snapshot (#9259). Mounted READ-ONLY at {@link CANDIDATE_TASK_MOUNT}. */ + taskInputPath: string; + /** Host path the run writes its proposals to. The ONLY writable mount. */ + outputPath: string; + limits?: Partial | undefined; +}; + +/** Where the frozen snapshot appears inside the container. Fixed, so a candidate agent's entrypoint can be + * written against a stable path and the runner never has to pass a configurable one it could be tricked + * into pointing elsewhere. */ +export const CANDIDATE_TASK_MOUNT = "/task/snapshot.json"; +export const CANDIDATE_OUTPUT_MOUNT = "/out/proposals.json"; + +export function resolveCandidateLimits(overrides?: Partial | undefined): CandidateResourceLimits { + const limits = { ...DEFAULT_CANDIDATE_LIMITS, ...(overrides ?? {}) }; + // A non-positive or non-finite limit is a caller bug that would REMOVE a bound rather than set one, so it + // falls back to the default instead of being honored. Failing open on a resource bound is the one + // direction that cannot be allowed here. + const sane = (value: number, fallback: number) => (Number.isFinite(value) && value > 0 ? value : fallback); + return { + memoryMb: sane(limits.memoryMb, DEFAULT_CANDIDATE_LIMITS.memoryMb), + cpus: sane(limits.cpus, DEFAULT_CANDIDATE_LIMITS.cpus), + wallClockSeconds: sane(limits.wallClockSeconds, DEFAULT_CANDIDATE_LIMITS.wallClockSeconds), + pidsLimit: sane(limits.pidsLimit, DEFAULT_CANDIDATE_LIMITS.pidsLimit), + }; +} + +/** + * The exact container arguments that enforce every property in this module's header. + * + * Returned as an array rather than a shell string on purpose: a string would have to be quoted by the + * caller, and a snapshot path containing a space or a quote would then be a command-injection vector from + * data we do not control. An argv array has no such seam. + */ +export function candidateSandboxArgs(spec: CandidateSandboxSpec): string[] { + const limits = resolveCandidateLimits(spec.limits); + return [ + "run", + "--rm", + // The control that matters most: no egress, so the realized future is unreachable. + "--network", + "none", + "--read-only", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--user", + "10001:10001", + "--pids-limit", + String(limits.pidsLimit), + "--memory", + `${limits.memoryMb}m`, + "--cpus", + String(limits.cpus), + // The task is mounted read-only: an agent may read the question and may not edit it. + "--mount", + `type=bind,source=${spec.taskInputPath},target=${CANDIDATE_TASK_MOUNT},readonly`, + // The single writable path, and it is a bind of one file, not a directory of ours. + "--mount", + `type=bind,source=${spec.outputPath},target=${CANDIDATE_OUTPUT_MOUNT}`, + spec.image, + ]; +} + +/** What a runner observed about the isolation it actually established, as opposed to what it intended. */ +export type EstablishedIsolation = { + networkDisabled: boolean; + rootfsReadOnly: boolean; + taskInputReadOnly: boolean; + nonRoot: boolean; + limitsApplied: boolean; + /** A structurally-valid attestation envelope was assembled AND its technology is genuine SNP/TDX + * hardware — not the sample attester. The runner decides this; this module only labels it. */ + genuineAttestation: boolean; +}; + +export type CandidateRunVerdict = + | { scoreable: true; trustTier: "attested" | "reproducible" } + | { scoreable: false; trustTier: null; failures: string[] }; + +/** + * Decide whether a completed candidate run may be SCORED, and what it may claim. + * + * FAIL-CLOSED, matching the attested-run path's own posture: a run that could not establish its claimed + * isolation does not score at all. It is not scored-and-flagged, and not scored at a lower tier — a + * networked agent's score is not a weaker measurement of the same thing, it is a measurement of something + * else entirely, and admitting it at any tier would put a fabricated number on the leaderboard. + * + * The tier is read from what was ESTABLISHED, never from what was configured: `attested` requires genuine + * hardware attestation, and everything else is `reproducible`. There is deliberately no path by which + * "the plumbing supports attestation" becomes an `attested` label. + */ +export function decideCandidateRunVerdict(established: EstablishedIsolation): CandidateRunVerdict { + const failures: string[] = []; + if (!established.networkDisabled) failures.push("network egress was not disabled — a candidate could fetch the realized future"); + if (!established.rootfsReadOnly) failures.push("root filesystem was not read-only — a candidate could persist state across work units"); + if (!established.taskInputReadOnly) failures.push("task input was not mounted read-only — a candidate could edit the question"); + if (!established.nonRoot) failures.push("container ran as root"); + if (!established.limitsApplied) failures.push("resource limits were not applied"); + if (failures.length > 0) return { scoreable: false, trustTier: null, failures }; + return { scoreable: true, trustTier: established.genuineAttestation ? "attested" : "reproducible" }; +} + +/** Render the limits as documentation rows — the issue asks for the limits AND the reasoning to be + * documented, and generating that from the same constants the runner uses is what keeps the doc from + * drifting away from what is actually enforced. */ +export function describeCandidateLimits(limits: CandidateResourceLimits = resolveCandidateLimits()): Array<{ limit: string; value: string; why: string }> { + return [ + { limit: "network", value: "none", why: "A candidate that can reach the network can fetch the realized future and score perfectly without predicting anything." }, + { limit: "rootfs", value: "read-only", why: "Prevents a candidate from persisting state between work units it is meant to answer independently." }, + { limit: "task input", value: "read-only bind", why: "A candidate may read the question and may not edit it." }, + { limit: "user", value: "10001:10001", why: "Non-root, so a container escape lands unprivileged rather than as root on the host." }, + { limit: "capabilities", value: "all dropped", why: "A scoring run needs none of them." }, + { limit: "privileges", value: "no-new-privileges", why: "Blocks setuid escalation inside the container." }, + { limit: "memory", value: `${limits.memoryMb} MiB`, why: "Bounded so one submission cannot exhaust the host and deny service to the queue." }, + { limit: "cpus", value: String(limits.cpus), why: "Bounded for the same reason, and so scoring throughput stays predictable." }, + { limit: "wall clock", value: `${limits.wallClockSeconds}s`, why: "An agent that never returns would otherwise hold a queue slot forever." }, + { limit: "pids", value: String(limits.pidsLimit), why: "Caps fork-bomb style resource exhaustion inside the container." }, + ]; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 0ddea29fb..09a2155c0 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -183,6 +183,7 @@ export * from "./calibration/provider-track-record.js"; export * from "./calibration/reliability-curve.js"; export * from "./calibration/attestation-envelope.js"; export * from "./calibration/attester.js"; +export * from "./calibration/candidate-sandbox.js"; export * from "./calibration/benchmark-proposal.js"; export * from "./calibration/benchmark-ground-truth.js"; export { diff --git a/packages/loopover-engine/test/candidate-sandbox.test.ts b/packages/loopover-engine/test/candidate-sandbox.test.ts new file mode 100644 index 000000000..d8ce8f93a --- /dev/null +++ b/packages/loopover-engine/test/candidate-sandbox.test.ts @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + CANDIDATE_OUTPUT_MOUNT, + CANDIDATE_TASK_MOUNT, + candidateSandboxArgs, + decideCandidateRunVerdict, + DEFAULT_CANDIDATE_LIMITS, + describeCandidateLimits, + resolveCandidateLimits, + type EstablishedIsolation, +} from "../dist/index.js"; + +// #9264 (harness #9216, epic #8534): candidate agents are untrusted by construction. Two properties carry +// the weight here and both are asserted directly: the sandbox args really do encode every claimed control +// (attestation is not a sandbox, so these must hold with or without SNP hardware), and the run verdict is +// FAIL-CLOSED — a run that could not establish its isolation does not score at any tier. + +const SPEC = { image: "loopover/replay-runner:pinned", taskInputPath: "/host/snapshot.json", outputPath: "/host/out.json" }; + +/** Everything established — the baseline a test then breaks one property at a time from. */ +const FULLY_ISOLATED: EstablishedIsolation = { + networkDisabled: true, + rootfsReadOnly: true, + taskInputReadOnly: true, + nonRoot: true, + limitsApplied: true, + genuineAttestation: false, +}; + +/** Adjacent argv pair lookup — the args are a flat array, so a flag's value is the element after it. */ +function valueOf(args: readonly string[], flag: string): string | undefined { + const index = args.indexOf(flag); + return index >= 0 ? args[index + 1] : undefined; +} + +test("REGRESSION: the sandbox args encode EVERY claimed control, network first", () => { + const args = candidateSandboxArgs(SPEC); + // The control that matters most: a candidate that can reach the network fetches the realized future. + assert.equal(valueOf(args, "--network"), "none"); + assert.ok(args.includes("--read-only"), "root filesystem must be read-only"); + assert.equal(valueOf(args, "--security-opt"), "no-new-privileges"); + assert.equal(valueOf(args, "--cap-drop"), "ALL"); + assert.equal(valueOf(args, "--user"), "10001:10001"); + assert.equal(valueOf(args, "--memory"), `${DEFAULT_CANDIDATE_LIMITS.memoryMb}m`); + assert.equal(valueOf(args, "--cpus"), String(DEFAULT_CANDIDATE_LIMITS.cpus)); + assert.equal(valueOf(args, "--pids-limit"), String(DEFAULT_CANDIDATE_LIMITS.pidsLimit)); + // The image is the last argument, so nothing this function adds can be parsed as the image name. + assert.equal(args[args.length - 1], SPEC.image); +}); + +test("REGRESSION: the task input is mounted READ-ONLY and the output is the only writable path", () => { + const args = candidateSandboxArgs(SPEC); + const mounts = args.filter((_, index) => args[index - 1] === "--mount"); + const taskMount = mounts.find((mount) => mount.includes(CANDIDATE_TASK_MOUNT)); + const outputMount = mounts.find((mount) => mount.includes(CANDIDATE_OUTPUT_MOUNT)); + assert.ok(taskMount?.endsWith(",readonly"), `task mount must be readonly, got ${taskMount}`); + assert.ok(taskMount?.includes(`source=${SPEC.taskInputPath}`)); + // Exactly one writable mount, and it is a single output file rather than a directory of ours. + assert.ok(outputMount && !outputMount.includes("readonly"), "output must be writable"); + assert.equal(mounts.filter((mount) => !mount.includes("readonly")).length, 1); + // No host filesystem beyond those two. + assert.equal(mounts.length, 2); +}); + +test("the args are an argv ARRAY, so a path containing shell metacharacters is inert", () => { + // A snapshot path is data we do not fully control. As a shell string this would be an injection seam; + // as argv elements it is just a (weird) path. + const hostile = { ...SPEC, taskInputPath: "/host/a b'; rm -rf /; echo '.json" }; + const args = candidateSandboxArgs(hostile); + const taskMount = args.find((arg) => arg.includes(CANDIDATE_TASK_MOUNT)); + assert.ok(taskMount?.includes(hostile.taskInputPath), "the path travels verbatim as one argv element"); + // It is ONE element, not split into several — nothing became a separate flag. + assert.equal(args.filter((arg) => arg === "rm").length, 0); +}); + +test("INVARIANT: a non-positive or non-finite limit falls back to the default — a bound never fails OPEN", () => { + for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + const resolved = resolveCandidateLimits({ memoryMb: bad, cpus: bad, wallClockSeconds: bad, pidsLimit: bad }); + assert.deepEqual(resolved, { ...DEFAULT_CANDIDATE_LIMITS }, `limit ${String(bad)} must not remove the bound`); + } + // A genuine override is honored in both directions. + const tightened = resolveCandidateLimits({ memoryMb: 512, wallClockSeconds: 60 }); + assert.equal(tightened.memoryMb, 512); + assert.equal(tightened.wallClockSeconds, 60); + assert.equal(tightened.cpus, DEFAULT_CANDIDATE_LIMITS.cpus, "unspecified limits keep their default"); + // Absent overrides resolve to the documented defaults. + assert.deepEqual(resolveCandidateLimits(), { ...DEFAULT_CANDIDATE_LIMITS }); + assert.deepEqual(resolveCandidateLimits(undefined), { ...DEFAULT_CANDIDATE_LIMITS }); +}); + +test("REGRESSION (fail-closed): a run missing ANY isolation property does not score, at any tier", () => { + const properties = ["networkDisabled", "rootfsReadOnly", "taskInputReadOnly", "nonRoot", "limitsApplied"] as const; + for (const property of properties) { + const verdict = decideCandidateRunVerdict({ ...FULLY_ISOLATED, [property]: false }); + assert.equal(verdict.scoreable, false, `${property} missing must not score`); + // Not scored-at-a-lower-tier: there is no tier at all, because the number would measure something else. + assert.equal(verdict.trustTier, null); + assert.ok(verdict.scoreable === false && verdict.failures.length === 1, `${property} should name exactly one failure`); + } + // Every failure is named at once, so an operator fixes the runner in one pass. + const broken = decideCandidateRunVerdict({ + networkDisabled: false, + rootfsReadOnly: false, + taskInputReadOnly: false, + nonRoot: false, + limitsApplied: false, + genuineAttestation: true, // even WITH attestation — attestation is not a sandbox + }); + assert.equal(broken.scoreable, false); + assert.ok(broken.scoreable === false && broken.failures.length === 5); + assert.ok(broken.scoreable === false && broken.failures[0]?.includes("realized future")); +}); + +test("REGRESSION: the tier comes from what was ESTABLISHED — 'attested' is never assumed from plumbing", () => { + const unattested = decideCandidateRunVerdict(FULLY_ISOLATED); + assert.deepEqual(unattested, { scoreable: true, trustTier: "reproducible" }); + const attested = decideCandidateRunVerdict({ ...FULLY_ISOLATED, genuineAttestation: true }); + assert.deepEqual(attested, { scoreable: true, trustTier: "attested" }); + // A fully-isolated run WITHOUT genuine hardware is honestly `reproducible`, never upgraded because the + // rest of the sandbox was perfect. + assert.notEqual(unattested.trustTier, "attested"); +}); + +test("the documented limits are generated FROM the enforced constants, so the doc cannot drift", () => { + const rows = describeCandidateLimits(); + assert.ok(rows.length >= 10, "every control is documented"); + const memory = rows.find((row) => row.limit === "memory"); + assert.equal(memory?.value, `${DEFAULT_CANDIDATE_LIMITS.memoryMb} MiB`); + // Every row carries a reason, not just a value — the issue asks for the reasoning, not a table of numbers. + for (const row of rows) assert.ok(row.why.length > 20, `${row.limit} needs a real rationale`); + // A tightened policy is described with the TIGHTENED numbers, proving the generation is real. + const tightened = describeCandidateLimits(resolveCandidateLimits({ memoryMb: 256 })); + assert.equal(tightened.find((row) => row.limit === "memory")?.value, "256 MiB"); +});