feat(benchmark): untrusted candidate sandbox policy and fail-closed run verdict (#9264)

packages/loopover-engine/src/calibration/candidate-sandbox.ts

@@ -0,0 +1,171 @@
+// Untrusted candidate-agent sandbox policy (#9264, harness #9216, epic #8534) — the pure half.
+//
+// Candidate agents are untrusted BY CONSTRUCTION: arbitrary third-party code, submitted to be scored. Two
+// facts drive everything here.
+//
+//   ATTESTATION IS NOT A SANDBOX. Attestation proves WHAT ran; it does nothing to constrain what that code
+//   can reach while running. So every property below must hold with or without SNP hardware, and none of
+//   them may be skipped because attestation is present. They are enforced by the container runtime, which
+//   exists today, rather than waiting on the TEE fleet (#8535/#8536).
+//
+//   THE NETWORK IS THE WHOLE BALLGAME. A candidate agent that can reach the network can simply fetch the
+//   realized future — the very outcomes it is being asked to predict — and score perfectly while having
+//   learned nothing. #9259 leak-proofs the DATA; this module leak-proofs the RUNTIME. Both are required:
+//   a perfect snapshot handed to a networked process is not a benchmark.
+//
+// This module is pure policy + honest labeling. It does not spawn anything: it produces the exact argument
+// list a runner must use, and it decides what a completed run is allowed to CLAIM. Keeping those two pure
+// means both are testable without a container runtime, and means the CLI cannot quietly weaken a limit — it
+// has no limits of its own to weaken.
+//
+// ── THE LIMITS, AND WHY EACH ONE ─────────────────────────────────────────────────────────────────────
+//   network: none        — see above. The single most important control here.
+//   read-only rootfs     — a candidate cannot persist anything between work units, so it cannot accumulate
+//                          state across a benchmark it is supposed to answer one unit at a time.
+//   task input read-only — the frozen snapshot is mounted read-only, so an agent cannot edit the question.
+//   no host filesystem   — no bind mounts beyond the read-only task input; nothing of ours is reachable.
+//   non-root             — the runner image already creates uid 10001; running as it means a container
+//                          escape lands as an unprivileged user rather than root on the host.
+//   memory / cpu / wall  — bounded so one submission cannot deny service to the queue. Wall-clock matters
+//                          most: an agent that never returns would otherwise hold a slot forever.
+//   no new privileges    — blocks setuid escalation inside the container.
+//   all capabilities dropped — a scoring run needs none of them.
+
+/** Defaults, chosen to be generous for honest work and far too small to be useful for anything else. */
+export const DEFAULT_CANDIDATE_LIMITS = {
+  memoryMb: 2048,
+  cpus: 2,
+  wallClockSeconds: 600,
+  pidsLimit: 256,
+} as const;
+
+export type CandidateResourceLimits = {
+  memoryMb: number;
+  cpus: number;
+  wallClockSeconds: number;
+  pidsLimit: number;
+};
+
+export type CandidateSandboxSpec = {
+  image: string;
+  /** Host path of the frozen snapshot (#9259). Mounted READ-ONLY at {@link CANDIDATE_TASK_MOUNT}. */
+  taskInputPath: string;
+  /** Host path the run writes its proposals to. The ONLY writable mount. */
+  outputPath: string;
+  limits?: Partial<CandidateResourceLimits> | undefined;
+};
+
+/** Where the frozen snapshot appears inside the container. Fixed, so a candidate agent's entrypoint can be
+ *  written against a stable path and the runner never has to pass a configurable one it could be tricked
+ *  into pointing elsewhere. */
+export const CANDIDATE_TASK_MOUNT = "/task/snapshot.json";
+export const CANDIDATE_OUTPUT_MOUNT = "/out/proposals.json";
+
+export function resolveCandidateLimits(overrides?: Partial<CandidateResourceLimits> | undefined): CandidateResourceLimits {
+  const limits = { ...DEFAULT_CANDIDATE_LIMITS, ...(overrides ?? {}) };
+  // A non-positive or non-finite limit is a caller bug that would REMOVE a bound rather than set one, so it
+  // falls back to the default instead of being honored. Failing open on a resource bound is the one
+  // direction that cannot be allowed here.
+  const sane = (value: number, fallback: number) => (Number.isFinite(value) && value > 0 ? value : fallback);
+  return {
+    memoryMb: sane(limits.memoryMb, DEFAULT_CANDIDATE_LIMITS.memoryMb),
+    cpus: sane(limits.cpus, DEFAULT_CANDIDATE_LIMITS.cpus),
+    wallClockSeconds: sane(limits.wallClockSeconds, DEFAULT_CANDIDATE_LIMITS.wallClockSeconds),
+    pidsLimit: sane(limits.pidsLimit, DEFAULT_CANDIDATE_LIMITS.pidsLimit),
+  };
+}
+
+/**
+ * The exact container arguments that enforce every property in this module's header.
+ *
+ * Returned as an array rather than a shell string on purpose: a string would have to be quoted by the
+ * caller, and a snapshot path containing a space or a quote would then be a command-injection vector from
+ * data we do not control. An argv array has no such seam.
+ */
+export function candidateSandboxArgs(spec: CandidateSandboxSpec): string[] {
+  const limits = resolveCandidateLimits(spec.limits);
+  return [
+    "run",
+    "--rm",
+    // The control that matters most: no egress, so the realized future is unreachable.
+    "--network",
+    "none",
+    "--read-only",
+    "--security-opt",
+    "no-new-privileges",
+    "--cap-drop",
+    "ALL",
+    "--user",
+    "10001:10001",
+    "--pids-limit",
+    String(limits.pidsLimit),
+    "--memory",
+    `${limits.memoryMb}m`,
+    "--cpus",
+    String(limits.cpus),
+    // The task is mounted read-only: an agent may read the question and may not edit it.
+    "--mount",
+    `type=bind,source=${spec.taskInputPath},target=${CANDIDATE_TASK_MOUNT},readonly`,
+    // The single writable path, and it is a bind of one file, not a directory of ours.
+    "--mount",
+    `type=bind,source=${spec.outputPath},target=${CANDIDATE_OUTPUT_MOUNT}`,
+    spec.image,
+  ];
+}
+
+/** What a runner observed about the isolation it actually established, as opposed to what it intended. */
+export type EstablishedIsolation = {
+  networkDisabled: boolean;
+  rootfsReadOnly: boolean;
+  taskInputReadOnly: boolean;
+  nonRoot: boolean;
+  limitsApplied: boolean;
+  /** A structurally-valid attestation envelope was assembled AND its technology is genuine SNP/TDX
+   *  hardware — not the sample attester. The runner decides this; this module only labels it. */
+  genuineAttestation: boolean;
+};
+
+export type CandidateRunVerdict =
+  | { scoreable: true; trustTier: "attested" | "reproducible" }
+  | { scoreable: false; trustTier: null; failures: string[] };
+
+/**
+ * Decide whether a completed candidate run may be SCORED, and what it may claim.
+ *
+ * FAIL-CLOSED, matching the attested-run path's own posture: a run that could not establish its claimed
+ * isolation does not score at all. It is not scored-and-flagged, and not scored at a lower tier — a
+ * networked agent's score is not a weaker measurement of the same thing, it is a measurement of something
+ * else entirely, and admitting it at any tier would put a fabricated number on the leaderboard.
+ *
+ * The tier is read from what was ESTABLISHED, never from what was configured: `attested` requires genuine
+ * hardware attestation, and everything else is `reproducible`. There is deliberately no path by which
+ * "the plumbing supports attestation" becomes an `attested` label.
+ */
+export function decideCandidateRunVerdict(established: EstablishedIsolation): CandidateRunVerdict {
+  const failures: string[] = [];
+  if (!established.networkDisabled) failures.push("network egress was not disabled — a candidate could fetch the realized future");
+  if (!established.rootfsReadOnly) failures.push("root filesystem was not read-only — a candidate could persist state across work units");
+  if (!established.taskInputReadOnly) failures.push("task input was not mounted read-only — a candidate could edit the question");
+  if (!established.nonRoot) failures.push("container ran as root");
+  if (!established.limitsApplied) failures.push("resource limits were not applied");
+  if (failures.length > 0) return { scoreable: false, trustTier: null, failures };
+  return { scoreable: true, trustTier: established.genuineAttestation ? "attested" : "reproducible" };
+}
+
+/** Render the limits as documentation rows — the issue asks for the limits AND the reasoning to be
+ *  documented, and generating that from the same constants the runner uses is what keeps the doc from
+ *  drifting away from what is actually enforced. */
+export function describeCandidateLimits(limits: CandidateResourceLimits = resolveCandidateLimits()): Array<{ limit: string; value: string; why: string }> {
+  return [
+    { limit: "network", value: "none", why: "A candidate that can reach the network can fetch the realized future and score perfectly without predicting anything." },
+    { limit: "rootfs", value: "read-only", why: "Prevents a candidate from persisting state between work units it is meant to answer independently." },
+    { limit: "task input", value: "read-only bind", why: "A candidate may read the question and may not edit it." },
+    { limit: "user", value: "10001:10001", why: "Non-root, so a container escape lands unprivileged rather than as root on the host." },
+    { limit: "capabilities", value: "all dropped", why: "A scoring run needs none of them." },
+    { limit: "privileges", value: "no-new-privileges", why: "Blocks setuid escalation inside the container." },
+    { limit: "memory", value: `${limits.memoryMb} MiB`, why: "Bounded so one submission cannot exhaust the host and deny service to the queue." },
+    { limit: "cpus", value: String(limits.cpus), why: "Bounded for the same reason, and so scoring throughput stays predictable." },
+    { limit: "wall clock", value: `${limits.wallClockSeconds}s`, why: "An agent that never returns would otherwise hold a queue slot forever." },
+    { limit: "pids", value: String(limits.pidsLimit), why: "Caps fork-bomb style resource exhaustion inside the container." },
+  ];
+}

packages/loopover-engine/src/index.ts

@@ -183,6 +183,7 @@ export * from "./calibration/provider-track-record.js";
 export * from "./calibration/reliability-curve.js";
 export * from "./calibration/attestation-envelope.js";
 export * from "./calibration/attester.js";
+export * from "./calibration/candidate-sandbox.js";
 export * from "./calibration/benchmark-proposal.js";
 export * from "./calibration/benchmark-ground-truth.js";
 export {

packages/loopover-engine/test/candidate-sandbox.test.ts

@@ -0,0 +1,136 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  CANDIDATE_OUTPUT_MOUNT,
+  CANDIDATE_TASK_MOUNT,
+  candidateSandboxArgs,
+  decideCandidateRunVerdict,
+  DEFAULT_CANDIDATE_LIMITS,
+  describeCandidateLimits,
+  resolveCandidateLimits,
+  type EstablishedIsolation,
+} from "../dist/index.js";
+
+// #9264 (harness #9216, epic #8534): candidate agents are untrusted by construction. Two properties carry
+// the weight here and both are asserted directly: the sandbox args really do encode every claimed control
+// (attestation is not a sandbox, so these must hold with or without SNP hardware), and the run verdict is
+// FAIL-CLOSED — a run that could not establish its isolation does not score at any tier.
+
+const SPEC = { image: "loopover/replay-runner:pinned", taskInputPath: "/host/snapshot.json", outputPath: "/host/out.json" };
+
+/** Everything established — the baseline a test then breaks one property at a time from. */
+const FULLY_ISOLATED: EstablishedIsolation = {
+  networkDisabled: true,
+  rootfsReadOnly: true,
+  taskInputReadOnly: true,
+  nonRoot: true,
+  limitsApplied: true,
+  genuineAttestation: false,
+};
+
+/** Adjacent argv pair lookup — the args are a flat array, so a flag's value is the element after it. */
+function valueOf(args: readonly string[], flag: string): string | undefined {
+  const index = args.indexOf(flag);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+test("REGRESSION: the sandbox args encode EVERY claimed control, network first", () => {
+  const args = candidateSandboxArgs(SPEC);
+  // The control that matters most: a candidate that can reach the network fetches the realized future.
+  assert.equal(valueOf(args, "--network"), "none");
+  assert.ok(args.includes("--read-only"), "root filesystem must be read-only");
+  assert.equal(valueOf(args, "--security-opt"), "no-new-privileges");
+  assert.equal(valueOf(args, "--cap-drop"), "ALL");
+  assert.equal(valueOf(args, "--user"), "10001:10001");
+  assert.equal(valueOf(args, "--memory"), `${DEFAULT_CANDIDATE_LIMITS.memoryMb}m`);
+  assert.equal(valueOf(args, "--cpus"), String(DEFAULT_CANDIDATE_LIMITS.cpus));
+  assert.equal(valueOf(args, "--pids-limit"), String(DEFAULT_CANDIDATE_LIMITS.pidsLimit));
+  // The image is the last argument, so nothing this function adds can be parsed as the image name.
+  assert.equal(args[args.length - 1], SPEC.image);
+});
+
+test("REGRESSION: the task input is mounted READ-ONLY and the output is the only writable path", () => {
+  const args = candidateSandboxArgs(SPEC);
+  const mounts = args.filter((_, index) => args[index - 1] === "--mount");
+  const taskMount = mounts.find((mount) => mount.includes(CANDIDATE_TASK_MOUNT));
+  const outputMount = mounts.find((mount) => mount.includes(CANDIDATE_OUTPUT_MOUNT));
+  assert.ok(taskMount?.endsWith(",readonly"), `task mount must be readonly, got ${taskMount}`);
+  assert.ok(taskMount?.includes(`source=${SPEC.taskInputPath}`));
+  // Exactly one writable mount, and it is a single output file rather than a directory of ours.
+  assert.ok(outputMount && !outputMount.includes("readonly"), "output must be writable");
+  assert.equal(mounts.filter((mount) => !mount.includes("readonly")).length, 1);
+  // No host filesystem beyond those two.
+  assert.equal(mounts.length, 2);
+});
+
+test("the args are an argv ARRAY, so a path containing shell metacharacters is inert", () => {
+  // A snapshot path is data we do not fully control. As a shell string this would be an injection seam;
+  // as argv elements it is just a (weird) path.
+  const hostile = { ...SPEC, taskInputPath: "/host/a b'; rm -rf /; echo '.json" };
+  const args = candidateSandboxArgs(hostile);
+  const taskMount = args.find((arg) => arg.includes(CANDIDATE_TASK_MOUNT));
+  assert.ok(taskMount?.includes(hostile.taskInputPath), "the path travels verbatim as one argv element");
+  // It is ONE element, not split into several — nothing became a separate flag.
+  assert.equal(args.filter((arg) => arg === "rm").length, 0);
+});
+
+test("INVARIANT: a non-positive or non-finite limit falls back to the default — a bound never fails OPEN", () => {
+  for (const bad of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
+    const resolved = resolveCandidateLimits({ memoryMb: bad, cpus: bad, wallClockSeconds: bad, pidsLimit: bad });
+    assert.deepEqual(resolved, { ...DEFAULT_CANDIDATE_LIMITS }, `limit ${String(bad)} must not remove the bound`);
+  }
+  // A genuine override is honored in both directions.
+  const tightened = resolveCandidateLimits({ memoryMb: 512, wallClockSeconds: 60 });
+  assert.equal(tightened.memoryMb, 512);
+  assert.equal(tightened.wallClockSeconds, 60);
+  assert.equal(tightened.cpus, DEFAULT_CANDIDATE_LIMITS.cpus, "unspecified limits keep their default");
+  // Absent overrides resolve to the documented defaults.
+  assert.deepEqual(resolveCandidateLimits(), { ...DEFAULT_CANDIDATE_LIMITS });
+  assert.deepEqual(resolveCandidateLimits(undefined), { ...DEFAULT_CANDIDATE_LIMITS });
+});
+
+test("REGRESSION (fail-closed): a run missing ANY isolation property does not score, at any tier", () => {
+  const properties = ["networkDisabled", "rootfsReadOnly", "taskInputReadOnly", "nonRoot", "limitsApplied"] as const;
+  for (const property of properties) {
+    const verdict = decideCandidateRunVerdict({ ...FULLY_ISOLATED, [property]: false });
+    assert.equal(verdict.scoreable, false, `${property} missing must not score`);
+    // Not scored-at-a-lower-tier: there is no tier at all, because the number would measure something else.
+    assert.equal(verdict.trustTier, null);
+    assert.ok(verdict.scoreable === false && verdict.failures.length === 1, `${property} should name exactly one failure`);
+  }
+  // Every failure is named at once, so an operator fixes the runner in one pass.
+  const broken = decideCandidateRunVerdict({
+    networkDisabled: false,
+    rootfsReadOnly: false,
+    taskInputReadOnly: false,
+    nonRoot: false,
+    limitsApplied: false,
+    genuineAttestation: true, // even WITH attestation — attestation is not a sandbox
+  });
+  assert.equal(broken.scoreable, false);
+  assert.ok(broken.scoreable === false && broken.failures.length === 5);
+  assert.ok(broken.scoreable === false && broken.failures[0]?.includes("realized future"));
+});
+
+test("REGRESSION: the tier comes from what was ESTABLISHED — 'attested' is never assumed from plumbing", () => {
+  const unattested = decideCandidateRunVerdict(FULLY_ISOLATED);
+  assert.deepEqual(unattested, { scoreable: true, trustTier: "reproducible" });
+  const attested = decideCandidateRunVerdict({ ...FULLY_ISOLATED, genuineAttestation: true });
+  assert.deepEqual(attested, { scoreable: true, trustTier: "attested" });
+  // A fully-isolated run WITHOUT genuine hardware is honestly `reproducible`, never upgraded because the
+  // rest of the sandbox was perfect.
+  assert.notEqual(unattested.trustTier, "attested");
+});
+
+test("the documented limits are generated FROM the enforced constants, so the doc cannot drift", () => {
+  const rows = describeCandidateLimits();
+  assert.ok(rows.length >= 10, "every control is documented");
+  const memory = rows.find((row) => row.limit === "memory");
+  assert.equal(memory?.value, `${DEFAULT_CANDIDATE_LIMITS.memoryMb} MiB`);
+  // Every row carries a reason, not just a value — the issue asks for the reasoning, not a table of numbers.
+  for (const row of rows) assert.ok(row.why.length > 20, `${row.limit} needs a real rationale`);
+  // A tightened policy is described with the TIGHTENED numbers, proving the generation is real.
+  const tightened = describeCandidateLimits(resolveCandidateLimits({ memoryMb: 256 }));
+  assert.equal(tightened.find((row) => row.limit === "memory")?.value, "256 MiB");
+});