From 8823f580a6cd0abd74159a2fa6c86925dd0df757 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:33:07 -0700 Subject: [PATCH 01/16] Declare what a method does, and what its answers must obey A manifest could describe a method's shape but not its meaning: nothing said whether calling it changes the world, what its output must look like, or which properties every answer has to satisfy. The fitness gate needs all three, so methods can now declare effects (read|act), a JSON Schema for their output, metamorphic relations, and a known entity that must appear in a healthy answer. All fields are optional; existing manifests are untouched. --- src/core/manifest-contract.test.ts | 33 ++++++++++++++++++++++++++++++ src/core/types.ts | 21 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 src/core/manifest-contract.test.ts diff --git a/src/core/manifest-contract.test.ts b/src/core/manifest-contract.test.ts new file mode 100644 index 0000000..c27c335 --- /dev/null +++ b/src/core/manifest-contract.test.ts @@ -0,0 +1,33 @@ +/** + * C2 manifest contract fields are optional and preserved. + * Run: pnpm tsx --test src/core/manifest-contract.test.ts + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import type { MethodDeclaration, RelationDeclaration, MethodEffect } from './types.js'; + +test('a legacy method declaration without contract fields is valid', () => { + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + }; + assert.equal(m.effects, undefined); +}); + +test('contract fields round-trip', () => { + const relations: RelationDeclaration[] = [ + { kind: 'subset-on-tighter-filter', field: 'from' }, + { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, + ]; + const effects: MethodEffect = 'read'; + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + effects, + outputSchema: { type: 'array', items: { type: 'object' } }, + relations, + knownEntity: 'Weekly Standup', + }; + assert.equal(m.effects, 'read'); + assert.equal(m.relations?.length, 3); + assert.equal(m.knownEntity, 'Weekly Standup'); +}); diff --git a/src/core/types.ts b/src/core/types.ts index 9619934..cd9da92 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -63,11 +63,32 @@ export type ErrorMessage = AbjectMessage; // Interface Declaration // ============================================================================= +/** C2 contract: what calling a method does to the world. Reads are safe to + * generate and heal autonomously; acts must cross a hand-written gate. */ +export type MethodEffect = 'read' | 'act'; + +/** C2 contract: a metamorphic relation the method's outputs must satisfy. + * Checked by the fitness gate (src/protocol/fitness.ts) — never by an LLM. */ +export interface RelationDeclaration { + kind: 'subset-on-tighter-filter' | 'idempotent' | 'no-duplicates' + | 'sorted-by' | 'non-empty-for-known-entity'; + /** 'sorted-by': output field to be non-decreasing on. + * 'subset-on-tighter-filter': the argument that narrows the result. */ + field?: string; +} + export interface MethodDeclaration { name: string; description: string; parameters: ParameterDeclaration[]; returns?: TypeDeclaration; + /** C2 contract fields — all optional; legacy manifests are untouched. */ + effects?: MethodEffect; + /** JSON Schema the method's return value must validate against. */ + outputSchema?: Record; + relations?: RelationDeclaration[]; + /** A value that must appear somewhere in a healthy output (known-entity probe). */ + knownEntity?: string; } export interface ParameterDeclaration { From f39e4c23dbce9515c3cd6d2ccdb7c24afc457c20 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:35:35 -0700 Subject: [PATCH 02/16] Remember what the world said, so a fix can be held to it A cassette records one exchange: the request an object made, the answer the world gave, and what the object made of it. Capped per method, LRU evicted, credentials stripped at the door. This is the memory the fitness gate replays against every candidate. --- src/protocol/cassette.test.ts | 45 ++++++++++++++++ src/protocol/cassette.ts | 97 +++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 src/protocol/cassette.test.ts create mode 100644 src/protocol/cassette.ts diff --git a/src/protocol/cassette.test.ts b/src/protocol/cassette.test.ts new file mode 100644 index 0000000..6afb936 --- /dev/null +++ b/src/protocol/cassette.test.ts @@ -0,0 +1,45 @@ +/** Run: pnpm tsx --test src/protocol/cassette.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { CassetteStore, redactRequest, CASSETTE_CAP_PER_METHOD, type Cassette } from './cassette.js'; + +function mk(n: number, method = 'listEvents'): Cassette { + return { + method, args: { q: n }, + request: { method: 'GET', url: `https://example.test/events?q=${n}` }, + response: { status: 200, body: [{ id: n }] }, + parsedOutput: [{ id: n }], + recordedAt: n, + }; +} + +test('redactRequest strips credential headers case-insensitively', () => { + const r = redactRequest({ + method: 'GET', url: 'https://example.test/x', + headers: { Authorization: 'Bearer s3cret', 'X-Ok': 'yes', COOKIE: 'a=1', 'set-cookie': 'b=2' }, + }); + assert.deepEqual(r.headers, { 'X-Ok': 'yes' }); +}); + +test('store caps per method with LRU eviction', () => { + const s = new CassetteStore(); + for (let i = 0; i < CASSETTE_CAP_PER_METHOD + 5; i++) s.add(mk(i)); + const kept = s.byMethod('listEvents'); + assert.equal(kept.length, CASSETTE_CAP_PER_METHOD); + assert.equal(kept[0].recordedAt, 5); // 0..4 evicted +}); + +test('matchRequest finds exact url, then host+path fallback', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=1' })); + assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=other' })); + assert.equal(s.matchRequest({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + +test('toJSON/fromJSON round-trips and skips malformed entries', () => { + const s = new CassetteStore([mk(1), mk(2)]); + const back = CassetteStore.fromJSON(JSON.parse(JSON.stringify(s.toJSON()))); + assert.equal(back.all().length, 2); + const dirty = CassetteStore.fromJSON([mk(3), { junk: true }, 42]); + assert.equal(dirty.all().length, 1); +}); diff --git a/src/protocol/cassette.ts b/src/protocol/cassette.ts new file mode 100644 index 0000000..d07da01 --- /dev/null +++ b/src/protocol/cassette.ts @@ -0,0 +1,97 @@ +/** + * Cassette -- the ratchet's memory. + * + * A cassette is one recorded truth: a request the object made, the response + * the world gave, and the parsed answer the object produced from it. The + * fitness gate replays cassettes against every candidate source; a candidate + * that cannot reproduce recorded meaning does not deploy. Requests are + * redacted before storage so a cassette can never leak a credential. + */ + +export interface CassetteRequest { + method: string; + url: string; + headers?: Record; + body?: unknown; +} + +export interface Cassette { + method: string; + args: Record; + request: CassetteRequest; + response: { status: number; body: unknown }; + parsedOutput: unknown; + recordedAt: number; +} + +export const CASSETTE_CAP_PER_METHOD = 20; + +const REDACTED_HEADERS = new Set(['authorization', 'cookie', 'set-cookie']); + +export function redactRequest(req: CassetteRequest): CassetteRequest { + if (!req.headers) return req; + const headers: Record = {}; + for (const [k, v] of Object.entries(req.headers)) { + if (!REDACTED_HEADERS.has(k.toLowerCase())) headers[k] = v; + } + return { ...req, headers }; +} + +function hostPath(url: string): string | undefined { + try { const u = new URL(url); return `${u.host}${u.pathname}`; } catch { return undefined; } +} + +function isCassette(c: unknown): c is Cassette { + if (c === null || typeof c !== 'object') return false; + const x = c as Record; + return typeof x.method === 'string' + && x.args !== null && typeof x.args === 'object' + && x.request !== null && typeof x.request === 'object' + && typeof (x.request as Record).url === 'string' + && x.response !== null && typeof x.response === 'object' + && typeof x.recordedAt === 'number'; +} + +export class CassetteStore { + private cassettes: Cassette[] = []; + + constructor(initial?: Cassette[]) { + for (const c of initial ?? []) this.add(c); + } + + add(c: Cassette): void { + this.cassettes.push({ ...c, request: redactRequest(c.request) }); + const forMethod = this.cassettes.filter(x => x.method === c.method); + if (forMethod.length > CASSETTE_CAP_PER_METHOD) { + const evict = forMethod + .sort((a, b) => a.recordedAt - b.recordedAt) + .slice(0, forMethod.length - CASSETTE_CAP_PER_METHOD); + this.cassettes = this.cassettes.filter(x => !evict.includes(x)); + } + } + + byMethod(method: string): Cassette[] { + return this.cassettes + .filter(c => c.method === method) + .sort((a, b) => a.recordedAt - b.recordedAt); + } + + all(): Cassette[] { return [...this.cassettes]; } + + matchRequest(req: CassetteRequest): Cassette | undefined { + const exact = this.cassettes.find( + c => c.request.method === req.method && c.request.url === req.url); + if (exact) return exact; + const hp = hostPath(req.url); + if (!hp) return undefined; + return this.cassettes.find( + c => c.request.method === req.method && hostPath(c.request.url) === hp); + } + + toJSON(): Cassette[] { return this.all(); } + + static fromJSON(json: unknown): CassetteStore { + const arr = Array.isArray(json) ? json.filter(isCassette) : []; + return new CassetteStore(arr); + } +} From 26e3fc92944a487ecb9c4f54c66ab88c5f85bace Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:38:47 -0700 Subject: [PATCH 03/16] Judge a candidate by replaying what the world already said evaluate() is the gate a generated source must pass before it deploys: every cassette must be reproduced in meaning, every declared output schema satisfied, with all I/O served from the recording. No LLM, no network, nothing the candidate can edit. Relations and the mutation gate land next. --- src/protocol/fitness.test.ts | 64 +++++++++++++++++++++ src/protocol/fitness.ts | 107 +++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 src/protocol/fitness.test.ts create mode 100644 src/protocol/fitness.ts diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts new file mode 100644 index 0000000..ed6cf09 --- /dev/null +++ b/src/protocol/fitness.test.ts @@ -0,0 +1,64 @@ +/** Run: pnpm tsx --test src/protocol/fitness.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { evaluate, type Invoker } from './fitness.js'; +import { CassetteStore, type Cassette } from './cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +/** Test invoker: the "source" is the body of an async JS function + * (args, http) => output. Real sandboxing arrives with op_fitness (Task 6). */ +const testInvoker: Invoker = async (source, _method, args, http) => { + const fn = new Function('args', 'http', `"use strict"; return (async () => { ${source} })();`); + return fn(args, http); +}; + +const GOOD_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events?q=' + args.q }); + if (!res) throw new Error('no stub'); + return res.body; +`; +const WRONG_SOURCE = `return [];`; + +const cassette: Cassette = { + method: 'listEvents', args: { q: 1 }, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, +}; + +const methods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + effects: 'read', + outputSchema: { + type: 'array', + items: { type: 'object', required: ['id'], properties: { id: { type: 'number' } } }, + }, +}]; + +test('replay passes when the candidate reproduces recorded meaning', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('replay fails and short-circuits when output diverges from the cassette', async () => { + const v = await evaluate({ source: WRONG_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, false); + assert.equal(v.checks.some(c => c.check === 'schema'), false); // short-circuit +}); + +test('schema fails on schema-invalid output even when there is no cassette for it', async () => { + const badSchemaSource = `return [{ notId: true }];`; + const v = await evaluate({ source: badSchemaSource }, + { cassettes: new CassetteStore([]), methods }, testInvoker); + // no cassettes -> replay vacuously passes with a detail note; schema probe runs on empty args + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); + assert.equal(v.pass, false); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts new file mode 100644 index 0000000..8197f4f --- /dev/null +++ b/src/protocol/fitness.ts @@ -0,0 +1,107 @@ +/** + * Fitness -- the judge the object cannot edit. + * + * evaluate() decides whether a candidate source is fit to deploy. It never + * calls an LLM and never touches the network: all I/O is served from + * recorded cassettes through the caller-supplied Invoker. Checks run in + * order -- replay, schema, relations, mutation -- and the first hard + * failure short-circuits. + */ +import Ajv from 'ajv'; +import { CassetteStore } from './cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +export interface HttpExchange { status: number; body: unknown; } +export type HttpStub = (req: { method: string; url: string }) => HttpExchange | undefined; +export type Invoker = (source: string, method: string, + args: Record, http: HttpStub) => Promise; + +export interface CheckResult { + check: 'replay' | 'schema' | 'relations' | 'mutation'; + pass: boolean; + detail: string; +} +export interface Verdict { pass: boolean; checks: CheckResult[]; killRatio?: number; } +export interface FitnessEvidence { cassettes: CassetteStore; methods: MethodDeclaration[]; } +export interface FitnessOptions { maxMutants?: number; killThreshold?: number; } + +function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + const ka = Object.keys(a as object), kb = Object.keys(b as object); + if (ka.length !== kb.length) return false; + return ka.every(k => + deepEqual((a as Record)[k], (b as Record)[k])); +} + +function stubFor(cassettes: CassetteStore): HttpStub { + return req => { + const hit = cassettes.matchRequest({ method: req.method, url: req.url }); + return hit ? { status: hit.response.status, body: hit.response.body } : undefined; + }; +} + +async function checkReplay(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const all = ev.cassettes.all(); + if (all.length === 0) { + return { check: 'replay', pass: true, detail: 'no cassettes yet (first create); probe required by caller' }; + } + for (const c of all) { + let out: unknown; + try { + out = await invoke(source, c.method, c.args, stubFor(ev.cassettes)); + } catch (err) { + return { check: 'replay', pass: false, + detail: `${c.method}(${JSON.stringify(c.args)}) threw: ${err instanceof Error ? err.message : String(err)}` }; + } + if (!deepEqual(out, c.parsedOutput)) { + return { check: 'replay', pass: false, + detail: `${c.method}(${JSON.stringify(c.args)}) diverged from cassette recorded at ${c.recordedAt}` }; + } + } + return { check: 'replay', pass: true, detail: `${all.length} cassette(s) reproduced` }; +} + +async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const ajv = new Ajv({ allErrors: true, strict: false }); + for (const m of ev.methods) { + if (!m.outputSchema) continue; + const validate = ajv.compile(m.outputSchema); + const probes = ev.cassettes.byMethod(m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + for (const args of probes) { + let out: unknown; + try { + out = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch { + continue; // replay already judges throwing; schema judges shape of what returns + } + if (!validate(out)) { + return { check: 'schema', pass: false, + detail: `${m.name}: ${ajv.errorsText(validate.errors)}` }; + } + } + } + return { check: 'schema', pass: true, detail: 'all outputs validate' }; +} + +export async function evaluate(candidate: { source: string }, + evidence: FitnessEvidence, + invoker: Invoker, + opts?: FitnessOptions): Promise { + const checks: CheckResult[] = []; + + const replay = await checkReplay(candidate.source, evidence, invoker); + checks.push(replay); + if (!replay.pass) return { pass: false, checks }; + + const schema = await checkSchema(candidate.source, evidence, invoker); + checks.push(schema); + if (!schema.pass) return { pass: false, checks }; + + // Tasks 4 & 5 replace these: + checks.push({ check: 'relations', pass: true, detail: 'not yet checked' }); + checks.push({ check: 'mutation', pass: true, detail: 'not yet checked' }); + return { pass: true, checks }; +} From eba7ab9e883ef98b7c8c2592185344a36b00ef8d Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:42:56 -0700 Subject: [PATCH 04/16] Fail the schema check when nothing could be validated Track validation success per method; if a method with outputSchema has zero successful validations and at least one probe threw, fail schema with a detail noting the method threw on all probes. This prevents first-create candidates that always throw from spuriously passing schema validation. --- src/protocol/fitness.test.ts | 9 +++++++++ src/protocol/fitness.ts | 12 +++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index ed6cf09..963cf19 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -62,3 +62,12 @@ test('schema fails on schema-invalid output even when there is no cassette for i assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); assert.equal(v.pass, false); }); + +test('schema fails when every probe throws and a schema is declared', async () => { + const throwing = `throw new Error('not implemented');`; + const v = await evaluate({ source: throwing }, + { cassettes: new CassetteStore([]), methods }, testInvoker); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); + assert.match(v.checks.find(c => c.check === 'schema')!.detail, /no output could be validated/); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 8197f4f..4f191c4 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -70,17 +70,27 @@ async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker) const validate = ajv.compile(m.outputSchema); const probes = ev.cassettes.byMethod(m.name).map(c => c.args); if (probes.length === 0) probes.push({}); + let validatedCount = 0; + let firstError: string | undefined; for (const args of probes) { let out: unknown; try { out = await invoke(source, m.name, args, stubFor(ev.cassettes)); - } catch { + } catch (err) { + if (!firstError) { + firstError = err instanceof Error ? err.message : String(err); + } continue; // replay already judges throwing; schema judges shape of what returns } if (!validate(out)) { return { check: 'schema', pass: false, detail: `${m.name}: ${ajv.errorsText(validate.errors)}` }; } + validatedCount++; + } + if (validatedCount === 0 && firstError) { + return { check: 'schema', pass: false, + detail: `${m.name}: no output could be validated (all probes threw: ${firstError})` }; } } return { check: 'schema', pass: true, detail: 'all outputs validate' }; From 09d13ee84c7fd99d531654543fceb16a19246ca8 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:45:25 -0700 Subject: [PATCH 05/16] Check the properties no single assertion can Metamorphic relations judge a method by how its answers relate: same call twice agrees, no element repeats, order holds, a tighter filter returns a subset, and the entity every healthy answer contains is there. Declared in the manifest, checked by the gate, invisible to the LLM. --- src/protocol/fitness.test.ts | 48 +++++++++++++++++++++++ src/protocol/fitness.ts | 76 +++++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index 963cf19..e4e02c1 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -71,3 +71,51 @@ test('schema fails when every probe throws and a schema is declared', async () = assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); assert.match(v.checks.find(c => c.check === 'schema')!.detail, /no output could be validated/); }); + +const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }], + knownEntity: 'Weekly Standup', +}]; + +const relCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: null }, // body unused: sources below ignore http + parsedOutput: null as unknown, // parsedOutput unused: set per-test below + recordedAt: 1, +}; + +test('relations: duplicates and disorder are caught', async () => { + const dupSource = `return [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }];`; + // make replay vacuous: cassette parsedOutput matches the source's constant output + const c = { ...relCassette, parsedOutput: [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }] }; + const v = await evaluate({ source: dupSource }, + { cassettes: new CassetteStore([c]), methods: relMethods }, testInvoker); + const rel = v.checks.find(x => x.check === 'relations'); + assert.equal(rel?.pass, false); + assert.match(rel!.detail, /no-duplicates|sorted-by/); +}); + +test('relations: known entity must appear', async () => { + const noEntity = `return [{ startsAt: 'a', name: 'Other Thing' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Other Thing' }] }; + const v = await evaluate({ source: noEntity }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'non-empty-for-known-entity' }] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, false); +}); + +test('relations: a clean output passes all declared relations', async () => { + const clean = `return [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }] }; + const v = await evaluate({ source: clean }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [ + { kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, { kind: 'idempotent' }, + ] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 4f191c4..c1642cd 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -96,6 +96,76 @@ async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker) return { check: 'schema', pass: true, detail: 'all outputs validate' }; } +function fieldValue(el: unknown, field: string): unknown { + return el !== null && typeof el === 'object' + ? (el as Record)[field] : undefined; +} + +async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + for (const m of ev.methods) { + if (!m.relations?.length) continue; + const probes = ev.cassettes.byMethod(m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + for (const args of probes) { + let out: unknown; + try { out = await invoke(source, m.name, args, stubFor(ev.cassettes)); } + catch { continue; } // throwing is replay's failure, not relations' + for (const rel of m.relations) { + const fail = (why: string): CheckResult => + ({ check: 'relations', pass: false, detail: `${m.name} ${rel.kind}: ${why}` }); + switch (rel.kind) { + case 'idempotent': { + const again = await invoke(source, m.name, args, stubFor(ev.cassettes)); + if (!deepEqual(out, again)) return fail('two identical calls disagreed'); + break; + } + case 'no-duplicates': { + if (!Array.isArray(out)) return fail('output is not an array'); + for (let i = 0; i < out.length; i++) + for (let j = i + 1; j < out.length; j++) + if (deepEqual(out[i], out[j])) return fail(`elements ${i} and ${j} are equal`); + break; + } + case 'sorted-by': { + if (!Array.isArray(out)) return fail('output is not an array'); + if (!rel.field) return fail('sorted-by declared without a field'); + for (let i = 1; i < out.length; i++) { + const a = fieldValue(out[i - 1], rel.field), b = fieldValue(out[i], rel.field); + if (a === undefined || b === undefined) return fail(`element missing field '${rel.field}'`); + const ok = typeof a === 'string' && typeof b === 'string' + ? a.localeCompare(b) <= 0 : (a as number) <= (b as number); + if (!ok) return fail(`not sorted at index ${i}`); + } + break; + } + case 'subset-on-tighter-filter': { + if (!rel.field) return fail('declared without a field'); + const cs = ev.cassettes.byMethod(m.name) + .filter(c => c.args[rel.field!] !== undefined); + if (cs.length < 2) break; // insufficient cassettes: vacuous + const sorted = [...cs].sort((a, b) => + String(a.args[rel.field!]).localeCompare(String(b.args[rel.field!]))); + const loose = await invoke(source, m.name, sorted[0].args, stubFor(ev.cassettes)); + const tight = await invoke(source, m.name, sorted[sorted.length - 1].args, stubFor(ev.cassettes)); + if (!Array.isArray(loose) || !Array.isArray(tight)) return fail('outputs are not arrays'); + for (const t of tight) + if (!loose.some(l => deepEqual(l, t))) + return fail('tighter filter returned an element the looser one lacks'); + break; + } + case 'non-empty-for-known-entity': { + if (!m.knownEntity) break; // vacuous without a declared entity + if (!JSON.stringify(out ?? '').includes(m.knownEntity)) + return fail(`'${m.knownEntity}' absent from output`); + break; + } + } + } + } + } + return { check: 'relations', pass: true, detail: 'all declared relations hold' }; +} + export async function evaluate(candidate: { source: string }, evidence: FitnessEvidence, invoker: Invoker, @@ -110,8 +180,10 @@ export async function evaluate(candidate: { source: string }, checks.push(schema); if (!schema.pass) return { pass: false, checks }; - // Tasks 4 & 5 replace these: - checks.push({ check: 'relations', pass: true, detail: 'not yet checked' }); + const relations = await checkRelations(candidate.source, evidence, invoker); + checks.push(relations); + if (!relations.pass) return { pass: false, checks }; + checks.push({ check: 'mutation', pass: true, detail: 'not yet checked' }); return { pass: true, checks }; } From c9707806cc60f92b8cb59cdf8b4eaf7052c659d8 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:52:09 -0700 Subject: [PATCH 06/16] Hold relation re-invocations to the same discipline Secondary invocations (idempotent re-call, subset-on-tighter-filter loose/tight) must be wrapped in try/catch to fail the relation instead of rejecting the promise. Numeric filter arguments now sort numerically (a - b) instead of lexicographically; string sorts via localeCompare as before. Covers both cases in two new integration tests: stateful invoker failure on idempotent, and numeric argument ordering for tighter/looser comparisons. --- src/protocol/fitness.test.ts | 34 ++++++++++++++++++++++++++++++++++ src/protocol/fitness.ts | 30 +++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index e4e02c1..82115a7 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -119,3 +119,37 @@ test('relations: a clean output passes all declared relations', async () => { testInvoker); assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); }); + +test('relations: a throwing second invocation fails idempotent instead of rejecting evaluate', async () => { + // stateful source: first call returns [], second call throws + let calls = 0; + const flakyInvoker: Invoker = async (source, method, args, http) => { + calls++; + if (calls > 2) throw new Error('flaky'); + return []; + }; + const c = { ...relCassette, parsedOutput: [] as unknown }; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'idempotent' as const }] }] }, + flakyInvoker); + assert.equal(v.pass, false); + assert.match(v.checks.find(x => x.check === 'relations')!.detail, /second call threw/); +}); + +test('relations: subset-on-tighter-filter orders numeric filter args numerically', async () => { + const mk = (q: number, out: unknown[]) => ({ + method: 'listEvents', args: { q }, + request: { method: 'GET', url: `https://example.test/events?q=${q}` }, + response: { status: 200, body: out }, parsedOutput: out, recordedAt: q, + }); + // q=2 (looser, returns 2 items), q=10 (tighter, returns subset of 1) + const outputs: Record = { 2: [{ id: 1 }, { id: 2 }], 10: [{ id: 1 }] }; + const numInvoker: Invoker = async (_s, _m, args) => outputs[args.q as number]; + const v = await evaluate({ source: 'irrelevant' }, + { cassettes: new CassetteStore([mk(2, outputs[2]), mk(10, outputs[10])]), + methods: [{ name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'subset-on-tighter-filter' as const, field: 'q' }] }] }, + numInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index c1642cd..955280d 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -115,7 +115,13 @@ async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invok ({ check: 'relations', pass: false, detail: `${m.name} ${rel.kind}: ${why}` }); switch (rel.kind) { case 'idempotent': { - const again = await invoke(source, m.name, args, stubFor(ev.cassettes)); + let again: unknown; + try { + again = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`two identical calls disagreed (second call threw: ${msg})`); + } if (!deepEqual(out, again)) return fail('two identical calls disagreed'); break; } @@ -143,10 +149,24 @@ async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invok const cs = ev.cassettes.byMethod(m.name) .filter(c => c.args[rel.field!] !== undefined); if (cs.length < 2) break; // insufficient cassettes: vacuous - const sorted = [...cs].sort((a, b) => - String(a.args[rel.field!]).localeCompare(String(b.args[rel.field!]))); - const loose = await invoke(source, m.name, sorted[0].args, stubFor(ev.cassettes)); - const tight = await invoke(source, m.name, sorted[sorted.length - 1].args, stubFor(ev.cassettes)); + const sorted = [...cs].sort((a, b) => { + const aVal = a.args[rel.field!], bVal = b.args[rel.field!]; + if (typeof aVal === 'number' && typeof bVal === 'number') return aVal - bVal; + return String(aVal).localeCompare(String(bVal)); + }); + let loose: unknown, tight: unknown; + try { + loose = await invoke(source, m.name, sorted[0].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } + try { + tight = await invoke(source, m.name, sorted[sorted.length - 1].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } if (!Array.isArray(loose) || !Array.isArray(tight)) return fail('outputs are not arrays'); for (const t of tight) if (!loose.some(l => deepEqual(l, t))) From c5327cfed6148deffa4b3f6532912672984fc74e Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 14:55:19 -0700 Subject: [PATCH 07/16] Test the tests by breaking the candidate on purpose A fixed set of deterministic mutations -- flipped comparisons, dropped filters, emptied returns, renamed keys -- is applied to every candidate. If the cassettes, schemas, and relations cannot tell most mutants from the original, the evidence is too weak to certify a deploy, and the gate says so with a kill ratio. --- src/protocol/fitness.test.ts | 18 ++++++++ src/protocol/fitness.ts | 28 +++++++++++- src/protocol/mutants.test.ts | 28 ++++++++++++ src/protocol/mutants.ts | 82 ++++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 src/protocol/mutants.test.ts create mode 100644 src/protocol/mutants.ts diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index 82115a7..84c2f43 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -153,3 +153,21 @@ test('relations: subset-on-tighter-filter orders numeric filter args numerically numInvoker); assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); }); + +test('mutation gate kills mutants of a well-tested source', async () => { + // GOOD_SOURCE returns the cassette body verbatim; flipping its logic breaks replay. + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 12, killThreshold: 0.5 }); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + if (v.killRatio !== undefined && mut!.detail !== 'no mutation points') { + assert.ok(v.killRatio >= 0 && v.killRatio <= 1); + } +}); + +test('maxMutants: 0 skips the mutation gate', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'mutation')?.detail, 'skipped'); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 955280d..7f76735 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -9,6 +9,7 @@ */ import Ajv from 'ajv'; import { CassetteStore } from './cassette.js'; +import { generateMutants } from './mutants.js'; import type { MethodDeclaration } from '../core/types.js'; export interface HttpExchange { status: number; body: unknown; } @@ -204,6 +205,29 @@ export async function evaluate(candidate: { source: string }, checks.push(relations); if (!relations.pass) return { pass: false, checks }; - checks.push({ check: 'mutation', pass: true, detail: 'not yet checked' }); - return { pass: true, checks }; + const maxMutants = opts?.maxMutants ?? 12; + const killThreshold = opts?.killThreshold ?? 0.8; + if (maxMutants === 0) { + checks.push({ check: 'mutation', pass: true, detail: 'skipped' }); + return { pass: true, checks }; + } + const mutants = generateMutants(candidate.source, maxMutants); + if (mutants.length === 0) { + checks.push({ check: 'mutation', pass: true, detail: 'no mutation points' }); + return { pass: true, checks }; + } + let killed = 0; + for (const m of mutants) { + const r = await checkReplay(m.source, evidence, invoker); + if (!r.pass) { killed++; continue; } + const s = await checkSchema(m.source, evidence, invoker); + if (!s.pass) { killed++; continue; } + const rel = await checkRelations(m.source, evidence, invoker); + if (!rel.pass) killed++; + } + const killRatio = killed / mutants.length; + const pass = killRatio >= killThreshold; + checks.push({ check: 'mutation', pass, + detail: `${killed}/${mutants.length} mutants killed (threshold ${killThreshold})` }); + return { pass, checks, killRatio }; } diff --git a/src/protocol/mutants.test.ts b/src/protocol/mutants.test.ts new file mode 100644 index 0000000..60ed9ad --- /dev/null +++ b/src/protocol/mutants.test.ts @@ -0,0 +1,28 @@ +/** Run: pnpm tsx --test src/protocol/mutants.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { generateMutants } from './mutants.js'; + +const SRC = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + const items = res.body.filter(e => e.kind === 'event'); + if (items.length > 0) { return items; } + return []; +`; + +test('generates deterministic, distinct mutants up to max', () => { + const a = generateMutants(SRC, 12); + const b = generateMutants(SRC, 12); + assert.deepEqual(a.map(m => m.source), b.map(m => m.source)); + assert.ok(a.length >= 3, `expected >=3 mutants, got ${a.length}`); + assert.equal(new Set(a.map(m => m.source)).size, a.length); + for (const m of a) assert.notEqual(m.source, SRC); +}); + +test('respects max', () => { + assert.ok(generateMutants(SRC, 2).length <= 2); +}); + +test('source with no mutation points yields none', () => { + assert.equal(generateMutants(`return 42;`, 12).length, 0); +}); diff --git a/src/protocol/mutants.ts b/src/protocol/mutants.ts new file mode 100644 index 0000000..361a5c0 --- /dev/null +++ b/src/protocol/mutants.ts @@ -0,0 +1,82 @@ +/** + * Mutants -- deliberately broken copies of a candidate, used to test the + * tests. If the fitness evidence cannot tell a mutant from the real thing, + * the evidence is too weak to certify a heal. + */ +import * as acorn from 'acorn'; + +export interface Mutant { source: string; description: string; } + +interface Site { start: number; end: number; replacement: string; description: string; } + +const FLIP: Record = { + '<': '>=', '>': '<=', '<=': '>', '>=': '<', '===': '!==', '!==': '===', '==': '!=', '!=': '==', +}; + +export function generateMutants(source: string, max: number): Mutant[] { + if (max <= 0) return []; + let ast: acorn.Node; + try { + // Handler sources are statement lists; wrap so 'return' parses. + ast = acorn.parse(`async function __m__(args, http) {${source}\n}`, + { ecmaVersion: 'latest', allowAwaitOutsideFunction: true }); + } catch { + return []; + } + const offset = 'async function __m__(args, http) {'.length; + const sites: Site[] = []; + + (function walk(node: unknown): void { + if (node === null || typeof node !== 'object') return; + const n = node as acorn.Node & Record; + if (typeof n.type === 'string') { + if (n.type === 'BinaryExpression' && FLIP[(n as { operator?: string }).operator ?? '']) { + const op = (n as unknown as { operator: string; left: acorn.Node; right: acorn.Node }); + sites.push({ + start: op.left.end, end: op.right.start, + replacement: ` ${FLIP[op.operator]} `, + description: `flip '${op.operator}' to '${FLIP[op.operator]}'`, + }); + } + if (n.type === 'CallExpression') { + const callee = n.callee as (acorn.Node & { type: string; property?: { name?: string }; object?: acorn.Node }); + if (callee?.type === 'MemberExpression' && callee.property?.name === 'filter' && callee.object) { + sites.push({ + start: (n as acorn.Node).start, end: (n as acorn.Node).end, + replacement: source.slice(callee.object.start - offset, callee.object.end - offset), + description: 'drop a .filter(...)', + }); + } + } + if (n.type === 'ReturnStatement') { + const arg = n.argument as acorn.Node & { type?: string } | null; + if (arg && arg.type === 'ArrayExpression' && arg.end > arg.start + 2) { + sites.push({ start: arg.start, end: arg.end, replacement: '[]', + description: 'return [] instead of the array literal' }); + } + } + if (n.type === 'Property') { + const key = n.key as acorn.Node & { type?: string; value?: unknown }; + if (key?.type === 'Literal' && typeof key.value === 'string') { + sites.push({ start: key.start, end: key.end, replacement: `'__mutated__'`, + description: `swap property key '${key.value}'` }); + } + } + } + for (const v of Object.values(n)) { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object' && 'type' in (v as object)) walk(v); + } + })(ast); + + const seen = new Set(); + const mutants: Mutant[] = []; + for (const s of sites.sort((a, b) => a.start - b.start)) { + const mutated = source.slice(0, s.start - offset) + s.replacement + source.slice(s.end - offset); + if (mutated === source || seen.has(mutated)) continue; + seen.add(mutated); + mutants.push({ source: mutated, description: s.description }); + if (mutants.length >= max) break; + } + return mutants; +} From 19f11c0575fa08849b825b1161c3ab09a6e31eca Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:00:47 -0700 Subject: [PATCH 08/16] Prove the kill loop kills Verify that the mutation-gate kill-counting logic is exercised with a source that has real mutation points (comparisons, filter calls), confirming mutants are correctly identified and counted as killed when replay fails. --- src/protocol/fitness.test.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index 84c2f43..d639027 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -171,3 +171,30 @@ test('maxMutants: 0 skips the mutation gate', async () => { { cassettes: new CassetteStore([cassette]), methods }, testInvoker, { maxMutants: 0 }); assert.equal(v.checks.find(c => c.check === 'mutation')?.detail, 'skipped'); }); + +test('mutation gate counts kills on a source with real mutation points', async () => { + const KILLABLE_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.kind === 'event'); + return items; +`; + const killCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ kind: 'event', id: 1 }, { kind: 'other', id: 2 }] }, + parsedOutput: [{ kind: 'event', id: 1 }], + recordedAt: 1, + }; + const v = await evaluate({ source: KILLABLE_SOURCE }, + { cassettes: new CassetteStore([killCassette]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + assert.notEqual(mut!.detail, 'no mutation points'); + // expected mutants: flip '===' -> '!==' (returns the wrong item), drop .filter (returns both) — both killed by replay + assert.equal(v.killRatio, 1); + assert.equal(mut!.pass, true); + assert.match(mut!.detail, /2\/2 mutants killed/); +}); From c885d12b5849113add10442708b2e62830a10a47 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:06:26 -0700 Subject: [PATCH 09/16] Refuse to deploy what the gate has not passed ObjectCreator gains a fitness action: the draft runs in the sandbox with every call shimmed, HTTP served from the object's own cassettes, and the verdict recorded against a digest of the judged source. deploy_spawn and deploy_update now consult that gate -- no verdict, failed verdict, or edited-since-judged draft all refuse. The semantic reviewer stays advice; this is the part that is not. --- src/objects/object-creator-fitness.test.ts | 51 ++++++++++++++++ src/objects/object-creator.ts | 67 +++++++++++++++++++++- src/protocol/fitness.ts | 19 ++++++ src/protocol/sandbox-invoker.ts | 50 ++++++++++++++++ 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/objects/object-creator-fitness.test.ts create mode 100644 src/protocol/sandbox-invoker.ts diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts new file mode 100644 index 0000000..7091e8d --- /dev/null +++ b/src/objects/object-creator-fitness.test.ts @@ -0,0 +1,51 @@ +// Run: pnpm tsx --test src/objects/object-creator-fitness.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; +import { deployGate, evaluate } from '../protocol/fitness.js'; +import { CassetteStore, type Cassette } from '../protocol/cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + return res.body; + } +}`; + +const cassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; +const methods: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [] }]; + +test('sandbox invoker runs a handler map with HTTP served from cassettes', async () => { + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: HANDLER_MAP }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, true); +}); + +test('sandbox invoker refuses unstubbed I/O', async () => { + const leaky = `{ + async listEvents(msg) { return call('ShellExecutor', 'run', { command: 'ls' }); } + }`; + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: leaky }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O/); +}); + +test('deployGate refuses without a verdict, with a failed verdict, and on a stale digest', () => { + const src = 'return 1;'; + const digest = createHash('sha256').update(src).digest('hex'); + assert.equal(deployGate({}, src).ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: false, checks: [] }, fitnessSourceDigest: digest }, src).ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }, 'return 2;').ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }, src).ok, true); +}); diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index e8f0fc9..5e801f8 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -32,6 +32,9 @@ import { Log } from '../core/timed-log.js'; import { applyDiff, parseSearchReplaceBlocks, levenshtein } from './source-diff.js'; import { withKeyedLock } from '../core/keyed-lock.js'; import * as acorn from 'acorn'; +import { evaluate, deployGate, sourceDigest, type Verdict } from '../protocol/fitness.js'; +import { CassetteStore } from '../protocol/cassette.js'; +import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; const log = new Log('OBJECT-CREATOR'); @@ -287,6 +290,11 @@ interface LoopState { baselineCallKeys?: Set; /** Source the semantic reviewer has already seen — never review the same draft twice. */ semanticReviewedSource?: string; + /** The fitness gate's most recent verdict on a draft, and a digest of the + * source it judged. A verdict is only valid for the exact source it was + * computed against — see `deployGate` in `../protocol/fitness.js`. */ + fitnessVerdict?: Verdict; + fitnessSourceDigest?: string; terminal?: { kind: 'done' | 'fail'; result?: unknown; error?: string }; spawnedObjectId?: AbjectId; @@ -338,6 +346,8 @@ export class ObjectCreator extends Abject { private systemRegistryId?: AbjectId; private factoryId?: AbjectId; private abjectStoreId?: AbjectId; + /** Storage abject, for reading a target object's persisted fitness cassettes. Optional: absent means fitness judges on schema+relations+mutation alone. */ + private storageId?: AbjectId; private agentAbjectId?: AbjectId; private goalManagerId?: AbjectId; private knowledgeBaseId?: AbjectId; @@ -449,6 +459,7 @@ export class ObjectCreator extends Abject { this.factoryId = await this.requireDep('Factory'); this.systemRegistryId = (await this.discoverDep('SystemRegistry')) ?? undefined; this.abjectStoreId = (await this.discoverDep('AbjectStore')) ?? undefined; + this.storageId = (await this.discoverDep('Storage')) ?? undefined; this.agentAbjectId = (await this.discoverDep('AgentAbject')) ?? undefined; this.goalManagerId = (await this.discoverDep('GoalManager')) ?? undefined; this.knowledgeBaseId = (await this.discoverDep('KnowledgeBase')) ?? undefined; @@ -659,7 +670,7 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I private static readonly VALID_ACTIONS = [ 'call', 'draft_manifest', 'draft_source', 'edit_source', 'draft_diff', 'read_draft', 'replace_handler', 'add_handler', 'remove_handler', 'load_target', 'clone_object', 'draft_via_llm', - 'compile', 'validate_calls', 'review_semantics', 'deploy_spawn', 'deploy_update', + 'compile', 'validate_calls', 'review_semantics', 'fitness', 'deploy_spawn', 'deploy_update', 'compose_organism', 'extract_organelle', 'reply', 'ask_user', 'done', 'fail', ]; @@ -1795,6 +1806,50 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: result.verified, summary, result, data: advisory, error: result.verified ? undefined : this.formatSemanticIssues(result) }; } + /** + * Load the target object's persisted fitness cassettes from Storage + * (key `cassettes:`). Absent target, absent Storage, a missing + * key, or a corrupt payload all fall back to an empty store rather than + * failing the fitness op — an empty store still exercises schema, + * relations, and mutation; only replay is vacuous without recordings. + */ + private async loadCassettes(targetId?: AbjectId): Promise { + if (!targetId || !this.storageId) return new CassetteStore(); + try { + const raw = await this.sendRequest(this.storageId, 'get', { key: `cassettes:${targetId}` }); + return CassetteStore.fromJSON(raw ?? []); + } catch { + return new CassetteStore(); // no store, corrupt store, storage down: judge on schema+relations+mutation + } + } + + /** + * The hard gate: replays the target's recorded cassettes, validates output + * schemas and relations, and mutation-tests the staged draft — entirely + * inside the sandbox, with every I/O call shimmed to the object's own + * cassettes. Records the verdict and a digest of the judged source; + * deploy_spawn/deploy_update consult both via `deployGate` and refuse when + * the draft has changed since this ran. + */ + private async opFitness(state: LoopState): Promise<{ ok: boolean; summary: string; error?: string; data?: unknown }> { + if (!state.draftSource) return { ok: false, summary: 'fitness: no draft source', error: 'draft a source first' }; + const methods = state.draftManifest?.interface?.methods ?? []; + const cassettes = await this.loadCassettes(state.targetObjectId); + const verdict = await evaluate({ source: state.draftSource }, { cassettes, methods }, + buildSandboxInvoker()); + state.fitnessVerdict = verdict; + state.fitnessSourceDigest = sourceDigest(state.draftSource); + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + return { + ok: verdict.pass, + summary: verdict.pass + ? `fitness: PASS (${verdict.checks.map(c => c.check).join(', ')}${verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''})` + : `fitness: FAIL — ${failed}`, + error: verdict.pass ? undefined : failed, + data: verdict, + }; + } + // ── The deploy gate ─────────────────────────────────────────────────── // // Deploy is the ONLY place a check refuses to proceed, and it refuses on @@ -1903,6 +1958,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_spawn'); if (refusal) return refusal; + const gate = deployGate(state, state.draftSource!); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + const spawnReq: SpawnRequest = { manifest: state.draftManifest, source: state.draftSource, @@ -2000,6 +2058,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_update'); if (refusal) return refusal; + const gate = deployGate(state, state.draftSource!); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + // Resolve target: explicit objectId / targetName from action wins, else // fall back to the kind:modify state target. let targetId: AbjectId | undefined; @@ -3355,6 +3416,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I case 'review_semantics': res = await this.opReviewSemantics(state); break; + case 'fitness': + res = await this.opFitness(state); + break; case 'deploy_spawn': res = await this.opDeploySpawn(state); break; @@ -3714,6 +3778,7 @@ Your local actions are the supported way to create and modify Abjects: they carr - \`read_draft({handler?, lineRange?, grep?})\` — read the CURRENT source exactly as it stands. No args → the member outline. \`{handler:"name"}\` → that member's exact text, line-numbered. \`{lineRange:"a-b"}\` / \`{grep:"pattern"}\` → those lines. Read-only: it stages nothing and makes no progress on its own. Use it to look at a member you are about to rewrite, or one a check named. Never two turns in a row without an edit in between. - \`draft_via_llm({kind: "manifest" | "source", instructions})\` — ask an LLM to draft for you. It sees current loop state. Use when authoring a brand-new manifest or source from scratch is too large for one think-step. Do NOT use this for modifications of existing objects — use \`edit_source\` instead, since the LLM consistently truncates "preserve everything else" rewrites. - \`compile()\` / \`validate_calls()\` / \`review_semantics()\` — the checks, available explicitly but **rarely worth a step**: they run on their own (see *Checks run themselves*, below). +- \`fitness()\` — hard gate: replays recorded cassettes, validates output schemas and relations, and mutation-tests the draft. \`deploy_spawn\`/\`deploy_update\` refuse until fitness passes on the current draft. - \`deploy_spawn({})\` — deploy the staged drafts as a NEW Abject. Internally messages Factory.spawn with the manifest, source, and the right owner / parent / registryHint. Use for create flows. No payload: the staged drafts are read from loop state. - \`deploy_update({objectId?, targetName?})\` — deploy the staged source onto an EXISTING object. Internally hot-swaps the live object via its \`updateSource\` handler, then updates Registry's cached source + manifest, then persists via AbjectStore so the change survives a restart. The target is taken from \`objectId\` (UUID) or \`targetName\` (registered name) in the action payload, or from the task's target if it was started as a modify. If you investigated and discovered you should be modifying an existing object even though the loop kind is \`create\`, pass \`{objectId: ""}\` here. - \`compose_organism({name, description, organelleNames, interfaceSource?})\` packages EXISTING source-backed objects into ONE Organism: a composite Abject whose organelles (independent internal copies of the named objects) cooperate behind a membrane interface, while external callers see a single object with a single curated surface. The staged drafts define the membrane: \`draft_manifest\` is the organism's public surface and \`draft_source\` (or the explicit \`interfaceSource\`) is the forwarding handler map; when either is missing it is drafted automatically from the organelle manifests. The originals keep running, so remove them afterwards (or tell the user) when the organism replaces them. diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 7f76735..a78df10 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -8,6 +8,7 @@ * failure short-circuits. */ import Ajv from 'ajv'; +import { createHash } from 'node:crypto'; import { CassetteStore } from './cassette.js'; import { generateMutants } from './mutants.js'; import type { MethodDeclaration } from '../core/types.js'; @@ -231,3 +232,21 @@ export async function evaluate(candidate: { source: string }, detail: `${killed}/${mutants.length} mutants killed (threshold ${killThreshold})` }); return { pass, checks, killRatio }; } + +export function sourceDigest(source: string): string { + return createHash('sha256').update(source).digest('hex'); +} + +/** The hard gate deploy ops consult. A deploy may proceed only when the + * CURRENT draft has a passing verdict -- verdicts do not survive edits. */ +export function deployGate(state: { fitnessVerdict?: Verdict; fitnessSourceDigest?: string }, + draftSource: string): { ok: true } | { ok: false; error: string } { + if (!state.fitnessVerdict || state.fitnessSourceDigest !== sourceDigest(draftSource)) { + return { ok: false, error: 'deploy refused: no passing fitness verdict for this draft — run fitness' }; + } + if (!state.fitnessVerdict.pass) { + const failed = state.fitnessVerdict.checks.find(c => !c.pass); + return { ok: false, error: `deploy refused: fitness failed (${failed?.check}: ${failed?.detail})` }; + } + return { ok: true }; +} diff --git a/src/protocol/sandbox-invoker.ts b/src/protocol/sandbox-invoker.ts new file mode 100644 index 0000000..14344bf --- /dev/null +++ b/src/protocol/sandbox-invoker.ts @@ -0,0 +1,50 @@ +/** + * SandboxInvoker -- runs a ScriptableAbject handler-map source under + * runSandboxed with every inter-object call shimmed. HTTP is served from the + * fitness gate's HttpStub; anything else an object tries to reach throws, so + * a candidate cannot pass judgment by phoning the real world. + */ +import { runSandboxed } from '../core/sandbox.js'; +import type { Invoker, HttpStub } from './fitness.js'; + +const HTTP_TARGETS = new Set(['HttpClient', 'WebFetch']); + +/** A sandboxed candidate gets a bounded synchronous timeout -- generous for + * real handler logic, cheap insurance against a mutant that spins. It does + * not cover awaited Promises (see runSandboxed's docs), only the + * synchronous portions of the compile + call. */ +const SANDBOX_TIMEOUT_MS = 5000; + +function makeCall(http: HttpStub) { + return async (target: string, method: string, payload: Record) => { + if (!HTTP_TARGETS.has(target)) { + throw new Error(`fitness: unstubbed I/O -- call('${target}', '${method}')`); + } + const url = String(payload?.url ?? ''); + const httpMethod = method === 'post' || method === 'postJson' ? 'POST' + : String(payload?.method ?? 'GET').toUpperCase(); + const hit = http({ method: httpMethod, url }); + if (!hit) throw new Error(`fitness: unstubbed I/O -- no cassette for ${httpMethod} ${url}`); + return { status: hit.status, body: hit.body }; + }; +} + +export function buildSandboxInvoker(): Invoker { + return async (source, method, args, http) => { + const call = makeCall(http); + // runSandboxed wraps its code in `(async () => { CODE })()`; a bare + // parenthesized object expression as a statement evaluates and discards + // itself, so the handler map must be explicitly returned to escape the + // wrapper. + const handlers = await runSandboxed(`return (${source});`, { + call, + dep: (name: string) => name, + find: () => { throw new Error('fitness: unstubbed I/O -- find()'); }, + }, { timeout: SANDBOX_TIMEOUT_MS }) as Record }) => Promise>; + const handler = handlers?.[method] ?? handlers?.['*']; + if (typeof handler !== 'function') { + throw new Error(`fitness: source has no handler for '${method}'`); + } + return handler({ payload: args }); + }; +} From 11c68ddb0784f9518d9295dc46a6fc4ff3e875e8 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:15:09 -0700 Subject: [PATCH 10/16] Let an object's own traffic become its evidence HttpClient consults a per-object recorder before and after each request: record mode keeps every successful exchange as a cassette, replay mode serves recorded answers and refuses to improvise. Objects nobody registered pass through untouched. This is how the gate's memory grows without anyone authoring fixtures. --- src/objects/capabilities/http-client.ts | 36 ++++++++++++++--- src/protocol/cassette-recorder.test.ts | 33 ++++++++++++++++ src/protocol/cassette-recorder.ts | 51 +++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 src/protocol/cassette-recorder.test.ts create mode 100644 src/protocol/cassette-recorder.ts diff --git a/src/objects/capabilities/http-client.ts b/src/objects/capabilities/http-client.ts index 0600123..bc346fd 100644 --- a/src/objects/capabilities/http-client.ts +++ b/src/objects/capabilities/http-client.ts @@ -6,6 +6,7 @@ import { AbjectId, AbjectMessage } from '../../core/types.js'; import { Abject, DEFERRED_REPLY } from '../../core/abject.js'; import { error } from '../../core/message.js'; import { Capabilities } from '../../core/capability.js'; +import { beforeRequest, afterResponse } from '../../protocol/cassette-recorder.js'; const HTTP_INTERFACE = 'abjects:http'; @@ -190,7 +191,7 @@ export class HttpClient extends Abject { // for health pings during long-running fetches (e.g. LLM API calls). this.on('request', async (msg: AbjectMessage) => { const req = msg.payload as HttpRequest; - this.makeRequest(req).then( + this.makeRequest(req, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -206,7 +207,7 @@ export class HttpClient extends Abject { url: string; headers?: Record; }; - this.makeRequest({ method: 'GET', url, headers }).then( + this.makeRequest({ method: 'GET', url, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -223,7 +224,7 @@ export class HttpClient extends Abject { body: string; headers?: Record; }; - this.makeRequest({ method: 'POST', url, body, headers }).then( + this.makeRequest({ method: 'POST', url, body, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -260,7 +261,7 @@ export class HttpClient extends Abject { url, body: data, headers: { 'Content-Type': 'application/json' }, - }).then( + }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -300,13 +301,27 @@ export class HttpClient extends Abject { /** * Make an HTTP request with retry for transient errors. */ - async makeRequest(req: HttpRequest): Promise { + async makeRequest(req: HttpRequest, callerId?: string): Promise { if (this.webDisabled) throw new Error('Web access is disabled. Enable it in Settings > Permissions.'); // Validate URL const url = new URL(req.url); this.validateScheme(url.protocol); this.validateDomain(url.hostname); + // Replay seam: a registered replay-mode caller is served from its + // cassette store and never touches the network. A miss throws. + const replayed = beforeRequest(callerId, { method: req.method, url: req.url, headers: req.headers }); + if (replayed) { + const body = typeof replayed.body === 'string' ? replayed.body : JSON.stringify(replayed.body); + return { + status: replayed.status, + statusText: '', + headers: replayed.headers, + body, + ok: replayed.status >= 200 && replayed.status < 300, + }; + } + // Build fetch options const options: RequestInit = { method: req.method, @@ -345,6 +360,17 @@ export class HttpClient extends Abject { // Read body const body = await response.text(); + // Record seam: JSON-parse the body when possible so the cassette + // carries a matchable structure; fall back to the raw text. + let parsedBody: unknown = body; + try { + parsedBody = JSON.parse(body); + } catch { + // not JSON — keep parsedBody as the raw text + } + afterResponse(callerId, { method: req.method, url: req.url, headers: req.headers }, + { status: response.status, body: parsedBody }); + return { status: response.status, statusText: response.statusText, diff --git a/src/protocol/cassette-recorder.test.ts b/src/protocol/cassette-recorder.test.ts new file mode 100644 index 0000000..623731e --- /dev/null +++ b/src/protocol/cassette-recorder.test.ts @@ -0,0 +1,33 @@ +/** Run: pnpm tsx --test src/protocol/cassette-recorder.test.ts */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { setRecorder, clearRecorder, beforeRequest, afterResponse } from './cassette-recorder.js'; +import { CassetteStore } from './cassette.js'; + +afterEach(() => clearRecorder('obj-1')); + +test('record mode captures 2xx and calls onRecord; non-2xx is not recorded', () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder('obj-1', { mode: 'record', store, onRecord: () => persisted++ }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, { status: 200, body: { ok: 1 } }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/b' }, { status: 500, body: 'boom' }); + assert.equal(store.all().length, 1); + assert.equal(persisted, 1); +}); + +test('replay mode serves recorded responses and throws on a miss', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, { status: 200, body: { ok: 1 } }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/a' }); + assert.deepEqual(hit?.body, { ok: 1 }); + assert.throws(() => beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/miss' }), + /replay miss/); +}); + +test('unknown object id and live mode pass through', () => { + assert.equal(beforeRequest(undefined, { method: 'GET', url: 'https://x.test/' }), undefined); + assert.equal(beforeRequest('never-registered', { method: 'GET', url: 'https://x.test/' }), undefined); +}); diff --git a/src/protocol/cassette-recorder.ts b/src/protocol/cassette-recorder.ts new file mode 100644 index 0000000..ed04abb --- /dev/null +++ b/src/protocol/cassette-recorder.ts @@ -0,0 +1,51 @@ +/** + * CassetteRecorder -- the per-object seam between HttpClient and the + * cassette store. HttpClient asks it two questions: "should this request be + * served from a recording?" (replay) and "should this response be kept?" + * (record). Objects not registered here pass through untouched, so the + * seam costs nothing for the rest of the system. + */ +import { CassetteStore, type CassetteRequest, redactRequest } from './cassette.js'; + +export type RecorderMode = 'record' | 'replay' | 'live'; +export interface RecorderEntry { + mode: RecorderMode; + store: CassetteStore; + onRecord?: (store: CassetteStore) => void; +} + +const recorders = new Map(); + +export function setRecorder(objectId: string, entry: RecorderEntry): void { + recorders.set(objectId, entry); +} +export function clearRecorder(objectId: string): void { recorders.delete(objectId); } +export function getRecorder(objectId: string): RecorderEntry | undefined { + return recorders.get(objectId); +} + +export function beforeRequest(objectId: string | undefined, req: CassetteRequest): + { status: number; body: unknown; headers: Record } | undefined { + if (!objectId) return undefined; + const r = recorders.get(objectId); + if (!r || r.mode !== 'replay') return undefined; + const hit = r.store.matchRequest(req); + if (!hit) throw new Error(`replay miss: no cassette for ${req.method} ${req.url}`); + return { status: hit.response.status, body: hit.response.body, headers: {} }; +} + +export function afterResponse(objectId: string | undefined, req: CassetteRequest, + res: { status: number; body: unknown }): void { + if (!objectId) return; + const r = recorders.get(objectId); + if (!r || r.mode !== 'record') return; + if (res.status < 200 || res.status >= 300) return; + r.store.add({ + method: '_http', args: {}, + request: redactRequest(req), + response: res, + parsedOutput: res.body, + recordedAt: Date.now(), + }); + r.onRecord?.(r.store); +} From 208bd1fe1b63ab27dfc2179712275581f75aebc9 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:39:32 -0700 Subject: [PATCH 11/16] Hand candidates the response shape objects are taught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HttpClient's ask guide promises every caller { status, statusText, headers, body, ok } with body ALWAYS a raw string — that is the contract an LLM writes its handler against. The fitness invoker was returning { status, body:parsed }, so a candidate that did the documented JSON.parse(res.body) failed judgment while one written against a shape the runtime does not produce passed. Cassettes now keep rawBody (the response text verbatim) alongside parsedOutput, so replay can return the same characters the world sent — JSON.stringify of a parsed JSON string primitive is not the same text. Entries recorded before rawBody existed derive it on load. WebFetch leaves HTTP_TARGETS: its live FetchResult is nothing like an HttpResponse, so shimming it taught candidates a second fictional contract. A WebFetch-using candidate now fails with a plain unstubbed-I/O message. matchRequest is exact method+url only. The host+path fallback served ?q=1's recording to ?q=other, which defeats argument-dependent replay outright; the loose behaviour survives as matchRequestLoose for callers that want a sample of an endpoint rather than an answer to a question. --- .../capabilities/http-client-recorder.test.ts | 72 +++++++++++++++++++ src/objects/capabilities/http-client.ts | 7 +- src/objects/object-creator-fitness.test.ts | 34 ++++++++- src/protocol/cassette-recorder.test.ts | 31 ++++++-- src/protocol/cassette-recorder.ts | 19 ++--- src/protocol/cassette.test.ts | 22 +++++- src/protocol/cassette.ts | 29 +++++++- src/protocol/fitness.test.ts | 6 +- src/protocol/fitness.ts | 10 ++- src/protocol/sandbox-invoker.ts | 17 ++++- 10 files changed, 219 insertions(+), 28 deletions(-) create mode 100644 src/objects/capabilities/http-client-recorder.test.ts diff --git a/src/objects/capabilities/http-client-recorder.test.ts b/src/objects/capabilities/http-client-recorder.test.ts new file mode 100644 index 0000000..97cc0b4 --- /dev/null +++ b/src/objects/capabilities/http-client-recorder.test.ts @@ -0,0 +1,72 @@ +/** Run: pnpm tsx --test src/objects/capabilities/http-client-recorder.test.ts + * + * The wiring between HttpClient and the cassette seam. Hermetic: the global + * fetch is replaced for the duration of each test and restored afterwards — + * the replay test proves the network is never touched by making fetch throw. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { HttpClient } from './http-client.js'; +import { CassetteStore, type Cassette } from '../../protocol/cassette.js'; +import { setRecorder, clearRecorder } from '../../protocol/cassette-recorder.js'; + +const CALLER = 'obj-under-test'; +const realFetch = globalThis.fetch; + +afterEach(() => { + clearRecorder(CALLER); + globalThis.fetch = realFetch; +}); + +const recorded: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; + +test('replay mode returns the full recorded HttpResponse without calling fetch', async () => { + globalThis.fetch = (() => { throw new Error('fetch must not be called in replay mode'); }) as typeof fetch; + setRecorder(CALLER, { mode: 'replay', store: new CassetteStore([recorded]) }); + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/events' }, CALLER); + + assert.equal(res.status, 200); + assert.equal(res.ok, true); + assert.equal(res.statusText, ''); + assert.deepEqual(res.headers, {}); + assert.equal(res.body, '[{"id":1}]'); // raw text, verbatim + assert.deepEqual(JSON.parse(res.body), [{ id: 1 }]); +}); + +test('record mode lands one cassette carrying the raw body', async () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder(CALLER, { mode: 'record', store, onRecord: () => persisted++ }); + globalThis.fetch = (async () => new Response('{"hello":"world"}', { + status: 200, statusText: 'OK', headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/hello' }, CALLER); + + assert.equal(res.body, '{"hello":"world"}'); + assert.equal(persisted, 1); + assert.equal(store.all().length, 1); + const [c] = store.all(); + assert.equal(c.request.url, 'https://example.test/hello'); + assert.equal(c.rawBody, '{"hello":"world"}'); + assert.deepEqual(c.parsedOutput, { hello: 'world' }); +}); + +test('an unregistered caller neither replays nor records', async () => { + const store = new CassetteStore(); + setRecorder(CALLER, { mode: 'record', store }); + globalThis.fetch = (async () => new Response('ok', { status: 200 })) as typeof fetch; + + await new HttpClient().makeRequest({ method: 'GET', url: 'https://example.test/x' }, 'someone-else'); + assert.equal(store.all().length, 0); +}); diff --git a/src/objects/capabilities/http-client.ts b/src/objects/capabilities/http-client.ts index bc346fd..c890335 100644 --- a/src/objects/capabilities/http-client.ts +++ b/src/objects/capabilities/http-client.ts @@ -312,12 +312,13 @@ export class HttpClient extends Abject { // cassette store and never touches the network. A miss throws. const replayed = beforeRequest(callerId, { method: req.method, url: req.url, headers: req.headers }); if (replayed) { - const body = typeof replayed.body === 'string' ? replayed.body : JSON.stringify(replayed.body); + // A full HttpResponse, with the recorded body text verbatim — the + // caller must not be able to tell replay from the live network. return { status: replayed.status, statusText: '', headers: replayed.headers, - body, + body: replayed.rawBody, ok: replayed.status >= 200 && replayed.status < 300, }; } @@ -369,7 +370,7 @@ export class HttpClient extends Abject { // not JSON — keep parsedBody as the raw text } afterResponse(callerId, { method: req.method, url: req.url, headers: req.headers }, - { status: response.status, body: parsedBody }); + { status: response.status, body: parsedBody, rawBody: body }); return { status: response.status, diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts index 7091e8d..cec0b7d 100644 --- a/src/objects/object-creator-fitness.test.ts +++ b/src/objects/object-creator-fitness.test.ts @@ -10,7 +10,8 @@ import type { MethodDeclaration } from '../core/types.js'; const HANDLER_MAP = `{ async listEvents(msg) { const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); - return res.body; + if (!res.ok) throw new Error('http ' + res.status); + return JSON.parse(res.body); } }`; @@ -18,6 +19,7 @@ const cassette: Cassette = { method: 'listEvents', args: {}, request: { method: 'GET', url: 'https://example.test/events' }, response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', parsedOutput: [{ id: 1 }], recordedAt: 1, }; @@ -30,6 +32,36 @@ test('sandbox invoker runs a handler map with HTTP served from cassettes', async assert.equal(v.pass, true); }); +test("the HttpClient shim returns the shape HttpClient's ask guide teaches", async () => { + // { status, statusText, headers, body, ok } with body ALWAYS a raw string. + const echoShape = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + return { keys: Object.keys(res).sort(), bodyType: typeof res.body, body: res.body, ok: res.ok, status: res.status }; + } + }`; + const out = await buildSandboxInvoker()(echoShape, 'listEvents', {}, + () => ({ status: 200, body: [{ id: 1 }], rawBody: '[{"id":1}]' })); + // Structural, not deepEqual: the value crosses out of the vm realm, so its + // prototype is not this realm's Object.prototype. + const got = out as Record; + assert.deepEqual([...(got.keys as string[])], ['body', 'headers', 'ok', 'status', 'statusText']); + assert.equal(got.bodyType, 'string'); + assert.equal(got.body, '[{"id":1}]'); + assert.equal(got.ok, true); + assert.equal(got.status, 200); +}); + +test('WebFetch is not stubbed: its live shape is not an HttpResponse', async () => { + const webFetcher = `{ + async listEvents(msg) { return call('WebFetch', 'fetch', { url: 'https://example.test/events' }); } + }`; + const v = await evaluate({ source: webFetcher }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O -- call\('WebFetch'/); +}); + test('sandbox invoker refuses unstubbed I/O', async () => { const leaky = `{ async listEvents(msg) { return call('ShellExecutor', 'run', { command: 'ls' }); } diff --git a/src/protocol/cassette-recorder.test.ts b/src/protocol/cassette-recorder.test.ts index 623731e..08b59e7 100644 --- a/src/protocol/cassette-recorder.test.ts +++ b/src/protocol/cassette-recorder.test.ts @@ -2,27 +2,46 @@ import { test, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { setRecorder, clearRecorder, beforeRequest, afterResponse } from './cassette-recorder.js'; -import { CassetteStore } from './cassette.js'; +import { CassetteStore, HTTP_CASSETTE_METHOD } from './cassette.js'; afterEach(() => clearRecorder('obj-1')); -test('record mode captures 2xx and calls onRecord; non-2xx is not recorded', () => { +test('record mode captures 2xx with raw AND parsed body; non-2xx is not recorded', () => { const store = new CassetteStore(); let persisted = 0; setRecorder('obj-1', { mode: 'record', store, onRecord: () => persisted++ }); - afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, { status: 200, body: { ok: 1 } }); - afterResponse('obj-1', { method: 'GET', url: 'https://example.test/b' }, { status: 500, body: 'boom' }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, body: { ok: 1 }, rawBody: '{"ok":1}' }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/b' }, + { status: 500, body: 'boom', rawBody: 'boom' }); assert.equal(store.all().length, 1); assert.equal(persisted, 1); + const [c] = store.all(); + assert.equal(c.method, HTTP_CASSETTE_METHOD); + assert.equal(c.rawBody, '{"ok":1}'); + assert.deepEqual(c.parsedOutput, { ok: 1 }); +}); + +test('a JSON string primitive survives the record/replay round-trip verbatim', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + // The world sent the four characters `"hi"`; JSON.parse of that is `hi`. + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/s' }, + { status: 200, body: 'hi', rawBody: '"hi"' }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/s' }); + assert.equal(hit?.rawBody, '"hi"'); }); test('replay mode serves recorded responses and throws on a miss', () => { const store = new CassetteStore(); setRecorder('obj-1', { mode: 'record', store }); - afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, { status: 200, body: { ok: 1 } }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, body: { ok: 1 }, rawBody: '{"ok":1}' }); setRecorder('obj-1', { mode: 'replay', store }); const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/a' }); - assert.deepEqual(hit?.body, { ok: 1 }); + assert.equal(hit?.status, 200); + assert.equal(hit?.rawBody, '{"ok":1}'); assert.throws(() => beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/miss' }), /replay miss/); }); diff --git a/src/protocol/cassette-recorder.ts b/src/protocol/cassette-recorder.ts index ed04abb..a3d175d 100644 --- a/src/protocol/cassette-recorder.ts +++ b/src/protocol/cassette-recorder.ts @@ -5,7 +5,7 @@ * (record). Objects not registered here pass through untouched, so the * seam costs nothing for the rest of the system. */ -import { CassetteStore, type CassetteRequest, redactRequest } from './cassette.js'; +import { CassetteStore, HTTP_CASSETTE_METHOD, type CassetteRequest, redactRequest } from './cassette.js'; export type RecorderMode = 'record' | 'replay' | 'live'; export interface RecorderEntry { @@ -20,30 +20,31 @@ export function setRecorder(objectId: string, entry: RecorderEntry): void { recorders.set(objectId, entry); } export function clearRecorder(objectId: string): void { recorders.delete(objectId); } -export function getRecorder(objectId: string): RecorderEntry | undefined { - return recorders.get(objectId); -} +/** `rawBody` is the response text verbatim: HttpClient's contract promises + * callers a raw string body, so replay must return the same characters the + * world sent rather than a re-stringified parse of them. */ export function beforeRequest(objectId: string | undefined, req: CassetteRequest): - { status: number; body: unknown; headers: Record } | undefined { + { status: number; rawBody: string; headers: Record } | undefined { if (!objectId) return undefined; const r = recorders.get(objectId); if (!r || r.mode !== 'replay') return undefined; const hit = r.store.matchRequest(req); if (!hit) throw new Error(`replay miss: no cassette for ${req.method} ${req.url}`); - return { status: hit.response.status, body: hit.response.body, headers: {} }; + return { status: hit.response.status, rawBody: hit.rawBody, headers: {} }; } export function afterResponse(objectId: string | undefined, req: CassetteRequest, - res: { status: number; body: unknown }): void { + res: { status: number; body: unknown; rawBody: string }): void { if (!objectId) return; const r = recorders.get(objectId); if (!r || r.mode !== 'record') return; if (res.status < 200 || res.status >= 300) return; r.store.add({ - method: '_http', args: {}, + method: HTTP_CASSETTE_METHOD, args: {}, request: redactRequest(req), - response: res, + response: { status: res.status, body: res.body }, + rawBody: res.rawBody, parsedOutput: res.body, recordedAt: Date.now(), }); diff --git a/src/protocol/cassette.test.ts b/src/protocol/cassette.test.ts index 6afb936..c432c73 100644 --- a/src/protocol/cassette.test.ts +++ b/src/protocol/cassette.test.ts @@ -8,6 +8,7 @@ function mk(n: number, method = 'listEvents'): Cassette { method, args: { q: n }, request: { method: 'GET', url: `https://example.test/events?q=${n}` }, response: { status: 200, body: [{ id: n }] }, + rawBody: `[{"id":${n}}]`, parsedOutput: [{ id: n }], recordedAt: n, }; @@ -29,17 +30,34 @@ test('store caps per method with LRU eviction', () => { assert.equal(kept[0].recordedAt, 5); // 0..4 evicted }); -test('matchRequest finds exact url, then host+path fallback', () => { +test('matchRequest is exact: a different query string is a different request', () => { const s = new CassetteStore([mk(1)]); assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=1' })); - assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=other' })); + // ?q=other is NOT ?q=1 — serving it would defeat argument-dependent replay. + assert.equal(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=other' }), undefined); assert.equal(s.matchRequest({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); }); +test('matchRequestLoose falls back to host+path when no exact match exists', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=1' })); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=other' })); + assert.equal(s.matchRequestLoose({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + test('toJSON/fromJSON round-trips and skips malformed entries', () => { const s = new CassetteStore([mk(1), mk(2)]); const back = CassetteStore.fromJSON(JSON.parse(JSON.stringify(s.toJSON()))); assert.equal(back.all().length, 2); + assert.equal(back.all()[0].rawBody, '[{"id":1}]'); const dirty = CassetteStore.fromJSON([mk(3), { junk: true }, 42]); assert.equal(dirty.all().length, 1); }); + +test('fromJSON derives rawBody for entries recorded before it existed', () => { + const legacy = { ...mk(7) } as Partial; + delete legacy.rawBody; + const store = CassetteStore.fromJSON([legacy]); + assert.equal(store.all().length, 1); + assert.equal(store.all()[0].rawBody, JSON.stringify([{ id: 7 }])); +}); diff --git a/src/protocol/cassette.ts b/src/protocol/cassette.ts index d07da01..88c1a36 100644 --- a/src/protocol/cassette.ts +++ b/src/protocol/cassette.ts @@ -20,12 +20,21 @@ export interface Cassette { args: Record; request: CassetteRequest; response: { status: number; body: unknown }; + /** The response body EXACTLY as the world sent it, before any parsing. + * HttpClient's contract says `body` is always a raw string, so replay must + * hand back the same characters — `JSON.stringify(parsed)` is not the same + * text for a JSON string primitive, and the round-trip loses meaning. */ + rawBody: string; parsedOutput: unknown; recordedAt: number; } export const CASSETTE_CAP_PER_METHOD = 20; +/** The method name recorded for raw HTTP traffic. These cassettes are stubs + * for the object's own calls, never a method the fitness gate can replay. */ +export const HTTP_CASSETTE_METHOD = '_http'; + const REDACTED_HEADERS = new Set(['authorization', 'cookie', 'set-cookie']); export function redactRequest(req: CassetteRequest): CassetteRequest { @@ -41,6 +50,12 @@ function hostPath(url: string): string | undefined { try { const u = new URL(url); return `${u.host}${u.pathname}`; } catch { return undefined; } } +/** Cassettes recorded before `rawBody` existed derive it from the parsed + * body. Lossy for a JSON string primitive, but honest and never undefined. */ +function rawBodyOf(c: Cassette): string { + return typeof c.rawBody === 'string' ? c.rawBody : (JSON.stringify(c.response.body) ?? ''); +} + function isCassette(c: unknown): c is Cassette { if (c === null || typeof c !== 'object') return false; const x = c as Record; @@ -60,7 +75,7 @@ export class CassetteStore { } add(c: Cassette): void { - this.cassettes.push({ ...c, request: redactRequest(c.request) }); + this.cassettes.push({ ...c, request: redactRequest(c.request), rawBody: rawBodyOf(c) }); const forMethod = this.cassettes.filter(x => x.method === c.method); if (forMethod.length > CASSETTE_CAP_PER_METHOD) { const evict = forMethod @@ -78,9 +93,19 @@ export class CassetteStore { all(): Cassette[] { return [...this.cassettes]; } + /** Exact method+url only. Replay is argument-dependent: `?q=1` and + * `?q=other` are different questions, and answering one with the other's + * recording would let a candidate "reproduce" traffic it never made. */ matchRequest(req: CassetteRequest): Cassette | undefined { - const exact = this.cassettes.find( + return this.cassettes.find( c => c.request.method === req.method && c.request.url === req.url); + } + + /** Exact match, else any recording of the same host+path. Deliberately NOT + * used by the replay seam — kept for callers that want a representative + * sample of an endpoint rather than an answer to a specific question. */ + matchRequestLoose(req: CassetteRequest): Cassette | undefined { + const exact = this.matchRequest(req); if (exact) return exact; const hp = hostPath(req.url); if (!hp) return undefined; diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index d639027..0913b2e 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -23,6 +23,7 @@ const cassette: Cassette = { method: 'listEvents', args: { q: 1 }, request: { method: 'GET', url: 'https://example.test/events?q=1' }, response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], recordedAt: 1, }; @@ -82,6 +83,7 @@ const relCassette: Cassette = { method: 'listEvents', args: {}, request: { method: 'GET', url: 'https://example.test/events' }, response: { status: 200, body: null }, // body unused: sources below ignore http + rawBody: 'null', parsedOutput: null as unknown, // parsedOutput unused: set per-test below recordedAt: 1, }; @@ -141,7 +143,8 @@ test('relations: subset-on-tighter-filter orders numeric filter args numerically const mk = (q: number, out: unknown[]) => ({ method: 'listEvents', args: { q }, request: { method: 'GET', url: `https://example.test/events?q=${q}` }, - response: { status: 200, body: out }, parsedOutput: out, recordedAt: q, + response: { status: 200, body: out }, rawBody: JSON.stringify(out), + parsedOutput: out, recordedAt: q, }); // q=2 (looser, returns 2 items), q=10 (tighter, returns subset of 1) const outputs: Record = { 2: [{ id: 1 }, { id: 2 }], 10: [{ id: 1 }] }; @@ -183,6 +186,7 @@ test('mutation gate counts kills on a source with real mutation points', async ( method: 'listEvents', args: {}, request: { method: 'GET', url: 'https://example.test/events' }, response: { status: 200, body: [{ kind: 'event', id: 1 }, { kind: 'other', id: 2 }] }, + rawBody: '[{"kind":"event","id":1},{"kind":"other","id":2}]', parsedOutput: [{ kind: 'event', id: 1 }], recordedAt: 1, }; diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index a78df10..97e711f 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -13,7 +13,11 @@ import { CassetteStore } from './cassette.js'; import { generateMutants } from './mutants.js'; import type { MethodDeclaration } from '../core/types.js'; -export interface HttpExchange { status: number; body: unknown; } +/** One recorded response, as the fitness gate hands it to an invoker's HTTP + * shim. `rawBody` is the response text verbatim -- the shim must return it + * unchanged, because HttpClient promises objects a raw string body. `body` + * is the same response already parsed, for invokers that want it. */ +export interface HttpExchange { status: number; body: unknown; rawBody: string; } export type HttpStub = (req: { method: string; url: string }) => HttpExchange | undefined; export type Invoker = (source: string, method: string, args: Record, http: HttpStub) => Promise; @@ -40,7 +44,9 @@ function deepEqual(a: unknown, b: unknown): boolean { function stubFor(cassettes: CassetteStore): HttpStub { return req => { const hit = cassettes.matchRequest({ method: req.method, url: req.url }); - return hit ? { status: hit.response.status, body: hit.response.body } : undefined; + return hit + ? { status: hit.response.status, body: hit.response.body, rawBody: hit.rawBody } + : undefined; }; } diff --git a/src/protocol/sandbox-invoker.ts b/src/protocol/sandbox-invoker.ts index 14344bf..a6a9d91 100644 --- a/src/protocol/sandbox-invoker.ts +++ b/src/protocol/sandbox-invoker.ts @@ -7,7 +7,11 @@ import { runSandboxed } from '../core/sandbox.js'; import type { Invoker, HttpStub } from './fitness.js'; -const HTTP_TARGETS = new Set(['HttpClient', 'WebFetch']); +// HttpClient only. WebFetch's live return shape (FetchResult) is nothing +// like an HttpResponse, so shimming it here would teach a candidate a +// contract the runtime does not honour; a WebFetch-using candidate fails +// with a plain unstubbed-I/O message until a real stub exists. +const HTTP_TARGETS = new Set(['HttpClient']); /** A sandboxed candidate gets a bounded synchronous timeout -- generous for * real handler logic, cheap insurance against a mutant that spins. It does @@ -25,7 +29,16 @@ function makeCall(http: HttpStub) { : String(payload?.method ?? 'GET').toUpperCase(); const hit = http({ method: httpMethod, url }); if (!hit) throw new Error(`fitness: unstubbed I/O -- no cassette for ${httpMethod} ${url}`); - return { status: hit.status, body: hit.body }; + // The exact shape HttpClient's ask guide teaches objects: body is ALWAYS + // a raw string (`JSON.parse(result.body)`), ok is 2xx. A candidate judged + // against any other shape is judged against a runtime that does not exist. + return { + status: hit.status, + statusText: '', + headers: {} as Record, + body: hit.rawBody, + ok: hit.status >= 200 && hit.status < 300, + }; }; } From dc2031ac33b01e95fd5f300b8b072bcf1f57fd7d Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:42:06 -0700 Subject: [PATCH 12/16] Let the gate say what it does not know MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With zero cassettes — the only state this branch actually produces at runtime, since nothing calls setRecorder yet — the gate failed honest objects for reasons that were about the evidence, not the candidate: a declared outputSchema failed on a {} probe, and a real handler map scored 0/N killed. Only contract-free sources passed, which is exactly backwards. An empty store now returns an unverified PASS with each check saying so, and probes nothing. The mutation gate also failed open on the one input it could not read. generateMutants wrapped every source as a function body, so the canonical handler map did not parse and a syntax error and a clean source were both "no mutation points". It now returns null for "parsed under no dialect", and that fails the mutation check outright; zero sites from a real parse still passes. '_http' cassettes are raw traffic captured on the object's behalf, not method calls. Replaying them asked the invoker for a handler no object can have, so any object that had ever recorded could never be healed again. They now participate only as HTTP stubs. --- src/protocol/fitness.test.ts | 90 ++++++++++++++++++++++++++++++++---- src/protocol/fitness.ts | 42 +++++++++++++++-- src/protocol/mutants.test.ts | 37 +++++++++++++-- src/protocol/mutants.ts | 45 ++++++++++++++---- 4 files changed, 186 insertions(+), 28 deletions(-) diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index 0913b2e..b754be2 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -54,25 +54,99 @@ test('replay fails and short-circuits when output diverges from the cassette', a assert.equal(v.checks.some(c => c.check === 'schema'), false); // short-circuit }); -test('schema fails on schema-invalid output even when there is no cassette for it', async () => { +test('an empty cassette store yields an honest unverified pass, probing nothing', async () => { + const neverInvoked: Invoker = async () => { + throw new Error('the gate probed a candidate it has no evidence for'); + }; + const v = await evaluate({ source: 'return [{ notId: true }];' }, + { cassettes: new CassetteStore([]), methods }, neverInvoked); + assert.equal(v.pass, true); + const detail = (name: string) => v.checks.find(c => c.check === name)!; + assert.equal(detail('replay').pass, true); + assert.equal(detail('schema').pass, true); + assert.equal(detail('schema').detail, 'no cassettes — schema unverified'); + assert.equal(detail('relations').pass, true); + assert.equal(detail('relations').detail, 'no cassettes — relations unverified'); + assert.equal(detail('mutation').pass, true); + assert.equal(detail('mutation').detail, 'no cassettes — mutation gate requires evidence'); + assert.equal(v.killRatio, undefined); +}); + +test('schema fails on schema-invalid output', async () => { const badSchemaSource = `return [{ notId: true }];`; + const c: Cassette = { + ...cassette, + response: { status: 200, body: [{ notId: true }] }, + rawBody: '[{"notId":true}]', + parsedOutput: [{ notId: true }], + }; const v = await evaluate({ source: badSchemaSource }, - { cassettes: new CassetteStore([]), methods }, testInvoker); - // no cassettes -> replay vacuously passes with a detail note; schema probe runs on empty args - assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); - assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); + { cassettes: new CassetteStore([c]), methods }, testInvoker); + // replay passes (the source reproduces the recording); schema still refuses it + assert.equal(v.checks.find(x => x.check === 'replay')?.pass, true); + assert.equal(v.checks.find(x => x.check === 'schema')?.pass, false); assert.equal(v.pass, false); }); test('schema fails when every probe throws and a schema is declared', async () => { - const throwing = `throw new Error('not implemented');`; - const v = await evaluate({ source: throwing }, - { cassettes: new CassetteStore([]), methods }, testInvoker); + // `listEvents` has evidence and reproduces it. `countEvents` declares a + // schema, owns no cassette, and throws on its one {} probe — so nothing + // about its contract was ever validated, which is a failure, not a pass. + const source = `return [{ id: 1, startsAt: '2026-08-23' }];`; + const throwsForCount: Invoker = async (src, method, args, http) => { + if (method === 'countEvents') throw new Error('not implemented'); + return testInvoker(src, method, args, http); + }; + const v = await evaluate({ source }, + { cassettes: new CassetteStore([cassette]), + methods: [...methods, + { name: 'countEvents', description: '', parameters: [], outputSchema: { type: 'number' } }] }, + throwsForCount, { maxMutants: 0 }); assert.equal(v.pass, false); assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); assert.match(v.checks.find(c => c.check === 'schema')!.detail, /no output could be validated/); }); +test('_http cassettes stub HTTP but are never replayed as methods', async () => { + // The recorder writes every captured response under the method '_http'. + // Feeding those to the invoker asks it for a handler no object has. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, + }; + const methodAware: Invoker = async (src, method, args, http) => { + if (method !== 'listEvents') throw new Error(`fitness: source has no handler for '${method}'`); + return testInvoker(src, method, args, http); + }; + const fixedUrl = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + return res.body; + `; + const v = await evaluate({ source: fixedUrl }, + { cassettes: new CassetteStore([httpOnly]), methods }, methodAware, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + // and stubFor still served the recording: the schema probe fetched it + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('mutation fails when the candidate parses under no dialect', async () => { + // An invoker that ignores the source: replay/schema/relations all pass, so + // the verdict turns entirely on whether the gate admits it cannot read it. + const blind: Invoker = async () => [{ id: 1, startsAt: '2026-08-23' }]; + const v = await evaluate({ source: `{ async listEvents(msg) { return [ }` }, + { cassettes: new CassetteStore([cassette]), methods }, blind); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.equal(mut.detail, 'candidate does not parse'); +}); + const relMethods: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [], relations: [{ kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }], diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 97e711f..0499457 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -9,8 +9,9 @@ */ import Ajv from 'ajv'; import { createHash } from 'node:crypto'; -import { CassetteStore } from './cassette.js'; +import { HTTP_CASSETTE_METHOD, type CassetteStore } from './cassette.js'; import { generateMutants } from './mutants.js'; +import type { Cassette } from './cassette.js'; import type { MethodDeclaration } from '../core/types.js'; /** One recorded response, as the fitness gate hands it to an invoker's HTTP @@ -50,8 +51,18 @@ function stubFor(cassettes: CassetteStore): HttpStub { }; } +/** Cassettes the gate may replay as a METHOD call. '_http' cassettes are raw + * traffic the recorder captured on the object's behalf: they are stubs for + * the candidate's own outbound calls (see `stubFor`), not invocations any + * object has a handler for. Replaying them asks for a handler that cannot + * exist, which would brick every heal of an object that ever recorded. */ +function replayable(ev: FitnessEvidence, method?: string): Cassette[] { + const all = method === undefined ? ev.cassettes.all() : ev.cassettes.byMethod(method); + return all.filter(c => c.method !== HTTP_CASSETTE_METHOD); +} + async function checkReplay(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { - const all = ev.cassettes.all(); + const all = replayable(ev); if (all.length === 0) { return { check: 'replay', pass: true, detail: 'no cassettes yet (first create); probe required by caller' }; } @@ -76,7 +87,7 @@ async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker) for (const m of ev.methods) { if (!m.outputSchema) continue; const validate = ajv.compile(m.outputSchema); - const probes = ev.cassettes.byMethod(m.name).map(c => c.args); + const probes = replayable(ev, m.name).map(c => c.args); if (probes.length === 0) probes.push({}); let validatedCount = 0; let firstError: string | undefined; @@ -112,7 +123,7 @@ function fieldValue(el: unknown, field: string): unknown { async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { for (const m of ev.methods) { if (!m.relations?.length) continue; - const probes = ev.cassettes.byMethod(m.name).map(c => c.args); + const probes = replayable(ev, m.name).map(c => c.args); if (probes.length === 0) probes.push({}); for (const args of probes) { let out: unknown; @@ -154,7 +165,7 @@ async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invok } case 'subset-on-tighter-filter': { if (!rel.field) return fail('declared without a field'); - const cs = ev.cassettes.byMethod(m.name) + const cs = replayable(ev, m.name) .filter(c => c.args[rel.field!] !== undefined); if (cs.length < 2) break; // insufficient cassettes: vacuous const sorted = [...cs].sort((a, b) => { @@ -200,6 +211,21 @@ export async function evaluate(candidate: { source: string }, opts?: FitnessOptions): Promise { const checks: CheckResult[] = []; + // No evidence at all. Every check below would then be a probe with invented + // arguments against a candidate nothing has ever exercised: a declared + // outputSchema would fail on the {} probe, and a handler map would be + // mutation-tested with nothing able to kill a single mutant. Both are + // verdicts about the EVIDENCE, not the candidate, so say so and pass — + // an unverified pass, honestly labelled, beats a fabricated failure. + if (evidence.cassettes.all().length === 0) { + return { pass: true, checks: [ + await checkReplay(candidate.source, evidence, invoker), // vacuous: invokes nothing + { check: 'schema', pass: true, detail: 'no cassettes — schema unverified' }, + { check: 'relations', pass: true, detail: 'no cassettes — relations unverified' }, + { check: 'mutation', pass: true, detail: 'no cassettes — mutation gate requires evidence' }, + ] }; + } + const replay = await checkReplay(candidate.source, evidence, invoker); checks.push(replay); if (!replay.pass) return { pass: false, checks }; @@ -219,6 +245,12 @@ export async function evaluate(candidate: { source: string }, return { pass: true, checks }; } const mutants = generateMutants(candidate.source, maxMutants); + if (mutants === null) { + // Not "nothing to break" — the gate could not read the candidate under any + // dialect it knows, so the mutation evidence is absent rather than empty. + checks.push({ check: 'mutation', pass: false, detail: 'candidate does not parse' }); + return { pass: false, checks }; + } if (mutants.length === 0) { checks.push({ check: 'mutation', pass: true, detail: 'no mutation points' }); return { pass: true, checks }; diff --git a/src/protocol/mutants.test.ts b/src/protocol/mutants.test.ts index 60ed9ad..f646ccc 100644 --- a/src/protocol/mutants.test.ts +++ b/src/protocol/mutants.test.ts @@ -10,9 +10,19 @@ const SRC = ` return []; `; +/** The house dialect: a handler map, which is an EXPRESSION, not a statement + * list. This is what the invoker actually runs (`return (${source});`). */ +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + const items = JSON.parse(res.body).filter(e => e.kind === 'event'); + return items; + } +}`; + test('generates deterministic, distinct mutants up to max', () => { - const a = generateMutants(SRC, 12); - const b = generateMutants(SRC, 12); + const a = generateMutants(SRC, 12)!; + const b = generateMutants(SRC, 12)!; assert.deepEqual(a.map(m => m.source), b.map(m => m.source)); assert.ok(a.length >= 3, `expected >=3 mutants, got ${a.length}`); assert.equal(new Set(a.map(m => m.source)).size, a.length); @@ -20,9 +30,26 @@ test('generates deterministic, distinct mutants up to max', () => { }); test('respects max', () => { - assert.ok(generateMutants(SRC, 2).length <= 2); + assert.ok(generateMutants(SRC, 2)!.length <= 2); +}); + +test('source with no mutation points yields an empty list, not null', () => { + assert.deepEqual(generateMutants(`return 42;`, 12), []); +}); + +test('a handler map yields sites: it is parsed as the expression it is', () => { + const m = generateMutants(HANDLER_MAP, 12); + assert.notEqual(m, null); + assert.deepEqual(m!.map(x => x.description).sort(), [ + `drop a .filter(...)`, + `flip '===' to '!=='`, + ]); + for (const x of m!) assert.notEqual(x.source, HANDLER_MAP); + // every mutant is still a parseable handler map + for (const x of m!) assert.notEqual(generateMutants(x.source, 1), null); }); -test('source with no mutation points yields none', () => { - assert.equal(generateMutants(`return 42;`, 12).length, 0); +test('a source that parses under no dialect returns null, not an empty list', () => { + assert.equal(generateMutants(`{ async listEvents(msg) { return [ }`, 12), null); + assert.equal(generateMutants(`function ( {`, 12), null); }); diff --git a/src/protocol/mutants.ts b/src/protocol/mutants.ts index 361a5c0..bca4bd6 100644 --- a/src/protocol/mutants.ts +++ b/src/protocol/mutants.ts @@ -13,17 +13,42 @@ const FLIP: Record = { '<': '>=', '>': '<=', '<=': '>', '>=': '<', '===': '!==', '!==': '===', '==': '!=', '!=': '==', }; -export function generateMutants(source: string, max: number): Mutant[] { - if (max <= 0) return []; - let ast: acorn.Node; - try { - // Handler sources are statement lists; wrap so 'return' parses. - ast = acorn.parse(`async function __m__(args, http) {${source}\n}`, - { ecmaVersion: 'latest', allowAwaitOutsideFunction: true }); - } catch { - return []; +const PARSE_OPTS: acorn.Options = { ecmaVersion: 'latest', allowAwaitOutsideFunction: true }; + +/** The dialects a candidate may be written in, most canonical first. + * + * The invoker runs a candidate as `return (${source});` — so the house-style + * handler map is an EXPRESSION, and a bare-brace map (`{ async m(){} }`) is a + * valid object literal but NOT a valid block. Parsing only the statement-list + * dialect therefore found zero mutation sites in exactly the sources the gate + * exists to judge. Expression first, statement list second. */ +const WRAPS: ReadonlyArray<{ prefix: string; suffix: string }> = [ + { prefix: '(', suffix: ')' }, + { prefix: 'async function __m__(args, http) {', suffix: '\n}' }, +]; + +interface Parsed { ast: acorn.Node; offset: number; } + +function parseCandidate(source: string): Parsed | null { + for (const w of WRAPS) { + try { + return { ast: acorn.parse(`${w.prefix}${source}${w.suffix}`, PARSE_OPTS), offset: w.prefix.length }; + } catch { /* not this dialect — try the next */ } } - const offset = 'async function __m__(args, http) {'.length; + return null; +} + +/** + * Mutants of `source`, or `null` when the source parses under NO supported + * dialect. The distinction is load-bearing: an empty list means "nothing here + * to break", while null means the gate could not read the candidate at all — + * which must fail, not pass. + */ +export function generateMutants(source: string, max: number): Mutant[] | null { + const parsed = parseCandidate(source); + if (!parsed) return null; + if (max <= 0) return []; + const { ast, offset } = parsed; const sites: Site[] = []; (function walk(node: unknown): void { From 25b618568fbb1387c1e726146a9c257dec0029a1 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:43:24 -0700 Subject: [PATCH 13/16] Give a judged handler the `this` the runtime gives it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handlers were invoked unbound, so `this` was undefined inside them. The house style is a thin handler over a private helper — `return this.shape(rows)` — and every such object failed judgment on a `this` the gate itself withheld. Handlers are now bound to a minimal stand-in for ScriptableAbject's handler proxy: sibling handlers, the call/dep/find shims, an inert data/saveData/emit/ changed/observe, assert-like ensure/invariant, and an id. Nothing could interrupt a candidate that never returned, either: the vm's timeout is synchronous-only, so a mutant that flips a loop guard and awaits inside it hung evaluate forever. Compile and invocation now race a deadline, default 5s and injectable for tests. A timing-out mutant is killed by replay's catch; a timing-out candidate fails it. --- src/objects/object-creator-fitness.test.ts | 56 +++++++++++++++- src/protocol/sandbox-invoker.ts | 78 ++++++++++++++++++++-- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts index cec0b7d..c76d729 100644 --- a/src/objects/object-creator-fitness.test.ts +++ b/src/objects/object-creator-fitness.test.ts @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; +import { buildSandboxInvoker, FITNESS_INVOCATION_TIMEOUT_MS } from '../protocol/sandbox-invoker.js'; import { deployGate, evaluate } from '../protocol/fitness.js'; import { CassetteStore, type Cassette } from '../protocol/cassette.js'; import type { MethodDeclaration } from '../core/types.js'; @@ -73,6 +73,60 @@ test('sandbox invoker refuses unstubbed I/O', async () => { assert.match(v.checks[0].detail, /unstubbed I\/O/); }); +test('a handler reaches its siblings through `this`, as it does at runtime', async () => { + const withHelper = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + if (!res.ok) throw new Error('http ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(items) { return items.map(e => ({ id: e.id })); } + }`; + const v = await evaluate({ source: withHelper }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('the handler proxy carries the members the runtime proxy carries', async () => { + const usesProxy = `{ + async listEvents(msg) { + this.ensure(typeof this.id === 'string', 'id must be a string'); + this.invariant(this.data && typeof this.data === 'object', 'data must be an object'); + this.data.seen = true; + await this.saveData(); + this.emit('Somewhere', 'looked', {}); + this.changed('items'); + this.observe('Somewhere'); + return [{ id: this.data.seen ? 1 : 0 }]; + } + }`; + const v = await evaluate({ source: usesProxy }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('ensure/invariant throw on a falsy condition', async () => { + const breach = `{ async listEvents(msg) { this.ensure(false, 'nope'); return []; } }`; + const v = await evaluate({ source: breach }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /ContractViolation \(ensure\): nope/); +}); + +test('an invocation that never returns is killed by the deadline', async () => { + // runSandboxed's own timeout is synchronous-only, so an await inside a + // flipped loop used to hang evaluate forever. + assert.equal(FITNESS_INVOCATION_TIMEOUT_MS, 5000); + const hangs = `{ async listEvents(msg) { await new Promise(() => {}); return []; } }`; + const started = Date.now(); + const v = await evaluate({ source: hangs }, + { cassettes: new CassetteStore([cassette]), methods }, + buildSandboxInvoker({ timeoutMs: 100 }), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /fitness: invocation timeout/); + assert.ok(Date.now() - started < 4000, 'the gate must not wait on a hung candidate'); +}); + test('deployGate refuses without a verdict, with a failed verdict, and on a stale digest', () => { const src = 'return 1;'; const digest = createHash('sha256').update(src).digest('hex'); diff --git a/src/protocol/sandbox-invoker.ts b/src/protocol/sandbox-invoker.ts index a6a9d91..63b0f00 100644 --- a/src/protocol/sandbox-invoker.ts +++ b/src/protocol/sandbox-invoker.ts @@ -19,6 +19,19 @@ const HTTP_TARGETS = new Set(['HttpClient']); * synchronous portions of the compile + call. */ const SANDBOX_TIMEOUT_MS = 5000; +/** Wall-clock ceiling on ONE judged invocation, compile included. The vm's + * own timeout is synchronous-only, so a mutant that flips a loop guard and + * awaits inside it hangs evaluate forever with nothing to interrupt it. + * A timing-out mutant is thereby killed; a timing-out candidate fails. */ +export const FITNESS_INVOCATION_TIMEOUT_MS = 5000; + +/** Members the handler proxy owns; user members never shadow them. + * Mirrors ScriptableAbject.PROXY_BUILTINS. */ +const PROXY_BUILTINS = new Set([ + 'call', 'dep', 'find', 'changed', 'emit', 'observe', 'id', + 'data', 'saveData', 'ensure', 'invariant', +]); + function makeCall(http: HttpStub) { return async (target: string, method: string, payload: Record) => { if (!HTTP_TARGETS.has(target)) { @@ -42,8 +55,47 @@ function makeCall(http: HttpStub) { }; } -export function buildSandboxInvoker(): Invoker { - return async (source, method, args, http) => { +/** Reject if `p` has not settled within `ms`. The hung promise is abandoned, + * not cancelled -- nothing in a vm can be cancelled -- but it holds no timer + * of its own, so it never keeps the process alive. */ +function withDeadline(p: Promise, ms: number): Promise { + let timer: ReturnType; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('fitness: invocation timeout')), ms); + }); + return Promise.race([p, deadline]).finally(() => clearTimeout(timer)); +} + +/** + * A minimal stand-in for ScriptableAbject's handler proxy. Handlers are bound + * to it, so `this.sibling(...)` resolves the way it does in the live runtime + * -- without one, a two-method object (the house style: a thin handler over a + * private helper) failed judgment on a `this` the gate itself withheld. + * State-mutating members are inert: judgment must not persist anything. + */ +function buildHandlerProxy(call: ReturnType): Record { + return { + call, + dep: (name: string) => name, + find: () => { throw new Error('fitness: unstubbed I/O -- find()'); }, + data: {}, + saveData: async () => {}, + emit: () => {}, + changed: () => {}, + observe: () => {}, + ensure: (cond: unknown, message?: string) => { + if (!cond) throw new Error(`ContractViolation (ensure): ${message ?? 'condition failed'}`); + }, + invariant: (cond: unknown, message?: string) => { + if (!cond) throw new Error(`ContractViolation (invariant): ${message ?? 'invariant failed'}`); + }, + id: 'fitness-candidate', + }; +} + +export function buildSandboxInvoker(opts?: { timeoutMs?: number }): Invoker { + const timeoutMs = opts?.timeoutMs ?? FITNESS_INVOCATION_TIMEOUT_MS; + return (source, method, args, http) => withDeadline((async () => { const call = makeCall(http); // runSandboxed wraps its code in `(async () => { CODE })()`; a bare // parenthesized object expression as a statement evaluates and discards @@ -53,11 +105,23 @@ export function buildSandboxInvoker(): Invoker { call, dep: (name: string) => name, find: () => { throw new Error('fitness: unstubbed I/O -- find()'); }, - }, { timeout: SANDBOX_TIMEOUT_MS }) as Record }) => Promise>; - const handler = handlers?.[method] ?? handlers?.['*']; - if (typeof handler !== 'function') { - throw new Error(`fitness: source has no handler for '${method}'`); + }, { timeout: SANDBOX_TIMEOUT_MS }) as Record | null; + + const proxy = buildHandlerProxy(call); + const bound = new Map }) => Promise>(); + for (const [key, value] of Object.entries(handlers ?? {})) { + if (typeof value === 'function') { + const fn = (value as (...a: unknown[]) => unknown).bind(proxy) as + (msg: { payload: Record }) => Promise; + bound.set(key, fn); + if (!PROXY_BUILTINS.has(key)) proxy[key] = fn; + } else if (!PROXY_BUILTINS.has(key)) { + proxy[key] = value; // state property, same as the runtime does + } } + + const handler = bound.get(method) ?? bound.get('*'); + if (!handler) throw new Error(`fitness: source has no handler for '${method}'`); return handler({ payload: args }); - }; + })(), timeoutMs); } From 8034b87eecd6bbbc1cf0312718d1d53dc0049d21 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:45:08 -0700 Subject: [PATCH 14/16] Bind a verdict to the object and the contract it judged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A verdict was keyed on the source alone, so redrafting the manifest kept it valid — even though the schema and relation checks are judgments of the source AGAINST those declarations. sourceDigest becomes verdictDigest(source, methods). It was also keyed on nothing at all where the object was concerned. opFitness read cassettes for state.targetObjectId, while deploy_update can resolve a different explicit objectId/targetName — one it stamps AFTER the old gate ran. A verdict built from one object's recorded traffic could therefore wave a deploy onto another. The judged target is recorded and the gate refuses a mismatch; deploy_update now resolves its target before consulting the gate. On a heal the agent edits source without redrafting a manifest, so methods came back empty and schema/relations were vacuous. opFitness now reads the live target's manifest via describe, cached so the deploy gate recomputes the same digest, with a note in the op summary when the read comes back empty or fails. --- src/objects/object-creator-fitness.test.ts | 39 ++++++++++-- src/objects/object-creator.ts | 72 +++++++++++++++++----- src/protocol/fitness.ts | 24 ++++++-- 3 files changed, 109 insertions(+), 26 deletions(-) diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts index c76d729..91339df 100644 --- a/src/objects/object-creator-fitness.test.ts +++ b/src/objects/object-creator-fitness.test.ts @@ -3,7 +3,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { buildSandboxInvoker, FITNESS_INVOCATION_TIMEOUT_MS } from '../protocol/sandbox-invoker.js'; -import { deployGate, evaluate } from '../protocol/fitness.js'; +import { deployGate, evaluate, verdictDigest } from '../protocol/fitness.js'; import { CassetteStore, type Cassette } from '../protocol/cassette.js'; import type { MethodDeclaration } from '../core/types.js'; @@ -127,11 +127,38 @@ test('an invocation that never returns is killed by the deadline', async () => { assert.ok(Date.now() - started < 4000, 'the gate must not wait on a hung candidate'); }); +test('verdictDigest covers the declarations, not just the source', () => { + const src = 'return 1;'; + const more: MethodDeclaration[] = [...methods, { name: 'countEvents', description: '', parameters: [] }]; + assert.notEqual(verdictDigest(src, methods), verdictDigest(src, more)); + assert.equal(verdictDigest(src, methods), verdictDigest(src, [...methods])); + assert.equal(verdictDigest(src, methods), + createHash('sha256').update(src + '\0' + JSON.stringify(methods)).digest('hex')); +}); + test('deployGate refuses without a verdict, with a failed verdict, and on a stale digest', () => { const src = 'return 1;'; - const digest = createHash('sha256').update(src).digest('hex'); - assert.equal(deployGate({}, src).ok, false); - assert.equal(deployGate({ fitnessVerdict: { pass: false, checks: [] }, fitnessSourceDigest: digest }, src).ok, false); - assert.equal(deployGate({ fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }, 'return 2;').ok, false); - assert.equal(deployGate({ fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }, src).ok, true); + const digest = verdictDigest(src, methods); + const passing = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }; + assert.equal(deployGate({}, src, methods).ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: false, checks: [] }, fitnessSourceDigest: digest }, src, methods).ok, false); + assert.equal(deployGate(passing, 'return 2;', methods).ok, false); + // a re-drafted manifest invalidates the verdict too: schema and relations + // were judged against the declarations as they stood + assert.equal(deployGate(passing, src, + [...methods, { name: 'countEvents', description: '', parameters: [] }]).ok, false); + assert.equal(deployGate(passing, src, methods).ok, true); +}); + +test('deployGate refuses a verdict earned against a different object', () => { + const src = 'return 1;'; + const digest = verdictDigest(src, methods); + const judged = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest, fitnessTargetId: 'obj-a' }; + const refusal = deployGate(judged, src, methods, 'obj-b'); + assert.equal(refusal.ok, false); + assert.match((refusal as { error: string }).error, /fitness verdict is for a different object/); + assert.equal(deployGate(judged, src, methods, 'obj-a').ok, true); + // a create has no target on either side, and an unresolved target does not refuse + assert.equal(deployGate(judged, src, methods).ok, true); + assert.equal(deployGate({ ...judged, fitnessTargetId: undefined }, src, methods, 'obj-b').ok, true); }); diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index 5e801f8..54737b3 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -32,7 +32,7 @@ import { Log } from '../core/timed-log.js'; import { applyDiff, parseSearchReplaceBlocks, levenshtein } from './source-diff.js'; import { withKeyedLock } from '../core/keyed-lock.js'; import * as acorn from 'acorn'; -import { evaluate, deployGate, sourceDigest, type Verdict } from '../protocol/fitness.js'; +import { evaluate, deployGate, verdictDigest, type Verdict } from '../protocol/fitness.js'; import { CassetteStore } from '../protocol/cassette.js'; import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; @@ -290,11 +290,16 @@ interface LoopState { baselineCallKeys?: Set; /** Source the semantic reviewer has already seen — never review the same draft twice. */ semanticReviewedSource?: string; - /** The fitness gate's most recent verdict on a draft, and a digest of the - * source it judged. A verdict is only valid for the exact source it was - * computed against — see `deployGate` in `../protocol/fitness.js`. */ + /** The fitness gate's most recent verdict on a draft, a digest of what it + * judged (source AND declarations), and the object it judged it for. A + * verdict is only valid for that exact triple — see `deployGate` in + * `../protocol/fitness.js`. */ fitnessVerdict?: Verdict; fitnessSourceDigest?: string; + fitnessTargetId?: AbjectId; + /** Live-manifest methods read for the heal path, cached so the deploy gate + * recomputes the same digest without a second round trip. */ + fitnessLiveMethods?: { targetId: AbjectId; methods: MethodDeclaration[]; note?: string }; terminal?: { kind: 'done' | 'fail'; result?: unknown; error?: string }; spawnedObjectId?: AbjectId; @@ -1823,6 +1828,39 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I } } + /** + * The method declarations the fitness gate judges against. + * + * Normally the staged draft manifest. On a HEAL the agent edits source + * without redrafting a manifest, so `draftManifest` is unset and the schema + * and relation checks would be vacuous against an empty method list — read + * the live target's manifest instead, the same `describe` a dependency + * lookup uses. Cached per target: the deploy gate must recompute the SAME + * digest, and must not fail a deploy because a best-effort read that + * succeeded at judgment time happens to fail now. + */ + private async fitnessMethods(state: LoopState): Promise<{ methods: MethodDeclaration[]; note?: string }> { + const drafted = state.draftManifest?.interface?.methods; + if (drafted) return { methods: drafted }; + const targetId = state.targetObjectId; + if (!targetId) return { methods: [] }; + const cached = state.fitnessLiveMethods; + if (cached?.targetId === targetId) return { methods: cached.methods, note: cached.note }; + + let methods: MethodDeclaration[] = []; + let note: string | undefined; + try { + const ir = await this.sendRequest>(targetId, 'describe', {}, 5000); + methods = (ir?.manifest as AbjectManifest | undefined)?.interface?.methods ?? []; + if (methods.length === 0) note = 'live manifest declares no methods — schema/relations vacuous'; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + note = `live manifest unreadable (${msg.slice(0, 60)}) — schema/relations vacuous`; + } + state.fitnessLiveMethods = { targetId, methods, note }; + return { methods, note }; + } + /** * The hard gate: replays the target's recorded cassettes, validates output * schemas and relations, and mutation-tests the staged draft — entirely @@ -1833,18 +1871,20 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I */ private async opFitness(state: LoopState): Promise<{ ok: boolean; summary: string; error?: string; data?: unknown }> { if (!state.draftSource) return { ok: false, summary: 'fitness: no draft source', error: 'draft a source first' }; - const methods = state.draftManifest?.interface?.methods ?? []; + const { methods, note } = await this.fitnessMethods(state); const cassettes = await this.loadCassettes(state.targetObjectId); const verdict = await evaluate({ source: state.draftSource }, { cassettes, methods }, buildSandboxInvoker()); state.fitnessVerdict = verdict; - state.fitnessSourceDigest = sourceDigest(state.draftSource); + state.fitnessSourceDigest = verdictDigest(state.draftSource, methods); + state.fitnessTargetId = state.targetObjectId; const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + const caveat = note ? ` [${note}]` : ''; return { ok: verdict.pass, - summary: verdict.pass + summary: (verdict.pass ? `fitness: PASS (${verdict.checks.map(c => c.check).join(', ')}${verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''})` - : `fitness: FAIL — ${failed}`, + : `fitness: FAIL — ${failed}`) + caveat, error: verdict.pass ? undefined : failed, data: verdict, }; @@ -1958,7 +1998,8 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_spawn'); if (refusal) return refusal; - const gate = deployGate(state, state.draftSource!); + const gate = deployGate(state, state.draftSource, + (await this.fitnessMethods(state)).methods); if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; const spawnReq: SpawnRequest = { @@ -2058,11 +2099,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_update'); if (refusal) return refusal; - const gate = deployGate(state, state.draftSource!); - if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; - - // Resolve target: explicit objectId / targetName from action wins, else - // fall back to the kind:modify state target. + // Resolve target BEFORE the fitness gate: the action can name an object + // the gate never saw (the gate judged `state.targetObjectId`, which an + // explicit objectId/targetName overrides), and a verdict earned against + // another object's cassettes says nothing about this one. let targetId: AbjectId | undefined; let targetLabel: string | undefined; const explicitId = this.actionField(action, ['objectId']); @@ -2084,6 +2124,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: false, summary: 'deploy_update: no target', error: 'pass {objectId} or {targetName} in the action payload, or use deploy_spawn for new objects' }; } + const gate = deployGate(state, state.draftSource, + (await this.fitnessMethods(state)).methods, targetId); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + // Everything from here is one write to one object. Two modify loops // running at once would otherwise interleave their four steps and leave // the live object, the Registry's cache, and the store each holding a diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 0499457..4e39dcd 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -271,17 +271,29 @@ export async function evaluate(candidate: { source: string }, return { pass, checks, killRatio }; } -export function sourceDigest(source: string): string { - return createHash('sha256').update(source).digest('hex'); +/** What a verdict is ABOUT. Not the source alone: the schema and relation + * checks are judgments of the source AGAINST the declarations, so a redrafted + * manifest invalidates a verdict exactly as a redrafted source does. */ +export function verdictDigest(source: string, methods: MethodDeclaration[]): string { + return createHash('sha256').update(source + '\0' + JSON.stringify(methods)).digest('hex'); } /** The hard gate deploy ops consult. A deploy may proceed only when the - * CURRENT draft has a passing verdict -- verdicts do not survive edits. */ -export function deployGate(state: { fitnessVerdict?: Verdict; fitnessSourceDigest?: string }, - draftSource: string): { ok: true } | { ok: false; error: string } { - if (!state.fitnessVerdict || state.fitnessSourceDigest !== sourceDigest(draftSource)) { + * CURRENT draft has a passing verdict -- verdicts do not survive edits -- + * and only onto the object that verdict was earned against: deploy_update + * can resolve an explicit target the gate never saw, and a verdict built + * from another object's cassettes says nothing about this one. */ +export function deployGate(state: { fitnessVerdict?: Verdict; fitnessSourceDigest?: string; fitnessTargetId?: string }, + draftSource: string, + methods: MethodDeclaration[], + resolvedTargetId?: string): { ok: true } | { ok: false; error: string } { + if (!state.fitnessVerdict || state.fitnessSourceDigest !== verdictDigest(draftSource, methods)) { return { ok: false, error: 'deploy refused: no passing fitness verdict for this draft — run fitness' }; } + if (state.fitnessTargetId !== undefined && resolvedTargetId !== undefined + && state.fitnessTargetId !== resolvedTargetId) { + return { ok: false, error: 'deploy refused: fitness verdict is for a different object' }; + } if (!state.fitnessVerdict.pass) { const failed = state.fitnessVerdict.checks.find(c => !c.pass); return { ok: false, error: `deploy refused: fitness failed (${failed?.check}: ${failed?.detail})` }; From f4392b700f3757808cb7c4599c5c2921e6b4137f Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Sun, 23 Aug 2026 15:47:19 -0700 Subject: [PATCH 15/16] Cover the weak-evidence and real-dialect cases, and say where the gate is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two verdicts nothing exercised: a source whose recording kills only one of its four mutants (0.25, below the 0.8 threshold) must fail, and the whole dialect an LLM actually writes — parenthesized handler map, a helper reached through `this`, an HttpClient call checked with res.ok and parsed out of res.body, against a cassette with a raw body — must pass every check with the mutation gate armed. The comments still described a world with two mechanical checks and no fitness gate, which is now the only non-mechanical reason a deploy is refused. --- src/objects/object-creator-fitness.test.ts | 46 ++++++++++++++++++++++ src/objects/object-creator.ts | 25 ++++++++++-- src/protocol/fitness.test.ts | 32 +++++++++++++++ src/protocol/fitness.ts | 3 +- 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts index 91339df..d6d68b3 100644 --- a/src/objects/object-creator-fitness.test.ts +++ b/src/objects/object-creator-fitness.test.ts @@ -162,3 +162,49 @@ test('deployGate refuses a verdict earned against a different object', () => { assert.equal(deployGate(judged, src, methods).ok, true); assert.equal(deployGate({ ...judged, fitnessTargetId: undefined }, src, methods, 'obj-b').ok, true); }); + +test('end to end, in the dialect an LLM actually writes', async () => { + // Everything the house style puts in one handler: a parenthesized handler + // map, a thin handler over a private helper reached through `this`, an + // HttpClient call whose response is checked with `res.ok` and parsed out of + // the raw `res.body` string — judged against one recorded cassette, through + // the real sandbox invoker, with every check armed. + const REAL = `({ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events?q=' + msg.payload.q }); + if (!res.ok) throw new Error('HTTP ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(rows) { + return rows.map(r => ({ id: r.id, name: r.name })); + } + })`; + const evidence: Cassette = { + method: 'listEvents', args: { q: 'all' }, + request: { method: 'GET', url: 'https://example.test/events?q=all' }, + response: { status: 200, body: [{ id: 1, name: 'Weekly Standup', extra: 9 }] }, + rawBody: '[{"id":1,"name":"Weekly Standup","extra":9}]', + parsedOutput: [{ id: 1, name: 'Weekly Standup' }], + recordedAt: 1, + }; + const declared: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], effects: 'read', + outputSchema: { + type: 'array', + items: { + type: 'object', required: ['id', 'name'], + properties: { id: { type: 'number' }, name: { type: 'string' } }, + }, + }, + relations: [{ kind: 'no-duplicates' }, { kind: 'idempotent' }, { kind: 'non-empty-for-known-entity' }], + knownEntity: 'Weekly Standup', + }]; + + const v = await evaluate({ source: REAL }, + { cassettes: new CassetteStore([evidence]), methods: declared }, buildSandboxInvoker()); + + assert.equal(v.pass, true, JSON.stringify(v.checks, null, 2)); + assert.equal(v.checks.find(c => c.check === 'replay')?.detail, '1 cassette(s) reproduced'); + assert.equal(v.checks.find(c => c.check === 'schema')?.detail, 'all outputs validate'); + assert.equal(v.checks.find(c => c.check === 'relations')?.detail, 'all declared relations hold'); +}); diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index 54737b3..3144207 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -683,7 +683,8 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // ── Staging, and when each check runs ───────────────────────────────── // // Every op that writes `state.draftSource` ends in `finishEdit`. Two checks - // exist and they have very different natures, so they run at different times: + // run themselves off an edit, and they have very different natures, so they + // run at different times: // // SYNTAX — runs on EVERY edit, and is load-bearing. Members are addressed by // name through the parsed object literal, so an unparseable draft cannot be @@ -702,6 +703,12 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // of the same LLM response are queued, or the agent passed `more: true`) it // does not run. It runs once, on the closing edit. // + // A THIRD gate — FITNESS — also refuses a deploy, but it is not an edit-time + // check and it is not mechanical: it is explicit (`fitness()`), it judges the + // whole draft against the target's RECORDED TRAFFIC (replay, declared output + // schemas, declared relations, mutation), and it costs a step. See + // `opFitness` and `deployGate` in `../protocol/fitness.js`. + // // A staging op never fails merely because the object is incomplete. That // matters mechanically: AgentAbject discards the rest of a batched response // when one action fails, so a spurious failure would throw away the very edits @@ -1892,9 +1899,19 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // ── The deploy gate ─────────────────────────────────────────────────── // - // Deploy is the ONLY place a check refuses to proceed, and it refuses on - // mechanical grounds only: a draft that does not parse, or that calls a method - // a dependency's live manifest does not have, must never reach the live object. + // Deploy is the ONLY place a check refuses to proceed, and it refuses on two + // different kinds of ground. + // + // MECHANICAL (`gateDeploy`, below): a draft that does not parse, or that + // calls a method a dependency's live manifest does not have, must never + // reach the live object. + // + // EVIDENTIAL (`deployGate`, ../protocol/fitness.js): the draft must carry a + // PASSING fitness verdict — recorded traffic replayed, declared schemas and + // relations upheld, mutants killed — for exactly this source, these + // declarations, and this target. Not mechanical, and the only non-mechanical + // refusal there is. + // // LLM judgments (the semantic reviewer) advise but never block — a deployed // object answering real calls teaches more per step than another blind pass. diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index b754be2..3548fed 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -276,3 +276,35 @@ test('mutation gate counts kills on a source with real mutation points', async ( assert.equal(mut!.pass, true); assert.match(mut!.detail, /2\/2 mutants killed/); }); + +test('mutation fails the verdict when the evidence cannot kill enough mutants', async () => { + // Four mutation sites, only one of them observable through the recording: + // the live filter excludes nothing the cassette contains, and `spare` is + // dead code. Weak evidence must read as a failure, not a pass. + const WEAK_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.id > 0); + const spare = res.body.filter(e => e.id > 100); + return items; +`; + const weak: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }, { id: 2 }] }, + rawBody: '[{"id":1},{"id":2}]', + parsedOutput: [{ id: 1 }, { id: 2 }], + recordedAt: 1, + }; + const v = await evaluate({ source: WEAK_SOURCE }, + { cassettes: new CassetteStore([weak]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.ok(v.killRatio !== undefined && v.killRatio < 0.8, `killRatio was ${v.killRatio}`); + // measured: only the live filter's flipped guard changes what comes back + assert.equal(v.killRatio, 0.25); + assert.equal(mut.detail, '1/4 mutants killed (threshold 0.8)'); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 4e39dcd..559a129 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -9,9 +9,8 @@ */ import Ajv from 'ajv'; import { createHash } from 'node:crypto'; -import { HTTP_CASSETTE_METHOD, type CassetteStore } from './cassette.js'; +import { HTTP_CASSETTE_METHOD, type Cassette, type CassetteStore } from './cassette.js'; import { generateMutants } from './mutants.js'; -import type { Cassette } from './cassette.js'; import type { MethodDeclaration } from '../core/types.js'; /** One recorded response, as the fitness gate hands it to an invoker's HTTP From fd54b067b07600b59aff4c013fbe84a340864680 Mon Sep 17 00:00:00 2001 From: Andre Burnt Date: Mon, 24 Aug 2026 11:31:57 -0700 Subject: [PATCH 16/16] Say which checks judged nothing, instead of claiming they held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An object whose only recorded traffic is `_http` has a store that is not empty but is unattributed: `replayable` filters those cassettes out, so every method-level check probes with a single `{}` call. A method that needs arguments throws on it, `checkRelations` swallowed the throw, and the function fell through to "all declared relations hold" having evaluated nothing — on data whose own recording violated the relations it declared. checkSchema already had the guard (`validatedCount === 0` fails closed); relations did not. It now counts probes that produced an output and, when none did, reports the methods it could not judge. The pass is kept rather than inverted, matching `evaluate`'s no-evidence path: an unverified pass, honestly labelled, and the mutation gate still refuses to certify a candidate nothing can kill. The verdict line the loop reads listed check NAMES, so three vacuous passes rendered identically to three earned ones. `summarizeVerdict` now names the checks that verified nothing. Nothing on this branch can reach that state — `setRecorder` has no non-test callers and no writer for `cassettes:` exists yet — but the recorder wiring in the next PR makes `_http`-only the normal shape of a populated store. Co-Authored-By: Claude Opus 5 (1M context) --- src/objects/object-creator.ts | 13 ++++++----- src/protocol/fitness.test.ts | 41 ++++++++++++++++++++++++++++++++++- src/protocol/fitness.ts | 34 ++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index 3144207..299d8c0 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -32,7 +32,7 @@ import { Log } from '../core/timed-log.js'; import { applyDiff, parseSearchReplaceBlocks, levenshtein } from './source-diff.js'; import { withKeyedLock } from '../core/keyed-lock.js'; import * as acorn from 'acorn'; -import { evaluate, deployGate, verdictDigest, type Verdict } from '../protocol/fitness.js'; +import { evaluate, deployGate, verdictDigest, summarizeVerdict, type Verdict } from '../protocol/fitness.js'; import { CassetteStore } from '../protocol/cassette.js'; import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; @@ -1822,8 +1822,11 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I * Load the target object's persisted fitness cassettes from Storage * (key `cassettes:`). Absent target, absent Storage, a missing * key, or a corrupt payload all fall back to an empty store rather than - * failing the fitness op — an empty store still exercises schema, - * relations, and mutation; only replay is vacuous without recordings. + * failing the fitness op. An empty store is judged by `evaluate`'s + * no-evidence path: every check returns an unverified pass, honestly + * labelled. A store holding only `_http` traffic is not empty but is + * equally unattributed, so replay and relations report unverified and a + * declared outputSchema fails closed. */ private async loadCassettes(targetId?: AbjectId): Promise { if (!targetId || !this.storageId) return new CassetteStore(); @@ -1889,9 +1892,7 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const caveat = note ? ` [${note}]` : ''; return { ok: verdict.pass, - summary: (verdict.pass - ? `fitness: PASS (${verdict.checks.map(c => c.check).join(', ')}${verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''})` - : `fitness: FAIL — ${failed}`) + caveat, + summary: summarizeVerdict(verdict) + caveat, error: verdict.pass ? undefined : failed, data: verdict, }; diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts index 3548fed..675341a 100644 --- a/src/protocol/fitness.test.ts +++ b/src/protocol/fitness.test.ts @@ -1,7 +1,7 @@ /** Run: pnpm tsx --test src/protocol/fitness.test.ts */ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { evaluate, type Invoker } from './fitness.js'; +import { evaluate, summarizeVerdict, type Invoker } from './fitness.js'; import { CassetteStore, type Cassette } from './cassette.js'; import type { MethodDeclaration } from '../core/types.js'; @@ -308,3 +308,42 @@ test('mutation fails the verdict when the evidence cannot kill enough mutants', assert.equal(v.killRatio, 0.25); assert.equal(mut.detail, '1/4 mutants killed (threshold 0.8)'); }); + +test('relations say so when no cassette is attributed to the method', async () => { + // Only _http traffic was captured, so `replayable` finds nothing to probe + // with and falls back to a single {} call. A method that needs arguments + // throws on it, and the check must not report success it never earned. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1 }, { id: 1 }] }, + rawBody: '[{"id":1},{"id":1}]', + parsedOutput: [{ id: 1 }, { id: 1 }], + recordedAt: 1, + }; + const needsArgs: Invoker = async (_src, _method, args) => { + if ((args as { q?: unknown }).q === undefined) throw new Error('q is required'); + return [{ id: 1 }, { id: 1 }]; // duplicates: no-duplicates would FAIL if ever evaluated + }; + const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], effects: 'read', + relations: [{ kind: 'no-duplicates' }], + }]; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([httpOnly]), methods: relMethods }, needsArgs, { maxMutants: 0 }); + const rel = v.checks.find(c => c.check === 'relations')!; + assert.match(rel.detail, /unverified/, + 'relations must report unverified evidence, not "all declared relations hold"'); +}); + +test('a passing verdict names the checks that verified nothing', () => { + // The loop's LLM reads this line. Listing check NAMES alone reads as + // "four checks passed" when three of them judged no evidence at all. + const summary = summarizeVerdict({ pass: true, checks: [ + { check: 'replay', pass: true, detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }, + { check: 'schema', pass: true, detail: 'all outputs validate' }, + { check: 'relations', pass: true, detail: 'relations unverified (no replayable cassettes for listEvents)' }, + { check: 'mutation', pass: true, detail: 'no mutation points' }, + ] }); + assert.match(summary, /unverified/, 'a pass built on no evidence must say so'); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts index 559a129..48e268a 100644 --- a/src/protocol/fitness.ts +++ b/src/protocol/fitness.ts @@ -63,7 +63,8 @@ function replayable(ev: FitnessEvidence, method?: string): Cassette[] { async function checkReplay(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { const all = replayable(ev); if (all.length === 0) { - return { check: 'replay', pass: true, detail: 'no cassettes yet (first create); probe required by caller' }; + return { check: 'replay', pass: true, + detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }; } for (const c of all) { let out: unknown; @@ -120,14 +121,20 @@ function fieldValue(el: unknown, field: string): unknown { } async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + /** Methods whose relations nothing could be evaluated against: every probe + * threw, so the loop below judged nothing about them. Saying the relations + * hold would be a claim the evidence never supported. */ + const unverified: string[] = []; for (const m of ev.methods) { if (!m.relations?.length) continue; const probes = replayable(ev, m.name).map(c => c.args); if (probes.length === 0) probes.push({}); + let evaluated = 0; for (const args of probes) { let out: unknown; try { out = await invoke(source, m.name, args, stubFor(ev.cassettes)); } catch { continue; } // throwing is replay's failure, not relations' + evaluated++; for (const rel of m.relations) { const fail = (why: string): CheckResult => ({ check: 'relations', pass: false, detail: `${m.name} ${rel.kind}: ${why}` }); @@ -200,6 +207,14 @@ async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invok } } } + if (evaluated === 0) unverified.push(m.name); + } + // Mirrors checkSchema's `validatedCount === 0` guard. Unlike schema, an + // unverified pass (not a failure) keeps faith with `evaluate`'s no-evidence + // path: the mutation gate still refuses to certify what nothing can kill. + if (unverified.length > 0) { + return { check: 'relations', pass: true, + detail: `relations unverified (no replayable cassettes for ${unverified.join(', ')})` }; } return { check: 'relations', pass: true, detail: 'all declared relations hold' }; } @@ -270,6 +285,23 @@ export async function evaluate(candidate: { source: string }, return { pass, checks, killRatio }; } +/** The one line the loop's driver reads. A verdict whose checks judged no + * evidence must not read like one that judged plenty, so checks that verified + * nothing are named rather than silently counted among the passes. */ +export function summarizeVerdict(verdict: Verdict): string { + if (!verdict.pass) { + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + return `fitness: FAIL — ${failed}`; + } + const kill = verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''; + const names = verdict.checks.map(c => c.check).join(', '); + const unverified = verdict.checks + .filter(c => /unverified|nothing replayed|requires evidence/.test(c.detail)) + .map(c => c.check); + const caveat = unverified.length > 0 ? ` — unverified: ${unverified.join(', ')}` : ''; + return `fitness: PASS (${names}${kill})${caveat}`; +} + /** What a verdict is ABOUT. Not the source alone: the schema and relation * checks are judgments of the source AGAINST the declarations, so a redrafted * manifest invalidates a verdict exactly as a redrafted source does. */