diff --git a/src/core/manifest-contract.test.ts b/src/core/manifest-contract.test.ts new file mode 100644 index 0000000..2d98d99 --- /dev/null +++ b/src/core/manifest-contract.test.ts @@ -0,0 +1,33 @@ +/** + * Manifest contract fields are optional and preserved. + * Run: pnpm tsx --test src/core/manifest-contract.test.ts + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import type { MethodDeclaration, RelationDeclaration, SideEffect } from './types.js'; + +test('a legacy method declaration without contract fields is valid', () => { + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + }; + assert.equal(m.sideEffects, undefined); +}); + +test('contract fields round-trip', () => { + const relations: RelationDeclaration[] = [ + { kind: 'subset-on-tighter-filter', field: 'from' }, + { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, + ]; + const sideEffects: SideEffect = 'read-only'; + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + sideEffects, + outputSchema: { type: 'array', items: { type: 'object' } }, + relations, + entityRef: 'Weekly Standup', + }; + assert.equal(m.sideEffects, 'read-only'); + assert.equal(m.relations?.length, 3); + assert.equal(m.entityRef, 'Weekly Standup'); +}); diff --git a/src/core/sandbox.ts b/src/core/sandbox.ts index cd541be..67eacee 100644 --- a/src/core/sandbox.ts +++ b/src/core/sandbox.ts @@ -49,6 +49,10 @@ export const BLOCKED_CODE_PATTERNS: ReadonlyArray<{ pattern: RegExp; label: stri { pattern: /\bfetch\s*\(/, label: 'fetch()' }, { pattern: /\bXMLHttpRequest\b/, label: 'XMLHttpRequest' }, { pattern: /\bWebSocket\b/, label: 'WebSocket' }, + // Realm intrinsics, not injected globals: Atomics.wait blocks the thread + // past V8's interrupt check, so no vm timeout can recover from it. + { pattern: /\bAtomics\b/, label: 'Atomics' }, + { pattern: /\bSharedArrayBuffer\b/, label: 'SharedArrayBuffer' }, ]; /** diff --git a/src/core/types.ts b/src/core/types.ts index 9619934..4a62590 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -63,11 +63,37 @@ export type ErrorMessage = AbjectMessage; // Interface Declaration // ============================================================================= +/** What calling a method does to the world. Reads are safe to + * generate and heal autonomously; acts must cross a hand-written gate. */ +export type SideEffect = 'read-only' | 'mutating'; + +/** A metamorphic relation the method's outputs must satisfy. + * Checked by the fitness gate (src/protocol/fitness.ts) — never by an LLM. */ +export interface RelationDeclaration { + kind: 'subset-on-tighter-filter' | 'idempotent' | 'no-duplicates' + | 'sorted-by' | 'non-empty-for-known-entity'; + /** 'sorted-by': output field to be non-decreasing on. + * 'subset-on-tighter-filter': the argument that narrows the result. */ + field?: string; +} + export interface MethodDeclaration { name: string; description: string; parameters: ParameterDeclaration[]; returns?: TypeDeclaration; + /** Contract fields the fitness gate judges against. All optional, so a + * manifest written before they existed stays valid. */ + sideEffects?: SideEffect; + /** JSON Schema the method's return value must validate against. */ + outputSchema?: Record; + relations?: RelationDeclaration[]; + /** A value that must appear somewhere in a healthy output (known-entity probe). */ + entityRef?: string; + /** Dot-paths masked in this method's recorded response bodies before they + * reach cassette storage (e.g. 'user.ssn'). For payload fields the header + * and query redaction cannot know about. */ + redactPaths?: string[]; } export interface ParameterDeclaration { diff --git a/src/objects/capabilities/http-client-recorder.test.ts b/src/objects/capabilities/http-client-recorder.test.ts new file mode 100644 index 0000000..97cc0b4 --- /dev/null +++ b/src/objects/capabilities/http-client-recorder.test.ts @@ -0,0 +1,72 @@ +/** Run: pnpm tsx --test src/objects/capabilities/http-client-recorder.test.ts + * + * The wiring between HttpClient and the cassette seam. Hermetic: the global + * fetch is replaced for the duration of each test and restored afterwards — + * the replay test proves the network is never touched by making fetch throw. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { HttpClient } from './http-client.js'; +import { CassetteStore, type Cassette } from '../../protocol/cassette.js'; +import { setRecorder, clearRecorder } from '../../protocol/cassette-recorder.js'; + +const CALLER = 'obj-under-test'; +const realFetch = globalThis.fetch; + +afterEach(() => { + clearRecorder(CALLER); + globalThis.fetch = realFetch; +}); + +const recorded: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; + +test('replay mode returns the full recorded HttpResponse without calling fetch', async () => { + globalThis.fetch = (() => { throw new Error('fetch must not be called in replay mode'); }) as typeof fetch; + setRecorder(CALLER, { mode: 'replay', store: new CassetteStore([recorded]) }); + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/events' }, CALLER); + + assert.equal(res.status, 200); + assert.equal(res.ok, true); + assert.equal(res.statusText, ''); + assert.deepEqual(res.headers, {}); + assert.equal(res.body, '[{"id":1}]'); // raw text, verbatim + assert.deepEqual(JSON.parse(res.body), [{ id: 1 }]); +}); + +test('record mode lands one cassette carrying the raw body', async () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder(CALLER, { mode: 'record', store, onRecord: () => persisted++ }); + globalThis.fetch = (async () => new Response('{"hello":"world"}', { + status: 200, statusText: 'OK', headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/hello' }, CALLER); + + assert.equal(res.body, '{"hello":"world"}'); + assert.equal(persisted, 1); + assert.equal(store.all().length, 1); + const [c] = store.all(); + assert.equal(c.request.url, 'https://example.test/hello'); + assert.equal(c.rawBody, '{"hello":"world"}'); + assert.deepEqual(c.parsedOutput, { hello: 'world' }); +}); + +test('an unregistered caller neither replays nor records', async () => { + const store = new CassetteStore(); + setRecorder(CALLER, { mode: 'record', store }); + globalThis.fetch = (async () => new Response('ok', { status: 200 })) as typeof fetch; + + await new HttpClient().makeRequest({ method: 'GET', url: 'https://example.test/x' }, 'someone-else'); + assert.equal(store.all().length, 0); +}); diff --git a/src/objects/capabilities/http-client.ts b/src/objects/capabilities/http-client.ts index 0600123..27567fc 100644 --- a/src/objects/capabilities/http-client.ts +++ b/src/objects/capabilities/http-client.ts @@ -6,6 +6,7 @@ import { AbjectId, AbjectMessage } from '../../core/types.js'; import { Abject, DEFERRED_REPLY } from '../../core/abject.js'; import { error } from '../../core/message.js'; import { Capabilities } from '../../core/capability.js'; +import { beforeRequest, afterResponse } from '../../protocol/cassette-recorder.js'; const HTTP_INTERFACE = 'abjects:http'; @@ -190,7 +191,7 @@ export class HttpClient extends Abject { // for health pings during long-running fetches (e.g. LLM API calls). this.on('request', async (msg: AbjectMessage) => { const req = msg.payload as HttpRequest; - this.makeRequest(req).then( + this.makeRequest(req, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -206,7 +207,7 @@ export class HttpClient extends Abject { url: string; headers?: Record; }; - this.makeRequest({ method: 'GET', url, headers }).then( + this.makeRequest({ method: 'GET', url, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -223,7 +224,7 @@ export class HttpClient extends Abject { body: string; headers?: Record; }; - this.makeRequest({ method: 'POST', url, body, headers }).then( + this.makeRequest({ method: 'POST', url, body, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -260,7 +261,7 @@ export class HttpClient extends Abject { url, body: data, headers: { 'Content-Type': 'application/json' }, - }).then( + }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -300,13 +301,28 @@ export class HttpClient extends Abject { /** * Make an HTTP request with retry for transient errors. */ - async makeRequest(req: HttpRequest): Promise { + async makeRequest(req: HttpRequest, callerId?: string): Promise { if (this.webDisabled) throw new Error('Web access is disabled. Enable it in Settings > Permissions.'); // Validate URL const url = new URL(req.url); this.validateScheme(url.protocol); this.validateDomain(url.hostname); + // Replay seam: a registered replay-mode caller is served from its + // cassette store and never touches the network. A miss throws. + const replayed = beforeRequest(callerId, { method: req.method, url: req.url, headers: req.headers, body: req.body }); + if (replayed) { + // A full HttpResponse, with the recorded body text verbatim — the + // caller must not be able to tell replay from the live network. + return { + status: replayed.status, + statusText: '', + headers: replayed.headers, + body: replayed.rawBody, + ok: replayed.status >= 200 && replayed.status < 300, + }; + } + // Build fetch options const options: RequestInit = { method: req.method, @@ -345,6 +361,9 @@ export class HttpClient extends Abject { // Read body const body = await response.text(); + afterResponse(callerId, { method: req.method, url: req.url, headers: req.headers, body: req.body }, + { status: response.status, rawBody: body }); + return { status: response.status, statusText: response.statusText, diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts new file mode 100644 index 0000000..2fd23ff --- /dev/null +++ b/src/objects/object-creator-fitness.test.ts @@ -0,0 +1,259 @@ +// Run: pnpm tsx --test src/objects/object-creator-fitness.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { buildSandboxInvoker, FITNESS_INVOCATION_TIMEOUT_MS } from '../protocol/sandbox-invoker.js'; +import { deployGate, evaluate, verdictDigest } from '../protocol/fitness.js'; +import { CassetteStore, type Cassette } from '../protocol/cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + if (!res.ok) throw new Error('http ' + res.status); + return JSON.parse(res.body); + } +}`; + +const cassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; +const methods: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [] }]; + +test('sandbox invoker runs a handler map with HTTP served from cassettes', async () => { + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: HANDLER_MAP }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, true); +}); + +test("the HttpClient shim returns the shape HttpClient's ask guide teaches", async () => { + // { status, statusText, headers, body, ok } with body ALWAYS a raw string. + const echoShape = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + return { keys: Object.keys(res).sort(), bodyType: typeof res.body, body: res.body, ok: res.ok, status: res.status }; + } + }`; + const out = await buildSandboxInvoker()(echoShape, 'listEvents', {}, + () => ({ status: 200, body: [{ id: 1 }], rawBody: '[{"id":1}]' })); + // Structural, not deepEqual: the value crosses out of the vm realm, so its + // prototype is not this realm's Object.prototype. + const got = out as Record; + assert.deepEqual([...(got.keys as string[])], ['body', 'headers', 'ok', 'status', 'statusText']); + assert.equal(got.bodyType, 'string'); + assert.equal(got.body, '[{"id":1}]'); + assert.equal(got.ok, true); + assert.equal(got.status, 200); +}); + +test('WebFetch is not stubbed: its live shape is not an HttpResponse', async () => { + const webFetcher = `{ + async listEvents(msg) { return call('WebFetch', 'fetch', { url: 'https://example.test/events' }); } + }`; + const v = await evaluate({ source: webFetcher }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O -- call\('WebFetch'/); +}); + +test('sandbox invoker refuses unstubbed I/O', async () => { + const leaky = `{ + async listEvents(msg) { return call('ShellExecutor', 'run', { command: 'ls' }); } + }`; + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: leaky }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O/); +}); + +test('a handler reaches its siblings through `this`, as it does at runtime', async () => { + const withHelper = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + if (!res.ok) throw new Error('http ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(items) { return items.map(e => ({ id: e.id })); } + }`; + const v = await evaluate({ source: withHelper }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('the handler proxy carries the members the runtime proxy carries', async () => { + const usesProxy = `{ + async listEvents(msg) { + this.ensure(typeof this.id === 'string', 'id must be a string'); + this.invariant(this.data && typeof this.data === 'object', 'data must be an object'); + this.data.seen = true; + await this.saveData(); + this.emit('Somewhere', 'looked', {}); + this.changed('items'); + this.observe('Somewhere'); + return [{ id: this.data.seen ? 1 : 0 }]; + } + }`; + const v = await evaluate({ source: usesProxy }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('ensure/invariant throw on a falsy condition', async () => { + const breach = `{ async listEvents(msg) { this.ensure(false, 'nope'); return []; } }`; + const v = await evaluate({ source: breach }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /ContractViolation \(ensure\): nope/); +}); + +test('an invocation that never settles is killed by the deadline', async () => { + // The worker is terminated on expiry, so nothing a candidate does can make + // the gate wait on it. + assert.equal(FITNESS_INVOCATION_TIMEOUT_MS, 5000); + const hangs = `{ async listEvents(msg) { await new Promise(() => {}); return []; } }`; + const started = Date.now(); + const v = await evaluate({ source: hangs }, + { cassettes: new CassetteStore([cassette]), methods }, + buildSandboxInvoker({ timeoutMs: 100 }), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /fitness: invocation timeout/); + assert.ok(Date.now() - started < 4000, 'the gate must not wait on a hung candidate'); +}); + +test('verdictDigest covers the declarations and the target, not just the source', () => { + const src = 'return 1;'; + const more: MethodDeclaration[] = [...methods, { name: 'countEvents', description: '', parameters: [] }]; + assert.notEqual(verdictDigest(src, methods), verdictDigest(src, more)); + assert.equal(verdictDigest(src, methods), verdictDigest(src, [...methods])); + assert.notEqual(verdictDigest(src, methods), verdictDigest(src, methods, 'obj-a')); + assert.notEqual(verdictDigest(src, methods, 'obj-a'), verdictDigest(src, methods, 'obj-b')); +}); + +test('deployGate refuses without a verdict, with a failed verdict, and on a stale digest', () => { + const src = 'return 1;'; + const digest = verdictDigest(src, methods); + const passing = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }; + assert.equal(deployGate({}, src, methods).ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: false, checks: [] }, fitnessSourceDigest: digest }, src, methods).ok, false); + assert.equal(deployGate(passing, 'return 2;', methods).ok, false); + // a re-drafted manifest invalidates the verdict too: schema and relations + // were judged against the declarations as they stood + assert.equal(deployGate(passing, src, + [...methods, { name: 'countEvents', description: '', parameters: [] }]).ok, false); + assert.equal(deployGate(passing, src, methods).ok, true); +}); + +test('deployGate refuses a verdict earned against a different object', () => { + const src = 'return 1;'; + const judged = { fitnessVerdict: { pass: true, checks: [] }, + fitnessSourceDigest: verdictDigest(src, methods, 'obj-a') }; + const refusal = deployGate(judged, src, methods, 'obj-b'); + assert.equal(refusal.ok, false); + assert.equal(deployGate(judged, src, methods, 'obj-a').ok, true); + // the target lives in the digest, so a targetless deploy cannot use a + // targeted verdict — and a targeted deploy cannot use a targetless one + assert.equal(deployGate(judged, src, methods).ok, false); + const targetless = { fitnessVerdict: { pass: true, checks: [] }, + fitnessSourceDigest: verdictDigest(src, methods) }; + assert.equal(deployGate(targetless, src, methods, 'obj-b').ok, false); + assert.equal(deployGate(targetless, src, methods).ok, true); +}); + +test('end to end, in the dialect an LLM actually writes', async () => { + // Everything the house style puts in one handler: a parenthesized handler + // map, a thin handler over a private helper reached through `this`, an + // HttpClient call whose response is checked with `res.ok` and parsed out of + // the raw `res.body` string — judged against one recorded cassette, through + // the real sandbox invoker, with every check armed. + const REAL = `({ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events?q=' + msg.payload.q }); + if (!res.ok) throw new Error('HTTP ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(rows) { + return rows.map(r => ({ id: r.id, name: r.name })); + } + })`; + const evidence: Cassette = { + method: 'listEvents', args: { q: 'all' }, + request: { method: 'GET', url: 'https://example.test/events?q=all' }, + response: { status: 200, body: [{ id: 1, name: 'Weekly Standup', extra: 9 }] }, + rawBody: '[{"id":1,"name":"Weekly Standup","extra":9}]', + parsedOutput: [{ id: 1, name: 'Weekly Standup' }], + recordedAt: 1, + }; + const declared: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], sideEffects: 'read-only', + outputSchema: { + type: 'array', + items: { + type: 'object', required: ['id', 'name'], + properties: { id: { type: 'number' }, name: { type: 'string' } }, + }, + }, + relations: [{ kind: 'no-duplicates' }, { kind: 'idempotent' }, { kind: 'non-empty-for-known-entity' }], + entityRef: 'Weekly Standup', + }]; + + const v = await evaluate({ source: REAL }, + { cassettes: new CassetteStore([evidence]), methods: declared }, buildSandboxInvoker()); + + assert.equal(v.pass, true, JSON.stringify(v.checks, null, 2)); + assert.equal(v.checks.find(c => c.check === 'replay')?.detail, '1 cassette(s) reproduced'); + assert.equal(v.checks.find(c => c.check === 'schema')?.detail, 'all outputs validate'); + assert.equal(v.checks.find(c => c.check === 'relations')?.detail, 'all declared relations hold'); +}); + +test('a synchronous spin in a handler is killed by the watchdog, not hung', async () => { + const spin = `({ async listEvents() { while (true) {} } })`; + const invoker = buildSandboxInvoker({ timeoutMs: 300 }); + await assert.rejects(() => invoker(spin, 'listEvents', {}, () => undefined), + /timed out|timeout/i); +}); + +test('a spin AFTER awaiting the stub is also killed', async () => { + const spin = `({ async listEvents() { + await this.call('HttpClient', 'get', { url: 'https://example.test/events?q=1' }); + while (true) {} + } })`; + const invoker = buildSandboxInvoker({ timeoutMs: 300 }); + await assert.rejects(() => invoker(spin, 'listEvents', {}, + () => ({ status: 200, body: [1], rawBody: '[1]' })), /timed out|timeout/i); +}); + +test('a candidate legitimately using timers is judged, not killed', async () => { + // The worker has its own event loop, so retry-with-backoff style handlers + // resolve normally instead of being mistaken for hostile code. + const timerUser = `({ async listEvents() { + await new Promise(r => setTimeout(r, 10)); return []; + } })`; + const invoker = buildSandboxInvoker({ timeoutMs: 3000 }); + const out = await invoker(timerUser, 'listEvents', {}, () => undefined); + assert.deepEqual(out, []); +}); + +test('a candidate reaching for Atomics is refused outright', async () => { + const atomicsUser = `({ async listEvents() { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); + } })`; + const invoker = buildSandboxInvoker({ timeoutMs: 300 }); + await assert.rejects(() => invoker(atomicsUser, 'listEvents', {}, () => undefined), + /Atomics|blocked/i); +}); + +test('every Factory spawn goes through the gated helper', async () => { + const { readFile } = await import('node:fs/promises'); + const src = await readFile(new URL('./object-creator.ts', import.meta.url), 'utf8'); + const direct = src.split('\n') + .filter(l => l.includes("'spawn'") && l.includes('sendRequest')); + assert.equal(direct.length, 1, + `Factory.spawn call sites outside gatedSpawn: ${direct.length - 1} too many`); +}); diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index e8f0fc9..ae40cbc 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -32,6 +32,9 @@ import { Log } from '../core/timed-log.js'; import { applyDiff, parseSearchReplaceBlocks, levenshtein } from './source-diff.js'; import { withKeyedLock } from '../core/keyed-lock.js'; import * as acorn from 'acorn'; +import { evaluate, deployGate, verdictDigest, summarizeVerdict, type Verdict } from '../protocol/fitness.js'; +import { CassetteStore } from '../protocol/cassette.js'; +import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; const log = new Log('OBJECT-CREATOR'); @@ -287,6 +290,14 @@ interface LoopState { baselineCallKeys?: Set; /** Source the semantic reviewer has already seen — never review the same draft twice. */ semanticReviewedSource?: string; + /** The fitness gate's most recent verdict on a draft, and a digest of what + * it judged: target, source, AND declarations. A verdict is only valid for + * that exact triple — see `deployGate` in `../protocol/fitness.js`. */ + fitnessVerdict?: Verdict; + fitnessSourceDigest?: string; + /** Live-manifest methods read for the heal path, cached so the deploy gate + * recomputes the same digest without a second round trip. */ + fitnessLiveMethods?: { targetId: AbjectId; methods: MethodDeclaration[]; note?: string }; terminal?: { kind: 'done' | 'fail'; result?: unknown; error?: string }; spawnedObjectId?: AbjectId; @@ -338,6 +349,8 @@ export class ObjectCreator extends Abject { private systemRegistryId?: AbjectId; private factoryId?: AbjectId; private abjectStoreId?: AbjectId; + /** Storage abject, for reading a target object's persisted fitness cassettes. Optional: absent means fitness judges on schema+relations+mutation alone. */ + private storageId?: AbjectId; private agentAbjectId?: AbjectId; private goalManagerId?: AbjectId; private knowledgeBaseId?: AbjectId; @@ -449,6 +462,7 @@ export class ObjectCreator extends Abject { this.factoryId = await this.requireDep('Factory'); this.systemRegistryId = (await this.discoverDep('SystemRegistry')) ?? undefined; this.abjectStoreId = (await this.discoverDep('AbjectStore')) ?? undefined; + this.storageId = (await this.discoverDep('Storage')) ?? undefined; this.agentAbjectId = (await this.discoverDep('AgentAbject')) ?? undefined; this.goalManagerId = (await this.discoverDep('GoalManager')) ?? undefined; this.knowledgeBaseId = (await this.discoverDep('KnowledgeBase')) ?? undefined; @@ -659,7 +673,7 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I private static readonly VALID_ACTIONS = [ 'call', 'draft_manifest', 'draft_source', 'edit_source', 'draft_diff', 'read_draft', 'replace_handler', 'add_handler', 'remove_handler', 'load_target', 'clone_object', 'draft_via_llm', - 'compile', 'validate_calls', 'review_semantics', 'deploy_spawn', 'deploy_update', + 'compile', 'validate_calls', 'review_semantics', 'fitness', 'deploy_spawn', 'deploy_update', 'compose_organism', 'extract_organelle', 'reply', 'ask_user', 'done', 'fail', ]; @@ -667,7 +681,8 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // ── Staging, and when each check runs ───────────────────────────────── // // Every op that writes `state.draftSource` ends in `finishEdit`. Two checks - // exist and they have very different natures, so they run at different times: + // run themselves off an edit, and they have very different natures, so they + // run at different times: // // SYNTAX — runs on EVERY edit, and is load-bearing. Members are addressed by // name through the parsed object literal, so an unparseable draft cannot be @@ -686,6 +701,12 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // of the same LLM response are queued, or the agent passed `more: true`) it // does not run. It runs once, on the closing edit. // + // A THIRD gate — FITNESS — also refuses a deploy, but it is not an edit-time + // check and it is not mechanical: it is explicit (`fitness()`), it judges the + // whole draft against the target's RECORDED TRAFFIC (replay, declared output + // schemas, declared relations, mutation), and it costs a step. See + // `opFitness` and `deployGate` in `../protocol/fitness.js`. + // // A staging op never fails merely because the object is incomplete. That // matters mechanically: AgentAbject discards the rest of a batched response // when one action fails, so a spurious failure would throw away the very edits @@ -1440,16 +1461,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I try { spawnReq.data = JSON.parse(JSON.stringify(registration.data)); } catch { spawnReq.data = {}; } } - let result: SpawnResult; - try { - result = await this.sendRequest(this.factoryId, 'spawn', spawnReq, 120000); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { ok: false, summary: `clone_object: spawn failed: ${msg.slice(0, 120)}`, error: msg }; - } - if (!result?.objectId) { - return { ok: false, summary: 'clone_object: Factory returned no objectId', error: 'unexpected Factory response' }; - } + const spawned = await this.gatedSpawn(state, spawnReq, 'clone_object', 'redeploys-live-source'); + if ('refusal' in spawned) return spawned.refusal; + const result = spawned.result; // Persist so the clone survives a restart (same as deploy_spawn). if (this.abjectStoreId) { @@ -1795,11 +1809,100 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: result.verified, summary, result, data: advisory, error: result.verified ? undefined : this.formatSemanticIssues(result) }; } + /** + * Load the target object's persisted fitness cassettes from Storage + * (key `cassettes:`). Absent target, absent Storage, a missing + * key, or a corrupt payload all fall back to an empty store rather than + * failing the fitness op. An empty store is judged by `evaluate`'s + * no-evidence path: every check returns an unverified pass, honestly + * labelled. A store holding only `_http` traffic is not empty but is + * equally unattributed, so replay and relations report unverified and a + * declared outputSchema fails closed. + */ + private async loadCassettes(targetId?: AbjectId): Promise { + if (!targetId || !this.storageId) return new CassetteStore(); + try { + const raw = await this.sendRequest(this.storageId, 'get', { key: `cassettes:${targetId}` }); + return CassetteStore.fromJSON(raw ?? []); + } catch { + return new CassetteStore(); // no store, corrupt store, storage down: judge on schema+relations+mutation + } + } + + /** + * The method declarations the fitness gate judges against. + * + * Normally the staged draft manifest. On a HEAL the agent edits source + * without redrafting a manifest, so `draftManifest` is unset and the schema + * and relation checks would be vacuous against an empty method list — read + * the live target's manifest instead, the same `describe` a dependency + * lookup uses. Cached per target: the deploy gate must recompute the SAME + * digest, and must not fail a deploy because a best-effort read that + * succeeded at judgment time happens to fail now. + */ + private async fitnessMethods(state: LoopState): Promise<{ methods: MethodDeclaration[]; note?: string }> { + const drafted = state.draftManifest?.interface?.methods; + if (drafted) return { methods: drafted }; + const targetId = state.targetObjectId; + if (!targetId) return { methods: [] }; + const cached = state.fitnessLiveMethods; + if (cached?.targetId === targetId) return { methods: cached.methods, note: cached.note }; + + let methods: MethodDeclaration[] = []; + let note: string | undefined; + try { + const ir = await this.sendRequest>(targetId, 'describe', {}, 5000); + methods = (ir?.manifest as AbjectManifest | undefined)?.interface?.methods ?? []; + if (methods.length === 0) note = 'live manifest declares no methods — schema/relations vacuous'; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + note = `live manifest unreadable (${msg.slice(0, 60)}) — schema/relations vacuous`; + } + state.fitnessLiveMethods = { targetId, methods, note }; + return { methods, note }; + } + + /** + * The hard gate: replays the target's recorded cassettes, validates output + * schemas and relations, and mutation-tests the staged draft — entirely + * inside the sandbox, with every I/O call shimmed to the object's own + * cassettes. Records the verdict and a digest of the judged source; + * deploy_spawn/deploy_update consult both via `deployGate` and refuse when + * the draft has changed since this ran. + */ + private async opFitness(state: LoopState): Promise<{ ok: boolean; summary: string; error?: string; data?: unknown }> { + if (!state.draftSource) return { ok: false, summary: 'fitness: no draft source', error: 'draft a source first' }; + const { methods, note } = await this.fitnessMethods(state); + const cassettes = await this.loadCassettes(state.targetObjectId); + const verdict = await evaluate({ source: state.draftSource }, { cassettes, methods }, + buildSandboxInvoker()); + state.fitnessVerdict = verdict; + state.fitnessSourceDigest = verdictDigest(state.draftSource, methods, state.targetObjectId); + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + const caveat = note ? ` [${note}]` : ''; + return { + ok: verdict.pass, + summary: summarizeVerdict(verdict) + caveat, + error: verdict.pass ? undefined : failed, + data: verdict, + }; + } + // ── The deploy gate ─────────────────────────────────────────────────── // - // Deploy is the ONLY place a check refuses to proceed, and it refuses on - // mechanical grounds only: a draft that does not parse, or that calls a method - // a dependency's live manifest does not have, must never reach the live object. + // Deploy is the ONLY place a check refuses to proceed, and it refuses on two + // different kinds of ground. + // + // MECHANICAL (`gateDeploy`, below): a draft that does not parse, or that + // calls a method a dependency's live manifest does not have, must never + // reach the live object. + // + // EVIDENTIAL (`deployGate`, ../protocol/fitness.js): the draft must carry a + // PASSING fitness verdict — recorded traffic replayed, declared schemas and + // relations upheld, mutants killed — for exactly this source, these + // declarations, and this target. Not mechanical, and the only non-mechanical + // refusal there is. + // // LLM judgments (the semantic reviewer) advise but never block — a deployed // object answering real calls teaches more per step than another blind pass. @@ -1888,6 +1991,53 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I } } + /** + * The one door to Factory.spawn. Every op that creates an object routes + * through here with an explicit policy: + * + * 'gate-draft' -- the source being deployed is a STAGED DRAFT; + * deployGate must hold a passing fitness verdict + * for it (digest-bound to source, declarations, + * and target) or the spawn is refused. + * 'redeploys-live-source' -- the source is copied VERBATIM from an object + * that is already live (clone, extract). There + * is no draft to judge and no cassettes exist + * under the not-yet-existing id. Exemption + * raised with the upstream maintainer on PR #11; + * tightening it is a one-line policy change. + * + * `judgeSource` is what the gate judges when it differs from what the + * Factory receives -- an organism deploys a JSON spec, but the code being + * shipped inside it is the staged membrane source. + */ + private async gatedSpawn(state: LoopState, spawnReq: SpawnRequest, label: string, + policy: 'gate-draft' | 'redeploys-live-source', + judgeSource?: string): + Promise<{ result: SpawnResult } | { refusal: { ok: false; summary: string; error: string } }> { + if (!this.factoryId) { + return { refusal: { ok: false, summary: `${label}: Factory unavailable`, error: 'Factory not resolved' } }; + } + if (policy === 'gate-draft') { + const judged = judgeSource ?? spawnReq.source; + if (typeof judged !== 'string' || judged.length === 0) { + return { refusal: { ok: false, summary: `${label}: no source to judge`, error: 'stage a source first' } }; + } + const gate = deployGate(state, judged, (await this.fitnessMethods(state)).methods); + if (!gate.ok) return { refusal: { ok: false, summary: gate.error, error: gate.error } }; + } + let result: SpawnResult; + try { + result = await this.sendRequest(this.factoryId, 'spawn', spawnReq, 120000); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { refusal: { ok: false, summary: `${label}: spawn failed: ${msg.slice(0, 120)}`, error: msg } }; + } + if (!result?.objectId) { + return { refusal: { ok: false, summary: `${label}: Factory returned no objectId`, error: 'unexpected Factory response' } }; + } + return { result }; + } + /** * Deploy a CREATE: read the staged manifest + source from the loop state * and send Factory.spawn server-side. Still pure message passing — this @@ -1911,17 +2061,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I registryHint: this.registryId, }; - let result: SpawnResult; - try { - result = await this.sendRequest(this.factoryId, 'spawn', spawnReq, 120000); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { ok: false, summary: `deploy_spawn: ${msg.slice(0, 120)}`, error: msg }; - } - - if (!result?.objectId) { - return { ok: false, summary: 'deploy_spawn: Factory returned no objectId', error: 'unexpected Factory response' }; - } + const spawned = await this.gatedSpawn(state, spawnReq, 'deploy_spawn', 'gate-draft'); + if ('refusal' in spawned) return spawned.refusal; + const result = spawned.result; state.spawnedObjectId = result.objectId; @@ -2000,8 +2142,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_update'); if (refusal) return refusal; - // Resolve target: explicit objectId / targetName from action wins, else - // fall back to the kind:modify state target. + // Resolve target BEFORE the fitness gate: the action can name an object + // the gate never saw (the gate judged `state.targetObjectId`, which an + // explicit objectId/targetName overrides), and a verdict earned against + // another object's cassettes says nothing about this one. let targetId: AbjectId | undefined; let targetLabel: string | undefined; const explicitId = this.actionField(action, ['objectId']); @@ -2023,6 +2167,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: false, summary: 'deploy_update: no target', error: 'pass {objectId} or {targetName} in the action payload, or use deploy_spawn for new objects' }; } + const gate = deployGate(state, state.draftSource, + (await this.fitnessMethods(state)).methods, targetId); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + // Everything from here is one write to one object. Two modify loops // running at once would otherwise interleave their four steps and leave // the live object, the Registry's cache, and the store each holding a @@ -2305,16 +2453,11 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I registryHint: this.registryId, }; - let result: SpawnResult; - try { - result = await this.sendRequest(this.factoryId, 'spawn', spawnReq, 120000); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { ok: false, summary: `compose_organism: spawn failed: ${msg.slice(0, 120)}`, error: msg }; - } - if (!result?.objectId) { - return { ok: false, summary: 'compose_organism: Factory returned no objectId', error: 'unexpected Factory response' }; - } + // The organism's Factory source is the JSON spec, but the code being + // shipped is the membrane draft -- that is what the gate judges. + const spawned = await this.gatedSpawn(state, spawnReq, 'compose_organism', 'gate-draft', membraneSource); + if ('refusal' in spawned) return spawned.refusal; + const result = spawned.result; state.spawnedObjectId = result.objectId; // The staged membrane drafts are now live inside the organism. @@ -2375,16 +2518,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I parentId: this.id, registryHint: this.registryId, }; - let result: SpawnResult; - try { - result = await this.sendRequest(this.factoryId, 'spawn', spawnReq, 120000); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { ok: false, summary: `extract_organelle: spawn failed: ${msg.slice(0, 120)}`, error: msg }; - } - if (!result?.objectId) { - return { ok: false, summary: 'extract_organelle: Factory returned no objectId', error: 'unexpected Factory response' }; - } + const spawned = await this.gatedSpawn(state, spawnReq, 'extract_organelle', 'redeploys-live-source'); + if ('refusal' in spawned) return spawned.refusal; + const result = spawned.result; state.spawnedObjectId = result.objectId; @@ -3355,6 +3491,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I case 'review_semantics': res = await this.opReviewSemantics(state); break; + case 'fitness': + res = await this.opFitness(state); + break; case 'deploy_spawn': res = await this.opDeploySpawn(state); break; @@ -3714,6 +3853,7 @@ Your local actions are the supported way to create and modify Abjects: they carr - \`read_draft({handler?, lineRange?, grep?})\` — read the CURRENT source exactly as it stands. No args → the member outline. \`{handler:"name"}\` → that member's exact text, line-numbered. \`{lineRange:"a-b"}\` / \`{grep:"pattern"}\` → those lines. Read-only: it stages nothing and makes no progress on its own. Use it to look at a member you are about to rewrite, or one a check named. Never two turns in a row without an edit in between. - \`draft_via_llm({kind: "manifest" | "source", instructions})\` — ask an LLM to draft for you. It sees current loop state. Use when authoring a brand-new manifest or source from scratch is too large for one think-step. Do NOT use this for modifications of existing objects — use \`edit_source\` instead, since the LLM consistently truncates "preserve everything else" rewrites. - \`compile()\` / \`validate_calls()\` / \`review_semantics()\` — the checks, available explicitly but **rarely worth a step**: they run on their own (see *Checks run themselves*, below). +- \`fitness()\` — hard gate: replays recorded cassettes, validates output schemas and relations, and mutation-tests the draft. \`deploy_spawn\`/\`deploy_update\` refuse until fitness passes on the current draft. - \`deploy_spawn({})\` — deploy the staged drafts as a NEW Abject. Internally messages Factory.spawn with the manifest, source, and the right owner / parent / registryHint. Use for create flows. No payload: the staged drafts are read from loop state. - \`deploy_update({objectId?, targetName?})\` — deploy the staged source onto an EXISTING object. Internally hot-swaps the live object via its \`updateSource\` handler, then updates Registry's cached source + manifest, then persists via AbjectStore so the change survives a restart. The target is taken from \`objectId\` (UUID) or \`targetName\` (registered name) in the action payload, or from the task's target if it was started as a modify. If you investigated and discovered you should be modifying an existing object even though the loop kind is \`create\`, pass \`{objectId: ""}\` here. - \`compose_organism({name, description, organelleNames, interfaceSource?})\` packages EXISTING source-backed objects into ONE Organism: a composite Abject whose organelles (independent internal copies of the named objects) cooperate behind a membrane interface, while external callers see a single object with a single curated surface. The staged drafts define the membrane: \`draft_manifest\` is the organism's public surface and \`draft_source\` (or the explicit \`interfaceSource\`) is the forwarding handler map; when either is missing it is drafted automatically from the organelle manifests. The originals keep running, so remove them afterwards (or tell the user) when the organism replaces them. diff --git a/src/protocol/canonical.ts b/src/protocol/canonical.ts new file mode 100644 index 0000000..4eaba09 --- /dev/null +++ b/src/protocol/canonical.ts @@ -0,0 +1,20 @@ +/** + * Canonical serialization for hashing. Two readings of the same value must + * hash identically regardless of property insertion order, or digests start + * disagreeing about facts that have not changed. + */ +import { createHash } from 'node:crypto'; + +/** JSON with object keys sorted recursively; array order is preserved + * (element order is meaning, key order is accident). */ +export function canonicalJson(v: unknown): string { + if (Array.isArray(v)) return `[${v.map(canonicalJson).join(',')}]`; + if (v !== null && typeof v === 'object') { + const keys = Object.keys(v as object).sort(); + return `{${keys.map(k => + `${JSON.stringify(k)}:${canonicalJson((v as Record)[k])}`).join(',')}}`; + } + return JSON.stringify(v) ?? 'null'; +} + +export const sha256 = (s: string): string => createHash('sha256').update(s).digest('hex'); diff --git a/src/protocol/cassette-recorder.test.ts b/src/protocol/cassette-recorder.test.ts new file mode 100644 index 0000000..fc48f6d --- /dev/null +++ b/src/protocol/cassette-recorder.test.ts @@ -0,0 +1,80 @@ +/** Run: pnpm tsx --test src/protocol/cassette-recorder.test.ts */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { setRecorder, clearRecorder, beforeRequest, afterResponse } from './cassette-recorder.js'; +import { CassetteStore, HTTP_CASSETTE_METHOD } from './cassette.js'; + +afterEach(() => clearRecorder('obj-1')); + +test('record mode captures 2xx with raw AND parsed body; non-2xx is not recorded', () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder('obj-1', { mode: 'record', store, onRecord: () => persisted++ }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, rawBody: '{"ok":1}' }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/b' }, + { status: 500, rawBody: 'boom' }); + assert.equal(store.all().length, 1); + assert.equal(persisted, 1); + const [c] = store.all(); + assert.equal(c.method, HTTP_CASSETTE_METHOD); + assert.equal(c.rawBody, '{"ok":1}'); + assert.deepEqual(c.parsedOutput, { ok: 1 }); +}); + +test('a JSON string primitive survives the record/replay round-trip verbatim', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + // The world sent the four characters `"hi"`; JSON.parse of that is `hi`. + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/s' }, + { status: 200, rawBody: '"hi"' }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/s' }); + assert.equal(hit?.rawBody, '"hi"'); +}); + +test('replay mode serves recorded responses and throws on a miss', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, rawBody: '{"ok":1}' }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/a' }); + assert.equal(hit?.status, 200); + assert.equal(hit?.rawBody, '{"ok":1}'); + assert.throws(() => beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/miss' }), + /replay miss/); +}); + +test('unknown object id and live mode pass through', () => { + assert.equal(beforeRequest(undefined, { method: 'GET', url: 'https://x.test/' }), undefined); + assert.equal(beforeRequest('never-registered', { method: 'GET', url: 'https://x.test/' }), undefined); +}); + +test('afterResponse parses rawBody itself, only when recording', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + afterResponse('obj-1', { method: 'GET', url: 'https://x.test/parse' }, + { status: 200, rawBody: '{"n":1}' }); + assert.deepEqual(store.all()[0].response.body, { n: 1 }); + assert.deepEqual(store.all()[0].parsedOutput, { n: 1 }); + assert.equal(store.all()[0].rawBody, '{"n":1}'); +}); + +test('non-JSON rawBody records as the raw text', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + afterResponse('obj-1', { method: 'GET', url: 'https://x.test/text' }, + { status: 200, rawBody: 'plain text' }); + assert.equal(store.all()[0].response.body, 'plain text'); +}); + +test('redactPaths masks recorded response bodies at the declared paths', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store, redactPaths: ['user.ssn'] }); + afterResponse('obj-1', { method: 'GET', url: 'https://x.test/u' }, + { status: 200, rawBody: '{"user":{"ssn":"123-45-6789","name":"A"}}' }); + const rec = store.all()[0]; + assert.deepEqual(rec.response.body, { user: { ssn: 'REDACTED', name: 'A' } }); + assert.doesNotMatch(rec.rawBody, /123-45-6789/); +}); diff --git a/src/protocol/cassette-recorder.ts b/src/protocol/cassette-recorder.ts new file mode 100644 index 0000000..144983b --- /dev/null +++ b/src/protocol/cassette-recorder.ts @@ -0,0 +1,80 @@ +/** + * CassetteRecorder -- the per-object seam between HttpClient and the + * cassette store. HttpClient asks it two questions: "should this request be + * served from a recording?" (replay) and "should this response be kept?" + * (record). Objects not registered here pass through untouched, so the + * seam costs nothing for the rest of the system. + */ +import { CassetteStore, HTTP_CASSETTE_METHOD, type CassetteRequest, redactRequest } from './cassette.js'; + +export type RecorderMode = 'record' | 'replay' | 'live'; +export interface RecorderEntry { + mode: RecorderMode; + store: CassetteStore; + onRecord?: (store: CassetteStore) => void; + /** Dot-paths masked in recorded response bodies (from the object's method + * declarations). Replay fidelity is deliberately sacrificed at these + * paths: rawBody is re-serialized post-redaction, because a verbatim raw + * body would defeat the point of masking. */ + redactPaths?: string[]; +} + +const recorders = new Map(); + +export function setRecorder(objectId: string, entry: RecorderEntry): void { + recorders.set(objectId, entry); +} +export function clearRecorder(objectId: string): void { recorders.delete(objectId); } + +/** `rawBody` is the response text verbatim: HttpClient's contract promises + * callers a raw string body, so replay must return the same characters the + * world sent rather than a re-stringified parse of them. */ +export function beforeRequest(objectId: string | undefined, req: CassetteRequest): + { status: number; rawBody: string; headers: Record } | undefined { + if (!objectId) return undefined; + const r = recorders.get(objectId); + if (!r || r.mode !== 'replay') return undefined; + const hit = r.store.matchRequest(req); + if (!hit) throw new Error(`replay miss: no cassette for ${req.method} ${req.url}`); + return { status: hit.response.status, rawBody: hit.rawBody, headers: {} }; +} + +export function afterResponse(objectId: string | undefined, req: CassetteRequest, + res: { status: number; rawBody: string }): void { + if (!objectId) return; + const r = recorders.get(objectId); + if (!r || r.mode !== 'record') return; + if (res.status < 200 || res.status >= 300) return; + // Parse HERE, not in HttpClient: the recorder is the only consumer of the + // parsed shape, and parsing every response in the system to feed a recorder + // that is almost never attached would tax the entire runtime's HTTP path. + let body: unknown = res.rawBody; + try { body = JSON.parse(res.rawBody); } catch { /* not JSON — keep the text */ } + let rawBody = res.rawBody; + if (r.redactPaths?.length && body !== null && typeof body === 'object') { + let touched = false; + for (const path of r.redactPaths) { + let node: unknown = body; + const keys = path.split('.'); + for (const key of keys.slice(0, -1)) { + node = node !== null && typeof node === 'object' + ? (node as Record)[key] : undefined; + } + const leaf = keys[keys.length - 1]; + if (node !== null && typeof node === 'object' && leaf in (node as object)) { + (node as Record)[leaf] = 'REDACTED'; + touched = true; + } + } + if (touched) rawBody = JSON.stringify(body); + } + r.store.add({ + method: HTTP_CASSETTE_METHOD, args: {}, + request: redactRequest(req), + response: { status: res.status, body }, + rawBody, + parsedOutput: body, + recordedAt: Date.now(), + }); + r.onRecord?.(r.store); +} diff --git a/src/protocol/cassette.test.ts b/src/protocol/cassette.test.ts new file mode 100644 index 0000000..19b6968 --- /dev/null +++ b/src/protocol/cassette.test.ts @@ -0,0 +1,105 @@ +/** Run: pnpm tsx --test src/protocol/cassette.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { CassetteStore, redactRequest, CASSETTE_CAP_PER_METHOD, type Cassette } from './cassette.js'; + +function mk(n: number, method = 'listEvents'): Cassette { + return { + method, args: { q: n }, + request: { method: 'GET', url: `https://example.test/events?q=${n}` }, + response: { status: 200, body: [{ id: n }] }, + rawBody: `[{"id":${n}}]`, + parsedOutput: [{ id: n }], + recordedAt: n, + }; +} + +test('redactRequest strips credential headers case-insensitively', () => { + const r = redactRequest({ + method: 'GET', url: 'https://example.test/x', + headers: { Authorization: 'Bearer s3cret', 'X-Ok': 'yes', COOKIE: 'a=1', 'set-cookie': 'b=2' }, + }); + assert.deepEqual(r.headers, { 'X-Ok': 'yes' }); +}); + +test('store caps per method with LRU eviction', () => { + const s = new CassetteStore(); + for (let i = 0; i < CASSETTE_CAP_PER_METHOD + 5; i++) s.add(mk(i)); + const kept = s.byMethod('listEvents'); + assert.equal(kept.length, CASSETTE_CAP_PER_METHOD); + assert.equal(kept[0].recordedAt, 5); // 0..4 evicted +}); + +test('matchRequest is exact: a different query string is a different request', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=1' })); + // ?q=other is NOT ?q=1 — serving it would defeat argument-dependent replay. + assert.equal(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=other' }), undefined); + assert.equal(s.matchRequest({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + +test('matchRequestLoose falls back to host+path when no exact match exists', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=1' })); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=other' })); + assert.equal(s.matchRequestLoose({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + +test('toJSON/fromJSON round-trips and skips malformed entries', () => { + const s = new CassetteStore([mk(1), mk(2)]); + const back = CassetteStore.fromJSON(JSON.parse(JSON.stringify(s.toJSON()))); + assert.equal(back.all().length, 2); + assert.equal(back.all()[0].rawBody, '[{"id":1}]'); + const dirty = CassetteStore.fromJSON([mk(3), { junk: true }, 42]); + assert.equal(dirty.all().length, 1); +}); + +test('fromJSON derives rawBody for entries recorded before it existed', () => { + const legacy = { ...mk(7) } as Partial; + delete legacy.rawBody; + const store = CassetteStore.fromJSON([legacy]); + assert.equal(store.all().length, 1); + assert.equal(store.all()[0].rawBody, JSON.stringify([{ id: 7 }])); +}); + +test('two POSTs to one url with different bodies are distinct cassettes', () => { + const store = new CassetteStore(); + const base = { method: 'send', args: {}, parsedOutput: 'x', recordedAt: 1 }; + store.add({ ...base, request: { method: 'POST', url: 'https://x.test/api', body: { amount: 1 } }, + response: { status: 200, body: 'a' }, rawBody: 'a' }); + store.add({ ...base, request: { method: 'POST', url: 'https://x.test/api', body: { amount: 2 } }, + response: { status: 200, body: 'b' }, rawBody: 'b' }); + const hit = store.matchRequest({ method: 'POST', url: 'https://x.test/api', body: { amount: 2 } }); + assert.equal(hit?.rawBody, 'b'); +}); + +test('omitting the body does not match a cassette recorded FOR a body', () => { + const store = new CassetteStore(); + store.add({ method: 'send', args: {}, + request: { method: 'POST', url: 'https://x.test/api', body: { amount: 1 } }, + response: { status: 200, body: 'a' }, rawBody: 'a', parsedOutput: 'a', recordedAt: 1 }); + assert.equal(store.matchRequest({ method: 'POST', url: 'https://x.test/api' }), undefined); +}); + +test('legacy body-less cassettes still match body-less requests', () => { + const store = new CassetteStore([mk(1)]); + assert.notEqual(store.matchRequest({ method: 'GET', url: 'https://example.test/events?q=1' }), undefined); +}); + +test('body key order does not decide a match', () => { + const store = new CassetteStore(); + store.add({ method: 'send', args: {}, + request: { method: 'POST', url: 'https://x.test/api', body: { a: 1, b: 2 } }, + response: { status: 200, body: 'a' }, rawBody: 'a', parsedOutput: 'a', recordedAt: 1 }); + assert.notEqual(store.matchRequest({ method: 'POST', url: 'https://x.test/api', body: { b: 2, a: 1 } }), undefined); +}); + +test('secret query params are redacted before storage', () => { + const store = new CassetteStore(); + store.add({ method: 'get', args: {}, + request: { method: 'GET', url: 'https://x.test/a?api_key=hunter2&q=1' }, + response: { status: 200, body: 1 }, rawBody: '1', parsedOutput: 1, recordedAt: 1 }); + const url = store.all()[0].request.url; + assert.doesNotMatch(url, /hunter2/); + assert.match(url, /q=1/); +}); diff --git a/src/protocol/cassette.ts b/src/protocol/cassette.ts new file mode 100644 index 0000000..7ebcacf --- /dev/null +++ b/src/protocol/cassette.ts @@ -0,0 +1,154 @@ +/** + * Cassette -- the evidence the fitness gate judges a candidate against. + * + * A cassette is one recorded truth: a request the object made, the response + * the world gave, and the parsed answer the object produced from it. The + * fitness gate replays cassettes against every candidate source; a candidate + * that cannot reproduce recorded meaning does not deploy. Requests are + * redacted before storage so a cassette can never leak a credential. + */ +import { canonicalJson, sha256 } from './canonical.js'; + +export interface CassetteRequest { + method: string; + url: string; + headers?: Record; + body?: unknown; +} + +export interface Cassette { + method: string; + args: Record; + request: CassetteRequest; + /** Canonical hash of `request.body`, set by the store when the request + * carried one. Matching is symmetric on it: a body-less request matches + * only body-less recordings, so a caller cannot skip the body and pick up + * an answer that was recorded for a specific payload. */ + bodyHash?: string; + response: { status: number; body: unknown }; + /** The response body EXACTLY as the world sent it, before any parsing. + * HttpClient's contract says `body` is always a raw string, so replay must + * hand back the same characters — `JSON.stringify(parsed)` is not the same + * text for a JSON string primitive, and the round-trip loses meaning. */ + rawBody: string; + parsedOutput: unknown; + recordedAt: number; +} + +export const CASSETTE_CAP_PER_METHOD = 20; + +/** The method name recorded for raw HTTP traffic. These cassettes are stubs + * for the object's own calls, never a method the fitness gate can replay. */ +export const HTTP_CASSETTE_METHOD = '_http'; + +const REDACTED_HEADERS = new Set(['authorization', 'cookie', 'set-cookie']); +const SECRET_QUERY_PARAM = /^(key|api_key|token|access_token|secret|auth|apikey)$/i; + +/** The canonical hash matching compares. `undefined` for a body-less + * request -- and only for one. */ +export function requestBodyHash(body: unknown): string | undefined { + return body === undefined ? undefined : sha256(canonicalJson(body)); +} + +export function redactRequest(req: CassetteRequest): CassetteRequest { + let out = req; + if (out.headers) { + const headers: Record = {}; + for (const [k, v] of Object.entries(out.headers)) { + if (!REDACTED_HEADERS.has(k.toLowerCase())) headers[k] = v; + } + out = { ...out, headers }; + } + // Credentials travel in query strings too (?api_key=...); a cassette must + // never store one. + try { + const u = new URL(out.url); + let touched = false; + for (const name of [...u.searchParams.keys()]) { + if (SECRET_QUERY_PARAM.test(name)) { u.searchParams.set(name, 'REDACTED'); touched = true; } + } + if (touched) out = { ...out, url: u.toString() }; + } catch { /* unparseable url -- store as given */ } + return out; +} + +function hostPath(url: string): string | undefined { + try { const u = new URL(url); return `${u.host}${u.pathname}`; } catch { return undefined; } +} + +/** Cassettes recorded before `rawBody` existed derive it from the parsed + * body. Lossy for a JSON string primitive, but honest and never undefined. */ +function rawBodyOf(c: Cassette): string { + return typeof c.rawBody === 'string' ? c.rawBody : (JSON.stringify(c.response.body) ?? ''); +} + +function isCassette(c: unknown): c is Cassette { + if (c === null || typeof c !== 'object') return false; + const x = c as Record; + return typeof x.method === 'string' + && x.args !== null && typeof x.args === 'object' + && x.request !== null && typeof x.request === 'object' + && typeof (x.request as Record).url === 'string' + && x.response !== null && typeof x.response === 'object' + && typeof x.recordedAt === 'number'; +} + +export class CassetteStore { + private cassettes: Cassette[] = []; + + constructor(initial?: Cassette[]) { + for (const c of initial ?? []) this.add(c); + } + + add(c: Cassette): void { + this.cassettes.push({ ...c, request: redactRequest(c.request), rawBody: rawBodyOf(c), + bodyHash: c.bodyHash ?? requestBodyHash(c.request.body) }); + const forMethod = this.cassettes.filter(x => x.method === c.method); + if (forMethod.length > CASSETTE_CAP_PER_METHOD) { + const evict = forMethod + .sort((a, b) => a.recordedAt - b.recordedAt) + .slice(0, forMethod.length - CASSETTE_CAP_PER_METHOD); + this.cassettes = this.cassettes.filter(x => !evict.includes(x)); + } + } + + byMethod(method: string): Cassette[] { + return this.cassettes + .filter(c => c.method === method) + .sort((a, b) => a.recordedAt - b.recordedAt); + } + + all(): Cassette[] { return [...this.cassettes]; } + + /** Exact method+url+body. Replay is argument-dependent: `?q=1` and + * `?q=other` are different questions, and answering one with the other's + * recording would let a candidate "reproduce" traffic it never made. The + * body comparison is symmetric -- a body-less request matches only + * body-less recordings -- so omitting the body is a miss, never a + * wildcard. */ + matchRequest(req: CassetteRequest): Cassette | undefined { + const hash = requestBodyHash(req.body); + return this.cassettes.find( + c => c.request.method === req.method && c.request.url === req.url + && c.bodyHash === hash); + } + + /** Exact match, else any recording of the same host+path. Deliberately NOT + * used by the replay seam — kept for callers that want a representative + * sample of an endpoint rather than an answer to a specific question. */ + matchRequestLoose(req: CassetteRequest): Cassette | undefined { + const exact = this.matchRequest(req); + if (exact) return exact; + const hp = hostPath(req.url); + if (!hp) return undefined; + return this.cassettes.find( + c => c.request.method === req.method && hostPath(c.request.url) === hp); + } + + toJSON(): Cassette[] { return this.all(); } + + static fromJSON(json: unknown): CassetteStore { + const arr = Array.isArray(json) ? json.filter(isCassette) : []; + return new CassetteStore(arr); + } +} diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts new file mode 100644 index 0000000..1aa085a --- /dev/null +++ b/src/protocol/fitness.test.ts @@ -0,0 +1,423 @@ +/** Run: pnpm tsx --test src/protocol/fitness.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { evaluate, summarizeVerdict, verdictDigest, deployGate, type Invoker } from './fitness.js'; +import { CassetteStore, type Cassette } from './cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +/** Test invoker: the "source" is the body of an async JS function + * (args, http) => output. The invoker the gate actually runs candidates + * under lives in ./sandbox-invoker.ts; this one keeps these tests on the + * checks themselves. */ +const testInvoker: Invoker = async (source, _method, args, http) => { + const fn = new Function('args', 'http', `"use strict"; return (async () => { ${source} })();`); + return fn(args, http); +}; + +const GOOD_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events?q=' + args.q }); + if (!res) throw new Error('no stub'); + return res.body; +`; +const WRONG_SOURCE = `return [];`; + +const cassette: Cassette = { + method: 'listEvents', args: { q: 1 }, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, +}; + +const methods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + sideEffects: 'read-only', + outputSchema: { + type: 'array', + items: { type: 'object', required: ['id'], properties: { id: { type: 'number' } } }, + }, +}]; + +test('replay passes when the candidate reproduces recorded meaning', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('replay fails and short-circuits when output diverges from the cassette', async () => { + const v = await evaluate({ source: WRONG_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, false); + assert.equal(v.checks.some(c => c.check === 'schema'), false); // short-circuit +}); + +test('an empty cassette store yields an honest unverified pass, probing nothing', async () => { + const neverInvoked: Invoker = async () => { + throw new Error('the gate probed a candidate it has no evidence for'); + }; + const v = await evaluate({ source: 'return [{ notId: true }];' }, + { cassettes: new CassetteStore([]), methods }, neverInvoked); + assert.equal(v.pass, true); + const detail = (name: string) => v.checks.find(c => c.check === name)!; + assert.equal(detail('replay').pass, true); + assert.equal(detail('schema').pass, true); + assert.equal(detail('schema').detail, 'no cassettes — schema unverified'); + assert.equal(detail('relations').pass, true); + assert.equal(detail('relations').detail, 'no cassettes — relations unverified'); + assert.equal(detail('mutation').pass, true); + assert.equal(detail('mutation').detail, 'no cassettes — mutation gate requires evidence'); + assert.equal(v.killRatio, undefined); +}); + +test('schema fails on schema-invalid output', async () => { + const badSchemaSource = `return [{ notId: true }];`; + const c: Cassette = { + ...cassette, + response: { status: 200, body: [{ notId: true }] }, + rawBody: '[{"notId":true}]', + parsedOutput: [{ notId: true }], + }; + const v = await evaluate({ source: badSchemaSource }, + { cassettes: new CassetteStore([c]), methods }, testInvoker); + // replay passes (the source reproduces the recording); schema still refuses it + assert.equal(v.checks.find(x => x.check === 'replay')?.pass, true); + assert.equal(v.checks.find(x => x.check === 'schema')?.pass, false); + assert.equal(v.pass, false); +}); + +test('schema fails when every probe throws and a schema is declared', async () => { + // `listEvents` has evidence and reproduces it. `countEvents` declares a + // schema, owns no cassette, and throws on its one {} probe — so nothing + // about its contract was ever validated, which is a failure, not a pass. + const source = `return [{ id: 1, startsAt: '2026-08-23' }];`; + const throwsForCount: Invoker = async (src, method, args, http) => { + if (method === 'countEvents') throw new Error('not implemented'); + return testInvoker(src, method, args, http); + }; + const v = await evaluate({ source }, + { cassettes: new CassetteStore([cassette]), + methods: [...methods, + { name: 'countEvents', description: '', parameters: [], outputSchema: { type: 'number' } }] }, + throwsForCount, { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); + assert.match(v.checks.find(c => c.check === 'schema')!.detail, /no output could be validated/); +}); + +test('_http cassettes stub HTTP but are never replayed as methods', async () => { + // The recorder writes every captured response under the method '_http'. + // Feeding those to the invoker asks it for a handler no object has. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, + }; + const methodAware: Invoker = async (src, method, args, http) => { + if (method !== 'listEvents') throw new Error(`fitness: source has no handler for '${method}'`); + return testInvoker(src, method, args, http); + }; + const fixedUrl = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + return res.body; + `; + const v = await evaluate({ source: fixedUrl }, + { cassettes: new CassetteStore([httpOnly]), methods }, methodAware, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + // and stubFor still served the recording: the schema probe fetched it + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('mutation fails when the candidate parses under no dialect', async () => { + // An invoker that ignores the source: replay/schema/relations all pass, so + // the verdict turns entirely on whether the gate admits it cannot read it. + const blind: Invoker = async () => [{ id: 1, startsAt: '2026-08-23' }]; + const v = await evaluate({ source: `{ async listEvents(msg) { return [ }` }, + { cassettes: new CassetteStore([cassette]), methods }, blind); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.equal(mut.detail, 'candidate does not parse'); +}); + +const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }], + entityRef: 'Weekly Standup', +}]; + +const relCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: null }, // body unused: sources below ignore http + rawBody: 'null', + parsedOutput: null as unknown, // parsedOutput unused: set per-test below + recordedAt: 1, +}; + +test('relations: duplicates and disorder are caught', async () => { + const dupSource = `return [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }];`; + // make replay vacuous: cassette parsedOutput matches the source's constant output + const c = { ...relCassette, parsedOutput: [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }] }; + const v = await evaluate({ source: dupSource }, + { cassettes: new CassetteStore([c]), methods: relMethods }, testInvoker); + const rel = v.checks.find(x => x.check === 'relations'); + assert.equal(rel?.pass, false); + assert.match(rel!.detail, /no-duplicates|sorted-by/); +}); + +test('relations: known entity must appear', async () => { + const noEntity = `return [{ startsAt: 'a', name: 'Other Thing' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Other Thing' }] }; + const v = await evaluate({ source: noEntity }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'non-empty-for-known-entity' }] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, false); +}); + +test('relations: a clean output passes all declared relations', async () => { + const clean = `return [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }] }; + const v = await evaluate({ source: clean }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [ + { kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, { kind: 'idempotent' }, + ] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); + +test('relations: a throwing second invocation fails idempotent instead of rejecting evaluate', async () => { + // stateful source: first call returns [], second call throws + let calls = 0; + const flakyInvoker: Invoker = async (source, method, args, http) => { + calls++; + if (calls > 2) throw new Error('flaky'); + return []; + }; + const c = { ...relCassette, parsedOutput: [] as unknown }; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'idempotent' as const }] }] }, + flakyInvoker); + assert.equal(v.pass, false); + assert.match(v.checks.find(x => x.check === 'relations')!.detail, /second call threw/); +}); + +test('relations: subset-on-tighter-filter orders numeric filter args numerically', async () => { + const mk = (q: number, out: unknown[]) => ({ + method: 'listEvents', args: { q }, + request: { method: 'GET', url: `https://example.test/events?q=${q}` }, + response: { status: 200, body: out }, rawBody: JSON.stringify(out), + parsedOutput: out, recordedAt: q, + }); + // q=2 (looser, returns 2 items), q=10 (tighter, returns subset of 1) + const outputs: Record = { 2: [{ id: 1 }, { id: 2 }], 10: [{ id: 1 }] }; + const numInvoker: Invoker = async (_s, _m, args) => outputs[args.q as number]; + const v = await evaluate({ source: 'irrelevant' }, + { cassettes: new CassetteStore([mk(2, outputs[2]), mk(10, outputs[10])]), + methods: [{ name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'subset-on-tighter-filter' as const, field: 'q' }] }] }, + numInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); + +test('mutation gate kills mutants of a well-tested source', async () => { + // GOOD_SOURCE returns the cassette body verbatim; flipping its logic breaks replay. + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 12, killThreshold: 0.5 }); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + if (v.killRatio !== undefined && mut!.detail !== 'no mutation points') { + assert.ok(v.killRatio >= 0 && v.killRatio <= 1); + } +}); + +test('maxMutants: 0 skips the mutation gate', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'mutation')?.detail, 'skipped'); +}); + +test('mutation gate counts kills on a source with real mutation points', async () => { + const KILLABLE_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.kind === 'event'); + return items; +`; + const killCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ kind: 'event', id: 1 }, { kind: 'other', id: 2 }] }, + rawBody: '[{"kind":"event","id":1},{"kind":"other","id":2}]', + parsedOutput: [{ kind: 'event', id: 1 }], + recordedAt: 1, + }; + const v = await evaluate({ source: KILLABLE_SOURCE }, + { cassettes: new CassetteStore([killCassette]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + assert.notEqual(mut!.detail, 'no mutation points'); + // expected mutants: flip '===' -> '!==' (returns the wrong item), drop .filter (returns both) — both killed by replay + assert.equal(v.killRatio, 1); + assert.equal(mut!.pass, true); + assert.match(mut!.detail, /2\/2 mutants killed/); +}); + +test('mutation fails the verdict when the evidence cannot kill enough mutants', async () => { + // Four mutation sites, only one of them observable through the recording: + // the live filter excludes nothing the cassette contains, and `spare` is + // dead code. Weak evidence must read as a failure, not a pass. + const WEAK_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.id > 0); + const spare = res.body.filter(e => e.id > 100); + return items; +`; + const weak: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }, { id: 2 }] }, + rawBody: '[{"id":1},{"id":2}]', + parsedOutput: [{ id: 1 }, { id: 2 }], + recordedAt: 1, + }; + const v = await evaluate({ source: WEAK_SOURCE }, + { cassettes: new CassetteStore([weak]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.ok(v.killRatio !== undefined && v.killRatio < 0.8, `killRatio was ${v.killRatio}`); + // measured: of six sites (two guard flips, two boundary nudges, two + // filter drops), only the live filter's flipped guard changes what + // comes back through this recording + assert.equal(v.killRatio, 1 / 6); + assert.equal(mut.detail, '1/6 mutants killed (threshold 0.8)'); +}); + +test('relations say so when no cassette is attributed to the method', async () => { + // Only _http traffic was captured, so `replayable` finds nothing to probe + // with and falls back to a single {} call. A method that needs arguments + // throws on it, and the check must not report success it never earned. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1 }, { id: 1 }] }, + rawBody: '[{"id":1},{"id":1}]', + parsedOutput: [{ id: 1 }, { id: 1 }], + recordedAt: 1, + }; + const needsArgs: Invoker = async (_src, _method, args) => { + if ((args as { q?: unknown }).q === undefined) throw new Error('q is required'); + return [{ id: 1 }, { id: 1 }]; // duplicates: no-duplicates would FAIL if ever evaluated + }; + const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], sideEffects: 'read-only', + relations: [{ kind: 'no-duplicates' }], + }]; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([httpOnly]), methods: relMethods }, needsArgs, { maxMutants: 0 }); + const rel = v.checks.find(c => c.check === 'relations')!; + assert.match(rel.detail, /unverified/, + 'relations must report unverified evidence, not "all declared relations hold"'); +}); + +test('a passing verdict names the checks that verified nothing', () => { + // The loop's LLM reads this line. Listing check NAMES alone reads as + // "four checks passed" when three of them judged no evidence at all. + const summary = summarizeVerdict({ pass: true, checks: [ + { check: 'replay', pass: true, verified: false, detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }, + { check: 'schema', pass: true, verified: true, detail: 'all outputs validate' }, + { check: 'relations', pass: true, verified: false, detail: 'relations unverified (no replayable cassettes for listEvents)' }, + { check: 'mutation', pass: true, verified: true, detail: 'no mutation points' }, + ] }); + assert.match(summary, /unverified/, 'a pass built on no evidence must say so'); +}); + +test('a targetless verdict does not authorize a targeted deploy', () => { + const digest = verdictDigest(GOOD_SOURCE, methods); // fitness ran with no target + const state = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }; + const gate = deployGate(state, GOOD_SOURCE, methods, 'object-B'); + assert.equal(gate.ok, false, 'an update to an explicit target must refuse a targetless verdict'); +}); + +test('a verdict earned against a target authorizes exactly that target', () => { + const digest = verdictDigest(GOOD_SOURCE, methods, 'object-A'); + const state = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }; + assert.equal(deployGate(state, GOOD_SOURCE, methods, 'object-A').ok, true); + assert.equal(deployGate(state, GOOD_SOURCE, methods, 'object-B').ok, false); +}); + +test('digest components cannot bleed across field boundaries', () => { + // A NUL inside the (generated) source must not collide with a NUL split + // placed in the targetId — the preimage must be canonical. + assert.notEqual(verdictDigest('B\0C', methods, 'A'), verdictDigest('C', methods, 'A\0B')); +}); + +test('digest is stable under object key order in methods', () => { + const reordered = methods.map(m => ({ outputSchema: m.outputSchema, name: m.name, + description: m.description, parameters: m.parameters, sideEffects: m.sideEffects })) as MethodDeclaration[]; + assert.equal(verdictDigest(GOOD_SOURCE, methods), verdictDigest(GOOD_SOURCE, reordered)); +}); + +test('recording-only evidence does not brick a candidate with no discriminators', async () => { + const httpOnly: Cassette = { ...cassette, method: '_http', args: {} }; + const bare: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [] }]; + const verdict = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([httpOnly]), methods: bare }, testInvoker); + assert.equal(verdict.pass, true, 'turning recording on must not fail working objects'); + const mutation = verdict.checks.find(c => c.check === 'mutation')!; + assert.equal(mutation.verified, false); + assert.match(mutation.detail, /evidence insufficient/); +}); + +test('relations-only candidate whose probes all throw is not bricked either', async () => { + const httpOnly: Cassette = { ...cassette, method: '_http', args: {} }; + const relOnly: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'no-duplicates' }] }]; + const throwing = `throw new Error('needs real args');`; + const verdict = await evaluate({ source: throwing }, + { cassettes: new CassetteStore([httpOnly]), methods: relOnly }, testInvoker); + assert.equal(verdict.pass, true); + assert.equal(verdict.checks.find(c => c.check === 'mutation')!.verified, false); +}); + +test('a verified discriminator still runs the mutation loop', async () => { + const withSite = ` + const res = http({ method: 'GET', url: 'https://example.test/events?q=' + args.q }); + if (!res) throw new Error('no stub'); + if (res.status >= 400) throw new Error('bad status'); + return res.body; + `; + const verdict = await evaluate({ source: withSite }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker); + assert.equal(verdict.checks.find(c => c.check === 'mutation')!.verified, true); + assert.notEqual(verdict.killRatio, undefined); +}); + +test('summarizeVerdict names unverified checks from the flag, not the wording', () => { + const s = summarizeVerdict({ pass: true, checks: [ + { check: 'replay', pass: true, verified: false, detail: 'anything at all' }, + { check: 'schema', pass: true, verified: true, detail: 'all outputs validate' }, + ] }); + assert.match(s, /unverified: replay/); + assert.doesNotMatch(s, /unverified:.*schema/); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts new file mode 100644 index 0000000..5dca4d3 --- /dev/null +++ b/src/protocol/fitness.ts @@ -0,0 +1,360 @@ +/** + * Fitness -- the judge the object cannot edit. + * + * evaluate() decides whether a candidate source is fit to deploy. It never + * calls an LLM and never touches the network: all I/O is served from + * recorded cassettes through the caller-supplied Invoker. Checks run in + * order -- replay, schema, relations, mutation -- and the first hard + * failure short-circuits. + */ +import Ajv from 'ajv'; +import { canonicalJson, sha256 } from './canonical.js'; +import { HTTP_CASSETTE_METHOD, type Cassette, type CassetteStore } from './cassette.js'; +import { generateMutants } from './mutants.js'; +import type { MethodDeclaration } from '../core/types.js'; + +/** One recorded response, as the fitness gate hands it to an invoker's HTTP + * shim. `rawBody` is the response text verbatim -- the shim must return it + * unchanged, because HttpClient promises objects a raw string body. `body` + * is the same response already parsed, for invokers that want it. */ +export interface HttpExchange { status: number; body: unknown; rawBody: string; } +export type HttpStub = (req: { method: string; url: string; body?: unknown }) => HttpExchange | undefined; +export type Invoker = (source: string, method: string, + args: Record, http: HttpStub) => Promise; + +export interface CheckResult { + check: 'replay' | 'schema' | 'relations' | 'mutation'; + pass: boolean; + /** Whether the check judged any actual invocation outcome. A pass with + * `verified: false` is an honest "nothing here could be judged", and the + * mutation gate refuses to run when no baseline check verified anything -- + * a loop over checks that cannot fail kills nothing and would report the + * candidate unfit for the evidence's failing. */ + verified: boolean; + detail: string; +} +export interface Verdict { pass: boolean; checks: CheckResult[]; killRatio?: number; } +export interface FitnessEvidence { cassettes: CassetteStore; methods: MethodDeclaration[]; } +export interface FitnessOptions { maxMutants?: number; killThreshold?: number; } + +function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + const ka = Object.keys(a as object), kb = Object.keys(b as object); + if (ka.length !== kb.length) return false; + return ka.every(k => + deepEqual((a as Record)[k], (b as Record)[k])); +} + +function stubFor(cassettes: CassetteStore): HttpStub { + return req => { + const hit = cassettes.matchRequest({ method: req.method, url: req.url, body: req.body }); + return hit + ? { status: hit.response.status, body: hit.response.body, rawBody: hit.rawBody } + : undefined; + }; +} + +/** Cassettes the gate may replay as a METHOD call. '_http' cassettes are raw + * traffic the recorder captured on the object's behalf: they are stubs for + * the candidate's own outbound calls (see `stubFor`), not invocations any + * object has a handler for. Replaying them asks for a handler that cannot + * exist, which would brick every heal of an object that ever recorded. */ +function replayable(ev: FitnessEvidence, method?: string): Cassette[] { + const all = method === undefined ? ev.cassettes.all() : ev.cassettes.byMethod(method); + return all.filter(c => c.method !== HTTP_CASSETTE_METHOD); +} + +async function checkReplay(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const all = replayable(ev); + if (all.length === 0) { + return { check: 'replay', pass: true, verified: false, + detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }; + } + for (const c of all) { + let out: unknown; + try { + out = await invoke(source, c.method, c.args, stubFor(ev.cassettes)); + } catch (err) { + return { check: 'replay', pass: false, verified: true, + detail: `${c.method}(${JSON.stringify(c.args)}) threw: ${err instanceof Error ? err.message : String(err)}` }; + } + if (!deepEqual(out, c.parsedOutput)) { + return { check: 'replay', pass: false, verified: true, + detail: `${c.method}(${JSON.stringify(c.args)}) diverged from cassette recorded at ${c.recordedAt}` }; + } + } + return { check: 'replay', pass: true, verified: true, detail: `${all.length} cassette(s) reproduced` }; +} + +async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const ajv = new Ajv({ allErrors: true, strict: false }); + let anyValidated = false; + for (const m of ev.methods) { + if (!m.outputSchema) continue; + const validate = ajv.compile(m.outputSchema); + const probes = replayable(ev, m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + let validatedCount = 0; + let firstError: string | undefined; + for (const args of probes) { + let out: unknown; + try { + out = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch (err) { + if (!firstError) { + firstError = err instanceof Error ? err.message : String(err); + } + continue; // replay already judges throwing; schema judges shape of what returns + } + if (!validate(out)) { + return { check: 'schema', pass: false, verified: true, + detail: `${m.name}: ${ajv.errorsText(validate.errors)}` }; + } + validatedCount++; + anyValidated = true; + } + if (validatedCount === 0 && firstError) { + return { check: 'schema', pass: false, verified: true, + detail: `${m.name}: no output could be validated (all probes threw: ${firstError})` }; + } + } + if (!anyValidated) { + return { check: 'schema', pass: true, verified: false, detail: 'no output schemas declared' }; + } + return { check: 'schema', pass: true, verified: true, detail: 'all outputs validate' }; +} + +function fieldValue(el: unknown, field: string): unknown { + return el !== null && typeof el === 'object' + ? (el as Record)[field] : undefined; +} + +async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + /** Methods whose relations nothing could be evaluated against: every probe + * threw, so the loop below judged nothing about them. Saying the relations + * hold would be a claim the evidence never supported. */ + const unverified: string[] = []; + let anyDeclared = false; + for (const m of ev.methods) { + if (!m.relations?.length) continue; + anyDeclared = true; + const probes = replayable(ev, m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + let evaluated = 0; + for (const args of probes) { + let out: unknown; + try { out = await invoke(source, m.name, args, stubFor(ev.cassettes)); } + catch { continue; } // throwing is replay's failure, not relations' + evaluated++; + for (const rel of m.relations) { + const fail = (why: string): CheckResult => + ({ check: 'relations', pass: false, verified: true, detail: `${m.name} ${rel.kind}: ${why}` }); + switch (rel.kind) { + case 'idempotent': { + let again: unknown; + try { + again = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`two identical calls disagreed (second call threw: ${msg})`); + } + if (!deepEqual(out, again)) return fail('two identical calls disagreed'); + break; + } + case 'no-duplicates': { + if (!Array.isArray(out)) return fail('output is not an array'); + for (let i = 0; i < out.length; i++) + for (let j = i + 1; j < out.length; j++) + if (deepEqual(out[i], out[j])) return fail(`elements ${i} and ${j} are equal`); + break; + } + case 'sorted-by': { + if (!Array.isArray(out)) return fail('output is not an array'); + if (!rel.field) return fail('sorted-by declared without a field'); + for (let i = 1; i < out.length; i++) { + const a = fieldValue(out[i - 1], rel.field), b = fieldValue(out[i], rel.field); + if (a === undefined || b === undefined) return fail(`element missing field '${rel.field}'`); + const ok = typeof a === 'string' && typeof b === 'string' + ? a.localeCompare(b) <= 0 : (a as number) <= (b as number); + if (!ok) return fail(`not sorted at index ${i}`); + } + break; + } + case 'subset-on-tighter-filter': { + if (!rel.field) return fail('declared without a field'); + const cs = replayable(ev, m.name) + .filter(c => c.args[rel.field!] !== undefined); + if (cs.length < 2) break; // insufficient cassettes: vacuous + const sorted = [...cs].sort((a, b) => { + const aVal = a.args[rel.field!], bVal = b.args[rel.field!]; + if (typeof aVal === 'number' && typeof bVal === 'number') return aVal - bVal; + return String(aVal).localeCompare(String(bVal)); + }); + let loose: unknown, tight: unknown; + try { + loose = await invoke(source, m.name, sorted[0].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } + try { + tight = await invoke(source, m.name, sorted[sorted.length - 1].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } + if (!Array.isArray(loose) || !Array.isArray(tight)) return fail('outputs are not arrays'); + for (const t of tight) + if (!loose.some(l => deepEqual(l, t))) + return fail('tighter filter returned an element the looser one lacks'); + break; + } + case 'non-empty-for-known-entity': { + if (!m.entityRef) break; // vacuous without a declared entity + if (!JSON.stringify(out ?? '').includes(m.entityRef)) + return fail(`'${m.entityRef}' absent from output`); + break; + } + } + } + } + if (evaluated === 0) unverified.push(m.name); + } + // Mirrors checkSchema's `validatedCount === 0` guard. Unlike schema, an + // unverified pass (not a failure) keeps faith with `evaluate`'s no-evidence + // path: the mutation gate still refuses to certify what nothing can kill. + if (unverified.length > 0) { + return { check: 'relations', pass: true, verified: false, + detail: `relations unverified (no replayable cassettes for ${unverified.join(', ')})` }; + } + if (!anyDeclared) { + return { check: 'relations', pass: true, verified: false, detail: 'no relations declared' }; + } + return { check: 'relations', pass: true, verified: true, detail: 'all declared relations hold' }; +} + +export async function evaluate(candidate: { source: string }, + evidence: FitnessEvidence, + invoker: Invoker, + opts?: FitnessOptions): Promise { + const checks: CheckResult[] = []; + + // No evidence at all. Every check below would then be a probe with invented + // arguments against a candidate nothing has ever exercised: a declared + // outputSchema would fail on the {} probe, and a handler map would be + // mutation-tested with nothing able to kill a single mutant. Both are + // verdicts about the EVIDENCE, not the candidate, so say so and pass — + // an unverified pass, honestly labelled, beats a fabricated failure. + if (evidence.cassettes.all().length === 0) { + return { pass: true, checks: [ + await checkReplay(candidate.source, evidence, invoker), // vacuous: invokes nothing + { check: 'schema', pass: true, verified: false, detail: 'no cassettes — schema unverified' }, + { check: 'relations', pass: true, verified: false, detail: 'no cassettes — relations unverified' }, + { check: 'mutation', pass: true, verified: false, detail: 'no cassettes — mutation gate requires evidence' }, + ] }; + } + + const replay = await checkReplay(candidate.source, evidence, invoker); + checks.push(replay); + if (!replay.pass) return { pass: false, checks }; + + const schema = await checkSchema(candidate.source, evidence, invoker); + checks.push(schema); + if (!schema.pass) return { pass: false, checks }; + + const relations = await checkRelations(candidate.source, evidence, invoker); + checks.push(relations); + if (!relations.pass) return { pass: false, checks }; + + const maxMutants = opts?.maxMutants ?? 12; + const killThreshold = opts?.killThreshold ?? 0.8; + if (maxMutants === 0) { + checks.push({ check: 'mutation', pass: true, verified: false, detail: 'skipped' }); + return { pass: true, checks }; + } + // Mutation testing asks: could this evidence tell a broken copy from the + // real thing? When no baseline check verified anything, the answer is + // already known -- no check can kill a mutant, and running the loop anyway + // would fail the candidate for the evidence's poverty. This is the state a + // store reaches when the recorder has captured raw traffic but no method + // calls have been attributed yet. + if (!checks.some(c => c.verified)) { + checks.push({ check: 'mutation', pass: true, verified: false, + detail: 'no check can kill a mutant — evidence insufficient' }); + return { pass: true, checks }; + } + const mutants = generateMutants(candidate.source, maxMutants); + if (mutants === null) { + // Not "nothing to break" — the gate could not read the candidate under any + // dialect it knows, so the mutation evidence is absent rather than empty. + checks.push({ check: 'mutation', pass: false, verified: true, detail: 'candidate does not parse' }); + return { pass: false, checks }; + } + if (mutants.length === 0) { + checks.push({ check: 'mutation', pass: true, verified: true, detail: 'no mutation points' }); + return { pass: true, checks }; + } + let killed = 0; + for (const m of mutants) { + const r = await checkReplay(m.source, evidence, invoker); + if (!r.pass) { killed++; continue; } + const s = await checkSchema(m.source, evidence, invoker); + if (!s.pass) { killed++; continue; } + const rel = await checkRelations(m.source, evidence, invoker); + if (!rel.pass) killed++; + } + const killRatio = killed / mutants.length; + const pass = killRatio >= killThreshold; + checks.push({ check: 'mutation', pass, verified: true, + detail: `${killed}/${mutants.length} mutants killed (threshold ${killThreshold})` }); + return { pass, checks, killRatio }; +} + +/** The one line the loop's driver reads. A verdict whose checks judged no + * evidence must not read like one that judged plenty, so checks that verified + * nothing are named rather than silently counted among the passes. */ +export function summarizeVerdict(verdict: Verdict): string { + if (!verdict.pass) { + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + return `fitness: FAIL — ${failed}`; + } + const kill = verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''; + const names = verdict.checks.map(c => c.check).join(', '); + const unverified = verdict.checks.filter(c => !c.verified).map(c => c.check); + const caveat = unverified.length > 0 ? ` — unverified: ${unverified.join(', ')}` : ''; + return `fitness: PASS (${names}${kill})${caveat}`; +} + +/** What a verdict is ABOUT: the target it was earned against, the source, and + * the declarations it was judged under. A redrafted manifest invalidates a + * verdict exactly as a redrafted source does, and a verdict earned with no + * target (a spawn) says nothing about any existing object. Components are + * hashed separately before the outer hash: `source` is generated text that + * may contain any byte, so field boundaries must not be reconstructible from + * a delimited concatenation. */ +export function verdictDigest(source: string, methods: MethodDeclaration[], targetId?: string): string { + return sha256(sha256(targetId ?? '') + sha256(source) + sha256(canonicalJson(methods))); +} + +/** The hard gate deploy ops consult. A deploy may proceed only when the + * CURRENT draft has a passing verdict -- verdicts do not survive edits -- + * and only onto the target inside that verdict's digest: deploy_update can + * resolve an explicit target the gate never saw, and a targetless (spawn) + * verdict earns nothing against any existing object. The target lives in + * the digest preimage rather than beside it, so there is no state to check + * separately and no unbound case to slip through. */ +export function deployGate(state: { fitnessVerdict?: Verdict; fitnessSourceDigest?: string }, + draftSource: string, + methods: MethodDeclaration[], + resolvedTargetId?: string): { ok: true } | { ok: false; error: string } { + if (!state.fitnessVerdict || state.fitnessSourceDigest !== verdictDigest(draftSource, methods, resolvedTargetId)) { + return { ok: false, error: 'deploy refused: no passing fitness verdict for this draft and target — run fitness' }; + } + if (!state.fitnessVerdict.pass) { + const failed = state.fitnessVerdict.checks.find(c => !c.pass); + return { ok: false, error: `deploy refused: fitness failed (${failed?.check}: ${failed?.detail})` }; + } + return { ok: true }; +} diff --git a/src/protocol/mutants.test.ts b/src/protocol/mutants.test.ts new file mode 100644 index 0000000..7dc2717 --- /dev/null +++ b/src/protocol/mutants.test.ts @@ -0,0 +1,73 @@ +/** Run: pnpm tsx --test src/protocol/mutants.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { generateMutants } from './mutants.js'; + +const SRC = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + const items = res.body.filter(e => e.kind === 'event'); + if (items.length > 0) { return items; } + return []; +`; + +/** The house dialect: a handler map, which is an EXPRESSION, not a statement + * list. This is what the invoker actually runs (`return (${source});`). */ +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + const items = JSON.parse(res.body).filter(e => e.kind === 'event'); + return items; + } +}`; + +test('generates deterministic, distinct mutants up to max', () => { + const a = generateMutants(SRC, 12)!; + const b = generateMutants(SRC, 12)!; + assert.deepEqual(a.map(m => m.source), b.map(m => m.source)); + assert.ok(a.length >= 3, `expected >=3 mutants, got ${a.length}`); + assert.equal(new Set(a.map(m => m.source)).size, a.length); + for (const m of a) assert.notEqual(m.source, SRC); +}); + +test('respects max', () => { + assert.ok(generateMutants(SRC, 2)!.length <= 2); +}); + +test('source with no mutation points yields an empty list, not null', () => { + assert.deepEqual(generateMutants(`return 42;`, 12), []); +}); + +test('a handler map yields sites: it is parsed as the expression it is', () => { + const m = generateMutants(HANDLER_MAP, 12); + assert.notEqual(m, null); + assert.deepEqual(m!.map(x => x.description).sort(), [ + `drop a .filter(...)`, + `flip '===' to '!=='`, + ]); + for (const x of m!) assert.notEqual(x.source, HANDLER_MAP); + // every mutant is still a parseable handler map + for (const x of m!) assert.notEqual(generateMutants(x.source, 1), null); +}); + +test('a source that parses under no dialect returns null, not an empty list', () => { + assert.equal(generateMutants(`{ async listEvents(msg) { return [ }`, 12), null); + assert.equal(generateMutants(`function ( {`, 12), null); +}); + +test('comparison operators yield boundary mutants, not just negations', () => { + const ms = generateMutants(`({ f(msg) { return msg.payload.n < 10; } })`, 20)!; + assert.ok(ms.some(m => m.description.includes("boundary '<' to '<='")), + JSON.stringify(ms.map(m => m.description))); + assert.ok(ms.some(m => m.source.includes('<= 10') && !m.source.includes('>='))); +}); + +test('arithmetic plus and minus are swapped', () => { + const ms = generateMutants(`({ f(msg) { return msg.payload.a + 1; } })`, 20)!; + assert.ok(ms.some(m => m.source.includes('- 1')), JSON.stringify(ms.map(m => m.source))); +}); + +test('string concatenation is not mistaken for arithmetic', () => { + const ms = generateMutants(`({ f(msg) { return 'HTTP ' + msg.payload.status; } })`, 20)!; + assert.ok(!ms.some(m => m.description.includes("swap '+'")), + JSON.stringify(ms.map(m => m.description))); +}); diff --git a/src/protocol/mutants.ts b/src/protocol/mutants.ts new file mode 100644 index 0000000..f9264aa --- /dev/null +++ b/src/protocol/mutants.ts @@ -0,0 +1,131 @@ +/** + * Mutants -- deliberately broken copies of a candidate, used to test the + * tests. If the fitness evidence cannot tell a mutant from the real thing, + * the evidence is too weak to certify a heal. + */ +import * as acorn from 'acorn'; + +export interface Mutant { source: string; description: string; } + +interface Site { start: number; end: number; replacement: string; description: string; } + +const FLIP: Record = { + '<': '>=', '>': '<=', '<=': '>', '>=': '<', '===': '!==', '!==': '===', '==': '!=', '!=': '==', +}; + +/** Off-by-one, not negation: a candidate whose evidence cannot tell `<` + * from `<=` cannot certify a boundary. */ +const BOUNDARY: Record = { + '<': '<=', '<=': '<', '>': '>=', '>=': '>', +}; + +/** Sign mistakes: pagination offsets, totals, deltas. `+` with a string + * literal or template operand is message formatting, not arithmetic -- + * its mutants would measure error text, never behavior. */ +const ARITH: Record = { '+': '-', '-': '+' }; + +function looksLikeConcat(node: { operator: string; left: acorn.Node; right: acorn.Node }): boolean { + if (node.operator !== '+') return false; + const stringy = (n: acorn.Node): boolean => + n.type === 'TemplateLiteral' + || (n.type === 'Literal' && typeof (n as unknown as { value?: unknown }).value === 'string'); + return stringy(node.left) || stringy(node.right); +} + +const PARSE_OPTS: acorn.Options = { ecmaVersion: 'latest', allowAwaitOutsideFunction: true }; + +/** The dialects a candidate may be written in, most canonical first. + * + * The invoker runs a candidate as `return (${source});` — so the house-style + * handler map is an EXPRESSION, and a bare-brace map (`{ async m(){} }`) is a + * valid object literal but NOT a valid block. Parsing only the statement-list + * dialect therefore found zero mutation sites in exactly the sources the gate + * exists to judge. Expression first, statement list second. */ +const WRAPS: ReadonlyArray<{ prefix: string; suffix: string }> = [ + { prefix: '(', suffix: ')' }, + { prefix: 'async function __m__(args, http) {', suffix: '\n}' }, +]; + +interface Parsed { ast: acorn.Node; offset: number; } + +function parseCandidate(source: string): Parsed | null { + for (const w of WRAPS) { + try { + return { ast: acorn.parse(`${w.prefix}${source}${w.suffix}`, PARSE_OPTS), offset: w.prefix.length }; + } catch { /* not this dialect — try the next */ } + } + return null; +} + +/** + * Mutants of `source`, or `null` when the source parses under NO supported + * dialect. The distinction is load-bearing: an empty list means "nothing here + * to break", while null means the gate could not read the candidate at all — + * which must fail, not pass. + */ +export function generateMutants(source: string, max: number): Mutant[] | null { + const parsed = parseCandidate(source); + if (!parsed) return null; + if (max <= 0) return []; + const { ast, offset } = parsed; + const sites: Site[] = []; + + (function walk(node: unknown): void { + if (node === null || typeof node !== 'object') return; + const n = node as acorn.Node & Record; + if (typeof n.type === 'string') { + if (n.type === 'BinaryExpression') { + const op = (n as unknown as { operator: string; left: acorn.Node; right: acorn.Node }); + for (const [table, verb] of [[FLIP, 'flip'], [BOUNDARY, 'boundary'], [ARITH, 'swap']] as const) { + const to = table[op.operator]; + if (!to) continue; + if (table === ARITH && looksLikeConcat(op)) continue; + sites.push({ + start: op.left.end, end: op.right.start, + replacement: ` ${to} `, + description: `${verb} '${op.operator}' to '${to}'`, + }); + } + } + if (n.type === 'CallExpression') { + const callee = n.callee as (acorn.Node & { type: string; property?: { name?: string }; object?: acorn.Node }); + if (callee?.type === 'MemberExpression' && callee.property?.name === 'filter' && callee.object) { + sites.push({ + start: (n as acorn.Node).start, end: (n as acorn.Node).end, + replacement: source.slice(callee.object.start - offset, callee.object.end - offset), + description: 'drop a .filter(...)', + }); + } + } + if (n.type === 'ReturnStatement') { + const arg = n.argument as acorn.Node & { type?: string } | null; + if (arg && arg.type === 'ArrayExpression' && arg.end > arg.start + 2) { + sites.push({ start: arg.start, end: arg.end, replacement: '[]', + description: 'return [] instead of the array literal' }); + } + } + if (n.type === 'Property') { + const key = n.key as acorn.Node & { type?: string; value?: unknown }; + if (key?.type === 'Literal' && typeof key.value === 'string') { + sites.push({ start: key.start, end: key.end, replacement: `'__mutated__'`, + description: `swap property key '${key.value}'` }); + } + } + } + for (const v of Object.values(n)) { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object' && 'type' in (v as object)) walk(v); + } + })(ast); + + const seen = new Set(); + const mutants: Mutant[] = []; + for (const s of sites.sort((a, b) => a.start - b.start)) { + const mutated = source.slice(0, s.start - offset) + s.replacement + source.slice(s.end - offset); + if (mutated === source || seen.has(mutated)) continue; + seen.add(mutated); + mutants.push({ source: mutated, description: s.description }); + if (mutants.length >= max) break; + } + return mutants; +} diff --git a/src/protocol/sandbox-invoker.ts b/src/protocol/sandbox-invoker.ts new file mode 100644 index 0000000..dcf49e8 --- /dev/null +++ b/src/protocol/sandbox-invoker.ts @@ -0,0 +1,251 @@ +/** + * SandboxInvoker -- runs a ScriptableAbject handler-map source with every + * inter-object call shimmed. HTTP is served from the fitness gate's HttpStub; + * anything else an object tries to reach throws, so a candidate cannot pass + * judgment by phoning the real world. + * + * Each invocation runs on a dedicated WORKER THREAD, inside a vm context that + * exposes only the sandbox builtins. The worker is what makes the timeout + * real: `worker.terminate()` preempts anything -- a synchronous spin, a spin + * after an await, even `Atomics.wait` -- where a vm-level timeout only covers + * the evaluation it was passed to, and an in-process microtask-mode drain + * corrupts async_hooks (observed as a native "async hook stack has become + * corrupted" crash under the test runner; any AsyncLocalStorage user is + * exposed the same way). `resourceLimits` bounds the candidate's heap, so an + * allocation bomb kills the worker, not the judge. Timers work normally -- + * the worker has its own event loop -- so a legitimate retry-with-backoff + * candidate is judged, not killed. + * + * The stub crosses the thread boundary synchronously: the worker posts the + * request and blocks on Atomics.wait while the host answers on its own event + * loop. Everything that crosses is structured-clone data; no host closure is + * reachable from the candidate. + * + * Residual risk, stated plainly: `vm` inside the worker is API hygiene, not a + * security boundary (Node's own docs). An escape lands in the worker -- which + * holds no secrets and dies with the invocation chain -- rather than in the + * judge's process. That containment is the reason judging pays a worker + * round-trip instead of running in-process. + */ +import { Worker, MessageChannel, type MessagePort } from 'node:worker_threads'; +import { validateCode } from '../core/sandbox.js'; +import type { Invoker, HttpStub } from './fitness.js'; + +/** Wall-clock ceiling on ONE judged invocation, compile included. On expiry + * the worker is terminated -- preempting even blocked threads -- and the + * invocation fails. A timing-out mutant is thereby killed; a timing-out + * candidate fails. */ +export const FITNESS_INVOCATION_TIMEOUT_MS = 5000; + +/** Candidate heap ceiling. Generous for handler logic; an allocation bomb + * (one `new Array(1e9)` needs no loop to OOM) kills the worker instead of + * the process that judges. */ +const WORKER_MAX_OLD_SPACE_MB = 256; + +/** The worker body, evaluated as CommonJS via `new Worker(code, {eval:true})` + * so it needs no loader. It executes one job at a time: build a vm context, + * compile the handler map, bind it to a runtime-shaped proxy, invoke, and + * post the outcome back. The `call` shim answers from the host's stub via + * the synchronous Atomics channel. */ +const WORKER_SOURCE = ` +'use strict'; +const { parentPort, workerData } = require('node:worker_threads'); +const { receiveMessageOnPort } = require('node:worker_threads'); +const vm = require('node:vm'); + +const signal = new Int32Array(workerData.signal); +const stubPort = workerData.stubPort; + +// Mirrors SANDBOX_BUILTINS in ../core/sandbox.js, rebuilt from this worker's +// realm (live host functions cannot cross the thread boundary). +const BUILTINS = { + Math, JSON, Date, Array, Object, String, Number, Boolean, RegExp, + Map, Set, Promise, Error, TypeError, RangeError, + parseInt, parseFloat, isNaN, isFinite, + encodeURIComponent, decodeURIComponent, encodeURI, decodeURI, + setTimeout, setInterval, clearTimeout, clearInterval, + console: { log() {}, warn() {}, error() {} }, +}; + +// Ask the host's HttpStub, synchronously: post the request, sleep on the +// signal until the host has written the reply to the port. +function askStub(req) { + Atomics.store(signal, 0, 0); + stubPort.postMessage(req); + Atomics.wait(signal, 0, 0); + const m = receiveMessageOnPort(stubPort); + return m ? m.message : undefined; +} + +function harness(source) { + return \` + const call = async (target, method, payload) => { + if (target !== 'HttpClient') { + throw new Error("fitness: unstubbed I/O -- call('" + target + "', '" + method + "')"); + } + const url = String((payload && payload.url) ?? ''); + const httpMethod = method === 'post' || method === 'postJson' ? 'POST' + : String((payload && payload.method) ?? 'GET').toUpperCase(); + const hit = __http({ method: httpMethod, url, body: payload && payload.body }); + if (!hit) throw new Error('fitness: unstubbed I/O -- no cassette for ' + httpMethod + ' ' + url); + // The exact shape HttpClient's ask guide teaches objects: body is + // ALWAYS a raw string, ok is 2xx. + return { status: hit.status, statusText: '', headers: {}, body: hit.rawBody, + ok: hit.status >= 200 && hit.status < 300 }; + }; + const dep = (name) => name; + const find = () => { throw new Error('fitness: unstubbed I/O -- find()'); }; + // A minimal stand-in for ScriptableAbject's handler proxy. Handlers are + // bound to it so \\\`this.sibling(...)\\\` resolves the way it does in the + // live runtime. State-mutating members are inert: judgment must not + // persist anything. + const proxy = { + call, dep, find, + data: {}, + saveData: async () => {}, + emit: () => {}, changed: () => {}, observe: () => {}, + ensure: (cond, message) => { + if (!cond) throw new Error('ContractViolation (ensure): ' + (message ?? 'condition failed')); + }, + invariant: (cond, message) => { + if (!cond) throw new Error('ContractViolation (invariant): ' + (message ?? 'invariant failed')); + }, + id: 'fitness-candidate', + }; + // Members the proxy owns; user members never shadow them. Mirrors + // ScriptableAbject.PROXY_BUILTINS. + const PROXY_BUILTINS = new Set(['call', 'dep', 'find', 'changed', 'emit', 'observe', 'id', + 'data', 'saveData', 'ensure', 'invariant']); + const handlers = (\${source}); + const bound = new Map(); + for (const [key, value] of Object.entries(handlers ?? {})) { + if (typeof value === 'function') { + const fn = value.bind(proxy); + bound.set(key, fn); + if (!PROXY_BUILTINS.has(key)) proxy[key] = fn; + } else if (!PROXY_BUILTINS.has(key)) { + proxy[key] = value; // state property, same as the runtime does + } + } + const handler = bound.get(__method) ?? bound.get('*'); + if (!handler) throw new Error("fitness: source has no handler for '" + __method + "'"); + return handler({ payload: __args }); + \`; +} + +parentPort.on('message', async (job) => { + try { + const ctx = vm.createContext({ + ...BUILTINS, + __http: askStub, + __method: job.method, + __args: job.args, + }); + const script = new vm.Script('(async () => {' + harness(job.source) + '})()', + { filename: 'fitness-invoker.js' }); + const value = await script.runInContext(ctx); + // Outcomes must survive structured clone; a candidate returning a + // function or symbol is returning something no manifest can declare. + parentPort.postMessage({ id: job.id, ok: true, value: JSON.parse(JSON.stringify(value ?? null)) }); + } catch (err) { + parentPort.postMessage({ id: job.id, ok: false, + error: String((err && err.message) || err) }); + } +}); +`; + +interface Pending { + resolve: (v: unknown) => void; + reject: (e: Error) => void; + http: HttpStub; + timer: ReturnType; +} + +/** One worker, reused across the sequential invocations of an evaluate() + * run; respawned lazily after a termination. */ +class WorkerInvoker { + private worker: Worker | undefined; + private signal: Int32Array | undefined; + private stubPort: MessagePort | undefined; + private pending = new Map(); + private nextId = 1; + + constructor(private timeoutMs: number) {} + + private spawn(): void { + const sab = new SharedArrayBuffer(4); + this.signal = new Int32Array(sab); + const { port1, port2 } = new MessageChannel(); + this.stubPort = port1; + this.worker = new Worker(WORKER_SOURCE, { + eval: true, + workerData: { signal: sab, stubPort: port2 }, + transferList: [port2], + resourceLimits: { maxOldGenerationSizeMb: WORKER_MAX_OLD_SPACE_MB }, + }); + this.worker.unref(); + port1.on('message', (req: { method: string; url: string; body?: unknown }) => { + // Exactly one invocation is in flight per worker, so the sole pending + // entry owns every stub request. + const inflight = [...this.pending.values()][0]; + let hit: unknown; + try { hit = inflight?.http(req) ?? null; } catch { hit = null; } + port1.postMessage(hit); + Atomics.store(this.signal!, 0, 1); + Atomics.notify(this.signal!, 0); + }); + // Attaching the listener re-refs the port; without this, an idle judged + // process never exits. + port1.unref(); + this.worker.on('message', (msg: { id: number; ok: boolean; value?: unknown; error?: string }) => { + const p = this.pending.get(msg.id); + if (!p) return; + this.pending.delete(msg.id); + clearTimeout(p.timer); + if (msg.ok) p.resolve(msg.value); + else p.reject(new Error(msg.error)); + }); + this.worker.on('error', (err: Error) => this.failAll(new Error(`fitness: worker error -- ${err.message}`))); + this.worker.on('exit', (code) => { + // An OOM-killed or crashed worker exits without answering; terminate() + // after a timeout lands here too, but its pending entry is already + // rejected and cleared. + if (code !== 0) this.failAll(new Error('fitness: worker died while judging (resource limit or crash)')); + this.worker = undefined; + }); + } + + private failAll(err: Error): void { + for (const [id, p] of this.pending) { + this.pending.delete(id); + clearTimeout(p.timer); + p.reject(err); + } + } + + invoke(source: string, method: string, args: Record, http: HttpStub): Promise { + const check = validateCode(source); + if (!check.valid) { + return Promise.reject(new Error(`fitness: blocked construct -- ${check.blocked}`)); + } + if (!this.worker) this.spawn(); + const worker = this.worker!; + worker.ref(); + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + this.worker = undefined; // respawn on next invocation + void worker.terminate(); + reject(new Error('fitness: invocation timeout')); + }, this.timeoutMs); + this.pending.set(id, { resolve, reject, http, timer }); + worker.postMessage({ id, source, method, args }); + }).finally(() => { if (this.worker === worker && this.pending.size === 0) worker.unref(); }); + } +} + +export function buildSandboxInvoker(opts?: { timeoutMs?: number }): Invoker { + const invoker = new WorkerInvoker(opts?.timeoutMs ?? FITNESS_INVOCATION_TIMEOUT_MS); + return (source, method, args, http) => invoker.invoke(source, method, args, http); +}