diff --git a/src/core/manifest-contract.test.ts b/src/core/manifest-contract.test.ts new file mode 100644 index 0000000..c27c335 --- /dev/null +++ b/src/core/manifest-contract.test.ts @@ -0,0 +1,33 @@ +/** + * C2 manifest contract fields are optional and preserved. + * Run: pnpm tsx --test src/core/manifest-contract.test.ts + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import type { MethodDeclaration, RelationDeclaration, MethodEffect } from './types.js'; + +test('a legacy method declaration without contract fields is valid', () => { + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + }; + assert.equal(m.effects, undefined); +}); + +test('contract fields round-trip', () => { + const relations: RelationDeclaration[] = [ + { kind: 'subset-on-tighter-filter', field: 'from' }, + { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, + ]; + const effects: MethodEffect = 'read'; + const m: MethodDeclaration = { + name: 'listEvents', description: 'list', parameters: [], + effects, + outputSchema: { type: 'array', items: { type: 'object' } }, + relations, + knownEntity: 'Weekly Standup', + }; + assert.equal(m.effects, 'read'); + assert.equal(m.relations?.length, 3); + assert.equal(m.knownEntity, 'Weekly Standup'); +}); diff --git a/src/core/types.ts b/src/core/types.ts index 9619934..cd9da92 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -63,11 +63,32 @@ export type ErrorMessage = AbjectMessage; // Interface Declaration // ============================================================================= +/** C2 contract: what calling a method does to the world. Reads are safe to + * generate and heal autonomously; acts must cross a hand-written gate. */ +export type MethodEffect = 'read' | 'act'; + +/** C2 contract: a metamorphic relation the method's outputs must satisfy. + * Checked by the fitness gate (src/protocol/fitness.ts) — never by an LLM. */ +export interface RelationDeclaration { + kind: 'subset-on-tighter-filter' | 'idempotent' | 'no-duplicates' + | 'sorted-by' | 'non-empty-for-known-entity'; + /** 'sorted-by': output field to be non-decreasing on. + * 'subset-on-tighter-filter': the argument that narrows the result. */ + field?: string; +} + export interface MethodDeclaration { name: string; description: string; parameters: ParameterDeclaration[]; returns?: TypeDeclaration; + /** C2 contract fields — all optional; legacy manifests are untouched. */ + effects?: MethodEffect; + /** JSON Schema the method's return value must validate against. */ + outputSchema?: Record; + relations?: RelationDeclaration[]; + /** A value that must appear somewhere in a healthy output (known-entity probe). */ + knownEntity?: string; } export interface ParameterDeclaration { diff --git a/src/objects/capabilities/http-client-recorder.test.ts b/src/objects/capabilities/http-client-recorder.test.ts new file mode 100644 index 0000000..97cc0b4 --- /dev/null +++ b/src/objects/capabilities/http-client-recorder.test.ts @@ -0,0 +1,72 @@ +/** Run: pnpm tsx --test src/objects/capabilities/http-client-recorder.test.ts + * + * The wiring between HttpClient and the cassette seam. Hermetic: the global + * fetch is replaced for the duration of each test and restored afterwards — + * the replay test proves the network is never touched by making fetch throw. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { HttpClient } from './http-client.js'; +import { CassetteStore, type Cassette } from '../../protocol/cassette.js'; +import { setRecorder, clearRecorder } from '../../protocol/cassette-recorder.js'; + +const CALLER = 'obj-under-test'; +const realFetch = globalThis.fetch; + +afterEach(() => { + clearRecorder(CALLER); + globalThis.fetch = realFetch; +}); + +const recorded: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; + +test('replay mode returns the full recorded HttpResponse without calling fetch', async () => { + globalThis.fetch = (() => { throw new Error('fetch must not be called in replay mode'); }) as typeof fetch; + setRecorder(CALLER, { mode: 'replay', store: new CassetteStore([recorded]) }); + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/events' }, CALLER); + + assert.equal(res.status, 200); + assert.equal(res.ok, true); + assert.equal(res.statusText, ''); + assert.deepEqual(res.headers, {}); + assert.equal(res.body, '[{"id":1}]'); // raw text, verbatim + assert.deepEqual(JSON.parse(res.body), [{ id: 1 }]); +}); + +test('record mode lands one cassette carrying the raw body', async () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder(CALLER, { mode: 'record', store, onRecord: () => persisted++ }); + globalThis.fetch = (async () => new Response('{"hello":"world"}', { + status: 200, statusText: 'OK', headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + + const res = await new HttpClient().makeRequest( + { method: 'GET', url: 'https://example.test/hello' }, CALLER); + + assert.equal(res.body, '{"hello":"world"}'); + assert.equal(persisted, 1); + assert.equal(store.all().length, 1); + const [c] = store.all(); + assert.equal(c.request.url, 'https://example.test/hello'); + assert.equal(c.rawBody, '{"hello":"world"}'); + assert.deepEqual(c.parsedOutput, { hello: 'world' }); +}); + +test('an unregistered caller neither replays nor records', async () => { + const store = new CassetteStore(); + setRecorder(CALLER, { mode: 'record', store }); + globalThis.fetch = (async () => new Response('ok', { status: 200 })) as typeof fetch; + + await new HttpClient().makeRequest({ method: 'GET', url: 'https://example.test/x' }, 'someone-else'); + assert.equal(store.all().length, 0); +}); diff --git a/src/objects/capabilities/http-client.ts b/src/objects/capabilities/http-client.ts index 0600123..c890335 100644 --- a/src/objects/capabilities/http-client.ts +++ b/src/objects/capabilities/http-client.ts @@ -6,6 +6,7 @@ import { AbjectId, AbjectMessage } from '../../core/types.js'; import { Abject, DEFERRED_REPLY } from '../../core/abject.js'; import { error } from '../../core/message.js'; import { Capabilities } from '../../core/capability.js'; +import { beforeRequest, afterResponse } from '../../protocol/cassette-recorder.js'; const HTTP_INTERFACE = 'abjects:http'; @@ -190,7 +191,7 @@ export class HttpClient extends Abject { // for health pings during long-running fetches (e.g. LLM API calls). this.on('request', async (msg: AbjectMessage) => { const req = msg.payload as HttpRequest; - this.makeRequest(req).then( + this.makeRequest(req, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -206,7 +207,7 @@ export class HttpClient extends Abject { url: string; headers?: Record; }; - this.makeRequest({ method: 'GET', url, headers }).then( + this.makeRequest({ method: 'GET', url, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -223,7 +224,7 @@ export class HttpClient extends Abject { body: string; headers?: Record; }; - this.makeRequest({ method: 'POST', url, body, headers }).then( + this.makeRequest({ method: 'POST', url, body, headers }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -260,7 +261,7 @@ export class HttpClient extends Abject { url, body: data, headers: { 'Content-Type': 'application/json' }, - }).then( + }, msg.routing.from).then( (result) => this.sendDeferredReply(msg, result), (err) => { this.send(error(msg, 'HTTP_ERROR', @@ -300,13 +301,28 @@ export class HttpClient extends Abject { /** * Make an HTTP request with retry for transient errors. */ - async makeRequest(req: HttpRequest): Promise { + async makeRequest(req: HttpRequest, callerId?: string): Promise { if (this.webDisabled) throw new Error('Web access is disabled. Enable it in Settings > Permissions.'); // Validate URL const url = new URL(req.url); this.validateScheme(url.protocol); this.validateDomain(url.hostname); + // Replay seam: a registered replay-mode caller is served from its + // cassette store and never touches the network. A miss throws. + const replayed = beforeRequest(callerId, { method: req.method, url: req.url, headers: req.headers }); + if (replayed) { + // A full HttpResponse, with the recorded body text verbatim — the + // caller must not be able to tell replay from the live network. + return { + status: replayed.status, + statusText: '', + headers: replayed.headers, + body: replayed.rawBody, + ok: replayed.status >= 200 && replayed.status < 300, + }; + } + // Build fetch options const options: RequestInit = { method: req.method, @@ -345,6 +361,17 @@ export class HttpClient extends Abject { // Read body const body = await response.text(); + // Record seam: JSON-parse the body when possible so the cassette + // carries a matchable structure; fall back to the raw text. + let parsedBody: unknown = body; + try { + parsedBody = JSON.parse(body); + } catch { + // not JSON — keep parsedBody as the raw text + } + afterResponse(callerId, { method: req.method, url: req.url, headers: req.headers }, + { status: response.status, body: parsedBody, rawBody: body }); + return { status: response.status, statusText: response.statusText, diff --git a/src/objects/object-creator-fitness.test.ts b/src/objects/object-creator-fitness.test.ts new file mode 100644 index 0000000..d6d68b3 --- /dev/null +++ b/src/objects/object-creator-fitness.test.ts @@ -0,0 +1,210 @@ +// Run: pnpm tsx --test src/objects/object-creator-fitness.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { buildSandboxInvoker, FITNESS_INVOCATION_TIMEOUT_MS } from '../protocol/sandbox-invoker.js'; +import { deployGate, evaluate, verdictDigest } from '../protocol/fitness.js'; +import { CassetteStore, type Cassette } from '../protocol/cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + if (!res.ok) throw new Error('http ' + res.status); + return JSON.parse(res.body); + } +}`; + +const cassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }] }, + rawBody: '[{"id":1}]', + parsedOutput: [{ id: 1 }], + recordedAt: 1, +}; +const methods: MethodDeclaration[] = [{ name: 'listEvents', description: '', parameters: [] }]; + +test('sandbox invoker runs a handler map with HTTP served from cassettes', async () => { + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: HANDLER_MAP }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, true); +}); + +test("the HttpClient shim returns the shape HttpClient's ask guide teaches", async () => { + // { status, statusText, headers, body, ok } with body ALWAYS a raw string. + const echoShape = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + return { keys: Object.keys(res).sort(), bodyType: typeof res.body, body: res.body, ok: res.ok, status: res.status }; + } + }`; + const out = await buildSandboxInvoker()(echoShape, 'listEvents', {}, + () => ({ status: 200, body: [{ id: 1 }], rawBody: '[{"id":1}]' })); + // Structural, not deepEqual: the value crosses out of the vm realm, so its + // prototype is not this realm's Object.prototype. + const got = out as Record; + assert.deepEqual([...(got.keys as string[])], ['body', 'headers', 'ok', 'status', 'statusText']); + assert.equal(got.bodyType, 'string'); + assert.equal(got.body, '[{"id":1}]'); + assert.equal(got.ok, true); + assert.equal(got.status, 200); +}); + +test('WebFetch is not stubbed: its live shape is not an HttpResponse', async () => { + const webFetcher = `{ + async listEvents(msg) { return call('WebFetch', 'fetch', { url: 'https://example.test/events' }); } + }`; + const v = await evaluate({ source: webFetcher }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O -- call\('WebFetch'/); +}); + +test('sandbox invoker refuses unstubbed I/O', async () => { + const leaky = `{ + async listEvents(msg) { return call('ShellExecutor', 'run', { command: 'ls' }); } + }`; + const invoker = buildSandboxInvoker(); + const v = await evaluate({ source: leaky }, + { cassettes: new CassetteStore([cassette]), methods }, invoker, { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /unstubbed I\/O/); +}); + +test('a handler reaches its siblings through `this`, as it does at runtime', async () => { + const withHelper = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + if (!res.ok) throw new Error('http ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(items) { return items.map(e => ({ id: e.id })); } + }`; + const v = await evaluate({ source: withHelper }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('the handler proxy carries the members the runtime proxy carries', async () => { + const usesProxy = `{ + async listEvents(msg) { + this.ensure(typeof this.id === 'string', 'id must be a string'); + this.invariant(this.data && typeof this.data === 'object', 'data must be an object'); + this.data.seen = true; + await this.saveData(); + this.emit('Somewhere', 'looked', {}); + this.changed('items'); + this.observe('Somewhere'); + return [{ id: this.data.seen ? 1 : 0 }]; + } + }`; + const v = await evaluate({ source: usesProxy }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, true, JSON.stringify(v.checks)); +}); + +test('ensure/invariant throw on a falsy condition', async () => { + const breach = `{ async listEvents(msg) { this.ensure(false, 'nope'); return []; } }`; + const v = await evaluate({ source: breach }, + { cassettes: new CassetteStore([cassette]), methods }, buildSandboxInvoker(), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /ContractViolation \(ensure\): nope/); +}); + +test('an invocation that never returns is killed by the deadline', async () => { + // runSandboxed's own timeout is synchronous-only, so an await inside a + // flipped loop used to hang evaluate forever. + assert.equal(FITNESS_INVOCATION_TIMEOUT_MS, 5000); + const hangs = `{ async listEvents(msg) { await new Promise(() => {}); return []; } }`; + const started = Date.now(); + const v = await evaluate({ source: hangs }, + { cassettes: new CassetteStore([cassette]), methods }, + buildSandboxInvoker({ timeoutMs: 100 }), { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.match(v.checks[0].detail, /fitness: invocation timeout/); + assert.ok(Date.now() - started < 4000, 'the gate must not wait on a hung candidate'); +}); + +test('verdictDigest covers the declarations, not just the source', () => { + const src = 'return 1;'; + const more: MethodDeclaration[] = [...methods, { name: 'countEvents', description: '', parameters: [] }]; + assert.notEqual(verdictDigest(src, methods), verdictDigest(src, more)); + assert.equal(verdictDigest(src, methods), verdictDigest(src, [...methods])); + assert.equal(verdictDigest(src, methods), + createHash('sha256').update(src + '\0' + JSON.stringify(methods)).digest('hex')); +}); + +test('deployGate refuses without a verdict, with a failed verdict, and on a stale digest', () => { + const src = 'return 1;'; + const digest = verdictDigest(src, methods); + const passing = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest }; + assert.equal(deployGate({}, src, methods).ok, false); + assert.equal(deployGate({ fitnessVerdict: { pass: false, checks: [] }, fitnessSourceDigest: digest }, src, methods).ok, false); + assert.equal(deployGate(passing, 'return 2;', methods).ok, false); + // a re-drafted manifest invalidates the verdict too: schema and relations + // were judged against the declarations as they stood + assert.equal(deployGate(passing, src, + [...methods, { name: 'countEvents', description: '', parameters: [] }]).ok, false); + assert.equal(deployGate(passing, src, methods).ok, true); +}); + +test('deployGate refuses a verdict earned against a different object', () => { + const src = 'return 1;'; + const digest = verdictDigest(src, methods); + const judged = { fitnessVerdict: { pass: true, checks: [] }, fitnessSourceDigest: digest, fitnessTargetId: 'obj-a' }; + const refusal = deployGate(judged, src, methods, 'obj-b'); + assert.equal(refusal.ok, false); + assert.match((refusal as { error: string }).error, /fitness verdict is for a different object/); + assert.equal(deployGate(judged, src, methods, 'obj-a').ok, true); + // a create has no target on either side, and an unresolved target does not refuse + assert.equal(deployGate(judged, src, methods).ok, true); + assert.equal(deployGate({ ...judged, fitnessTargetId: undefined }, src, methods, 'obj-b').ok, true); +}); + +test('end to end, in the dialect an LLM actually writes', async () => { + // Everything the house style puts in one handler: a parenthesized handler + // map, a thin handler over a private helper reached through `this`, an + // HttpClient call whose response is checked with `res.ok` and parsed out of + // the raw `res.body` string — judged against one recorded cassette, through + // the real sandbox invoker, with every check armed. + const REAL = `({ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events?q=' + msg.payload.q }); + if (!res.ok) throw new Error('HTTP ' + res.status); + return this.shape(JSON.parse(res.body)); + }, + shape(rows) { + return rows.map(r => ({ id: r.id, name: r.name })); + } + })`; + const evidence: Cassette = { + method: 'listEvents', args: { q: 'all' }, + request: { method: 'GET', url: 'https://example.test/events?q=all' }, + response: { status: 200, body: [{ id: 1, name: 'Weekly Standup', extra: 9 }] }, + rawBody: '[{"id":1,"name":"Weekly Standup","extra":9}]', + parsedOutput: [{ id: 1, name: 'Weekly Standup' }], + recordedAt: 1, + }; + const declared: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], effects: 'read', + outputSchema: { + type: 'array', + items: { + type: 'object', required: ['id', 'name'], + properties: { id: { type: 'number' }, name: { type: 'string' } }, + }, + }, + relations: [{ kind: 'no-duplicates' }, { kind: 'idempotent' }, { kind: 'non-empty-for-known-entity' }], + knownEntity: 'Weekly Standup', + }]; + + const v = await evaluate({ source: REAL }, + { cassettes: new CassetteStore([evidence]), methods: declared }, buildSandboxInvoker()); + + assert.equal(v.pass, true, JSON.stringify(v.checks, null, 2)); + assert.equal(v.checks.find(c => c.check === 'replay')?.detail, '1 cassette(s) reproduced'); + assert.equal(v.checks.find(c => c.check === 'schema')?.detail, 'all outputs validate'); + assert.equal(v.checks.find(c => c.check === 'relations')?.detail, 'all declared relations hold'); +}); diff --git a/src/objects/object-creator.ts b/src/objects/object-creator.ts index e8f0fc9..299d8c0 100644 --- a/src/objects/object-creator.ts +++ b/src/objects/object-creator.ts @@ -32,6 +32,9 @@ import { Log } from '../core/timed-log.js'; import { applyDiff, parseSearchReplaceBlocks, levenshtein } from './source-diff.js'; import { withKeyedLock } from '../core/keyed-lock.js'; import * as acorn from 'acorn'; +import { evaluate, deployGate, verdictDigest, summarizeVerdict, type Verdict } from '../protocol/fitness.js'; +import { CassetteStore } from '../protocol/cassette.js'; +import { buildSandboxInvoker } from '../protocol/sandbox-invoker.js'; const log = new Log('OBJECT-CREATOR'); @@ -287,6 +290,16 @@ interface LoopState { baselineCallKeys?: Set; /** Source the semantic reviewer has already seen — never review the same draft twice. */ semanticReviewedSource?: string; + /** The fitness gate's most recent verdict on a draft, a digest of what it + * judged (source AND declarations), and the object it judged it for. A + * verdict is only valid for that exact triple — see `deployGate` in + * `../protocol/fitness.js`. */ + fitnessVerdict?: Verdict; + fitnessSourceDigest?: string; + fitnessTargetId?: AbjectId; + /** Live-manifest methods read for the heal path, cached so the deploy gate + * recomputes the same digest without a second round trip. */ + fitnessLiveMethods?: { targetId: AbjectId; methods: MethodDeclaration[]; note?: string }; terminal?: { kind: 'done' | 'fail'; result?: unknown; error?: string }; spawnedObjectId?: AbjectId; @@ -338,6 +351,8 @@ export class ObjectCreator extends Abject { private systemRegistryId?: AbjectId; private factoryId?: AbjectId; private abjectStoreId?: AbjectId; + /** Storage abject, for reading a target object's persisted fitness cassettes. Optional: absent means fitness judges on schema+relations+mutation alone. */ + private storageId?: AbjectId; private agentAbjectId?: AbjectId; private goalManagerId?: AbjectId; private knowledgeBaseId?: AbjectId; @@ -449,6 +464,7 @@ export class ObjectCreator extends Abject { this.factoryId = await this.requireDep('Factory'); this.systemRegistryId = (await this.discoverDep('SystemRegistry')) ?? undefined; this.abjectStoreId = (await this.discoverDep('AbjectStore')) ?? undefined; + this.storageId = (await this.discoverDep('Storage')) ?? undefined; this.agentAbjectId = (await this.discoverDep('AgentAbject')) ?? undefined; this.goalManagerId = (await this.discoverDep('GoalManager')) ?? undefined; this.knowledgeBaseId = (await this.discoverDep('KnowledgeBase')) ?? undefined; @@ -659,7 +675,7 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I private static readonly VALID_ACTIONS = [ 'call', 'draft_manifest', 'draft_source', 'edit_source', 'draft_diff', 'read_draft', 'replace_handler', 'add_handler', 'remove_handler', 'load_target', 'clone_object', 'draft_via_llm', - 'compile', 'validate_calls', 'review_semantics', 'deploy_spawn', 'deploy_update', + 'compile', 'validate_calls', 'review_semantics', 'fitness', 'deploy_spawn', 'deploy_update', 'compose_organism', 'extract_organelle', 'reply', 'ask_user', 'done', 'fail', ]; @@ -667,7 +683,8 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // ── Staging, and when each check runs ───────────────────────────────── // // Every op that writes `state.draftSource` ends in `finishEdit`. Two checks - // exist and they have very different natures, so they run at different times: + // run themselves off an edit, and they have very different natures, so they + // run at different times: // // SYNTAX — runs on EVERY edit, and is load-bearing. Members are addressed by // name through the parsed object literal, so an unparseable draft cannot be @@ -686,6 +703,12 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I // of the same LLM response are queued, or the agent passed `more: true`) it // does not run. It runs once, on the closing edit. // + // A THIRD gate — FITNESS — also refuses a deploy, but it is not an edit-time + // check and it is not mechanical: it is explicit (`fitness()`), it judges the + // whole draft against the target's RECORDED TRAFFIC (replay, declared output + // schemas, declared relations, mutation), and it costs a step. See + // `opFitness` and `deployGate` in `../protocol/fitness.js`. + // // A staging op never fails merely because the object is incomplete. That // matters mechanically: AgentAbject discards the rest of a batched response // when one action fails, so a spurious failure would throw away the very edits @@ -1795,11 +1818,101 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: result.verified, summary, result, data: advisory, error: result.verified ? undefined : this.formatSemanticIssues(result) }; } + /** + * Load the target object's persisted fitness cassettes from Storage + * (key `cassettes:`). Absent target, absent Storage, a missing + * key, or a corrupt payload all fall back to an empty store rather than + * failing the fitness op. An empty store is judged by `evaluate`'s + * no-evidence path: every check returns an unverified pass, honestly + * labelled. A store holding only `_http` traffic is not empty but is + * equally unattributed, so replay and relations report unverified and a + * declared outputSchema fails closed. + */ + private async loadCassettes(targetId?: AbjectId): Promise { + if (!targetId || !this.storageId) return new CassetteStore(); + try { + const raw = await this.sendRequest(this.storageId, 'get', { key: `cassettes:${targetId}` }); + return CassetteStore.fromJSON(raw ?? []); + } catch { + return new CassetteStore(); // no store, corrupt store, storage down: judge on schema+relations+mutation + } + } + + /** + * The method declarations the fitness gate judges against. + * + * Normally the staged draft manifest. On a HEAL the agent edits source + * without redrafting a manifest, so `draftManifest` is unset and the schema + * and relation checks would be vacuous against an empty method list — read + * the live target's manifest instead, the same `describe` a dependency + * lookup uses. Cached per target: the deploy gate must recompute the SAME + * digest, and must not fail a deploy because a best-effort read that + * succeeded at judgment time happens to fail now. + */ + private async fitnessMethods(state: LoopState): Promise<{ methods: MethodDeclaration[]; note?: string }> { + const drafted = state.draftManifest?.interface?.methods; + if (drafted) return { methods: drafted }; + const targetId = state.targetObjectId; + if (!targetId) return { methods: [] }; + const cached = state.fitnessLiveMethods; + if (cached?.targetId === targetId) return { methods: cached.methods, note: cached.note }; + + let methods: MethodDeclaration[] = []; + let note: string | undefined; + try { + const ir = await this.sendRequest>(targetId, 'describe', {}, 5000); + methods = (ir?.manifest as AbjectManifest | undefined)?.interface?.methods ?? []; + if (methods.length === 0) note = 'live manifest declares no methods — schema/relations vacuous'; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + note = `live manifest unreadable (${msg.slice(0, 60)}) — schema/relations vacuous`; + } + state.fitnessLiveMethods = { targetId, methods, note }; + return { methods, note }; + } + + /** + * The hard gate: replays the target's recorded cassettes, validates output + * schemas and relations, and mutation-tests the staged draft — entirely + * inside the sandbox, with every I/O call shimmed to the object's own + * cassettes. Records the verdict and a digest of the judged source; + * deploy_spawn/deploy_update consult both via `deployGate` and refuse when + * the draft has changed since this ran. + */ + private async opFitness(state: LoopState): Promise<{ ok: boolean; summary: string; error?: string; data?: unknown }> { + if (!state.draftSource) return { ok: false, summary: 'fitness: no draft source', error: 'draft a source first' }; + const { methods, note } = await this.fitnessMethods(state); + const cassettes = await this.loadCassettes(state.targetObjectId); + const verdict = await evaluate({ source: state.draftSource }, { cassettes, methods }, + buildSandboxInvoker()); + state.fitnessVerdict = verdict; + state.fitnessSourceDigest = verdictDigest(state.draftSource, methods); + state.fitnessTargetId = state.targetObjectId; + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + const caveat = note ? ` [${note}]` : ''; + return { + ok: verdict.pass, + summary: summarizeVerdict(verdict) + caveat, + error: verdict.pass ? undefined : failed, + data: verdict, + }; + } + // ── The deploy gate ─────────────────────────────────────────────────── // - // Deploy is the ONLY place a check refuses to proceed, and it refuses on - // mechanical grounds only: a draft that does not parse, or that calls a method - // a dependency's live manifest does not have, must never reach the live object. + // Deploy is the ONLY place a check refuses to proceed, and it refuses on two + // different kinds of ground. + // + // MECHANICAL (`gateDeploy`, below): a draft that does not parse, or that + // calls a method a dependency's live manifest does not have, must never + // reach the live object. + // + // EVIDENTIAL (`deployGate`, ../protocol/fitness.js): the draft must carry a + // PASSING fitness verdict — recorded traffic replayed, declared schemas and + // relations upheld, mutants killed — for exactly this source, these + // declarations, and this target. Not mechanical, and the only non-mechanical + // refusal there is. + // // LLM judgments (the semantic reviewer) advise but never block — a deployed // object answering real calls teaches more per step than another blind pass. @@ -1903,6 +2016,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_spawn'); if (refusal) return refusal; + const gate = deployGate(state, state.draftSource, + (await this.fitnessMethods(state)).methods); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + const spawnReq: SpawnRequest = { manifest: state.draftManifest, source: state.draftSource, @@ -2000,8 +2117,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I const refusal = this.gateDeploy(state, 'deploy_update'); if (refusal) return refusal; - // Resolve target: explicit objectId / targetName from action wins, else - // fall back to the kind:modify state target. + // Resolve target BEFORE the fitness gate: the action can name an object + // the gate never saw (the gate judged `state.targetObjectId`, which an + // explicit objectId/targetName overrides), and a verdict earned against + // another object's cassettes says nothing about this one. let targetId: AbjectId | undefined; let targetLabel: string | undefined; const explicitId = this.actionField(action, ['objectId']); @@ -2023,6 +2142,10 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I return { ok: false, summary: 'deploy_update: no target', error: 'pass {objectId} or {targetName} in the action payload, or use deploy_spawn for new objects' }; } + const gate = deployGate(state, state.draftSource, + (await this.fitnessMethods(state)).methods, targetId); + if (!gate.ok) return { ok: false, summary: gate.error, error: gate.error }; + // Everything from here is one write to one object. Two modify loops // running at once would otherwise interleave their four steps and leave // the live object, the Registry's cache, and the store each holding a @@ -3355,6 +3478,9 @@ When invited to a Sprint Plan, describe the concrete authoring or modification I case 'review_semantics': res = await this.opReviewSemantics(state); break; + case 'fitness': + res = await this.opFitness(state); + break; case 'deploy_spawn': res = await this.opDeploySpawn(state); break; @@ -3714,6 +3840,7 @@ Your local actions are the supported way to create and modify Abjects: they carr - \`read_draft({handler?, lineRange?, grep?})\` — read the CURRENT source exactly as it stands. No args → the member outline. \`{handler:"name"}\` → that member's exact text, line-numbered. \`{lineRange:"a-b"}\` / \`{grep:"pattern"}\` → those lines. Read-only: it stages nothing and makes no progress on its own. Use it to look at a member you are about to rewrite, or one a check named. Never two turns in a row without an edit in between. - \`draft_via_llm({kind: "manifest" | "source", instructions})\` — ask an LLM to draft for you. It sees current loop state. Use when authoring a brand-new manifest or source from scratch is too large for one think-step. Do NOT use this for modifications of existing objects — use \`edit_source\` instead, since the LLM consistently truncates "preserve everything else" rewrites. - \`compile()\` / \`validate_calls()\` / \`review_semantics()\` — the checks, available explicitly but **rarely worth a step**: they run on their own (see *Checks run themselves*, below). +- \`fitness()\` — hard gate: replays recorded cassettes, validates output schemas and relations, and mutation-tests the draft. \`deploy_spawn\`/\`deploy_update\` refuse until fitness passes on the current draft. - \`deploy_spawn({})\` — deploy the staged drafts as a NEW Abject. Internally messages Factory.spawn with the manifest, source, and the right owner / parent / registryHint. Use for create flows. No payload: the staged drafts are read from loop state. - \`deploy_update({objectId?, targetName?})\` — deploy the staged source onto an EXISTING object. Internally hot-swaps the live object via its \`updateSource\` handler, then updates Registry's cached source + manifest, then persists via AbjectStore so the change survives a restart. The target is taken from \`objectId\` (UUID) or \`targetName\` (registered name) in the action payload, or from the task's target if it was started as a modify. If you investigated and discovered you should be modifying an existing object even though the loop kind is \`create\`, pass \`{objectId: ""}\` here. - \`compose_organism({name, description, organelleNames, interfaceSource?})\` packages EXISTING source-backed objects into ONE Organism: a composite Abject whose organelles (independent internal copies of the named objects) cooperate behind a membrane interface, while external callers see a single object with a single curated surface. The staged drafts define the membrane: \`draft_manifest\` is the organism's public surface and \`draft_source\` (or the explicit \`interfaceSource\`) is the forwarding handler map; when either is missing it is drafted automatically from the organelle manifests. The originals keep running, so remove them afterwards (or tell the user) when the organism replaces them. diff --git a/src/protocol/cassette-recorder.test.ts b/src/protocol/cassette-recorder.test.ts new file mode 100644 index 0000000..08b59e7 --- /dev/null +++ b/src/protocol/cassette-recorder.test.ts @@ -0,0 +1,52 @@ +/** Run: pnpm tsx --test src/protocol/cassette-recorder.test.ts */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { setRecorder, clearRecorder, beforeRequest, afterResponse } from './cassette-recorder.js'; +import { CassetteStore, HTTP_CASSETTE_METHOD } from './cassette.js'; + +afterEach(() => clearRecorder('obj-1')); + +test('record mode captures 2xx with raw AND parsed body; non-2xx is not recorded', () => { + const store = new CassetteStore(); + let persisted = 0; + setRecorder('obj-1', { mode: 'record', store, onRecord: () => persisted++ }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, body: { ok: 1 }, rawBody: '{"ok":1}' }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/b' }, + { status: 500, body: 'boom', rawBody: 'boom' }); + assert.equal(store.all().length, 1); + assert.equal(persisted, 1); + const [c] = store.all(); + assert.equal(c.method, HTTP_CASSETTE_METHOD); + assert.equal(c.rawBody, '{"ok":1}'); + assert.deepEqual(c.parsedOutput, { ok: 1 }); +}); + +test('a JSON string primitive survives the record/replay round-trip verbatim', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + // The world sent the four characters `"hi"`; JSON.parse of that is `hi`. + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/s' }, + { status: 200, body: 'hi', rawBody: '"hi"' }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/s' }); + assert.equal(hit?.rawBody, '"hi"'); +}); + +test('replay mode serves recorded responses and throws on a miss', () => { + const store = new CassetteStore(); + setRecorder('obj-1', { mode: 'record', store }); + afterResponse('obj-1', { method: 'GET', url: 'https://example.test/a' }, + { status: 200, body: { ok: 1 }, rawBody: '{"ok":1}' }); + setRecorder('obj-1', { mode: 'replay', store }); + const hit = beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/a' }); + assert.equal(hit?.status, 200); + assert.equal(hit?.rawBody, '{"ok":1}'); + assert.throws(() => beforeRequest('obj-1', { method: 'GET', url: 'https://example.test/miss' }), + /replay miss/); +}); + +test('unknown object id and live mode pass through', () => { + assert.equal(beforeRequest(undefined, { method: 'GET', url: 'https://x.test/' }), undefined); + assert.equal(beforeRequest('never-registered', { method: 'GET', url: 'https://x.test/' }), undefined); +}); diff --git a/src/protocol/cassette-recorder.ts b/src/protocol/cassette-recorder.ts new file mode 100644 index 0000000..a3d175d --- /dev/null +++ b/src/protocol/cassette-recorder.ts @@ -0,0 +1,52 @@ +/** + * CassetteRecorder -- the per-object seam between HttpClient and the + * cassette store. HttpClient asks it two questions: "should this request be + * served from a recording?" (replay) and "should this response be kept?" + * (record). Objects not registered here pass through untouched, so the + * seam costs nothing for the rest of the system. + */ +import { CassetteStore, HTTP_CASSETTE_METHOD, type CassetteRequest, redactRequest } from './cassette.js'; + +export type RecorderMode = 'record' | 'replay' | 'live'; +export interface RecorderEntry { + mode: RecorderMode; + store: CassetteStore; + onRecord?: (store: CassetteStore) => void; +} + +const recorders = new Map(); + +export function setRecorder(objectId: string, entry: RecorderEntry): void { + recorders.set(objectId, entry); +} +export function clearRecorder(objectId: string): void { recorders.delete(objectId); } + +/** `rawBody` is the response text verbatim: HttpClient's contract promises + * callers a raw string body, so replay must return the same characters the + * world sent rather than a re-stringified parse of them. */ +export function beforeRequest(objectId: string | undefined, req: CassetteRequest): + { status: number; rawBody: string; headers: Record } | undefined { + if (!objectId) return undefined; + const r = recorders.get(objectId); + if (!r || r.mode !== 'replay') return undefined; + const hit = r.store.matchRequest(req); + if (!hit) throw new Error(`replay miss: no cassette for ${req.method} ${req.url}`); + return { status: hit.response.status, rawBody: hit.rawBody, headers: {} }; +} + +export function afterResponse(objectId: string | undefined, req: CassetteRequest, + res: { status: number; body: unknown; rawBody: string }): void { + if (!objectId) return; + const r = recorders.get(objectId); + if (!r || r.mode !== 'record') return; + if (res.status < 200 || res.status >= 300) return; + r.store.add({ + method: HTTP_CASSETTE_METHOD, args: {}, + request: redactRequest(req), + response: { status: res.status, body: res.body }, + rawBody: res.rawBody, + parsedOutput: res.body, + recordedAt: Date.now(), + }); + r.onRecord?.(r.store); +} diff --git a/src/protocol/cassette.test.ts b/src/protocol/cassette.test.ts new file mode 100644 index 0000000..c432c73 --- /dev/null +++ b/src/protocol/cassette.test.ts @@ -0,0 +1,63 @@ +/** Run: pnpm tsx --test src/protocol/cassette.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { CassetteStore, redactRequest, CASSETTE_CAP_PER_METHOD, type Cassette } from './cassette.js'; + +function mk(n: number, method = 'listEvents'): Cassette { + return { + method, args: { q: n }, + request: { method: 'GET', url: `https://example.test/events?q=${n}` }, + response: { status: 200, body: [{ id: n }] }, + rawBody: `[{"id":${n}}]`, + parsedOutput: [{ id: n }], + recordedAt: n, + }; +} + +test('redactRequest strips credential headers case-insensitively', () => { + const r = redactRequest({ + method: 'GET', url: 'https://example.test/x', + headers: { Authorization: 'Bearer s3cret', 'X-Ok': 'yes', COOKIE: 'a=1', 'set-cookie': 'b=2' }, + }); + assert.deepEqual(r.headers, { 'X-Ok': 'yes' }); +}); + +test('store caps per method with LRU eviction', () => { + const s = new CassetteStore(); + for (let i = 0; i < CASSETTE_CAP_PER_METHOD + 5; i++) s.add(mk(i)); + const kept = s.byMethod('listEvents'); + assert.equal(kept.length, CASSETTE_CAP_PER_METHOD); + assert.equal(kept[0].recordedAt, 5); // 0..4 evicted +}); + +test('matchRequest is exact: a different query string is a different request', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=1' })); + // ?q=other is NOT ?q=1 — serving it would defeat argument-dependent replay. + assert.equal(s.matchRequest({ method: 'GET', url: 'https://example.test/events?q=other' }), undefined); + assert.equal(s.matchRequest({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + +test('matchRequestLoose falls back to host+path when no exact match exists', () => { + const s = new CassetteStore([mk(1)]); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=1' })); + assert.ok(s.matchRequestLoose({ method: 'GET', url: 'https://example.test/events?q=other' })); + assert.equal(s.matchRequestLoose({ method: 'GET', url: 'https://elsewhere.test/events' }), undefined); +}); + +test('toJSON/fromJSON round-trips and skips malformed entries', () => { + const s = new CassetteStore([mk(1), mk(2)]); + const back = CassetteStore.fromJSON(JSON.parse(JSON.stringify(s.toJSON()))); + assert.equal(back.all().length, 2); + assert.equal(back.all()[0].rawBody, '[{"id":1}]'); + const dirty = CassetteStore.fromJSON([mk(3), { junk: true }, 42]); + assert.equal(dirty.all().length, 1); +}); + +test('fromJSON derives rawBody for entries recorded before it existed', () => { + const legacy = { ...mk(7) } as Partial; + delete legacy.rawBody; + const store = CassetteStore.fromJSON([legacy]); + assert.equal(store.all().length, 1); + assert.equal(store.all()[0].rawBody, JSON.stringify([{ id: 7 }])); +}); diff --git a/src/protocol/cassette.ts b/src/protocol/cassette.ts new file mode 100644 index 0000000..88c1a36 --- /dev/null +++ b/src/protocol/cassette.ts @@ -0,0 +1,122 @@ +/** + * Cassette -- the ratchet's memory. + * + * A cassette is one recorded truth: a request the object made, the response + * the world gave, and the parsed answer the object produced from it. The + * fitness gate replays cassettes against every candidate source; a candidate + * that cannot reproduce recorded meaning does not deploy. Requests are + * redacted before storage so a cassette can never leak a credential. + */ + +export interface CassetteRequest { + method: string; + url: string; + headers?: Record; + body?: unknown; +} + +export interface Cassette { + method: string; + args: Record; + request: CassetteRequest; + response: { status: number; body: unknown }; + /** The response body EXACTLY as the world sent it, before any parsing. + * HttpClient's contract says `body` is always a raw string, so replay must + * hand back the same characters — `JSON.stringify(parsed)` is not the same + * text for a JSON string primitive, and the round-trip loses meaning. */ + rawBody: string; + parsedOutput: unknown; + recordedAt: number; +} + +export const CASSETTE_CAP_PER_METHOD = 20; + +/** The method name recorded for raw HTTP traffic. These cassettes are stubs + * for the object's own calls, never a method the fitness gate can replay. */ +export const HTTP_CASSETTE_METHOD = '_http'; + +const REDACTED_HEADERS = new Set(['authorization', 'cookie', 'set-cookie']); + +export function redactRequest(req: CassetteRequest): CassetteRequest { + if (!req.headers) return req; + const headers: Record = {}; + for (const [k, v] of Object.entries(req.headers)) { + if (!REDACTED_HEADERS.has(k.toLowerCase())) headers[k] = v; + } + return { ...req, headers }; +} + +function hostPath(url: string): string | undefined { + try { const u = new URL(url); return `${u.host}${u.pathname}`; } catch { return undefined; } +} + +/** Cassettes recorded before `rawBody` existed derive it from the parsed + * body. Lossy for a JSON string primitive, but honest and never undefined. */ +function rawBodyOf(c: Cassette): string { + return typeof c.rawBody === 'string' ? c.rawBody : (JSON.stringify(c.response.body) ?? ''); +} + +function isCassette(c: unknown): c is Cassette { + if (c === null || typeof c !== 'object') return false; + const x = c as Record; + return typeof x.method === 'string' + && x.args !== null && typeof x.args === 'object' + && x.request !== null && typeof x.request === 'object' + && typeof (x.request as Record).url === 'string' + && x.response !== null && typeof x.response === 'object' + && typeof x.recordedAt === 'number'; +} + +export class CassetteStore { + private cassettes: Cassette[] = []; + + constructor(initial?: Cassette[]) { + for (const c of initial ?? []) this.add(c); + } + + add(c: Cassette): void { + this.cassettes.push({ ...c, request: redactRequest(c.request), rawBody: rawBodyOf(c) }); + const forMethod = this.cassettes.filter(x => x.method === c.method); + if (forMethod.length > CASSETTE_CAP_PER_METHOD) { + const evict = forMethod + .sort((a, b) => a.recordedAt - b.recordedAt) + .slice(0, forMethod.length - CASSETTE_CAP_PER_METHOD); + this.cassettes = this.cassettes.filter(x => !evict.includes(x)); + } + } + + byMethod(method: string): Cassette[] { + return this.cassettes + .filter(c => c.method === method) + .sort((a, b) => a.recordedAt - b.recordedAt); + } + + all(): Cassette[] { return [...this.cassettes]; } + + /** Exact method+url only. Replay is argument-dependent: `?q=1` and + * `?q=other` are different questions, and answering one with the other's + * recording would let a candidate "reproduce" traffic it never made. */ + matchRequest(req: CassetteRequest): Cassette | undefined { + return this.cassettes.find( + c => c.request.method === req.method && c.request.url === req.url); + } + + /** Exact match, else any recording of the same host+path. Deliberately NOT + * used by the replay seam — kept for callers that want a representative + * sample of an endpoint rather than an answer to a specific question. */ + matchRequestLoose(req: CassetteRequest): Cassette | undefined { + const exact = this.matchRequest(req); + if (exact) return exact; + const hp = hostPath(req.url); + if (!hp) return undefined; + return this.cassettes.find( + c => c.request.method === req.method && hostPath(c.request.url) === hp); + } + + toJSON(): Cassette[] { return this.all(); } + + static fromJSON(json: unknown): CassetteStore { + const arr = Array.isArray(json) ? json.filter(isCassette) : []; + return new CassetteStore(arr); + } +} diff --git a/src/protocol/fitness.test.ts b/src/protocol/fitness.test.ts new file mode 100644 index 0000000..675341a --- /dev/null +++ b/src/protocol/fitness.test.ts @@ -0,0 +1,349 @@ +/** Run: pnpm tsx --test src/protocol/fitness.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { evaluate, summarizeVerdict, type Invoker } from './fitness.js'; +import { CassetteStore, type Cassette } from './cassette.js'; +import type { MethodDeclaration } from '../core/types.js'; + +/** Test invoker: the "source" is the body of an async JS function + * (args, http) => output. Real sandboxing arrives with op_fitness (Task 6). */ +const testInvoker: Invoker = async (source, _method, args, http) => { + const fn = new Function('args', 'http', `"use strict"; return (async () => { ${source} })();`); + return fn(args, http); +}; + +const GOOD_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events?q=' + args.q }); + if (!res) throw new Error('no stub'); + return res.body; +`; +const WRONG_SOURCE = `return [];`; + +const cassette: Cassette = { + method: 'listEvents', args: { q: 1 }, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, +}; + +const methods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + effects: 'read', + outputSchema: { + type: 'array', + items: { type: 'object', required: ['id'], properties: { id: { type: 'number' } } }, + }, +}]; + +test('replay passes when the candidate reproduces recorded meaning', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('replay fails and short-circuits when output diverges from the cassette', async () => { + const v = await evaluate({ source: WRONG_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, false); + assert.equal(v.checks.some(c => c.check === 'schema'), false); // short-circuit +}); + +test('an empty cassette store yields an honest unverified pass, probing nothing', async () => { + const neverInvoked: Invoker = async () => { + throw new Error('the gate probed a candidate it has no evidence for'); + }; + const v = await evaluate({ source: 'return [{ notId: true }];' }, + { cassettes: new CassetteStore([]), methods }, neverInvoked); + assert.equal(v.pass, true); + const detail = (name: string) => v.checks.find(c => c.check === name)!; + assert.equal(detail('replay').pass, true); + assert.equal(detail('schema').pass, true); + assert.equal(detail('schema').detail, 'no cassettes — schema unverified'); + assert.equal(detail('relations').pass, true); + assert.equal(detail('relations').detail, 'no cassettes — relations unverified'); + assert.equal(detail('mutation').pass, true); + assert.equal(detail('mutation').detail, 'no cassettes — mutation gate requires evidence'); + assert.equal(v.killRatio, undefined); +}); + +test('schema fails on schema-invalid output', async () => { + const badSchemaSource = `return [{ notId: true }];`; + const c: Cassette = { + ...cassette, + response: { status: 200, body: [{ notId: true }] }, + rawBody: '[{"notId":true}]', + parsedOutput: [{ notId: true }], + }; + const v = await evaluate({ source: badSchemaSource }, + { cassettes: new CassetteStore([c]), methods }, testInvoker); + // replay passes (the source reproduces the recording); schema still refuses it + assert.equal(v.checks.find(x => x.check === 'replay')?.pass, true); + assert.equal(v.checks.find(x => x.check === 'schema')?.pass, false); + assert.equal(v.pass, false); +}); + +test('schema fails when every probe throws and a schema is declared', async () => { + // `listEvents` has evidence and reproduces it. `countEvents` declares a + // schema, owns no cassette, and throws on its one {} probe — so nothing + // about its contract was ever validated, which is a failure, not a pass. + const source = `return [{ id: 1, startsAt: '2026-08-23' }];`; + const throwsForCount: Invoker = async (src, method, args, http) => { + if (method === 'countEvents') throw new Error('not implemented'); + return testInvoker(src, method, args, http); + }; + const v = await evaluate({ source }, + { cassettes: new CassetteStore([cassette]), + methods: [...methods, + { name: 'countEvents', description: '', parameters: [], outputSchema: { type: 'number' } }] }, + throwsForCount, { maxMutants: 0 }); + assert.equal(v.pass, false); + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, false); + assert.match(v.checks.find(c => c.check === 'schema')!.detail, /no output could be validated/); +}); + +test('_http cassettes stub HTTP but are never replayed as methods', async () => { + // The recorder writes every captured response under the method '_http'. + // Feeding those to the invoker asks it for a handler no object has. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1, startsAt: '2026-08-23' }] }, + rawBody: '[{"id":1,"startsAt":"2026-08-23"}]', + parsedOutput: [{ id: 1, startsAt: '2026-08-23' }], + recordedAt: 1, + }; + const methodAware: Invoker = async (src, method, args, http) => { + if (method !== 'listEvents') throw new Error(`fitness: source has no handler for '${method}'`); + return testInvoker(src, method, args, http); + }; + const fixedUrl = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + return res.body; + `; + const v = await evaluate({ source: fixedUrl }, + { cassettes: new CassetteStore([httpOnly]), methods }, methodAware, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'replay')?.pass, true); + // and stubFor still served the recording: the schema probe fetched it + assert.equal(v.checks.find(c => c.check === 'schema')?.pass, true); + assert.equal(v.pass, true); +}); + +test('mutation fails when the candidate parses under no dialect', async () => { + // An invoker that ignores the source: replay/schema/relations all pass, so + // the verdict turns entirely on whether the gate admits it cannot read it. + const blind: Invoker = async () => [{ id: 1, startsAt: '2026-08-23' }]; + const v = await evaluate({ source: `{ async listEvents(msg) { return [ }` }, + { cassettes: new CassetteStore([cassette]), methods }, blind); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.equal(mut.detail, 'candidate does not parse'); +}); + +const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }], + knownEntity: 'Weekly Standup', +}]; + +const relCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: null }, // body unused: sources below ignore http + rawBody: 'null', + parsedOutput: null as unknown, // parsedOutput unused: set per-test below + recordedAt: 1, +}; + +test('relations: duplicates and disorder are caught', async () => { + const dupSource = `return [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }];`; + // make replay vacuous: cassette parsedOutput matches the source's constant output + const c = { ...relCassette, parsedOutput: [{ startsAt: 'b' }, { startsAt: 'a' }, { startsAt: 'a' }] }; + const v = await evaluate({ source: dupSource }, + { cassettes: new CassetteStore([c]), methods: relMethods }, testInvoker); + const rel = v.checks.find(x => x.check === 'relations'); + assert.equal(rel?.pass, false); + assert.match(rel!.detail, /no-duplicates|sorted-by/); +}); + +test('relations: known entity must appear', async () => { + const noEntity = `return [{ startsAt: 'a', name: 'Other Thing' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Other Thing' }] }; + const v = await evaluate({ source: noEntity }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'non-empty-for-known-entity' }] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, false); +}); + +test('relations: a clean output passes all declared relations', async () => { + const clean = `return [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }];`; + const c = { ...relCassette, parsedOutput: [{ startsAt: 'a', name: 'Weekly Standup' }, { startsAt: 'b', name: 'Other' }] }; + const v = await evaluate({ source: clean }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [ + { kind: 'no-duplicates' }, { kind: 'sorted-by', field: 'startsAt' }, + { kind: 'non-empty-for-known-entity' }, { kind: 'idempotent' }, + ] }] }, + testInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); + +test('relations: a throwing second invocation fails idempotent instead of rejecting evaluate', async () => { + // stateful source: first call returns [], second call throws + let calls = 0; + const flakyInvoker: Invoker = async (source, method, args, http) => { + calls++; + if (calls > 2) throw new Error('flaky'); + return []; + }; + const c = { ...relCassette, parsedOutput: [] as unknown }; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([c]), + methods: [{ ...relMethods[0], relations: [{ kind: 'idempotent' as const }] }] }, + flakyInvoker); + assert.equal(v.pass, false); + assert.match(v.checks.find(x => x.check === 'relations')!.detail, /second call threw/); +}); + +test('relations: subset-on-tighter-filter orders numeric filter args numerically', async () => { + const mk = (q: number, out: unknown[]) => ({ + method: 'listEvents', args: { q }, + request: { method: 'GET', url: `https://example.test/events?q=${q}` }, + response: { status: 200, body: out }, rawBody: JSON.stringify(out), + parsedOutput: out, recordedAt: q, + }); + // q=2 (looser, returns 2 items), q=10 (tighter, returns subset of 1) + const outputs: Record = { 2: [{ id: 1 }, { id: 2 }], 10: [{ id: 1 }] }; + const numInvoker: Invoker = async (_s, _m, args) => outputs[args.q as number]; + const v = await evaluate({ source: 'irrelevant' }, + { cassettes: new CassetteStore([mk(2, outputs[2]), mk(10, outputs[10])]), + methods: [{ name: 'listEvents', description: '', parameters: [], + relations: [{ kind: 'subset-on-tighter-filter' as const, field: 'q' }] }] }, + numInvoker); + assert.equal(v.checks.find(x => x.check === 'relations')?.pass, true); +}); + +test('mutation gate kills mutants of a well-tested source', async () => { + // GOOD_SOURCE returns the cassette body verbatim; flipping its logic breaks replay. + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, + { maxMutants: 12, killThreshold: 0.5 }); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + if (v.killRatio !== undefined && mut!.detail !== 'no mutation points') { + assert.ok(v.killRatio >= 0 && v.killRatio <= 1); + } +}); + +test('maxMutants: 0 skips the mutation gate', async () => { + const v = await evaluate({ source: GOOD_SOURCE }, + { cassettes: new CassetteStore([cassette]), methods }, testInvoker, { maxMutants: 0 }); + assert.equal(v.checks.find(c => c.check === 'mutation')?.detail, 'skipped'); +}); + +test('mutation gate counts kills on a source with real mutation points', async () => { + const KILLABLE_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.kind === 'event'); + return items; +`; + const killCassette: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ kind: 'event', id: 1 }, { kind: 'other', id: 2 }] }, + rawBody: '[{"kind":"event","id":1},{"kind":"other","id":2}]', + parsedOutput: [{ kind: 'event', id: 1 }], + recordedAt: 1, + }; + const v = await evaluate({ source: KILLABLE_SOURCE }, + { cassettes: new CassetteStore([killCassette]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + const mut = v.checks.find(c => c.check === 'mutation'); + assert.ok(mut, 'mutation check ran'); + assert.notEqual(mut!.detail, 'no mutation points'); + // expected mutants: flip '===' -> '!==' (returns the wrong item), drop .filter (returns both) — both killed by replay + assert.equal(v.killRatio, 1); + assert.equal(mut!.pass, true); + assert.match(mut!.detail, /2\/2 mutants killed/); +}); + +test('mutation fails the verdict when the evidence cannot kill enough mutants', async () => { + // Four mutation sites, only one of them observable through the recording: + // the live filter excludes nothing the cassette contains, and `spare` is + // dead code. Weak evidence must read as a failure, not a pass. + const WEAK_SOURCE = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + if (!res) throw new Error('no stub'); + const items = res.body.filter(e => e.id > 0); + const spare = res.body.filter(e => e.id > 100); + return items; +`; + const weak: Cassette = { + method: 'listEvents', args: {}, + request: { method: 'GET', url: 'https://example.test/events' }, + response: { status: 200, body: [{ id: 1 }, { id: 2 }] }, + rawBody: '[{"id":1},{"id":2}]', + parsedOutput: [{ id: 1 }, { id: 2 }], + recordedAt: 1, + }; + const v = await evaluate({ source: WEAK_SOURCE }, + { cassettes: new CassetteStore([weak]), + methods: [{ name: 'listEvents', description: '', parameters: [] }] }, + testInvoker); + assert.equal(v.pass, false); + const mut = v.checks.find(c => c.check === 'mutation')!; + assert.equal(mut.pass, false); + assert.ok(v.killRatio !== undefined && v.killRatio < 0.8, `killRatio was ${v.killRatio}`); + // measured: only the live filter's flipped guard changes what comes back + assert.equal(v.killRatio, 0.25); + assert.equal(mut.detail, '1/4 mutants killed (threshold 0.8)'); +}); + +test('relations say so when no cassette is attributed to the method', async () => { + // Only _http traffic was captured, so `replayable` finds nothing to probe + // with and falls back to a single {} call. A method that needs arguments + // throws on it, and the check must not report success it never earned. + const httpOnly: Cassette = { + method: '_http', args: {}, + request: { method: 'GET', url: 'https://example.test/events?q=1' }, + response: { status: 200, body: [{ id: 1 }, { id: 1 }] }, + rawBody: '[{"id":1},{"id":1}]', + parsedOutput: [{ id: 1 }, { id: 1 }], + recordedAt: 1, + }; + const needsArgs: Invoker = async (_src, _method, args) => { + if ((args as { q?: unknown }).q === undefined) throw new Error('q is required'); + return [{ id: 1 }, { id: 1 }]; // duplicates: no-duplicates would FAIL if ever evaluated + }; + const relMethods: MethodDeclaration[] = [{ + name: 'listEvents', description: '', parameters: [], effects: 'read', + relations: [{ kind: 'no-duplicates' }], + }]; + const v = await evaluate({ source: 'return [];' }, + { cassettes: new CassetteStore([httpOnly]), methods: relMethods }, needsArgs, { maxMutants: 0 }); + const rel = v.checks.find(c => c.check === 'relations')!; + assert.match(rel.detail, /unverified/, + 'relations must report unverified evidence, not "all declared relations hold"'); +}); + +test('a passing verdict names the checks that verified nothing', () => { + // The loop's LLM reads this line. Listing check NAMES alone reads as + // "four checks passed" when three of them judged no evidence at all. + const summary = summarizeVerdict({ pass: true, checks: [ + { check: 'replay', pass: true, detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }, + { check: 'schema', pass: true, detail: 'all outputs validate' }, + { check: 'relations', pass: true, detail: 'relations unverified (no replayable cassettes for listEvents)' }, + { check: 'mutation', pass: true, detail: 'no mutation points' }, + ] }); + assert.match(summary, /unverified/, 'a pass built on no evidence must say so'); +}); diff --git a/src/protocol/fitness.ts b/src/protocol/fitness.ts new file mode 100644 index 0000000..48e268a --- /dev/null +++ b/src/protocol/fitness.ts @@ -0,0 +1,333 @@ +/** + * Fitness -- the judge the object cannot edit. + * + * evaluate() decides whether a candidate source is fit to deploy. It never + * calls an LLM and never touches the network: all I/O is served from + * recorded cassettes through the caller-supplied Invoker. Checks run in + * order -- replay, schema, relations, mutation -- and the first hard + * failure short-circuits. + */ +import Ajv from 'ajv'; +import { createHash } from 'node:crypto'; +import { HTTP_CASSETTE_METHOD, type Cassette, type CassetteStore } from './cassette.js'; +import { generateMutants } from './mutants.js'; +import type { MethodDeclaration } from '../core/types.js'; + +/** One recorded response, as the fitness gate hands it to an invoker's HTTP + * shim. `rawBody` is the response text verbatim -- the shim must return it + * unchanged, because HttpClient promises objects a raw string body. `body` + * is the same response already parsed, for invokers that want it. */ +export interface HttpExchange { status: number; body: unknown; rawBody: string; } +export type HttpStub = (req: { method: string; url: string }) => HttpExchange | undefined; +export type Invoker = (source: string, method: string, + args: Record, http: HttpStub) => Promise; + +export interface CheckResult { + check: 'replay' | 'schema' | 'relations' | 'mutation'; + pass: boolean; + detail: string; +} +export interface Verdict { pass: boolean; checks: CheckResult[]; killRatio?: number; } +export interface FitnessEvidence { cassettes: CassetteStore; methods: MethodDeclaration[]; } +export interface FitnessOptions { maxMutants?: number; killThreshold?: number; } + +function deepEqual(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if (typeof a !== 'object' || typeof b !== 'object' || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + const ka = Object.keys(a as object), kb = Object.keys(b as object); + if (ka.length !== kb.length) return false; + return ka.every(k => + deepEqual((a as Record)[k], (b as Record)[k])); +} + +function stubFor(cassettes: CassetteStore): HttpStub { + return req => { + const hit = cassettes.matchRequest({ method: req.method, url: req.url }); + return hit + ? { status: hit.response.status, body: hit.response.body, rawBody: hit.rawBody } + : undefined; + }; +} + +/** Cassettes the gate may replay as a METHOD call. '_http' cassettes are raw + * traffic the recorder captured on the object's behalf: they are stubs for + * the candidate's own outbound calls (see `stubFor`), not invocations any + * object has a handler for. Replaying them asks for a handler that cannot + * exist, which would brick every heal of an object that ever recorded. */ +function replayable(ev: FitnessEvidence, method?: string): Cassette[] { + const all = method === undefined ? ev.cassettes.all() : ev.cassettes.byMethod(method); + return all.filter(c => c.method !== HTTP_CASSETTE_METHOD); +} + +async function checkReplay(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const all = replayable(ev); + if (all.length === 0) { + return { check: 'replay', pass: true, + detail: 'no method-attributed cassettes; nothing replayed (probe required by caller)' }; + } + for (const c of all) { + let out: unknown; + try { + out = await invoke(source, c.method, c.args, stubFor(ev.cassettes)); + } catch (err) { + return { check: 'replay', pass: false, + detail: `${c.method}(${JSON.stringify(c.args)}) threw: ${err instanceof Error ? err.message : String(err)}` }; + } + if (!deepEqual(out, c.parsedOutput)) { + return { check: 'replay', pass: false, + detail: `${c.method}(${JSON.stringify(c.args)}) diverged from cassette recorded at ${c.recordedAt}` }; + } + } + return { check: 'replay', pass: true, detail: `${all.length} cassette(s) reproduced` }; +} + +async function checkSchema(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + const ajv = new Ajv({ allErrors: true, strict: false }); + for (const m of ev.methods) { + if (!m.outputSchema) continue; + const validate = ajv.compile(m.outputSchema); + const probes = replayable(ev, m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + let validatedCount = 0; + let firstError: string | undefined; + for (const args of probes) { + let out: unknown; + try { + out = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch (err) { + if (!firstError) { + firstError = err instanceof Error ? err.message : String(err); + } + continue; // replay already judges throwing; schema judges shape of what returns + } + if (!validate(out)) { + return { check: 'schema', pass: false, + detail: `${m.name}: ${ajv.errorsText(validate.errors)}` }; + } + validatedCount++; + } + if (validatedCount === 0 && firstError) { + return { check: 'schema', pass: false, + detail: `${m.name}: no output could be validated (all probes threw: ${firstError})` }; + } + } + return { check: 'schema', pass: true, detail: 'all outputs validate' }; +} + +function fieldValue(el: unknown, field: string): unknown { + return el !== null && typeof el === 'object' + ? (el as Record)[field] : undefined; +} + +async function checkRelations(source: string, ev: FitnessEvidence, invoke: Invoker): Promise { + /** Methods whose relations nothing could be evaluated against: every probe + * threw, so the loop below judged nothing about them. Saying the relations + * hold would be a claim the evidence never supported. */ + const unverified: string[] = []; + for (const m of ev.methods) { + if (!m.relations?.length) continue; + const probes = replayable(ev, m.name).map(c => c.args); + if (probes.length === 0) probes.push({}); + let evaluated = 0; + for (const args of probes) { + let out: unknown; + try { out = await invoke(source, m.name, args, stubFor(ev.cassettes)); } + catch { continue; } // throwing is replay's failure, not relations' + evaluated++; + for (const rel of m.relations) { + const fail = (why: string): CheckResult => + ({ check: 'relations', pass: false, detail: `${m.name} ${rel.kind}: ${why}` }); + switch (rel.kind) { + case 'idempotent': { + let again: unknown; + try { + again = await invoke(source, m.name, args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`two identical calls disagreed (second call threw: ${msg})`); + } + if (!deepEqual(out, again)) return fail('two identical calls disagreed'); + break; + } + case 'no-duplicates': { + if (!Array.isArray(out)) return fail('output is not an array'); + for (let i = 0; i < out.length; i++) + for (let j = i + 1; j < out.length; j++) + if (deepEqual(out[i], out[j])) return fail(`elements ${i} and ${j} are equal`); + break; + } + case 'sorted-by': { + if (!Array.isArray(out)) return fail('output is not an array'); + if (!rel.field) return fail('sorted-by declared without a field'); + for (let i = 1; i < out.length; i++) { + const a = fieldValue(out[i - 1], rel.field), b = fieldValue(out[i], rel.field); + if (a === undefined || b === undefined) return fail(`element missing field '${rel.field}'`); + const ok = typeof a === 'string' && typeof b === 'string' + ? a.localeCompare(b) <= 0 : (a as number) <= (b as number); + if (!ok) return fail(`not sorted at index ${i}`); + } + break; + } + case 'subset-on-tighter-filter': { + if (!rel.field) return fail('declared without a field'); + const cs = replayable(ev, m.name) + .filter(c => c.args[rel.field!] !== undefined); + if (cs.length < 2) break; // insufficient cassettes: vacuous + const sorted = [...cs].sort((a, b) => { + const aVal = a.args[rel.field!], bVal = b.args[rel.field!]; + if (typeof aVal === 'number' && typeof bVal === 'number') return aVal - bVal; + return String(aVal).localeCompare(String(bVal)); + }); + let loose: unknown, tight: unknown; + try { + loose = await invoke(source, m.name, sorted[0].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } + try { + tight = await invoke(source, m.name, sorted[sorted.length - 1].args, stubFor(ev.cassettes)); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return fail(`invocation threw during relation check: ${msg}`); + } + if (!Array.isArray(loose) || !Array.isArray(tight)) return fail('outputs are not arrays'); + for (const t of tight) + if (!loose.some(l => deepEqual(l, t))) + return fail('tighter filter returned an element the looser one lacks'); + break; + } + case 'non-empty-for-known-entity': { + if (!m.knownEntity) break; // vacuous without a declared entity + if (!JSON.stringify(out ?? '').includes(m.knownEntity)) + return fail(`'${m.knownEntity}' absent from output`); + break; + } + } + } + } + if (evaluated === 0) unverified.push(m.name); + } + // Mirrors checkSchema's `validatedCount === 0` guard. Unlike schema, an + // unverified pass (not a failure) keeps faith with `evaluate`'s no-evidence + // path: the mutation gate still refuses to certify what nothing can kill. + if (unverified.length > 0) { + return { check: 'relations', pass: true, + detail: `relations unverified (no replayable cassettes for ${unverified.join(', ')})` }; + } + return { check: 'relations', pass: true, detail: 'all declared relations hold' }; +} + +export async function evaluate(candidate: { source: string }, + evidence: FitnessEvidence, + invoker: Invoker, + opts?: FitnessOptions): Promise { + const checks: CheckResult[] = []; + + // No evidence at all. Every check below would then be a probe with invented + // arguments against a candidate nothing has ever exercised: a declared + // outputSchema would fail on the {} probe, and a handler map would be + // mutation-tested with nothing able to kill a single mutant. Both are + // verdicts about the EVIDENCE, not the candidate, so say so and pass — + // an unverified pass, honestly labelled, beats a fabricated failure. + if (evidence.cassettes.all().length === 0) { + return { pass: true, checks: [ + await checkReplay(candidate.source, evidence, invoker), // vacuous: invokes nothing + { check: 'schema', pass: true, detail: 'no cassettes — schema unverified' }, + { check: 'relations', pass: true, detail: 'no cassettes — relations unverified' }, + { check: 'mutation', pass: true, detail: 'no cassettes — mutation gate requires evidence' }, + ] }; + } + + const replay = await checkReplay(candidate.source, evidence, invoker); + checks.push(replay); + if (!replay.pass) return { pass: false, checks }; + + const schema = await checkSchema(candidate.source, evidence, invoker); + checks.push(schema); + if (!schema.pass) return { pass: false, checks }; + + const relations = await checkRelations(candidate.source, evidence, invoker); + checks.push(relations); + if (!relations.pass) return { pass: false, checks }; + + const maxMutants = opts?.maxMutants ?? 12; + const killThreshold = opts?.killThreshold ?? 0.8; + if (maxMutants === 0) { + checks.push({ check: 'mutation', pass: true, detail: 'skipped' }); + return { pass: true, checks }; + } + const mutants = generateMutants(candidate.source, maxMutants); + if (mutants === null) { + // Not "nothing to break" — the gate could not read the candidate under any + // dialect it knows, so the mutation evidence is absent rather than empty. + checks.push({ check: 'mutation', pass: false, detail: 'candidate does not parse' }); + return { pass: false, checks }; + } + if (mutants.length === 0) { + checks.push({ check: 'mutation', pass: true, detail: 'no mutation points' }); + return { pass: true, checks }; + } + let killed = 0; + for (const m of mutants) { + const r = await checkReplay(m.source, evidence, invoker); + if (!r.pass) { killed++; continue; } + const s = await checkSchema(m.source, evidence, invoker); + if (!s.pass) { killed++; continue; } + const rel = await checkRelations(m.source, evidence, invoker); + if (!rel.pass) killed++; + } + const killRatio = killed / mutants.length; + const pass = killRatio >= killThreshold; + checks.push({ check: 'mutation', pass, + detail: `${killed}/${mutants.length} mutants killed (threshold ${killThreshold})` }); + return { pass, checks, killRatio }; +} + +/** The one line the loop's driver reads. A verdict whose checks judged no + * evidence must not read like one that judged plenty, so checks that verified + * nothing are named rather than silently counted among the passes. */ +export function summarizeVerdict(verdict: Verdict): string { + if (!verdict.pass) { + const failed = verdict.checks.filter(c => !c.pass).map(c => `${c.check}: ${c.detail}`).join('; '); + return `fitness: FAIL — ${failed}`; + } + const kill = verdict.killRatio !== undefined ? `, kill ${verdict.killRatio.toFixed(2)}` : ''; + const names = verdict.checks.map(c => c.check).join(', '); + const unverified = verdict.checks + .filter(c => /unverified|nothing replayed|requires evidence/.test(c.detail)) + .map(c => c.check); + const caveat = unverified.length > 0 ? ` — unverified: ${unverified.join(', ')}` : ''; + return `fitness: PASS (${names}${kill})${caveat}`; +} + +/** What a verdict is ABOUT. Not the source alone: the schema and relation + * checks are judgments of the source AGAINST the declarations, so a redrafted + * manifest invalidates a verdict exactly as a redrafted source does. */ +export function verdictDigest(source: string, methods: MethodDeclaration[]): string { + return createHash('sha256').update(source + '\0' + JSON.stringify(methods)).digest('hex'); +} + +/** The hard gate deploy ops consult. A deploy may proceed only when the + * CURRENT draft has a passing verdict -- verdicts do not survive edits -- + * and only onto the object that verdict was earned against: deploy_update + * can resolve an explicit target the gate never saw, and a verdict built + * from another object's cassettes says nothing about this one. */ +export function deployGate(state: { fitnessVerdict?: Verdict; fitnessSourceDigest?: string; fitnessTargetId?: string }, + draftSource: string, + methods: MethodDeclaration[], + resolvedTargetId?: string): { ok: true } | { ok: false; error: string } { + if (!state.fitnessVerdict || state.fitnessSourceDigest !== verdictDigest(draftSource, methods)) { + return { ok: false, error: 'deploy refused: no passing fitness verdict for this draft — run fitness' }; + } + if (state.fitnessTargetId !== undefined && resolvedTargetId !== undefined + && state.fitnessTargetId !== resolvedTargetId) { + return { ok: false, error: 'deploy refused: fitness verdict is for a different object' }; + } + if (!state.fitnessVerdict.pass) { + const failed = state.fitnessVerdict.checks.find(c => !c.pass); + return { ok: false, error: `deploy refused: fitness failed (${failed?.check}: ${failed?.detail})` }; + } + return { ok: true }; +} diff --git a/src/protocol/mutants.test.ts b/src/protocol/mutants.test.ts new file mode 100644 index 0000000..f646ccc --- /dev/null +++ b/src/protocol/mutants.test.ts @@ -0,0 +1,55 @@ +/** Run: pnpm tsx --test src/protocol/mutants.test.ts */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { generateMutants } from './mutants.js'; + +const SRC = ` + const res = http({ method: 'GET', url: 'https://example.test/events' }); + const items = res.body.filter(e => e.kind === 'event'); + if (items.length > 0) { return items; } + return []; +`; + +/** The house dialect: a handler map, which is an EXPRESSION, not a statement + * list. This is what the invoker actually runs (`return (${source});`). */ +const HANDLER_MAP = `{ + async listEvents(msg) { + const res = await call('HttpClient', 'get', { url: 'https://example.test/events' }); + const items = JSON.parse(res.body).filter(e => e.kind === 'event'); + return items; + } +}`; + +test('generates deterministic, distinct mutants up to max', () => { + const a = generateMutants(SRC, 12)!; + const b = generateMutants(SRC, 12)!; + assert.deepEqual(a.map(m => m.source), b.map(m => m.source)); + assert.ok(a.length >= 3, `expected >=3 mutants, got ${a.length}`); + assert.equal(new Set(a.map(m => m.source)).size, a.length); + for (const m of a) assert.notEqual(m.source, SRC); +}); + +test('respects max', () => { + assert.ok(generateMutants(SRC, 2)!.length <= 2); +}); + +test('source with no mutation points yields an empty list, not null', () => { + assert.deepEqual(generateMutants(`return 42;`, 12), []); +}); + +test('a handler map yields sites: it is parsed as the expression it is', () => { + const m = generateMutants(HANDLER_MAP, 12); + assert.notEqual(m, null); + assert.deepEqual(m!.map(x => x.description).sort(), [ + `drop a .filter(...)`, + `flip '===' to '!=='`, + ]); + for (const x of m!) assert.notEqual(x.source, HANDLER_MAP); + // every mutant is still a parseable handler map + for (const x of m!) assert.notEqual(generateMutants(x.source, 1), null); +}); + +test('a source that parses under no dialect returns null, not an empty list', () => { + assert.equal(generateMutants(`{ async listEvents(msg) { return [ }`, 12), null); + assert.equal(generateMutants(`function ( {`, 12), null); +}); diff --git a/src/protocol/mutants.ts b/src/protocol/mutants.ts new file mode 100644 index 0000000..bca4bd6 --- /dev/null +++ b/src/protocol/mutants.ts @@ -0,0 +1,107 @@ +/** + * Mutants -- deliberately broken copies of a candidate, used to test the + * tests. If the fitness evidence cannot tell a mutant from the real thing, + * the evidence is too weak to certify a heal. + */ +import * as acorn from 'acorn'; + +export interface Mutant { source: string; description: string; } + +interface Site { start: number; end: number; replacement: string; description: string; } + +const FLIP: Record = { + '<': '>=', '>': '<=', '<=': '>', '>=': '<', '===': '!==', '!==': '===', '==': '!=', '!=': '==', +}; + +const PARSE_OPTS: acorn.Options = { ecmaVersion: 'latest', allowAwaitOutsideFunction: true }; + +/** The dialects a candidate may be written in, most canonical first. + * + * The invoker runs a candidate as `return (${source});` — so the house-style + * handler map is an EXPRESSION, and a bare-brace map (`{ async m(){} }`) is a + * valid object literal but NOT a valid block. Parsing only the statement-list + * dialect therefore found zero mutation sites in exactly the sources the gate + * exists to judge. Expression first, statement list second. */ +const WRAPS: ReadonlyArray<{ prefix: string; suffix: string }> = [ + { prefix: '(', suffix: ')' }, + { prefix: 'async function __m__(args, http) {', suffix: '\n}' }, +]; + +interface Parsed { ast: acorn.Node; offset: number; } + +function parseCandidate(source: string): Parsed | null { + for (const w of WRAPS) { + try { + return { ast: acorn.parse(`${w.prefix}${source}${w.suffix}`, PARSE_OPTS), offset: w.prefix.length }; + } catch { /* not this dialect — try the next */ } + } + return null; +} + +/** + * Mutants of `source`, or `null` when the source parses under NO supported + * dialect. The distinction is load-bearing: an empty list means "nothing here + * to break", while null means the gate could not read the candidate at all — + * which must fail, not pass. + */ +export function generateMutants(source: string, max: number): Mutant[] | null { + const parsed = parseCandidate(source); + if (!parsed) return null; + if (max <= 0) return []; + const { ast, offset } = parsed; + const sites: Site[] = []; + + (function walk(node: unknown): void { + if (node === null || typeof node !== 'object') return; + const n = node as acorn.Node & Record; + if (typeof n.type === 'string') { + if (n.type === 'BinaryExpression' && FLIP[(n as { operator?: string }).operator ?? '']) { + const op = (n as unknown as { operator: string; left: acorn.Node; right: acorn.Node }); + sites.push({ + start: op.left.end, end: op.right.start, + replacement: ` ${FLIP[op.operator]} `, + description: `flip '${op.operator}' to '${FLIP[op.operator]}'`, + }); + } + if (n.type === 'CallExpression') { + const callee = n.callee as (acorn.Node & { type: string; property?: { name?: string }; object?: acorn.Node }); + if (callee?.type === 'MemberExpression' && callee.property?.name === 'filter' && callee.object) { + sites.push({ + start: (n as acorn.Node).start, end: (n as acorn.Node).end, + replacement: source.slice(callee.object.start - offset, callee.object.end - offset), + description: 'drop a .filter(...)', + }); + } + } + if (n.type === 'ReturnStatement') { + const arg = n.argument as acorn.Node & { type?: string } | null; + if (arg && arg.type === 'ArrayExpression' && arg.end > arg.start + 2) { + sites.push({ start: arg.start, end: arg.end, replacement: '[]', + description: 'return [] instead of the array literal' }); + } + } + if (n.type === 'Property') { + const key = n.key as acorn.Node & { type?: string; value?: unknown }; + if (key?.type === 'Literal' && typeof key.value === 'string') { + sites.push({ start: key.start, end: key.end, replacement: `'__mutated__'`, + description: `swap property key '${key.value}'` }); + } + } + } + for (const v of Object.values(n)) { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object' && 'type' in (v as object)) walk(v); + } + })(ast); + + const seen = new Set(); + const mutants: Mutant[] = []; + for (const s of sites.sort((a, b) => a.start - b.start)) { + const mutated = source.slice(0, s.start - offset) + s.replacement + source.slice(s.end - offset); + if (mutated === source || seen.has(mutated)) continue; + seen.add(mutated); + mutants.push({ source: mutated, description: s.description }); + if (mutants.length >= max) break; + } + return mutants; +} diff --git a/src/protocol/sandbox-invoker.ts b/src/protocol/sandbox-invoker.ts new file mode 100644 index 0000000..63b0f00 --- /dev/null +++ b/src/protocol/sandbox-invoker.ts @@ -0,0 +1,127 @@ +/** + * SandboxInvoker -- runs a ScriptableAbject handler-map source under + * runSandboxed with every inter-object call shimmed. HTTP is served from the + * fitness gate's HttpStub; anything else an object tries to reach throws, so + * a candidate cannot pass judgment by phoning the real world. + */ +import { runSandboxed } from '../core/sandbox.js'; +import type { Invoker, HttpStub } from './fitness.js'; + +// HttpClient only. WebFetch's live return shape (FetchResult) is nothing +// like an HttpResponse, so shimming it here would teach a candidate a +// contract the runtime does not honour; a WebFetch-using candidate fails +// with a plain unstubbed-I/O message until a real stub exists. +const HTTP_TARGETS = new Set(['HttpClient']); + +/** A sandboxed candidate gets a bounded synchronous timeout -- generous for + * real handler logic, cheap insurance against a mutant that spins. It does + * not cover awaited Promises (see runSandboxed's docs), only the + * synchronous portions of the compile + call. */ +const SANDBOX_TIMEOUT_MS = 5000; + +/** Wall-clock ceiling on ONE judged invocation, compile included. The vm's + * own timeout is synchronous-only, so a mutant that flips a loop guard and + * awaits inside it hangs evaluate forever with nothing to interrupt it. + * A timing-out mutant is thereby killed; a timing-out candidate fails. */ +export const FITNESS_INVOCATION_TIMEOUT_MS = 5000; + +/** Members the handler proxy owns; user members never shadow them. + * Mirrors ScriptableAbject.PROXY_BUILTINS. */ +const PROXY_BUILTINS = new Set([ + 'call', 'dep', 'find', 'changed', 'emit', 'observe', 'id', + 'data', 'saveData', 'ensure', 'invariant', +]); + +function makeCall(http: HttpStub) { + return async (target: string, method: string, payload: Record) => { + if (!HTTP_TARGETS.has(target)) { + throw new Error(`fitness: unstubbed I/O -- call('${target}', '${method}')`); + } + const url = String(payload?.url ?? ''); + const httpMethod = method === 'post' || method === 'postJson' ? 'POST' + : String(payload?.method ?? 'GET').toUpperCase(); + const hit = http({ method: httpMethod, url }); + if (!hit) throw new Error(`fitness: unstubbed I/O -- no cassette for ${httpMethod} ${url}`); + // The exact shape HttpClient's ask guide teaches objects: body is ALWAYS + // a raw string (`JSON.parse(result.body)`), ok is 2xx. A candidate judged + // against any other shape is judged against a runtime that does not exist. + return { + status: hit.status, + statusText: '', + headers: {} as Record, + body: hit.rawBody, + ok: hit.status >= 200 && hit.status < 300, + }; + }; +} + +/** Reject if `p` has not settled within `ms`. The hung promise is abandoned, + * not cancelled -- nothing in a vm can be cancelled -- but it holds no timer + * of its own, so it never keeps the process alive. */ +function withDeadline(p: Promise, ms: number): Promise { + let timer: ReturnType; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('fitness: invocation timeout')), ms); + }); + return Promise.race([p, deadline]).finally(() => clearTimeout(timer)); +} + +/** + * A minimal stand-in for ScriptableAbject's handler proxy. Handlers are bound + * to it, so `this.sibling(...)` resolves the way it does in the live runtime + * -- without one, a two-method object (the house style: a thin handler over a + * private helper) failed judgment on a `this` the gate itself withheld. + * State-mutating members are inert: judgment must not persist anything. + */ +function buildHandlerProxy(call: ReturnType): Record { + return { + call, + dep: (name: string) => name, + find: () => { throw new Error('fitness: unstubbed I/O -- find()'); }, + data: {}, + saveData: async () => {}, + emit: () => {}, + changed: () => {}, + observe: () => {}, + ensure: (cond: unknown, message?: string) => { + if (!cond) throw new Error(`ContractViolation (ensure): ${message ?? 'condition failed'}`); + }, + invariant: (cond: unknown, message?: string) => { + if (!cond) throw new Error(`ContractViolation (invariant): ${message ?? 'invariant failed'}`); + }, + id: 'fitness-candidate', + }; +} + +export function buildSandboxInvoker(opts?: { timeoutMs?: number }): Invoker { + const timeoutMs = opts?.timeoutMs ?? FITNESS_INVOCATION_TIMEOUT_MS; + return (source, method, args, http) => withDeadline((async () => { + const call = makeCall(http); + // runSandboxed wraps its code in `(async () => { CODE })()`; a bare + // parenthesized object expression as a statement evaluates and discards + // itself, so the handler map must be explicitly returned to escape the + // wrapper. + const handlers = await runSandboxed(`return (${source});`, { + call, + dep: (name: string) => name, + find: () => { throw new Error('fitness: unstubbed I/O -- find()'); }, + }, { timeout: SANDBOX_TIMEOUT_MS }) as Record | null; + + const proxy = buildHandlerProxy(call); + const bound = new Map }) => Promise>(); + for (const [key, value] of Object.entries(handlers ?? {})) { + if (typeof value === 'function') { + const fn = (value as (...a: unknown[]) => unknown).bind(proxy) as + (msg: { payload: Record }) => Promise; + bound.set(key, fn); + if (!PROXY_BUILTINS.has(key)) proxy[key] = fn; + } else if (!PROXY_BUILTINS.has(key)) { + proxy[key] = value; // state property, same as the runtime does + } + } + + const handler = bound.get(method) ?? bound.get('*'); + if (!handler) throw new Error(`fitness: source has no handler for '${method}'`); + return handler({ payload: args }); + })(), timeoutMs); +}