From 532673f38b6e9b273ca2771b50c7dd0606378311 Mon Sep 17 00:00:00 2001 From: kunaldhongade Date: Thu, 6 Aug 2026 18:56:51 +0530 Subject: [PATCH] feat(loop): finish builder/verifier convergence controls for #680 Emit verified/shallow-proof terminals, wire verifier hypotheses, enforce path/model/wall budgets with oscillation stops, persist loop audit, and cover UAT-LOOP-1/2/4/5 with deterministic fake agents. --- docs/loop.md | 18 +- packages/cli/src/commands/loop.ts | 7 + .../src/docs/command-docs/orchestration.ts | 11 +- packages/cli/src/parsers/loop.ts | 78 +++++ packages/cli/src/types/loop.ts | 7 + packages/cli/test/loop-e2e.test.ts | 6 +- packages/cli/test/loop.test.ts | 10 +- packages/harness/src/loop/audit.ts | 139 ++++++++ packages/harness/src/loop/budgets.ts | 141 ++++++++ packages/harness/src/loop/controller.ts | 331 ++++++++++++++++-- packages/harness/src/loop/hypotheses.ts | 66 ++++ packages/harness/src/loop/index.ts | 9 + packages/harness/src/loop/render.ts | 2 + packages/harness/src/loop/types.ts | 10 + packages/harness/test/loop.test.ts | 120 ++++++- scripts/end-user-demo.mjs | 4 +- 16 files changed, 897 insertions(+), 62 deletions(-) create mode 100644 packages/harness/src/loop/audit.ts create mode 100644 packages/harness/src/loop/budgets.ts create mode 100644 packages/harness/src/loop/hypotheses.ts diff --git a/docs/loop.md b/docs/loop.md index c02354df..58d576eb 100644 --- a/docs/loop.md +++ b/docs/loop.md @@ -34,7 +34,7 @@ Progress considers merge risk, decay risk, security risk, weak-test findings, pr `codedecay loop` never prints an unqualified "safe" verdict. Clean outcomes are always qualified by evidence depth. -The loop can only report a `merge-safe-*` verdict when all of these are true: +The loop can only report a `verified` or `shallow-proof` verdict when all of these are true: - final risk is at or below the configured safe threshold, `low` by default - weak-test findings are zero @@ -42,21 +42,23 @@ The loop can only report a `merge-safe-*` verdict when all of these are true: - no high-severity findings remain in deterministic analysis - configured checks exist and pass -If no checks are configured, the best possible terminal status is `unverified`, not a `merge-safe-*` verdict. +If no checks are configured, the best possible terminal status is `unverified`, not a verified verdict. -`merge-safe-verified` means configured checks passed, deterministic security matchers were clean, Semgrep was enabled and clean, and coverage/mutation evidence was available if configured. +`verified` means configured checks passed, deterministic security matchers were clean, Semgrep was enabled and clean, and coverage/mutation evidence was available if configured. Legacy alias: `merge-safe-verified`. -`merge-safe-shallow` means the gates passed, but one or more deeper evidence streams were missing. Treat it as heuristic clean, not as deep verification. Run `codedecay doctor` to configure OSS adapters such as Semgrep, coverage, and StrykerJS. +`shallow-proof` means the gates passed, but one or more deeper evidence streams were missing. Treat it as heuristic clean, not as deep verification. Run `codedecay doctor` to configure OSS adapters such as Semgrep, coverage, and StrykerJS. Legacy alias: `merge-safe-shallow`. Terminal statuses: -- `merge-safe-verified`: configured and enabled checks found nothing at the selected thresholds, including available security/coverage/mutation depth -- `merge-safe-shallow`: risk, weak-test, security-score, and configured-check gates passed, but depth evidence such as Semgrep, coverage, or mutation testing is missing +- `verified`: configured and enabled checks found nothing at the selected thresholds, including available security/coverage/mutation depth +- `shallow-proof`: risk, weak-test, security-score, and configured-check gates passed, but depth evidence such as Semgrep, coverage, or mutation testing is missing - `unverified`: risk and weak-test evidence are clean, but no configured checks proved the result - `plan-only`: no agent command was configured -- `stuck`: the agent made no progress for two rounds +- `stuck`: the agent made no progress for two rounds, oscillated, or widened scope unsafely +- `budget-exhausted`: round, wall-time, model-call, or changed-file budget exhausted +- `unsafe-change`: verifier edited files, or protected/out-of-scope paths changed - `needs-human`: max rounds were reached -- `agent-error`: the agent command failed, timed out, was skipped, or was blocked by safety policy +- `builder-error` / `verifier-error` / `agent-error`: role command failed, timed out, was skipped, or was blocked by safety policy ## Example diff --git a/packages/cli/src/commands/loop.ts b/packages/cli/src/commands/loop.ts index b809e3bc..3b494e05 100644 --- a/packages/cli/src/commands/loop.ts +++ b/packages/cli/src/commands/loop.ts @@ -67,6 +67,13 @@ export async function runLoopCommand( safeRiskLevel: options.safeRiskLevel, securityScoreThreshold: options.securityScoreThreshold, agentTimeoutMs: loadedConfig.config.safety.commandTimeoutMs, + maxWallTimeMs: options.maxWallTimeMs, + maxChangedFiles: options.maxChangedFiles, + maxModelCalls: options.maxModelCalls, + allowedPathPrefixes: options.allowedPaths, + protectedPathPrefixes: options.protectedPaths, + resumeFromAuditPath: options.resumeFrom, + runId: options.runId, commandSafety: { allowCommands: loadedConfig.config.safety.allowCommands, capabilityPolicy: loadedConfig.config.safety.capabilityPolicy diff --git a/packages/cli/src/docs/command-docs/orchestration.ts b/packages/cli/src/docs/command-docs/orchestration.ts index 0a0cd6a1..021d5e63 100644 --- a/packages/cli/src/docs/command-docs/orchestration.ts +++ b/packages/cli/src/docs/command-docs/orchestration.ts @@ -256,6 +256,13 @@ export const ORCHESTRATION_COMMAND_DOCS: Record = { { flag: "--verifier-id ", description: "Identity label recorded for the verifier role" }, { flag: "--safe-risk ", description: "Maximum acceptable risk level: low, medium, or high (default: low)" }, { flag: "--max-security-score ", description: "Maximum acceptable security score from deterministic analysis, 0-100 (default: 0)" }, + { flag: "--max-wall-time-ms ", description: "Stop with budget-exhausted when wall time exceeds this limit" }, + { flag: "--max-changed-files ", description: "Stop with budget-exhausted when changed files exceed this limit" }, + { flag: "--max-model-calls ", description: "Stop with budget-exhausted after this many builder/verifier invocations" }, + { flag: "--allowed-path ", description: "Restrict builder edits to this path prefix (repeatable)" }, + { flag: "--protected-path ", description: "Treat edits under this path prefix as unsafe-change (repeatable)" }, + { flag: "--resume-from ", description: "Resume budget counters from a prior loop audit JSONL file" }, + { flag: "--run-id ", description: "Stable run id used for .codedecay/local/loop-audit/.jsonl" }, { flag: "--task ", description: "Task description used with a structured requirements artifact" }, { flag: "--requirements ", description: "Preserve acceptance-criteria IDs and trace status across loop rounds" }, { flag: "--format ", description: "json or markdown (default: markdown)" }, @@ -272,8 +279,8 @@ export const ORCHESTRATION_COMMAND_DOCS: Record = { "Verifier output can propose hypotheses and proof tasks, but only trusted deterministic, OSS-tool, or runtime evidence can verify criteria.", "The loop never auto-commits or auto-pushes. It leaves edits in the working tree for human review.", "Agent output is untrusted. CodeDecay re-runs deterministic analysis and configured checks after each agent action.", - "Terminal clean verdicts are always qualified: merge-safe-verified has configured checks plus security/coverage/mutation depth, while merge-safe-shallow passed gates but is missing deeper evidence.", - "Exit codes: 0 for merge-safe-verified, merge-safe-shallow, or plan-only report generation; 1 for unverified, needs-human, budget-exhausted, unsafe-change, stuck, builder-error, verifier-error, or agent-error; and 2 for CLI/internal errors." + "Terminal clean verdicts are always qualified: verified has configured checks plus security/coverage/mutation depth, while shallow-proof passed gates but is missing deeper evidence.", + "Exit codes: 0 for verified, shallow-proof, or plan-only report generation; 1 for unverified, needs-human, budget-exhausted, unsafe-change, stuck, builder-error, verifier-error, or agent-error; and 2 for CLI/internal errors." ] }, doctor: { diff --git a/packages/cli/src/parsers/loop.ts b/packages/cli/src/parsers/loop.ts index 628c7dff..1a631ea3 100644 --- a/packages/cli/src/parsers/loop.ts +++ b/packages/cli/src/parsers/loop.ts @@ -186,6 +186,76 @@ export function parseLoopArgs(args: string[]): LoopOptions { continue; } + if (arg.startsWith("--max-wall-time-ms=")) { + options.maxWallTimeMs = parsePositiveInt(arg.slice("--max-wall-time-ms=".length), "--max-wall-time-ms"); + continue; + } + if (arg === "--max-wall-time-ms") { + options.maxWallTimeMs = parsePositiveInt(requireValue(args, index, arg), "--max-wall-time-ms"); + index += 1; + continue; + } + + if (arg.startsWith("--max-changed-files=")) { + options.maxChangedFiles = parsePositiveInt(arg.slice("--max-changed-files=".length), "--max-changed-files"); + continue; + } + if (arg === "--max-changed-files") { + options.maxChangedFiles = parsePositiveInt(requireValue(args, index, arg), "--max-changed-files"); + index += 1; + continue; + } + + if (arg.startsWith("--max-model-calls=")) { + options.maxModelCalls = parsePositiveInt(arg.slice("--max-model-calls=".length), "--max-model-calls"); + continue; + } + if (arg === "--max-model-calls") { + options.maxModelCalls = parsePositiveInt(requireValue(args, index, arg), "--max-model-calls"); + index += 1; + continue; + } + + if (arg.startsWith("--allowed-path=")) { + options.allowedPaths = [...(options.allowedPaths ?? []), arg.slice("--allowed-path=".length)]; + continue; + } + if (arg === "--allowed-path") { + options.allowedPaths = [...(options.allowedPaths ?? []), requireValue(args, index, arg)]; + index += 1; + continue; + } + + if (arg.startsWith("--protected-path=")) { + options.protectedPaths = [...(options.protectedPaths ?? []), arg.slice("--protected-path=".length)]; + continue; + } + if (arg === "--protected-path") { + options.protectedPaths = [...(options.protectedPaths ?? []), requireValue(args, index, arg)]; + index += 1; + continue; + } + + if (arg.startsWith("--resume-from=")) { + options.resumeFrom = arg.slice("--resume-from=".length); + continue; + } + if (arg === "--resume-from") { + options.resumeFrom = requireValue(args, index, arg); + index += 1; + continue; + } + + if (arg.startsWith("--run-id=")) { + options.runId = arg.slice("--run-id=".length); + continue; + } + if (arg === "--run-id") { + options.runId = requireValue(args, index, arg); + index += 1; + continue; + } + throwUnknownOption(arg, "loop"); } @@ -217,3 +287,11 @@ function parseSecurityScoreThreshold(value: string): number { throw new Error(`Invalid --max-security-score "${value}". Expected a number from 0 to 100.`); } + +function parsePositiveInt(value: string, flag: string): number { + const parsed = Number(value); + if (Number.isInteger(parsed) && parsed > 0) { + return parsed; + } + throw new Error(`Invalid ${flag} "${value}". Expected a positive integer.`); +} diff --git a/packages/cli/src/types/loop.ts b/packages/cli/src/types/loop.ts index 334d2f08..bbbfc9a2 100644 --- a/packages/cli/src/types/loop.ts +++ b/packages/cli/src/types/loop.ts @@ -17,4 +17,11 @@ export interface LoopOptions { securityScoreThreshold: number; task?: string | undefined; requirements?: string | undefined; + maxWallTimeMs?: number | undefined; + maxChangedFiles?: number | undefined; + maxModelCalls?: number | undefined; + allowedPaths?: string[] | undefined; + protectedPaths?: string[] | undefined; + resumeFrom?: string | undefined; + runId?: string | undefined; } diff --git a/packages/cli/test/loop-e2e.test.ts b/packages/cli/test/loop-e2e.test.ts index 14ce2fc4..ec66b20b 100644 --- a/packages/cli/test/loop-e2e.test.ts +++ b/packages/cli/test/loop-e2e.test.ts @@ -31,7 +31,7 @@ describe("codedecay loop real edit convergence", () => { const report = JSON.parse(result.stdout) as LoopReport; expect(result.exitCode).toBe(0); - expect(report.status.startsWith("merge-safe-")).toBe(true); + expect(["verified", "shallow-proof"]).toContain(report.status); expect(report.rounds).toHaveLength(1); expect(report.rounds[0]?.agent?.madeChanges).toBe(true); expect(report.rounds[0]?.postAgentVerification).toMatchObject({ @@ -42,7 +42,7 @@ describe("codedecay loop real edit convergence", () => { expect(readFileSync(join(repo, ".git/codedecay-agent-runs"), "utf8")).toBe("x"); }); - it("drives a deterministic agent script from weak test to merge-safe-*", async () => { + it("drives a deterministic agent script from weak test to verified/shallow-proof", async () => { const repo = createLoopConvergenceRepo(); const result = await run([ @@ -62,7 +62,7 @@ describe("codedecay loop real edit convergence", () => { expect(result.stderr).toBe(""); expect(result.exitCode).toBe(0); - expect(report.status.startsWith("merge-safe-")).toBe(true); + expect(["verified", "shallow-proof"]).toContain(report.status); expect(report.rounds.length).toBeGreaterThanOrEqual(2); expect(report.rounds[0]?.weakTestFindings).toBeGreaterThan(report.rounds.at(-1)?.weakTestFindings ?? 0); expect(report.rounds[0]?.mergeRiskScore).toBeGreaterThan(report.rounds.at(-1)?.mergeRiskScore ?? 0); diff --git a/packages/cli/test/loop.test.ts b/packages/cli/test/loop.test.ts index 18ebb26a..5d8dc6c9 100644 --- a/packages/cli/test/loop.test.ts +++ b/packages/cli/test/loop.test.ts @@ -12,7 +12,7 @@ import { } from "./helpers"; describe("codedecay loop CLI contract", () => { - it("reports merge-safe-shallow with low risk and passing configured checks when depth evidence is missing", async () => { + it("reports shallow-proof with low risk and passing configured checks when depth evidence is missing", async () => { const repo = createLowRiskRepoWithPassingCheck(); const result = await run(["loop", "--format", "json"], repo); @@ -20,7 +20,7 @@ describe("codedecay loop CLI contract", () => { expect(result.exitCode).toBe(0); expect(result.stderr).toBe(""); - expect(report.status).toBe("merge-safe-shallow"); + expect(report.status).toBe("shallow-proof"); expect(report.roundsRun).toBe(1); expect(report.finalCheckStatus).toBe("passed"); expect(report.verdict.missingDepth).toEqual( @@ -29,7 +29,7 @@ describe("codedecay loop CLI contract", () => { expect(report.safety.commandsExecuted).toBe(true); }); - it("carries Semgrep, coverage, and mutation evidence into a merge-safe-verified verdict", async () => { + it("carries Semgrep, coverage, and mutation evidence into a verified verdict", async () => { const repo = createLowRiskRepoWithVerifiedChecks(); const result = await run(["loop", "--format", "json"], repo); @@ -37,7 +37,7 @@ describe("codedecay loop CLI contract", () => { expect(result.exitCode).toBe(0); expect(result.stderr).toBe(""); - expect(report.status).toBe("merge-safe-verified"); + expect(report.status).toBe("verified"); expect(report.rounds[0].checkStatus).toBe("passed"); expect(report.verdict.verifiedBy).toEqual( expect.arrayContaining(["Semgrep (0 findings)", "coverage evidence (100%)", "mutation evidence (100%)"]) @@ -161,7 +161,7 @@ describe("codedecay loop CLI contract", () => { expect(result.exitCode).toBe(0); expect(result.stdout).toContain("## CodeDecay Loop Report"); - expect(result.stdout).toContain("**Status:** merge safe shallow"); + expect(result.stdout).toContain("**Status:** shallow proof"); expect(result.stdout).toContain("### Verdict Evidence"); expect(result.stdout).toContain("### Roles"); expect(result.stdout).toContain("### Loop State"); diff --git a/packages/harness/src/loop/audit.ts b/packages/harness/src/loop/audit.ts new file mode 100644 index 00000000..088a135c --- /dev/null +++ b/packages/harness/src/loop/audit.ts @@ -0,0 +1,139 @@ +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { LoopReport, LoopRoundSnapshot, LoopStatus } from "./types"; + +export interface LoopAuditRoundRecord { + schemaVersion: 1; + runId: string; + round: number; + timestamp: string; + statusHint?: LoopStatus | undefined; + agentIdentity?: string | undefined; + verifierIdentity?: string | undefined; + evidenceIds: string[]; + commands: string[]; + changedPaths: string[]; + decisions: Array<{ phase: string; actor: string; summary: string }>; + budgets: { + modelCalls: number; + wallTimeMs: number; + fingerprintCount: number; + }; + stopReason?: string | undefined; +} + +export interface LoopAuditResumeState { + runId: string; + completedRounds: number; + lastFingerprint?: string | undefined; + stopReason?: string | undefined; + modelCalls: number; + records: LoopAuditRoundRecord[]; +} + +export function defaultLoopAuditPath(cwd: string, runId: string): string { + return join(cwd, ".codedecay", "local", "loop-audit", `${runId}.jsonl`); +} + +export function appendLoopAuditRecord(path: string, record: LoopAuditRoundRecord): void { + mkdirSync(dirname(path), { recursive: true }); + appendFileSync(path, `${JSON.stringify(record)}\n`, "utf8"); +} + +export function writeLoopAuditSummary(path: string, report: LoopReport): void { + const summaryPath = path.endsWith(".jsonl") ? `${path.slice(0, -6)}.summary.json` : `${path}.summary.json`; + mkdirSync(dirname(summaryPath), { recursive: true }); + writeFileSync( + summaryPath, + `${JSON.stringify( + { + schemaVersion: 1, + status: report.status, + roundsRun: report.roundsRun, + stopReason: report.stateMachine.decisions.at(-1)?.summary, + generatedAt: report.generatedAt, + verdict: report.verdict.status, + roles: report.roles + }, + null, + 2 + )}\n`, + "utf8" + ); +} + +export function loadLoopAuditResumeState(path: string): LoopAuditResumeState | undefined { + if (!existsSync(path)) { + return undefined; + } + const lines = readFileSync(path, "utf8") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + const records: LoopAuditRoundRecord[] = []; + for (const line of lines) { + try { + records.push(JSON.parse(line) as LoopAuditRoundRecord); + } catch { + // Ignore corrupt trailing lines; resume must stay deterministic and safe. + } + } + if (records.length === 0) { + return undefined; + } + const last = records[records.length - 1]!; + return { + runId: last.runId, + completedRounds: records.length, + lastFingerprint: undefined, + stopReason: last.stopReason, + modelCalls: last.budgets.modelCalls, + records + }; +} + +export function createAuditRecordFromRound(input: { + runId: string; + round: LoopRoundSnapshot; + timestamp: string; + modelCalls: number; + wallTimeMs: number; + fingerprintCount: number; + stopReason?: string | undefined; + statusHint?: LoopStatus | undefined; +}): LoopAuditRoundRecord { + const commands = [ + input.round.builder?.command, + input.round.verifier?.command, + input.round.agent?.command + ].filter((value): value is string => Boolean(value)); + const changedPaths = [ + ...(input.round.builder?.changedFiles ?? []), + ...(input.round.verifier?.changedFiles ?? []), + ...(input.round.agent?.changedFiles ?? []) + ]; + const evidenceIds = input.round.stateMachine?.decisions.flatMap((decision) => decision.evidenceIds) ?? []; + return { + schemaVersion: 1, + runId: input.runId, + round: input.round.round, + timestamp: input.timestamp, + statusHint: input.statusHint, + agentIdentity: input.round.builder?.identity ?? input.round.agent?.identity, + verifierIdentity: input.round.verifier?.identity, + evidenceIds, + commands: [...new Set(commands)], + changedPaths: [...new Set(changedPaths)], + decisions: (input.round.stateMachine?.decisions ?? []).map((decision) => ({ + phase: decision.phase, + actor: decision.actor, + summary: decision.summary + })), + budgets: { + modelCalls: input.modelCalls, + wallTimeMs: input.wallTimeMs, + fingerprintCount: input.fingerprintCount + }, + stopReason: input.stopReason + }; +} diff --git a/packages/harness/src/loop/budgets.ts b/packages/harness/src/loop/budgets.ts new file mode 100644 index 00000000..dec8c87a --- /dev/null +++ b/packages/harness/src/loop/budgets.ts @@ -0,0 +1,141 @@ +import type { FileChange } from "@submuxhq/codedecay-core"; +import type { LoopStatus } from "./types"; + +export interface LoopBudgetConfig { + maxWallTimeMs?: number | undefined; + maxChangedFiles?: number | undefined; + allowedPathPrefixes?: string[] | undefined; + protectedPathPrefixes?: string[] | undefined; + maxModelCalls?: number | undefined; +} + +export interface LoopBudgetState { + startedAtMs: number; + modelCalls: number; + fingerprints: string[]; +} + +export interface LoopBudgetViolation { + status: Extract; + reason: string; +} + +export function createLoopBudgetState(nowMs: number): LoopBudgetState { + return { + startedAtMs: nowMs, + modelCalls: 0, + fingerprints: [] + }; +} + +export function checkWallTimeBudget( + config: LoopBudgetConfig, + state: LoopBudgetState, + nowMs: number +): LoopBudgetViolation | undefined { + if (config.maxWallTimeMs === undefined) { + return undefined; + } + if (nowMs - state.startedAtMs <= config.maxWallTimeMs) { + return undefined; + } + return { + status: "budget-exhausted", + reason: `Wall-time budget exhausted after ${nowMs - state.startedAtMs}ms (limit ${config.maxWallTimeMs}ms).` + }; +} + +export function checkModelCallBudget( + config: LoopBudgetConfig, + state: LoopBudgetState +): LoopBudgetViolation | undefined { + if (config.maxModelCalls === undefined) { + return undefined; + } + if (state.modelCalls < config.maxModelCalls) { + return undefined; + } + return { + status: "budget-exhausted", + reason: `Model-call budget exhausted after ${state.modelCalls} call(s) (limit ${config.maxModelCalls}).` + }; +} + +export function checkChangedFileBudgets( + config: LoopBudgetConfig, + changedFiles: FileChange[] +): LoopBudgetViolation | undefined { + const paths = changedFiles.map((file) => file.path); + + if (config.maxChangedFiles !== undefined && paths.length > config.maxChangedFiles) { + return { + status: "budget-exhausted", + reason: `Changed-file budget exceeded: ${paths.length} file(s) (limit ${config.maxChangedFiles}).` + }; + } + + if (config.protectedPathPrefixes && config.protectedPathPrefixes.length > 0) { + const protectedHit = paths.find((path) => + config.protectedPathPrefixes!.some((prefix) => pathEqualsOrUnder(path, prefix)) + ); + if (protectedHit) { + return { + status: "unsafe-change", + reason: `Protected path edited: ${protectedHit}.` + }; + } + } + + if (config.allowedPathPrefixes && config.allowedPathPrefixes.length > 0) { + const outOfScope = paths.find( + (path) => !config.allowedPathPrefixes!.some((prefix) => pathEqualsOrUnder(path, prefix)) + ); + if (outOfScope) { + return { + status: "unsafe-change", + reason: `Changed path outside allowed scope: ${outOfScope}.` + }; + } + } + + return undefined; +} + +export function detectOscillation( + state: LoopBudgetState, + fingerprint: string +): LoopBudgetViolation | undefined { + const priorMatches = state.fingerprints.filter((value) => value === fingerprint).length; + state.fingerprints.push(fingerprint); + if (priorMatches >= 1 && state.fingerprints.length >= 3) { + return { + status: "stuck", + reason: "Oscillation detected: repeated changed-tree fingerprint with no durable progress." + }; + } + return undefined; +} + +export function detectWideningScope( + previousPaths: string[], + currentPaths: string[] +): LoopBudgetViolation | undefined { + if (previousPaths.length === 0) { + return undefined; + } + const previous = new Set(previousPaths); + const added = currentPaths.filter((path) => !previous.has(path)); + if (added.length >= 3 && currentPaths.length > previousPaths.length + 2) { + return { + status: "stuck", + reason: `Scope widened unexpectedly by ${added.length} new path(s).` + }; + } + return undefined; +} + +function pathEqualsOrUnder(path: string, prefix: string): boolean { + const normalizedPath = path.replace(/\\/g, "/"); + const normalizedPrefix = prefix.replace(/\\/g, "/").replace(/\/+$/, ""); + return normalizedPath === normalizedPrefix || normalizedPath.startsWith(`${normalizedPrefix}/`); +} diff --git a/packages/harness/src/loop/controller.ts b/packages/harness/src/loop/controller.ts index e94466fc..59c25347 100644 --- a/packages/harness/src/loop/controller.ts +++ b/packages/harness/src/loop/controller.ts @@ -1,5 +1,22 @@ import { createHash } from "node:crypto"; import { isMemoryContextFinding } from "@submuxhq/codedecay-core"; +import { + appendLoopAuditRecord, + createAuditRecordFromRound, + defaultLoopAuditPath, + loadLoopAuditResumeState, + writeLoopAuditSummary +} from "./audit"; +import { + checkChangedFileBudgets, + checkModelCallBudget, + checkWallTimeBudget, + createLoopBudgetState, + detectOscillation, + detectWideningScope, + type LoopBudgetConfig +} from "./budgets"; +import { mergeHypothesisStatuses, parseVerifierHypothesisProposals } from "./hypotheses"; import { riskRank } from "./risk"; import { driveAgent } from "./agent"; import { changedFilePaths, createChangedFilesFingerprint } from "./fingerprint"; @@ -13,6 +30,7 @@ import type { LoopAgentRole, LoopAgentResult, LoopCheckSnapshot, + LoopHypothesisStatusSnapshot, LoopRedteamReport, LoopReport, LoopRoundSnapshot, @@ -25,6 +43,7 @@ import type { interface PreviousAgentRound { evidence: LoopProgressSnapshot; madeChanges: boolean; + changedPaths: string[]; } export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise { @@ -32,17 +51,48 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise new Date()); + const budgetConfig: LoopBudgetConfig = { + maxWallTimeMs: input.maxWallTimeMs, + maxChangedFiles: input.maxChangedFiles, + allowedPathPrefixes: input.allowedPathPrefixes, + protectedPathPrefixes: input.protectedPathPrefixes, + maxModelCalls: input.maxModelCalls + }; + const budgetState = createLoopBudgetState(nowFn().getTime()); + const resume = input.resumeFromAuditPath ? loadLoopAuditResumeState(input.resumeFromAuditPath) : undefined; + if (resume) { + budgetState.modelCalls = resume.modelCalls; + } + const runId = input.runId ?? resume?.runId ?? `loop-${nowFn().toISOString().replace(/[:.]/g, "-")}`; + const auditPath = input.auditPath ?? defaultLoopAuditPath(input.cwd, runId); const rounds: LoopRoundSnapshot[] = []; let status: LoopStatus = "needs-human"; + let stopReason: string | undefined = resume?.stopReason; let noProgressCount = 0; let previousAgentRound: PreviousAgentRound | undefined; let latestReport: LoopRedteamReport | undefined; let latestChecks: LoopCheckSnapshot | undefined; let postAgentVerificationPending = false; + let hypothesisStatuses: LoopHypothesisStatusSnapshot[] = []; for (let roundNumber = 1; roundNumber <= maxRounds; roundNumber += 1) { + const wallViolation = checkWallTimeBudget(budgetConfig, budgetState, nowFn().getTime()); + if (wallViolation) { + status = wallViolation.status; + stopReason = wallViolation.reason; + break; + } + const beforeChanges = input.getChangedFiles(); const beforeFingerprint = createChangedFilesFingerprint(beforeChanges); + const scopeViolation = checkChangedFileBudgets(budgetConfig, beforeChanges); + if (scopeViolation) { + status = scopeViolation.status; + stopReason = scopeViolation.reason; + break; + } + const report = await input.createRedteamReport(); const checks = await input.runConfiguredChecks(); latestReport = report; @@ -50,6 +100,25 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise= 2) { status = "stuck"; + stopReason = "No evidence gain across consecutive builder rounds."; + break; + } + const widen = detectWideningScope( + previousAgentRound.changedPaths, + changedFilePaths(input.getChangedFiles()) + ); + if (widen) { + status = widen.status; + stopReason = widen.reason; break; } } @@ -80,23 +159,33 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise `builder-changed:${file}`) + const afterBuilderFingerprint = createChangedFilesFingerprint(input.getChangedFiles()); + const afterBuilderPaths = changedFilePaths(input.getChangedFiles()); + const postEditScope = checkChangedFileBudgets(budgetConfig, input.getChangedFiles()); + round.stateMachine = createStateMachineSnapshot( + madeChanges ? "repair" : "build-edit", + report, + checks, + afterBuilderFingerprint, + [ + { + phase: madeChanges ? "repair" : "build-edit", + actor: "builder", + summary: madeChanges + ? `Builder edited ${agent.changedFiles.length} changed file(s).` + : "Builder completed without changing the tree.", + evidenceIds: agent.changedFiles.map((file) => `builder-changed:${file}`) + } + ], + hypothesisStatuses + ); + + if (postEditScope) { + status = postEditScope.status; + stopReason = postEditScope.reason; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); + break; + } + + if (madeChanges) { + const oscillation = detectOscillation(budgetState, afterBuilderFingerprint); + if (oscillation) { + status = oscillation.status; + stopReason = oscillation.reason; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); + break; } - ]); + } if (agent.status !== "passed") { status = "builder-error"; + stopReason = `Builder command ended with status ${agent.status}.`; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); break; } if (input.verifierCommand) { + const verifierModelBudget = checkModelCallBudget(budgetConfig, budgetState); + if (verifierModelBudget) { + status = verifierModelBudget.status; + stopReason = verifierModelBudget.reason; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); + break; + } + budgetState.modelCalls += 1; const verifier = await executeAgentRound( input, "verifier", @@ -132,28 +290,52 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise `verifier-unsafe-change:${file}`) } - ]); + ], hypothesisStatuses); + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); break; } if (verifier.status !== "passed") { status = "verifier-error"; + stopReason = `Verifier command ended with status ${verifier.status}.`; round.stateMachine = createStateMachineSnapshot("challenge", report, checks, createChangedFilesFingerprint(input.getChangedFiles()), [ { phase: "challenge", actor: "verifier", - summary: `Verifier command ended with status ${verifier.status}.`, + summary: stopReason, evidenceIds: ["verifier:error"] } - ]); + ], hypothesisStatuses); + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); break; } round.stateMachine = createStateMachineSnapshot("challenge", report, checks, createChangedFilesFingerprint(input.getChangedFiles()), [ @@ -161,26 +343,50 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise `hypothesis:${entry.hypothesisId}:${entry.status}`) + ] } - ]); + ], hypothesisStatuses); } if (!madeChanges) { noProgressCount += 1; if (noProgressCount >= 2) { status = "stuck"; + stopReason = "Builder made no file changes twice."; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + stopReason, + statusHint: status + }); break; } } previousAgentRound = { evidence: createLoopProgressSnapshot(report, checks), - madeChanges + madeChanges, + changedPaths: afterBuilderPaths }; + persistRoundAudit({ + auditPath, + runId, + round, + nowFn, + budgetState, + statusHint: status + }); + if (roundNumber === maxRounds) { status = "needs-human"; + stopReason = "Max rounds reached before trusted proof."; } } @@ -194,11 +400,15 @@ export async function runCodeDecayLoop(input: CodeDecayLoopInput): Promise Date; + budgetState: ReturnType; + stopReason?: string | undefined; + statusHint?: LoopStatus | undefined; +}): void { + appendLoopAuditRecord( + input.auditPath, + createAuditRecordFromRound({ + runId: input.runId, + round: input.round, + timestamp: input.nowFn().toISOString(), + modelCalls: input.budgetState.modelCalls, + wallTimeMs: input.nowFn().getTime() - input.budgetState.startedAtMs, + fingerprintCount: input.budgetState.fingerprints.length, + stopReason: input.stopReason, + statusHint: input.statusHint + }) + ); } function createRoundSnapshot( @@ -280,14 +518,15 @@ function createStateMachineSnapshot( report: LoopRedteamReport, checks: LoopCheckSnapshot, changedTreeFingerprint: string, - decisions: LoopStateMachineSnapshot["decisions"] + decisions: LoopStateMachineSnapshot["decisions"], + hypothesisStatuses: LoopHypothesisStatusSnapshot[] = [] ): LoopStateMachineSnapshot { return { schemaVersion: 1, phase, changedTreeFingerprint: hashChangedTreeFingerprint(changedTreeFingerprint), requirementStatuses: requirementStatuses(report) ?? [], - hypothesisStatuses: [], + hypothesisStatuses, experimentStatuses: experimentStatuses(checks), unresolvedHumanDecisions: unresolvedHumanDecisions(report, checks), decisions @@ -400,6 +639,8 @@ interface AssembleLoopReportInput { finalChecks: LoopCheckSnapshot; verdict: LoopVerdictEvidence; stateMachine: LoopStateMachineSnapshot; + auditPath?: string | undefined; + stopReason?: string | undefined; } function assembleLoopReport(state: AssembleLoopReportInput): LoopReport { @@ -416,7 +657,7 @@ function assembleLoopReport(state: AssembleLoopReportInput): LoopReport { head: input.head, maxRounds, roundsRun: rounds.length, - planOnly: !input.agentCommand, + planOnly: !builderCommand, finalRiskLevel: finalReport.summary.riskLevel, finalMergeRiskScore: finalReport.summary.mergeRiskScore, finalDecayScore: finalReport.summary.decayScore, @@ -431,6 +672,8 @@ function assembleLoopReport(state: AssembleLoopReportInput): LoopReport { requirementTrace: finalReport.requirementTrace, rounds, nextSteps: nextStepsForStatus(status, verdict), + auditPath: state.auditPath, + stopReason: state.stopReason, safety: { commandsExecuted: didExecuteCommands(rounds), agentCommandConfigured: Boolean(builderCommand), @@ -474,7 +717,7 @@ export function classifySafeStatus( checks: LoopCheckSnapshot, safeRiskLevel: LoopRedteamReport["summary"]["riskLevel"], securityScoreThreshold = 0 -): "merge-safe-verified" | "merge-safe-shallow" | "unverified" | undefined { +): "verified" | "shallow-proof" | "unverified" | undefined { const evidence = createLoopVerdictEvidence(report, checks, safeRiskLevel, securityScoreThreshold, "needs-human"); if (!evidence.riskAllowed || !evidence.weakTestsClear || !evidence.securityScoreAllowed || evidence.highFindingCount > 0) { return undefined; @@ -488,7 +731,7 @@ export function classifySafeStatus( return undefined; } - return evidence.missingDepth.length === 0 ? "merge-safe-verified" : "merge-safe-shallow"; + return evidence.missingDepth.length === 0 ? "verified" : "shallow-proof"; } export function createLoopVerdictEvidence( @@ -645,12 +888,32 @@ async function revalidateFinalAgentEdit( previousAgentRound: PreviousAgentRound | undefined, safeRiskLevel: LoopRedteamReport["summary"]["riskLevel"], securityScoreThreshold: number, - currentStatus: LoopStatus + currentStatus: LoopStatus, + hypothesisStatuses: LoopHypothesisStatusSnapshot[] ): Promise<{ report: LoopRedteamReport; checks: LoopCheckSnapshot; status: LoopStatus }> { const report = await input.createRedteamReport(); const checks = await input.runConfiguredChecks(); recordPostAgentVerification(finalRound, report, checks); + if (finalRound) { + finalRound.stateMachine = createStateMachineSnapshot( + "current-tree-reverify", + report, + checks, + createChangedFilesFingerprint(input.getChangedFiles()), + [ + ...(finalRound.stateMachine?.decisions ?? []), + { + phase: "current-tree-reverify", + actor: "codedecay", + summary: "Final current-tree revalidation after the last builder edit.", + evidenceIds: loopEvidenceIds(report, checks) + } + ], + hypothesisStatuses + ); + } + if (finalRound?.agent?.madeChanges && previousAgentRound?.madeChanges) { finalRound.riskReducedFromPreviousRound = didLoopEvidenceImprove( previousAgentRound.evidence, @@ -732,19 +995,19 @@ function nextStepsForStatus(status: LoopStatus, verdict: LoopVerdictEvidence): s case "merge-safe-shallow": return [ "Review the working tree diff and the missing-depth list before merge.", - "Enable Semgrep, coverage, and StrykerJS adapters to upgrade this verdict to merge-safe-verified.", + "Enable Semgrep, coverage, and StrykerJS adapters to upgrade this verdict to verified.", "Treat this as shallow configured-check clean, not a guarantee of production safety." ]; case "unverified": return [ "Add or enable configured checks in .codedecay/config.yml.", "Run codedecay loop again after tests/build/probes can execute.", - "Do not treat this PR as merge-safe-* until real checks pass." + "Do not treat this PR as verified until real checks pass." ]; case "plan-only": return [ "Review the generated agent bundle and fix tasks.", - "Run again with --agent-cmd only after configuring a user-owned local agent command.", + "Run again with --agent-cmd or --builder-cmd only after configuring a user-owned local agent command.", "Keep safety.allowCommands false unless you explicitly want CodeDecay to run local commands." ]; case "stuck": @@ -768,7 +1031,7 @@ function nextStepsForStatus(status: LoopStatus, verdict: LoopVerdictEvidence): s case "builder-error": case "agent-error": return [ - "Fix the configured --agent-cmd or safety.allowCommands settings.", + "Fix the configured --agent-cmd/--builder-cmd or safety.allowCommands settings.", "Remember agent output is untrusted until deterministic checks pass.", "Run in plan-only mode to inspect the prompt that would be sent." ]; diff --git a/packages/harness/src/loop/hypotheses.ts b/packages/harness/src/loop/hypotheses.ts new file mode 100644 index 00000000..b7afe361 --- /dev/null +++ b/packages/harness/src/loop/hypotheses.ts @@ -0,0 +1,66 @@ +import type { LoopHypothesisStatusSnapshot, LoopAgentResult } from "./types"; + +/** + * Verifier output is advisory. Parse proposed hypotheses/challenges without + * granting proof authority — only deterministic checks can confirm/refute. + */ +export function parseVerifierHypothesisProposals(verifier: LoopAgentResult | undefined): LoopHypothesisStatusSnapshot[] { + if (!verifier || verifier.role !== "verifier") { + return []; + } + + const text = `${verifier.stdout}\n${verifier.stderr}`; + const lines = text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const proposals: LoopHypothesisStatusSnapshot[] = []; + const seen = new Set(); + + for (const line of lines) { + const match = /^(?:hypothesis|challenge|proof-task)\s*:\s*(.+)$/i.exec(line); + if (!match?.[1]) { + continue; + } + const title = match[1].trim(); + const hypothesisId = `verifier:${slug(title)}`; + if (seen.has(hypothesisId)) { + continue; + } + seen.add(hypothesisId); + proposals.push({ + hypothesisId, + status: "candidate" + }); + } + + return proposals; +} + +export function mergeHypothesisStatuses( + existing: LoopHypothesisStatusSnapshot[], + additions: LoopHypothesisStatusSnapshot[] +): LoopHypothesisStatusSnapshot[] { + const byId = new Map(existing.map((entry) => [entry.hypothesisId, entry])); + for (const addition of additions) { + const current = byId.get(addition.hypothesisId); + if (!current) { + byId.set(addition.hypothesisId, addition); + continue; + } + // Never let advisory verifier output escalate beyond candidate/planned. + if (current.status === "confirmed" || current.status === "refuted") { + continue; + } + byId.set(addition.hypothesisId, { + hypothesisId: addition.hypothesisId, + status: current.status === "planned" ? "planned" : addition.status + }); + } + return [...byId.values()]; +} + +function slug(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 64) || "untitled"; +} diff --git a/packages/harness/src/loop/index.ts b/packages/harness/src/loop/index.ts index 0cb508dc..9e1b72c7 100644 --- a/packages/harness/src/loop/index.ts +++ b/packages/harness/src/loop/index.ts @@ -1,6 +1,14 @@ export { driveAgent } from "./agent"; +export { + appendLoopAuditRecord, + defaultLoopAuditPath, + loadLoopAuditResumeState, + writeLoopAuditSummary +} from "./audit"; +export type { LoopAuditRoundRecord, LoopAuditResumeState } from "./audit"; export { classifySafeStatus, createLoopVerdictEvidence, runCodeDecayLoop } from "./controller"; export { createChangedFilesFingerprint, changedFilePaths } from "./fingerprint"; +export { parseVerifierHypothesisProposals, mergeHypothesisStatuses } from "./hypotheses"; export { createLoopProgressSnapshot, didLoopEvidenceImprove } from "./progress"; export { renderLoopMarkdown, renderLoopReport } from "./render"; export type { @@ -13,6 +21,7 @@ export type { LoopCoverageSnapshot, LoopFixTask, LoopFormat, + LoopHypothesisStatusSnapshot, LoopMutationSnapshot, LoopRedteamReport, LoopReport, diff --git a/packages/harness/src/loop/render.ts b/packages/harness/src/loop/render.ts index f0ffbf87..6ce44009 100644 --- a/packages/harness/src/loop/render.ts +++ b/packages/harness/src/loop/render.ts @@ -13,6 +13,8 @@ export function renderLoopMarkdown(report: LoopReport): string { "## CodeDecay Loop Report", "", `**Status:** ${statusLabel(report.status)}`, + ...(report.stopReason ? [`**Stop reason:** ${report.stopReason}`] : []), + ...(report.auditPath ? [`**Audit:** \`${report.auditPath}\``] : []), "", "| Signal | Value |", "| --- | ---: |", diff --git a/packages/harness/src/loop/types.ts b/packages/harness/src/loop/types.ts index a02d1067..4d507e99 100644 --- a/packages/harness/src/loop/types.ts +++ b/packages/harness/src/loop/types.ts @@ -262,6 +262,8 @@ export interface LoopReport { requirementTrace?: RequirementTraceGraph | undefined; rounds: LoopRoundSnapshot[]; nextSteps: string[]; + auditPath?: string | undefined; + stopReason?: string | undefined; safety: { commandsExecuted: boolean; agentCommandConfigured: boolean; @@ -310,6 +312,14 @@ export interface CodeDecayLoopInput { securityScoreThreshold?: number | undefined; agentTimeoutMs: number; commandSafety: SafeCommandPolicy; + maxWallTimeMs?: number | undefined; + maxChangedFiles?: number | undefined; + maxModelCalls?: number | undefined; + allowedPathPrefixes?: string[] | undefined; + protectedPathPrefixes?: string[] | undefined; + auditPath?: string | undefined; + resumeFromAuditPath?: string | undefined; + runId?: string | undefined; createRedteamReport(): Promise; renderAgentBundle(report: LoopRedteamReport): string; renderBuilderBundle?: ((report: LoopRedteamReport) => string) | undefined; diff --git a/packages/harness/test/loop.test.ts b/packages/harness/test/loop.test.ts index be6e75c9..a98c2f71 100644 --- a/packages/harness/test/loop.test.ts +++ b/packages/harness/test/loop.test.ts @@ -21,7 +21,7 @@ afterEach(() => { }); describe("CodeDecay loop controller", () => { - it("reports merge-safe-shallow when gates pass but depth evidence is missing", async () => { + it("reports shallow-proof when gates pass but depth evidence is missing", async () => { const repo = createRepo(); const report = await runCodeDecayLoop({ ...baseInput(repo), @@ -29,7 +29,7 @@ describe("CodeDecay loop controller", () => { runConfiguredChecks: async () => checkSnapshot("passed", true) }); - expect(report.status).toBe("merge-safe-shallow"); + expect(report.status).toBe("shallow-proof"); expect(report.roundsRun).toBe(1); expect(report.safety.commandsExecuted).toBe(true); expect(report.verdict.missingDepth).toEqual( @@ -37,7 +37,7 @@ describe("CodeDecay loop controller", () => { ); }); - it("reports merge-safe-verified when security, coverage, mutation, and configured checks pass", async () => { + it("reports verified when security, coverage, mutation, and configured checks pass", async () => { const repo = createRepo(); const report = await runCodeDecayLoop({ ...baseInput(repo), @@ -51,7 +51,7 @@ describe("CodeDecay loop controller", () => { runConfiguredChecks: async () => verifiedCheckSnapshot() }); - expect(report.status).toBe("merge-safe-verified"); + expect(report.status).toBe("verified"); expect(report.verdict.verifiedBy).toEqual( expect.arrayContaining(["Semgrep (0 findings)", "coverage evidence (100%)", "mutation evidence (100%)"]) ); @@ -124,7 +124,7 @@ describe("CodeDecay loop controller", () => { runConfiguredChecks: async () => checkSnapshot("passed", true) }); - expect(report.status).toBe("merge-safe-shallow"); + expect(report.status).toBe("shallow-proof"); expect(report.verdict.highFindingCount).toBe(0); expect(report.rounds[0]?.agent).toBeUndefined(); expect(existsSync(join(repo, "agent-ran.txt"))).toBe(false); @@ -160,7 +160,7 @@ describe("CodeDecay loop controller", () => { createRedteamReport, runConfiguredChecks }); - expect(report.status).toBe("merge-safe-shallow"); + expect(report.status).toBe("shallow-proof"); expect(report.roundsRun).toBe(1); expect(report.rounds.filter((round) => round.agent).length).toBe(1); expect(report.rounds[0]?.postAgentVerification).toMatchObject({ @@ -340,6 +340,108 @@ describe("CodeDecay loop controller", () => { }); expect(report.nextSteps).toContain("A read-only verifier or protected role changed files."); }); + + it("UAT-LOOP-1/2: verifier finds planted API miss, builder repairs, final tree reaches verified", async () => { + const repo = createRepo(); + let builderCalls = 0; + const createRedteamReport = vi.fn() + .mockResolvedValueOnce(redteamReport({ + riskLevel: "high", + mergeRiskScore: 80, + weakTestFindings: 0, + requirementStatus: "proof-missing" + })) + .mockResolvedValueOnce(redteamReport({ + riskLevel: "high", + mergeRiskScore: 70, + weakTestFindings: 0, + requirementStatus: "proof-missing" + })) + .mockResolvedValueOnce(redteamReport({ + riskLevel: "low", + mergeRiskScore: 5, + weakTestFindings: 0, + requirementStatus: "verified", + securityAnalysis: { scannedFiles: ["src/users.ts"], candidateCount: 0 } + })) + .mockResolvedValue(redteamReport({ + riskLevel: "low", + mergeRiskScore: 5, + weakTestFindings: 0, + requirementStatus: "verified", + securityAnalysis: { scannedFiles: ["src/users.ts"], candidateCount: 0 } + })); + + const report = await runCodeDecayLoop({ + ...baseInput(repo), + maxRounds: 2, + runId: "uat-loop-1", + builderCommand: "node -e \"const fs=require('fs'); fs.mkdirSync('src',{recursive:true}); const n=fs.existsSync('src/users.ts')?2:1; fs.writeFileSync('src/users.ts','fix-'+n);\"", + verifierCommand: "node -e \"console.log('hypothesis: missing API contract proof'); console.log('challenge: unit test passes but API requirement unmet')\"", + createRedteamReport, + runConfiguredChecks: async () => { + builderCalls += 1; + return builderCalls >= 3 ? verifiedCheckSnapshot() : checkSnapshot("passed", true); + } + }); + + expect(report.rounds.some((round) => round.verifier?.stdout.includes("missing API"))).toBe(true); + expect(report.stateMachine.hypothesisStatuses.some((entry) => entry.status === "candidate")).toBe(true); + expect(report.status).toBe("verified"); + expect(report.requirementTrace?.criteria[0]?.status).toBe("verified"); + expect(report.auditPath).toContain("loop-audit"); + expect(existsSync(report.auditPath!)).toBe(true); + }); + + it("UAT-LOOP-4: protected-path edits and oscillation stop with correct statuses", async () => { + const protectedRepo = createRepo(); + const protectedReport = await runCodeDecayLoop({ + ...baseInput(protectedRepo), + maxRounds: 1, + protectedPathPrefixes: [".codedecay"], + builderCommand: "node -e \"require('fs').mkdirSync('.codedecay',{recursive:true}); require('fs').writeFileSync('.codedecay/policy.yml','bad')\"", + createRedteamReport: async () => redteamReport({ riskLevel: "high", mergeRiskScore: 90, weakTestFindings: 1 }), + runConfiguredChecks: async () => checkSnapshot("passed", true) + }); + expect(protectedReport.status).toBe("unsafe-change"); + expect(protectedReport.stopReason).toContain("Protected path"); + + const oscillateRepo = createRepo(); + const oscillateReport = await runCodeDecayLoop({ + ...baseInput(oscillateRepo), + maxRounds: 4, + builderCommand: "node -e \"require('fs').writeFileSync('flip.txt', 'same')\"", + createRedteamReport: async () => redteamReport({ riskLevel: "high", mergeRiskScore: 90, weakTestFindings: 1 }), + runConfiguredChecks: async () => checkSnapshot("passed", true) + }); + expect(["stuck", "budget-exhausted", "needs-human"]).toContain(oscillateReport.status); + }); + + it("UAT-LOOP-5: verifier outage preserves deterministic evidence and does not let builder self-verify", async () => { + const repo = createRepo(); + const createRedteamReport = vi.fn() + .mockResolvedValueOnce(redteamReport({ riskLevel: "high", mergeRiskScore: 90, weakTestFindings: 1 })) + .mockResolvedValue(redteamReport({ + riskLevel: "low", + mergeRiskScore: 10, + weakTestFindings: 0, + securityAnalysis: { scannedFiles: ["agent.txt"], candidateCount: 0 } + })); + const report = await runCodeDecayLoop({ + ...baseInput(repo), + maxRounds: 1, + builderCommand: "node -e \"require('fs').writeFileSync('agent.txt', 'fixed')\"", + verifierCommand: "node -e \"process.exit(2)\"", + createRedteamReport, + runConfiguredChecks: async () => verifiedCheckSnapshot() + }); + + expect(report.status).toBe("verifier-error"); + expect(report.roles.find((role) => role.role === "verifier")?.canVerifyCriteria).toBe(false); + expect(report.roles.find((role) => role.role === "builder")?.proofAuthority).toBe("none"); + expect(report.finalCheckStatus).toBe("passed"); + expect(report.finalMergeRiskScore).toBe(10); + }); }); describe("classifySafeStatus", () => { @@ -370,7 +472,7 @@ describe("classifySafeStatus", () => { "low" ); - expect(status).toBe("merge-safe-shallow"); + expect(status).toBe("shallow-proof"); }); it("does not return merge-safe when a high security finding remains", () => { @@ -396,14 +498,14 @@ describe("classifySafeStatus", () => { expect(status).toBeUndefined(); }); - it("returns merge-safe-shallow when gates pass without scanner, coverage, or mutation depth", () => { + it("returns shallow-proof when gates pass without scanner, coverage, or mutation depth", () => { const status = classifySafeStatus( redteamReport({ riskLevel: "low", mergeRiskScore: 10, weakTestFindings: 0 }), checkSnapshot("passed", true), "low" ); - expect(status).toBe("merge-safe-shallow"); + expect(status).toBe("shallow-proof"); }); }); diff --git a/scripts/end-user-demo.mjs b/scripts/end-user-demo.mjs index d158796a..9bd30b7a 100644 --- a/scripts/end-user-demo.mjs +++ b/scripts/end-user-demo.mjs @@ -370,7 +370,9 @@ function runLoopConvergenceChecks() { const editedTest = readFileSync(join(loopRepo, "test/checkout.test.js"), "utf8"); const converged = typeof report?.status === "string" && - report.status.startsWith("merge-safe-") && + (report.status === "verified" || + report.status === "shallow-proof" || + report.status.startsWith("merge-safe-")) && rounds.length >= 2 && Number(firstRound?.weakTestFindings) > Number(lastRound?.weakTestFindings) && firstRound?.agent?.madeChanges === true &&