From 06f70102cb9c99c497b3e09c1a2b0035823413cb Mon Sep 17 00:00:00 2001 From: Jay Shen Date: Tue, 1 Sep 2026 11:32:31 +0800 Subject: [PATCH] fix: verify test data cleanup evidence --- src/server/runs/evidence.ts | 63 ++- src/server/runs/orchestrator.ts | 60 ++- src/server/runs/test-data.ts | 621 +++++++++++++++++++++++++++--- src/server/runs/workspace.ts | 17 + tests/acceptance/phase9.ts | 15 +- tests/closure3-test-data.test.ts | 349 +++++++++++++++++ tests/phase4-orchestrator.test.ts | 94 ++++- tests/phase4.test.ts | 69 +++- 8 files changed, 1198 insertions(+), 90 deletions(-) create mode 100644 tests/closure3-test-data.test.ts diff --git a/src/server/runs/evidence.ts b/src/server/runs/evidence.ts index 8a9cebc..3759441 100644 --- a/src/server/runs/evidence.ts +++ b/src/server/runs/evidence.ts @@ -3,6 +3,7 @@ import type { AgentToolResult, ToolDefinition } from '@earendil-works/pi-coding- import type { EvidenceReference } from '../../shared/types.js'; import { createTextResult } from './agent-session.js'; +import type { TestDataVerificationReceipt } from './test-data.js'; import type { OssAdapter } from '../storage/oss.js'; import { contentTypeFor } from '../storage/oss.js'; import { RunWorkspace, type RunEvidenceFile } from './workspace.js'; @@ -29,7 +30,14 @@ export interface RunEvidenceStore { readFailureCount?: () => number; recordReadFailure?: () => void; reviewReadCount?: () => number; - recordReviewRead?: () => void; + recordReviewRead?: (evidenceId?: string) => void; + captureCleanupQuery( + receipt: TestDataVerificationReceipt, + redactedContent: string, + ): Promise; + isCleanupClaimEvidence(evidenceId: string, runId: string, dataId: string): Promise; + readCleanupTextEvidence(evidenceId: string, runId: string, dataId: string): Promise; + isReviewedCleanupEvidence(evidenceId: string): boolean; } export interface EvidenceReadResult { @@ -48,6 +56,9 @@ class DefaultRunEvidenceStore implements RunEvidenceStore { private readonly references = new Map(); private readFailures = 0; private reviewReads = 0; + private cleanupSequence = 0; + private readonly cleanupEvidence = new Map(); + private readonly reviewedEvidence = new Set(); constructor( private readonly workspace: RunWorkspace, @@ -157,8 +168,54 @@ class DefaultRunEvidenceStore implements RunEvidenceStore { return this.reviewReads; } - recordReviewRead(): void { + recordReviewRead(evidenceId?: string): void { this.reviewReads += 1; + if (evidenceId) this.reviewedEvidence.add(evidenceId); + } + + async captureCleanupQuery( + receipt: TestDataVerificationReceipt, + redactedContent: string, + ): Promise { + this.cleanupSequence += 1; + const evidenceId = `cleanup-query-${receipt.sha256.slice(0, 16)}-${this.cleanupSequence}.json`; + await this.workspace.writeHarnessEvidence( + evidenceId, + `${JSON.stringify({ provenance: receipt, redactedContent }, null, 2)}\n`, + ); + this.cleanupEvidence.set(evidenceId, { ...receipt }); + return evidenceId; + } + + async isCleanupClaimEvidence( + evidenceId: string, + runId: string, + dataId: string, + ): Promise { + const captured = this.cleanupEvidence.get(evidenceId); + if (captured) return captured.runId === runId && captured.dataId === dataId; + if (!/\.(?:png|jpe?g|webp)$/i.test(evidenceId)) return false; + return (await this.list()).some((file) => file.name === evidenceId); + } + + async readCleanupTextEvidence( + evidenceId: string, + runId: string, + dataId: string, + ): Promise { + const captured = this.cleanupEvidence.get(evidenceId); + if (!captured || captured.runId !== runId || captured.dataId !== dataId) { + throw new Error('清理证据不属于当前 Run/data ID 或不是受控文本查询证据'); + } + const evidence = await this.readUploaded(evidenceId); + if (evidence.body.byteLength > 256 * 1024) throw new Error('清理证据超过审核大小限制'); + this.recordReviewRead(evidenceId); + return evidence.body.toString('utf8'); + } + + isReviewedCleanupEvidence(evidenceId: string): boolean { + const captured = this.cleanupEvidence.get(evidenceId); + return this.reviewedEvidence.has(evidenceId) && (captured ? captured.absent : true); } } @@ -256,7 +313,7 @@ export function createReviewerEvidenceTools(store: RunEvidenceStore): ToolDefini store.recordReadFailure?.(); return createTextResult('截图超过审核大小限制', { error: true }); } - store.recordReviewRead?.(); + store.recordReviewRead?.(params.filename); return { content: [ { diff --git a/src/server/runs/orchestrator.ts b/src/server/runs/orchestrator.ts index bd07529..61f3891 100644 --- a/src/server/runs/orchestrator.ts +++ b/src/server/runs/orchestrator.ts @@ -44,7 +44,12 @@ import { type RunEvidenceStore, } from './evidence.js'; import { createProviderAdapter, type ProviderAdapter } from './provider.js'; -import { createTestDataManager, createTestDataTools, type TestDataManager } from './test-data.js'; +import { + createReviewerTestDataTools, + createTestDataManager, + createTestDataTools, + type TestDataManager, +} from './test-data.js'; import { createRoleInstructionLoader, RoleInstructionError, @@ -402,6 +407,7 @@ class DefaultRunOrchestrator implements RunOrchestrator { 'Reviewer 未读取截图 evidence，无法完成独立视觉审核。', ]); } + await this.finalizeTestData(workspace, context); if (runnerCleanup.uploaded && !runnerCleanup.uploadFailed) { await this.cleanupRetainedEvidence(workspace, context, evidenceStore); } @@ -743,7 +749,11 @@ class DefaultRunOrchestrator implements RunOrchestrator { this.options.configuration.getRepository(), this.options.secretStore, ), - ...createTestDataTools(this.options.testData ?? createTestDataManager(), context.runId), + ...createTestDataTools( + this.options.testData ?? createTestDataManager(), + context.runId, + evidenceStore, + ), ...(evidenceStore ? createRunnerEvidenceTools(evidenceStore) : []), createArtifactWriterTool( 'write_execution', @@ -888,12 +898,7 @@ class DefaultRunOrchestrator implements RunOrchestrator { notes.push('UI 场景没有产生可审核的 evidence。'); } - let cleanup: { - ok: boolean; - attempted: number; - failed: string[]; - message: string; - }; + let cleanup: Awaited>; try { cleanup = await this.options.testData!.cleanup(context.runId); } catch { @@ -902,12 +907,16 @@ class DefaultRunOrchestrator implements RunOrchestrator { attempted: 0, failed: [], message: '测试数据清理适配器执行失败', + receipts: [], }; } - if (!cleanup.ok) { - this.addBlockingReason(context, `测试数据清理失败：${cleanup.message}`); - } - notes.push(`测试数据清理：${cleanup.message}`); + notes.push(`测试数据清理适配器：${cleanup.message}`); + notes.push( + ...cleanup.receipts.map( + (receipt) => + `清理核验 receipt：${receipt.dataId} · ${receipt.sourceId} · ${receipt.queriedAt} · ${receipt.statusCode === undefined ? `exit ${receipt.exitCode ?? 'n/a'}` : `HTTP ${receipt.statusCode}`} · summary ${receipt.summary} · sha256 ${receipt.sha256}`, + ), + ); if (cleanup.failed.length > 0) { notes.push(`测试数据清理失败项数量：${cleanup.failed.length}`); } @@ -918,6 +927,24 @@ class DefaultRunOrchestrator implements RunOrchestrator { return { uploaded, uploadFailed }; } + private async finalizeTestData(workspace: RunWorkspace, context: RunContext): Promise { + const result = (this.options.testData ?? createTestDataManager()).finalize(context.runId); + const notes = [`测试数据最终核验：${result.message}`]; + if (!result.ok) { + this.addBlockingReason(context, `测试数据清理失败：${result.message}`); + notes.push( + ...result.pending.map( + (entry) => + `测试数据残留：${entry.id}（${entry.status}${entry.rejectionReason ? `：${entry.rejectionReason}` : ''}）`, + ), + ); + } + if (context.blockingReasons.length > 0) { + notes.push(...context.blockingReasons.map((reason) => `Harness 阻塞：${reason}`)); + } + await this.appendExecutionNotes(workspace, notes); + } + private async cleanupRetainedEvidence( workspace: RunWorkspace, context: RunContext, @@ -976,6 +1003,11 @@ class DefaultRunOrchestrator implements RunOrchestrator { ]), ), ...(evidenceStore ? createReviewerEvidenceTools(evidenceStore) : []), + ...createReviewerTestDataTools( + this.options.testData ?? createTestDataManager(), + context.runId, + evidenceStore, + ), createArtifactWriterTool( 'write_review', '写入独立审核', @@ -1703,7 +1735,7 @@ ${JSON.stringify(runnerContext(context), null, 2)}`; function runnerOutputContract(): string { return `先读取 plan.md，再按计划使用受控 target、工作场景、命令、环境、测试数据和 evidence 工具。UI 场景只能使用受控的 headless、isolated Playwright MCP，优先使用 accessibility snapshot/ref；截图使用相对文件名并通过 list_evidence_files 确认存在。 -测试账号只用于当前操作，绝不能写入日志、命令输出、Markdown 或证据。使用 get_test_data_prefix 标记临时数据，登记后执行 cleanup_test_data。每个场景记录实际观察、命令退出码、决定性/辅助证据、偏差和清理结果。结束前分别通过 write_execution 和 write_draft_report 写完整工件；不可用条件记录为 blocked。`; +测试账号只用于当前操作，绝不能写入日志、命令输出、Markdown 或证据。使用 get_test_data_prefix 标记临时数据；创建后立即登记，删除后只能提交 Harness 捕获的受控查询证据或 Playwright 截图声明，并检查待核验列表。不能自填 evidence 正文、状态码、摘要或 hash。每个场景记录实际观察、命令退出码、决定性/辅助证据、偏差和清理结果。结束前分别通过 write_execution 和 write_draft_report 写完整工件；不可用条件记录为 blocked。`; } function reviewerUserMessage(context: RunContext): string { @@ -1714,7 +1746,7 @@ ${JSON.stringify(reviewerContext(context), null, 2)}`; } function reviewerOutputContract(): string { - return `依次读取 plan.md、execution.md、draft-report.md 和存在的 scenario-changes.patch；原始证据先于 Runner 草稿。查看截图只能使用 list_evidence_files 和 read_evidence_image，不能执行命令、读取测试账号或任意路径。 + return `依次读取 plan.md、execution.md、draft-report.md 和存在的 scenario-changes.patch；原始证据先于 Runner 草稿。清理声明必须先通过 read_test_data_cleanup_evidence 读取 Harness 捕获的受控文本证据，或通过 read_evidence_image 实际查看删除后截图，再调用 verify_test_data_cleanup 确认或拒绝；纯 Runner 声明不构成已清理。查看截图只能使用 list_evidence_files 和 read_evidence_image，不能执行命令、读取测试账号或任意路径。 截图不可访问、上传失败、视觉能力不足、清理未确认、场景缺失或影响不明时维持 blocked。零场景只有在 Main · 规划的计划确有依据时才能确认。结束前通过 write_review 写完整 review.md，并明确是否同意最终结果。`; } diff --git a/src/server/runs/test-data.ts b/src/server/runs/test-data.ts index 3c96b3d..d638a49 100644 --- a/src/server/runs/test-data.ts +++ b/src/server/runs/test-data.ts @@ -1,32 +1,122 @@ -import { randomUUID } from 'node:crypto'; +import { createHash } from 'node:crypto'; import { Type, type Static } from 'typebox'; import type { AgentToolResult, ToolDefinition } from '@earendil-works/pi-coding-agent'; import { createTextResult } from './agent-session.js'; +export type TestDataStatus = 'registered' | 'cleanup-claimed' | 'verified-cleaned' | 'rejected'; + export interface TestDataEntry { id: string; scenarioId?: string; description?: string; } +export interface TestDataVerificationReceipt { + sourceId: string; + sourceKind: 'cleanup-adapter' | 'api-query' | 'readonly-command'; + runId: string; + dataId: string; + queriedAt: string; + absent: boolean; + statusCode?: number; + exitCode?: number; + summary: string; + sha256: string; +} + +export interface TestDataAdapterObservation { + absent: boolean; + content: string; + statusCode?: number; + exitCode?: number; +} + +export interface TestDataCleanupAdapter { + id: string; + cleanupAndVerify( + input: Readonly<{ runId: string; entry: TestDataEntry }>, + ): Promise; +} + +export interface TestDataQueryAdapter { + id: string; + kind: 'api-query' | 'readonly-command'; + operations: Readonly>; + query( + input: Readonly<{ + runId: string; + entry: TestDataEntry; + operation: string; + parameters: Readonly>; + }>, + ): Promise; +} + +export interface TestDataRecord extends TestDataEntry { + status: TestDataStatus; + registeredAt: string; + claim?: { evidenceIds: string[]; claimedAt: string }; + verification?: TestDataVerificationReceipt; + rejectionReason?: string; +} + export interface TestDataCleanupResult { ok: boolean; attempted: number; failed: string[]; message: string; + receipts: TestDataVerificationReceipt[]; +} + +export interface TestDataFinalResult { + ok: boolean; + pending: Array>; + message: string; +} + +export interface TestDataEvidenceBoundary { + captureCleanupQuery( + receipt: TestDataVerificationReceipt, + redactedContent: string, + ): Promise; + isCleanupClaimEvidence(evidenceId: string, runId: string, dataId: string): Promise; + readCleanupTextEvidence(evidenceId: string, runId: string, dataId: string): Promise; + isReviewedCleanupEvidence(evidenceId: string): boolean; } export interface TestDataManager { prefix(runId: string): string; register(runId: string, entry: TestDataEntry): Promise; + submitClaim( + runId: string, + dataId: string, + evidenceIds: readonly string[], + ): Promise; + pending(runId: string): TestDataRecord[]; + query( + runId: string, + dataId: string, + adapterId: string, + operation: string, + parameters: Readonly>, + ): Promise<{ receipt: TestDataVerificationReceipt; redactedContent: string }>; + verify( + runId: string, + dataId: string, + decision: 'confirm' | 'reject', + reason: string | undefined, + wasRead: (evidenceId: string) => boolean, + ): Promise; cleanup(runId: string): Promise; + finalize(runId: string): TestDataFinalResult; } export interface TestDataManagerOptions { - cleanup?: (runId: string, entries: readonly TestDataEntry[]) => Promise; - id?: () => string; + cleanupAdapter?: TestDataCleanupAdapter; + queryAdapters?: readonly TestDataQueryAdapter[]; + now?: () => Date; } export function createTestDataManager(options: TestDataManagerOptions = {}): TestDataManager { @@ -34,11 +124,19 @@ export function createTestDataManager(options: TestDataManagerOptions = {}): Tes } class DefaultTestDataManager implements TestDataManager { - private readonly entries = new Map(); - private readonly id: () => string; + private readonly entries = new Map>(); + private readonly now: () => Date; + private readonly queryAdapters: Map; constructor(private readonly options: TestDataManagerOptions) { - this.id = options.id ?? randomUUID; + this.now = options.now ?? (() => new Date()); + this.queryAdapters = new Map(); + for (const adapter of options.queryAdapters ?? []) { + assertAdapterId(adapter.id); + if (this.queryAdapters.has(adapter.id)) throw new Error('测试数据查询适配器 ID 重复'); + this.queryAdapters.set(adapter.id, adapter); + } + if (options.cleanupAdapter) assertAdapterId(options.cleanupAdapter.id); } prefix(runId: string): string { @@ -46,78 +144,239 @@ class DefaultTestDataManager implements TestDataManager { } async register(runId: string, entry: TestDataEntry): Promise { - const id = entry.id.trim(); - if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(id)) { - throw new Error('测试数据标识无效'); + const id = normalizeDataId(entry.id); + if (!id.startsWith(this.prefix(runId))) { + throw new Error('测试数据标识必须使用当前 Run 前缀'); } - const entries = this.entries.get(runId) ?? []; - if (!entries.some((item) => item.id === id)) { - entries.push({ ...entry, id }); + const entries = this.entries.get(runId) ?? new Map(); + if (!entries.has(id)) { + entries.set(id, { + id, + ...(entry.scenarioId ? { scenarioId: normalizeShortText(entry.scenarioId, 200) } : {}), + ...(entry.description + ? { description: redactSensitiveText(normalizeShortText(entry.description, 500)) } + : {}), + status: 'registered', + registeredAt: this.now().toISOString(), + }); this.entries.set(runId, entries); } } + async submitClaim( + runId: string, + dataId: string, + evidenceIds: readonly string[], + ): Promise { + const record = this.requireRecord(runId, dataId); + const normalizedEvidence = [...new Set(evidenceIds.map(normalizeEvidenceId))]; + if (normalizedEvidence.length === 0 || normalizedEvidence.length > 8) { + throw new Error('清理声明必须引用 1–8 项当前 Run 证据'); + } + if (record.status === 'verified-cleaned') throw new Error('测试数据已经核验清理'); + if (record.status === 'cleanup-claimed') { + if (sameValues(record.claim?.evidenceIds ?? [], normalizedEvidence)) + return cloneRecord(record); + throw new Error('测试数据已有不同的清理声明'); + } + record.status = 'cleanup-claimed'; + record.claim = { evidenceIds: normalizedEvidence, claimedAt: this.now().toISOString() }; + delete record.rejectionReason; + return cloneRecord(record); + } + + pending(runId: string): TestDataRecord[] { + return [...(this.entries.get(runId)?.values() ?? [])] + .filter((entry) => entry.status !== 'verified-cleaned') + .map(cloneRecord) + .sort((left, right) => left.id.localeCompare(right.id)); + } + + async query( + runId: string, + dataId: string, + adapterId: string, + operation: string, + parameters: Readonly>, + ): Promise<{ receipt: TestDataVerificationReceipt; redactedContent: string }> { + const record = this.requireRecord(runId, dataId); + const adapter = this.queryAdapters.get(adapterId); + if (!adapter) throw new Error('测试数据查询适配器不在 allowlist'); + const allowedParameters = adapter.operations[operation]; + if (!allowedParameters) throw new Error('测试数据查询操作不在 allowlist'); + const normalizedParameters = normalizeQueryParameters(parameters, allowedParameters); + const observation = await adapter.query({ + runId, + entry: publicEntry(record), + operation, + parameters: normalizedParameters, + }); + return createReceipt(runId, record.id, adapter.id, adapter.kind, observation, this.now()); + } + + async verify( + runId: string, + dataId: string, + decision: 'confirm' | 'reject', + reason: string | undefined, + wasRead: (evidenceId: string) => boolean, + ): Promise { + const record = this.requireRecord(runId, dataId); + if (record.status === 'verified-cleaned') return cloneRecord(record); + if (record.status !== 'cleanup-claimed' || !record.claim) { + throw new Error('测试数据没有待审核的清理声明'); + } + if (!record.claim.evidenceIds.every(wasRead)) { + throw new Error('Reviewer 必须先读取全部证据，且受控查询必须确认数据不存在'); + } + if (decision === 'confirm') { + record.status = 'verified-cleaned'; + delete record.rejectionReason; + } else { + const normalizedReason = normalizeShortText(reason ?? '', 500); + if (normalizedReason === '') throw new Error('拒绝清理声明时必须提供原因'); + record.status = 'rejected'; + record.rejectionReason = redactSensitiveText(normalizedReason); + } + return cloneRecord(record); + } + async cleanup(runId: string): Promise { - const entries = this.entries.get(runId) ?? []; - if (entries.length > 0 && !this.options.cleanup) { + const entries = [...(this.entries.get(runId)?.values() ?? [])].filter( + (entry) => entry.status !== 'verified-cleaned', + ); + if (entries.length === 0) { return { - ok: false, - attempted: entries.length, - failed: entries.map((entry) => entry.id), - message: '没有配置测试数据清理适配器，无法确认测试数据已清理', + ok: true, + attempted: 0, + failed: [], + message: '没有待清理的测试数据', + receipts: [], }; } - try { - const failed = this.options.cleanup ? [...(await this.options.cleanup(runId, entries))] : []; - if (failed.length === 0) this.entries.delete(runId); - return { - ok: failed.length === 0, - attempted: entries.length, - failed, - message: - failed.length === 0 - ? entries.length === 0 - ? '没有登记需要清理的测试数据' - : `已清理 ${entries.length} 项测试数据` - : `有 ${failed.length} 项测试数据清理失败`, - }; - } catch { + const adapter = this.options.cleanupAdapter; + if (!adapter) { return { ok: false, - attempted: entries.length, + attempted: 0, failed: entries.map((entry) => entry.id), - message: '测试数据清理适配器执行失败', + message: '没有配置测试数据清理适配器，等待 Reviewer 核验清理声明', + receipts: [], }; } + + const failed: string[] = []; + const receipts: TestDataVerificationReceipt[] = []; + for (const entry of entries) { + try { + const observation = await adapter.cleanupAndVerify({ + runId, + entry: publicEntry(entry), + }); + const captured = createReceipt( + runId, + entry.id, + adapter.id, + 'cleanup-adapter', + observation, + this.now(), + ); + receipts.push(captured.receipt); + entry.verification = captured.receipt; + if (captured.receipt.absent) { + entry.status = 'verified-cleaned'; + delete entry.rejectionReason; + } else { + entry.status = 'rejected'; + entry.rejectionReason = '清理适配器独立查询确认数据仍存在'; + failed.push(entry.id); + } + } catch { + entry.status = 'rejected'; + entry.rejectionReason = '清理适配器执行或独立查询失败'; + failed.push(entry.id); + } + } + return { + ok: failed.length === 0, + attempted: entries.length, + failed, + message: + failed.length === 0 + ? `清理适配器已独立核验 ${entries.length} 项测试数据不存在` + : `有 ${failed.length} 项测试数据未通过清理适配器核验`, + receipts, + }; } - nextId(): string { - return this.id(); + finalize(runId: string): TestDataFinalResult { + const pending = this.pending(runId).map(({ id, scenarioId, status, rejectionReason }) => ({ + id, + ...(scenarioId ? { scenarioId } : {}), + status, + ...(rejectionReason ? { rejectionReason } : {}), + })); + return { + ok: pending.length === 0, + pending, + message: + pending.length === 0 + ? '全部登记测试数据均已独立核验清理' + : `仍有 ${pending.length} 项测试数据未确认清理`, + }; + } + + private requireRecord(runId: string, dataId: string): TestDataRecord { + const id = normalizeDataId(dataId); + const record = this.entries.get(runId)?.get(id); + if (!record) throw new Error('测试数据未在当前 Run 登记'); + return record; } } export function createTestDataTools( manager: TestDataManager, runId: string, + evidence: TestDataEvidenceBoundary | undefined, scenarioId?: string, ): ToolDefinition[] { - const registerParameters = Type.Object({ - id: Type.String({ description: '已创建测试数据的稳定标识，不要填写密码或 Token' }), - description: Type.Optional(Type.String({ description: '测试数据的简短说明' })), - }); - const cleanupParameters = Type.Object({}); + const registerParameters = Type.Object( + { + id: Type.String({ description: '已创建测试数据的稳定标识，不要填写密码或 Token' }), + description: Type.Optional(Type.String({ description: '测试数据的脱敏简短说明' })), + }, + { additionalProperties: false }, + ); + const claimParameters = Type.Object( + { + dataId: Type.String({ description: '当前 Run 已登记的测试数据 ID' }), + evidenceIds: Type.Array(Type.String(), { minItems: 1, maxItems: 8 }), + }, + { additionalProperties: false }, + ); + const captureParameters = Type.Object( + { + dataId: Type.String({ description: '当前 Run 已登记的测试数据 ID' }), + adapterId: Type.String({ description: 'Harness 配置的查询 adapter ID' }), + operation: Type.String({ description: 'adapter allowlist 内的只读查询操作' }), + parameters: Type.Optional( + Type.Record(Type.String({ pattern: '^[A-Za-z][A-Za-z0-9_]{0,63}$' }), Type.String()), + ), + }, + { additionalProperties: false }, + ); return [ { name: 'get_test_data_prefix', label: '获取测试数据标记', description: '获取当前 Run 的测试数据前缀。创建数据时必须使用该前缀，便于清理。', - parameters: Type.Object({}), + parameters: Type.Object({}, { additionalProperties: false }), execute: async () => createTextResult(manager.prefix(runId)), }, { name: 'register_test_data', label: '登记测试数据', - description: '登记本次场景创建的临时数据，场景结束时由 Harness 统一清理。', + description: '创建测试数据后立即登记；登记本身不代表数据已清理。', parameters: registerParameters, execute: async ( _toolCallId: string, @@ -136,18 +395,278 @@ export function createTestDataTools( }, }, { - name: 'cleanup_test_data', - label: '清理测试数据', - description: '在当前场景结束时执行一次测试数据清理，并返回清理结果。', - parameters: cleanupParameters, - execute: async (): Promise>> => { - const result = await manager.cleanup(runId); - return createTextResult(JSON.stringify(result), { cleanup: result }); + name: 'capture_test_data_cleanup_query', + label: '捕获清理后查询证据', + description: + '通过 Harness allowlist 内的 API/只读命令 adapter 查询已登记数据；响应正文、状态码和摘要由 Harness 直接捕获，不能由 Agent 提交。', + parameters: captureParameters, + execute: async ( + _toolCallId: string, + params: Static, + ): Promise>> => { + try { + if (!evidence) throw new Error('当前 Run 没有可用的 Evidence Store'); + const captured = await manager.query( + runId, + params.dataId, + params.adapterId, + params.operation, + params.parameters ?? {}, + ); + const evidenceId = await evidence.captureCleanupQuery( + captured.receipt, + captured.redactedContent, + ); + return createTextResult(JSON.stringify({ evidenceId, receipt: captured.receipt }), { + evidenceId, + }); + } catch (error) { + return createTextResult(errorMessage(error), { error: true }); + } + }, + }, + { + name: 'submit_test_data_cleanup_claim', + label: '提交测试数据清理声明', + description: + '引用当前 Run 中 Harness 管理的受控查询证据或 Playwright 截图，只提交声明；必须经 adapter 或 Reviewer 核验。', + parameters: claimParameters, + execute: async ( + _toolCallId: string, + params: Static, + ): Promise>> => { + try { + if (!evidence) throw new Error('当前 Run 没有可用的 Evidence Store'); + for (const evidenceId of params.evidenceIds) { + if (!(await evidence.isCleanupClaimEvidence(evidenceId, runId, params.dataId))) { + throw new Error('清理声明引用了不合格或不属于当前 Run/data ID 的证据'); + } + } + const record = await manager.submitClaim(runId, params.dataId, params.evidenceIds); + return createTextResult(JSON.stringify(publicRecord(record))); + } catch (error) { + return createTextResult(errorMessage(error), { error: true }); + } + }, + }, + { + name: 'list_pending_test_data', + label: '列出待核验测试数据', + description: '列出当前 Run 尚未独立核验清理的脱敏测试数据。', + parameters: Type.Object({}, { additionalProperties: false }), + execute: async () => + createTextResult(JSON.stringify(manager.pending(runId).map(publicRecord))), + }, + ]; +} + +export function createReviewerTestDataTools( + manager: TestDataManager, + runId: string, + evidence: TestDataEvidenceBoundary | undefined, +): ToolDefinition[] { + const readParameters = Type.Object( + { + dataId: Type.String(), + evidenceId: Type.String(), + }, + { additionalProperties: false }, + ); + const verifyParameters = Type.Object( + { + dataId: Type.String(), + decision: Type.Union([Type.Literal('confirm'), Type.Literal('reject')]), + reason: Type.Optional(Type.String({ maxLength: 500 })), + }, + { additionalProperties: false }, + ); + return [ + { + name: 'read_test_data_cleanup_evidence', + label: '读取测试数据清理证据', + description: + '读取 Harness 捕获并已上传的清理后文本查询证据。截图必须使用 read_evidence_image 查看。', + parameters: readParameters, + execute: async ( + _toolCallId: string, + params: Static, + ): Promise>> => { + try { + if (!evidence) throw new Error('当前 Run 没有可用的 Evidence Store'); + return createTextResult( + await evidence.readCleanupTextEvidence(params.evidenceId, runId, params.dataId), + ); + } catch (error) { + return createTextResult(errorMessage(error), { error: true }); + } + }, + }, + { + name: 'verify_test_data_cleanup', + label: '核验测试数据清理', + description: '仅在实际读取清理声明引用的全部受控证据后确认或拒绝；不能执行目标环境命令。', + parameters: verifyParameters, + execute: async ( + _toolCallId: string, + params: Static, + ): Promise>> => { + try { + if (!evidence) throw new Error('当前 Run 没有可用的 Evidence Store'); + const record = await manager.verify( + runId, + params.dataId, + params.decision, + params.reason, + (evidenceId) => evidence.isReviewedCleanupEvidence(evidenceId), + ); + return createTextResult(JSON.stringify(publicRecord(record))); + } catch (error) { + return createTextResult(errorMessage(error), { error: true }); + } }, }, ]; } +function createReceipt( + runId: string, + dataId: string, + sourceId: string, + sourceKind: TestDataVerificationReceipt['sourceKind'], + observation: TestDataAdapterObservation, + now: Date, +): { receipt: TestDataVerificationReceipt; redactedContent: string } { + if (typeof observation.content !== 'string') throw new Error('清理查询没有返回文本响应'); + if (Buffer.byteLength(observation.content, 'utf8') > 256 * 1024) { + throw new Error('清理查询响应超过大小限制'); + } + if (sourceKind === 'api-query' && !validStatusCode(observation.statusCode)) { + throw new Error('受控 API 查询缺少有效状态码'); + } + if (sourceKind === 'readonly-command' && !Number.isInteger(observation.exitCode)) { + throw new Error('受控只读命令缺少有效退出码'); + } + const redactedContent = redactSensitiveText(observation.content).slice(0, 64 * 1024); + const normalized = redactedContent.replace(/\s+/g, ' ').trim(); + const receipt: TestDataVerificationReceipt = { + sourceId, + sourceKind, + runId, + dataId, + queriedAt: now.toISOString(), + absent: observation.absent === true, + ...(validStatusCode(observation.statusCode) ? { statusCode: observation.statusCode } : {}), + ...(Number.isInteger(observation.exitCode) ? { exitCode: observation.exitCode } : {}), + summary: normalized.slice(0, 240), + sha256: createHash('sha256').update(redactedContent, 'utf8').digest('hex'), + }; + return { receipt, redactedContent }; +} + +function normalizeQueryParameters( + parameters: Readonly>, + allowed: readonly string[], +): Record { + const keys = Object.keys(parameters); + if (keys.length > 16 || keys.some((key) => !allowed.includes(key))) { + throw new Error('测试数据查询参数不在 allowlist'); + } + const result: Record = {}; + for (const key of keys) { + const value = parameters[key]; + if (typeof value !== 'string' || value.length > 200 || containsControlCharacters(value)) { + throw new Error('测试数据查询参数无效'); + } + result[key] = value; + } + return result; +} + +function containsControlCharacters(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 0x1f || code === 0x7f; + }); +} + +function normalizeDataId(value: string): string { + const id = value.trim(); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/.test(id) || looksLikeSecret(id)) { + throw new Error('测试数据标识无效'); + } + return id; +} + +function normalizeEvidenceId(value: string): string { + const id = value.trim(); + if (!/^[A-Za-z0-9][A-Za-z0-9._/-]{0,239}$/.test(id) || id.includes('..')) { + throw new Error('清理证据 ID 无效'); + } + return id; +} + +function normalizeShortText(value: string, maxLength: number): string { + const text = value.trim(); + if (text.length > maxLength || text.includes('\u0000')) throw new Error('测试数据文本无效'); + return text; +} + +function assertAdapterId(value: string): void { + if (!/^[a-z][a-z0-9-]{0,63}$/.test(value)) throw new Error('测试数据适配器 ID 无效'); +} + +function publicEntry(record: TestDataRecord): TestDataEntry { + return { + id: record.id, + ...(record.scenarioId ? { scenarioId: record.scenarioId } : {}), + ...(record.description ? { description: record.description } : {}), + }; +} + +function publicRecord(record: TestDataRecord): Record { + return { + id: record.id, + ...(record.scenarioId ? { scenarioId: record.scenarioId } : {}), + status: record.status, + evidenceIds: record.claim?.evidenceIds ?? [], + ...(record.rejectionReason ? { rejectionReason: record.rejectionReason } : {}), + }; +} + +function cloneRecord(record: TestDataRecord): TestDataRecord { + return { + ...record, + ...(record.claim + ? { claim: { ...record.claim, evidenceIds: [...record.claim.evidenceIds] } } + : {}), + ...(record.verification ? { verification: { ...record.verification } } : {}), + }; +} + +function sameValues(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function validStatusCode(value: number | undefined): value is number { + return Number.isInteger(value) && (value ?? 0) >= 100 && (value ?? 0) <= 599; +} + +function redactSensitiveText(value: string): string { + return value + .replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?[^,;\r\n}]+/gi, '$1[REDACTED]') + .replace(/(bearer\s+)[A-Za-z0-9._~+/-]+/gi, '$1[REDACTED]') + .replace( + /((?:password|passwd|token|secret|cookie|api[-_]?key)\s*["']?\s*[:=]\s*["']?)([^\s,"';}]+)/gi, + '$1[REDACTED]', + ) + .replace(/\b(?:github_pat_|gh[opsur]_|sk-)[A-Za-z0-9_-]+\b/g, '[REDACTED]') + .replace(/\bAKIA[0-9A-Z]{16}\b/g, '[REDACTED]'); +} + +function looksLikeSecret(value: string): boolean { + return /^(?:github_pat_|gh[opsur]_|sk-|AKIA)/.test(value); +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : '测试数据操作失败'; } diff --git a/src/server/runs/workspace.ts b/src/server/runs/workspace.ts index 3357436..364bdae 100644 --- a/src/server/runs/workspace.ts +++ b/src/server/runs/workspace.ts @@ -179,6 +179,23 @@ export class RunWorkspace implements RunArtifactReader { return files.sort((left, right) => left.name.localeCompare(right.name)); } + async writeHarnessEvidence(name: string, content: string): Promise { + if (!/^cleanup-query-[a-f0-9]{16}-[0-9]+\.json$/.test(name)) { + throw new RunWorkspaceError('ARTIFACT_NOT_ALLOWED', 'Harness 清理证据文件名无效'); + } + if (typeof content !== 'string' || content.includes('\u0000')) { + throw new RunWorkspaceError('ARTIFACT_INVALID', 'Harness 清理证据内容无效'); + } + if (Buffer.byteLength(content, 'utf8') > 256 * 1024) { + throw new RunWorkspaceError('ARTIFACT_INVALID', 'Harness 清理证据超出大小限制'); + } + await writeFile(this.evidencePath(name), content, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + } + async readEvidence(name: string): Promise { const path = this.evidencePath(name); try { diff --git a/tests/acceptance/phase9.ts b/tests/acceptance/phase9.ts index 3e1bedc..300a742 100644 --- a/tests/acceptance/phase9.ts +++ b/tests/acceptance/phase9.ts @@ -753,7 +753,16 @@ async function runRunProof(context: RepositoryProofContext): Promise { provider: {} as ProviderAdapter, sessions, commandRunner: createControlledCommandRunner({ ...process.env, PHASE9_SECRET: SAMPLE_SECRET }), - testData: createTestDataManager({ cleanup: async () => [] }), + testData: createTestDataManager({ + cleanupAdapter: { + id: 'acceptance-cleanup', + cleanupAndVerify: async () => ({ + absent: true, + content: 'fixture data not found', + statusCode: 404, + }), + }, + }), runStore: context.runStore, recoveryStore, logger: pino({ level: 'silent' }), @@ -1365,9 +1374,9 @@ class FixtureSessionFactory implements AgentSessionFactory { } if (input.role === 'runner') { await invokeTool(input, 'read_run_artifact', { name: 'plan.md' }); - await invokeTool(input, 'get_test_data_prefix', {}); + const dataPrefix = toolText(await invokeTool(input, 'get_test_data_prefix', {})); await invokeTool(input, 'register_test_data', { - id: 'phase9-login-fixture', + id: `${dataPrefix}phase9-login-fixture`, description: '本地样例登录数据', }); const command = await invokeTool(input, 'run_fixture_command', { diff --git a/tests/closure3-test-data.test.ts b/tests/closure3-test-data.test.ts new file mode 100644 index 0000000..ec0f5b7 --- /dev/null +++ b/tests/closure3-test-data.test.ts @@ -0,0 +1,349 @@ +import { strict as assert } from 'node:assert'; + +import { describe, it } from 'vitest'; +import type { AgentToolResult, ToolDefinition } from '@earendil-works/pi-coding-agent'; + +import { + createReviewerTestDataTools, + createTestDataManager, + createTestDataTools, + type TestDataEvidenceBoundary, + type TestDataVerificationReceipt, +} from '../src/server/runs/test-data.js'; + +const RUN_ID = '01K00000000000000000000001'; +const OTHER_RUN_ID = '01K00000000000000000000002'; +const DATA_ID = `luowang-${RUN_ID}-user-1`; + +describe('Closure 3 verified test data lifecycle', () => { + it('captures a real allowlisted API response, redacts it, and requires Reviewer read before confirmation', async () => { + let adapterCalls = 0; + const manager = createTestDataManager({ + now: () => new Date('2026-09-01T03:00:00.000Z'), + queryAdapters: [ + { + id: 'users-api', + kind: 'api-query', + operations: { 'lookup-by-id': [] }, + query: async ({ runId, entry, operation, parameters }) => { + adapterCalls += 1; + assert.equal(runId, RUN_ID); + assert.equal(entry.id, DATA_ID); + assert.equal(operation, 'lookup-by-id'); + assert.deepEqual(parameters, {}); + return { + absent: true, + statusCode: 404, + content: 'not found; token=adapter-secret-value; Authorization: Bearer hidden-value', + }; + }, + }, + ], + }); + const evidence = new MemoryEvidence(); + const runner = createTestDataTools(manager, RUN_ID, evidence); + const reviewer = createReviewerTestDataTools(manager, RUN_ID, evidence); + + await invoke(runner, 'register_test_data', { id: DATA_ID, description: 'temporary user' }); + const captured = await invokeJson(runner, 'capture_test_data_cleanup_query', { + dataId: DATA_ID, + adapterId: 'users-api', + operation: 'lookup-by-id', + parameters: {}, + content: 'Agent supplied fake body', + statusCode: 200, + sha256: 'Agent supplied fake hash', + }); + const evidenceId = captured.evidenceId as string; + assert.equal(adapterCalls, 1); + assert.match(evidenceId, /^controlled-1$/); + assert.doesNotMatch( + evidence.contents.get(evidenceId) ?? '', + /adapter-secret-value|hidden-value/, + ); + assert.doesNotMatch(evidence.contents.get(evidenceId) ?? '', /Agent supplied/); + const receipt = captured.receipt as TestDataVerificationReceipt; + assert.equal(receipt.sourceId, 'users-api'); + assert.equal(receipt.runId, RUN_ID); + assert.equal(receipt.dataId, DATA_ID); + assert.equal(receipt.statusCode, 404); + assert.equal(receipt.queriedAt, '2026-09-01T03:00:00.000Z'); + assert.match(receipt.sha256, /^[a-f0-9]{64}$/); + + await invoke(runner, 'submit_test_data_cleanup_claim', { + dataId: DATA_ID, + evidenceIds: [evidenceId], + }); + const unread = await invokeJson(reviewer, 'verify_test_data_cleanup', { + dataId: DATA_ID, + decision: 'confirm', + }); + assert.equal(unread.error, true); + assert.match(readText(unread), /先读取/); + + await invoke(reviewer, 'read_test_data_cleanup_evidence', { + dataId: DATA_ID, + evidenceId, + }); + const verified = await invokeJson(reviewer, 'verify_test_data_cleanup', { + dataId: DATA_ID, + decision: 'confirm', + }); + assert.equal(verified.status, 'verified-cleaned'); + assert.equal(manager.finalize(RUN_ID).ok, true); + }); + + it('does not let Reviewer confirm a controlled query that says data still exists', async () => { + const manager = createTestDataManager({ + queryAdapters: [ + { + id: 'users-api', + kind: 'api-query', + operations: { lookup: [] }, + query: async () => ({ absent: false, statusCode: 200, content: 'record exists' }), + }, + ], + }); + const evidence = new MemoryEvidence(); + const runner = createTestDataTools(manager, RUN_ID, evidence); + const reviewer = createReviewerTestDataTools(manager, RUN_ID, evidence); + await invoke(runner, 'register_test_data', { id: DATA_ID }); + const captured = await invokeJson(runner, 'capture_test_data_cleanup_query', { + dataId: DATA_ID, + adapterId: 'users-api', + operation: 'lookup', + }); + const evidenceId = captured.evidenceId as string; + await invoke(runner, 'submit_test_data_cleanup_claim', { + dataId: DATA_ID, + evidenceIds: [evidenceId], + }); + await invoke(reviewer, 'read_test_data_cleanup_evidence', { dataId: DATA_ID, evidenceId }); + const result = await invokeJson(reviewer, 'verify_test_data_cleanup', { + dataId: DATA_ID, + decision: 'confirm', + }); + assert.equal(result.error, true); + assert.equal(manager.finalize(RUN_ID).ok, false); + }); + + it('records readonly-command provenance and rejects operations or parameters outside the adapter allowlist', async () => { + const manager = createTestDataManager({ + queryAdapters: [ + { + id: 'fixture-query', + kind: 'readonly-command', + operations: { lookup: ['scope'] }, + query: async () => ({ absent: true, exitCode: 0, content: 'record absent' }), + }, + ], + }); + await manager.register(RUN_ID, { id: DATA_ID }); + + const captured = await manager.query(RUN_ID, DATA_ID, 'fixture-query', 'lookup', { + scope: 'synthetic', + }); + assert.equal(captured.receipt.sourceKind, 'readonly-command'); + assert.equal(captured.receipt.exitCode, 0); + assert.equal(captured.receipt.statusCode, undefined); + + await assert.rejects( + () => manager.query(RUN_ID, DATA_ID, 'fixture-query', 'echo', {}), + /操作不在 allowlist/, + ); + await assert.rejects( + () => + manager.query(RUN_ID, DATA_ID, 'fixture-query', 'lookup', { + path: '/arbitrary/path', + }), + /参数不在 allowlist/, + ); + await assert.rejects( + () => manager.query(RUN_ID, DATA_ID, 'unknown-adapter', 'lookup', {}), + /适配器不在 allowlist/, + ); + }); + + it('accepts only current managed evidence and rejects unregistered or cross-data claims', async () => { + const manager = createTestDataManager(); + const evidence = new MemoryEvidence(); + evidence.screenshots.add('cleanup-after.png'); + const tools = createTestDataTools(manager, RUN_ID, evidence); + await invoke(tools, 'register_test_data', { id: DATA_ID }); + + for (const evidenceId of ['../outside.png', 'https://example.test/evidence.png', 'notes.txt']) { + const result = await invokeJson(tools, 'submit_test_data_cleanup_claim', { + dataId: DATA_ID, + evidenceIds: [evidenceId], + }); + assert.equal(result.error, true, evidenceId); + } + const unknown = await invokeJson(tools, 'submit_test_data_cleanup_claim', { + dataId: `luowang-${RUN_ID}-unknown`, + evidenceIds: ['cleanup-after.png'], + }); + assert.equal(unknown.error, true); + + await manager.register(OTHER_RUN_ID, { id: `luowang-${OTHER_RUN_ID}-other` }); + const crossRun = createTestDataTools(manager, OTHER_RUN_ID, new MemoryEvidence()); + const cross = await invokeJson(crossRun, 'submit_test_data_cleanup_claim', { + dataId: `luowang-${OTHER_RUN_ID}-other`, + evidenceIds: ['cleanup-after.png'], + }); + assert.equal(cross.error, true); + assert.equal(manager.pending(RUN_ID)[0]?.status, 'registered'); + }); + + it('keeps pending and Reviewer-rejected records blocking with a redacted residual list', async () => { + const manager = createTestDataManager(); + const evidence = new MemoryEvidence(); + evidence.screenshots.add('cleanup-after.png'); + const runner = createTestDataTools(manager, RUN_ID, evidence); + const reviewer = createReviewerTestDataTools(manager, RUN_ID, evidence); + await invoke(runner, 'register_test_data', { id: DATA_ID }); + await invoke(runner, 'submit_test_data_cleanup_claim', { + dataId: DATA_ID, + evidenceIds: ['cleanup-after.png'], + }); + evidence.reviewed.add('cleanup-after.png'); + await invoke(reviewer, 'verify_test_data_cleanup', { + dataId: DATA_ID, + decision: 'reject', + reason: '截图不足；password=should-not-remain', + }); + + const final = manager.finalize(RUN_ID); + assert.equal(final.ok, false); + assert.equal(final.pending[0]?.status, 'rejected'); + assert.doesNotMatch(final.pending[0]?.rejectionReason ?? '', /should-not-remain/); + assert.match(final.pending[0]?.rejectionReason ?? '', /REDACTED/); + }); + + it('lets a trusted cleanup adapter mark only independently absent records verified', async () => { + const manager = createTestDataManager({ + cleanupAdapter: { + id: 'fixture-cleanup', + cleanupAndVerify: async ({ entry }) => ({ + absent: entry.id.endsWith('gone'), + statusCode: 200, + content: entry.id.endsWith('gone') ? 'not found' : 'still present', + }), + }, + }); + const gone = `luowang-${RUN_ID}-gone`; + const present = `luowang-${RUN_ID}-present`; + await manager.register(RUN_ID, { id: gone }); + await manager.register(RUN_ID, { id: present }); + await manager.submitClaim(RUN_ID, present, ['cleanup-after.png']); + + const result = await manager.cleanup(RUN_ID); + assert.equal(result.ok, false); + assert.equal(result.attempted, 2); + assert.deepEqual(result.failed, [present]); + assert.equal(result.receipts.length, 2); + assert.equal( + manager + .pending(RUN_ID) + .map((entry) => entry.id) + .join(','), + present, + ); + assert.equal(manager.pending(RUN_ID)[0]?.status, 'rejected'); + await assert.rejects( + () => manager.verify(RUN_ID, present, 'confirm', undefined, () => true), + /没有待审核/, + ); + assert.equal(manager.finalize(RUN_ID).ok, false); + }); + + it('requires no adapter or Reviewer action for a zero-data Run', async () => { + const manager = createTestDataManager(); + const cleanup = await manager.cleanup(RUN_ID); + assert.equal(cleanup.ok, true); + assert.equal(cleanup.attempted, 0); + assert.equal(manager.finalize(RUN_ID).ok, true); + }); +}); + +class MemoryEvidence implements TestDataEvidenceBoundary { + readonly contents = new Map(); + readonly bindings = new Map(); + readonly screenshots = new Set(); + readonly reviewed = new Set(); + readonly absent = new Map(); + + async captureCleanupQuery( + receipt: TestDataVerificationReceipt, + redactedContent: string, + ): Promise { + const id = `controlled-${this.contents.size + 1}`; + this.contents.set(id, JSON.stringify({ provenance: receipt, redactedContent })); + this.bindings.set(id, { runId: receipt.runId, dataId: receipt.dataId }); + this.absent.set(id, receipt.absent); + return id; + } + + async isCleanupClaimEvidence( + evidenceId: string, + runId: string, + dataId: string, + ): Promise { + const binding = this.bindings.get(evidenceId); + if (binding) return binding.runId === runId && binding.dataId === dataId; + return this.screenshots.has(evidenceId); + } + + async readCleanupTextEvidence( + evidenceId: string, + runId: string, + dataId: string, + ): Promise { + const binding = this.bindings.get(evidenceId); + if (!binding || binding.runId !== runId || binding.dataId !== dataId) { + throw new Error('not controlled'); + } + this.reviewed.add(evidenceId); + return this.contents.get(evidenceId) ?? ''; + } + + isReviewedCleanupEvidence(evidenceId: string): boolean { + return this.reviewed.has(evidenceId) && (this.absent.get(evidenceId) ?? true); + } +} + +async function invoke( + tools: readonly ToolDefinition[], + name: string, + params: Record, +): Promise>> { + const tool = tools.find((candidate) => candidate.name === name); + assert.ok(tool, `missing tool ${name}`); + return tool.execute( + 'closure3-tool', + params as never, + undefined, + undefined, + {} as never, + ) as Promise>>; +} + +async function invokeJson( + tools: readonly ToolDefinition[], + name: string, + params: Record, +): Promise> { + const result = await invoke(tools, name, params); + if (result.details.error === true) { + return { + error: true, + text: result.content.map((item) => (item.type === 'text' ? item.text : '')).join(''), + }; + } + const text = result.content.find((item) => item.type === 'text'); + assert.ok(text && text.type === 'text'); + return JSON.parse(text.text) as Record; +} + +function readText(value: Record): string { + return String(value.text ?? ''); +} diff --git a/tests/phase4-orchestrator.test.ts b/tests/phase4-orchestrator.test.ts index 38b6ccc..312ef6e 100644 --- a/tests/phase4-orchestrator.test.ts +++ b/tests/phase4-orchestrator.test.ts @@ -8,7 +8,7 @@ import { promisify } from 'node:util'; import pino from 'pino'; import { afterEach, describe, it } from 'vitest'; -import type { InlineExtension } from '@earendil-works/pi-coding-agent'; +import type { AgentToolResult, InlineExtension } from '@earendil-works/pi-coding-agent'; import { createConfigurationStore } from '../src/server/configuration.js'; import { loadConfig } from '../src/server/config.js'; import { initializeDatabase } from '../src/server/db/migrate.js'; @@ -57,6 +57,20 @@ describe('Phase 4 Run blocking boundaries', () => { } }); + it('lets a production Run finish without a cleanup adapter after Reviewer reads controlled query evidence', async () => { + const fixture = await createGitFixture(); + const context = await createRunContext(fixture, 'cleanup-review'); + + const result = await context.orchestrator.run({ + request: '验证 Reviewer 独立确认测试数据清理', + trigger: 'manual', + }); + + assert.equal(result.status, 'completed', JSON.stringify(result)); + assert.equal(result.result, 'passed', JSON.stringify(result)); + assert.match(result.artifacts['execution.md'] ?? '', /全部登记测试数据均已独立核验清理/); + }); + it('checks visual capability on the Reviewer instead of the text-only Runner', async () => { const fixture = await createGitFixture(); const context = await createRunContext(fixture, 'vision-reviewer'); @@ -97,6 +111,7 @@ describe('Phase 4 Run blocking boundaries', () => { type FailureMode = | 'upload-failure' | 'cleanup-failure' + | 'cleanup-review' | 'browser-missing' | 'review-read-failure' | 'vision-reviewer' @@ -114,6 +129,9 @@ interface RunContextFixture { } class FailureBoundarySessionFactory implements AgentSessionFactory { + private cleanupEvidenceId: string | undefined; + private cleanupDataId: string | undefined; + constructor(private readonly mode: FailureMode) {} async create(input: AgentSessionInput) { @@ -123,7 +141,7 @@ class FailureBoundarySessionFactory implements AgentSessionFactory { await invokeTool(input, 'get_run_context', {}); await invokeTool(input, 'write_plan', { content: - this.mode === 'cleanup-failure' + this.mode === 'cleanup-failure' || this.mode === 'cleanup-review' ? '# Plan\n\n无需场景测试：本次只验证非 UI 的清理边界。\n' : this.mode === 'vision-reviewer' ? '# Plan\n\nUI 登录场景：打开登录页面并核对截图差异。\n' @@ -135,11 +153,27 @@ class FailureBoundarySessionFactory implements AgentSessionFactory { if (input.role === 'runner') { await invokeTool(input, 'read_run_artifact', { name: 'plan.md' }); const context = parsePromptContext(input.userMessage); - if (this.mode === 'cleanup-failure') { + if (this.mode === 'cleanup-failure' || this.mode === 'cleanup-review') { + this.cleanupDataId = `luowang-${context.runId}-test-user-1`; await invokeTool(input, 'register_test_data', { - id: 'luowang-test-user-1', + id: this.cleanupDataId, description: 'fixture user', }); + if (this.mode === 'cleanup-review') { + const capture = await invokeTool(input, 'capture_test_data_cleanup_query', { + dataId: this.cleanupDataId, + adapterId: 'fixture-api', + operation: 'lookup-by-id', + parameters: {}, + content: 'Agent 不得覆盖 adapter 响应', + statusCode: 200, + }); + this.cleanupEvidenceId = toolJson(capture).evidenceId as string; + await invokeTool(input, 'submit_test_data_cleanup_claim', { + dataId: this.cleanupDataId, + evidenceIds: [this.cleanupEvidenceId], + }); + } } else if (this.mode !== 'browser-missing') { await mkdir(join(context.runDirectory, 'evidence'), { recursive: true }); await writeFile(join(context.runDirectory, 'evidence', 'login.png'), 'fixture image'); @@ -157,6 +191,18 @@ class FailureBoundarySessionFactory implements AgentSessionFactory { for (const name of ['plan.md', 'execution.md', 'draft-report.md']) { await invokeTool(input, 'read_run_artifact', { name }); } + if (this.mode === 'cleanup-review') { + assert.ok(this.cleanupEvidenceId); + assert.ok(this.cleanupDataId); + await invokeTool(input, 'read_test_data_cleanup_evidence', { + dataId: this.cleanupDataId, + evidenceId: this.cleanupEvidenceId, + }); + await invokeTool(input, 'verify_test_data_cleanup', { + dataId: this.cleanupDataId, + decision: 'confirm', + }); + } if ( this.mode === 'review-read-failure' || this.mode === 'vision-reviewer' || @@ -166,7 +212,10 @@ class FailureBoundarySessionFactory implements AgentSessionFactory { await invokeTool(input, 'read_evidence_image', { filename: 'login.png' }); } await invokeTool(input, 'write_review', { - content: '# Review\n\n独立审核完成。\n', + content: + this.mode === 'cleanup-review' + ? '# Review\n\n已读取受控查询证据并确认清理。无需场景测试。\n' + : '# Review\n\n独立审核完成。\n', }); return; } @@ -229,7 +278,33 @@ async function createRunContext(fixture: Fixture, mode: FailureMode): Promise ['luowang-test-user-1'] } : undefined, + mode === 'cleanup-failure' + ? { + cleanupAdapter: { + id: 'fixture-cleanup', + cleanupAndVerify: async () => ({ + absent: false, + content: 'fixture data still exists', + statusCode: 200, + }), + }, + } + : mode === 'cleanup-review' + ? { + queryAdapters: [ + { + id: 'fixture-api', + kind: 'api-query', + operations: { 'lookup-by-id': [] }, + query: async () => ({ + absent: true, + content: 'not found; token=must-not-appear', + statusCode: 404, + }), + }, + ], + } + : undefined, ), sessions: new FailureBoundarySessionFactory(mode), logger: pino({ level: 'silent' }), @@ -366,6 +441,13 @@ async function invokeTool( return tool.execute('phase4-fixture', params as never, undefined, undefined, {} as never); } +function toolJson(result: unknown): Record { + const toolResult = result as AgentToolResult>; + const text = toolResult.content.find((item) => item.type === 'text'); + assert.ok(text && text.type === 'text'); + return JSON.parse(text.text) as Record; +} + function parsePromptContext(prompt: string): { runId: string; baseCommit: string | null; diff --git a/tests/phase4.test.ts b/tests/phase4.test.ts index cf67ecc..cd09b04 100644 --- a/tests/phase4.test.ts +++ b/tests/phase4.test.ts @@ -19,7 +19,11 @@ import { createReviewerEvidenceTools, createRunEvidenceStore, } from '../src/server/runs/evidence.js'; -import { createTestDataManager } from '../src/server/runs/test-data.js'; +import { + createReviewerTestDataTools, + createTestDataManager, + createTestDataTools, +} from '../src/server/runs/test-data.js'; import { RunWorkspace } from '../src/server/runs/workspace.js'; import { createOssAdapter, type OssAdapter, type S3ClientLike } from '../src/server/storage/oss.js'; import type { SecretStore } from '../src/server/security/secret-store.js'; @@ -110,26 +114,32 @@ describe('Phase 4 browser and evidence boundaries', () => { it('blocks cleanup when data was registered without a real cleanup adapter', async () => { const manager = createTestDataManager(); - await manager.register('01K00000000000000000000001', { id: 'luowang-test-user-1' }); + const runId = '01K00000000000000000000001'; + const dataId = `${manager.prefix(runId)}test-user-1`; + await manager.register(runId, { id: dataId }); - const result = await manager.cleanup('01K00000000000000000000001'); + const result = await manager.cleanup(runId); assert.equal(result.ok, false); - assert.equal(result.attempted, 1); - assert.deepEqual(result.failed, ['luowang-test-user-1']); + assert.equal(result.attempted, 0); + assert.deepEqual(result.failed, [dataId]); assert.match(result.message, /清理适配器/); }); it('passes registered data to the cleanup adapter and forgets it after success', async () => { const calls: Array<{ runId: string; ids: string[] }> = []; const manager = createTestDataManager({ - cleanup: async (runId, entries) => { - calls.push({ runId, ids: entries.map((entry) => entry.id) }); - return []; + cleanupAdapter: { + id: 'fixture-cleanup', + cleanupAndVerify: async ({ runId, entry }) => { + calls.push({ runId, ids: [entry.id] }); + return { absent: true, content: 'not found', statusCode: 404 }; + }, }, }); const runId = '01K00000000000000000000001'; - await manager.register(runId, { id: 'luowang-test-user-1' }); + const dataId = `${manager.prefix(runId)}test-user-1`; + await manager.register(runId, { id: dataId }); const first = await manager.cleanup(runId); const second = await manager.cleanup(runId); @@ -137,10 +147,7 @@ describe('Phase 4 browser and evidence boundaries', () => { assert.equal(first.ok, true); assert.equal(first.attempted, 1); assert.equal(second.attempted, 0); - assert.deepEqual(calls, [ - { runId, ids: ['luowang-test-user-1'] }, - { runId, ids: [] }, - ]); + assert.deepEqual(calls, [{ runId, ids: [dataId] }]); }); it('lets Reviewer read only uploaded image evidence as image content', async () => { @@ -152,6 +159,33 @@ describe('Phase 4 browser and evidence boundaries', () => { const store = createRunEvidenceStore(workspace, fakeOss()); const upload = await store.uploadAll(); assert.equal(upload.failures.length, 0); + const manager = createTestDataManager(); + const dataId = `${manager.prefix(workspace.runId)}screenshot-user`; + await manager.register(workspace.runId, { id: dataId }); + const claimTool = createTestDataTools(manager, workspace.runId, store).find( + (candidate) => candidate.name === 'submit_test_data_cleanup_claim', + ); + assert.ok(claimTool); + await claimTool.execute( + 'claim', + { dataId, evidenceIds: ['login.png'] } as never, + undefined, + undefined, + {} as never, + ); + const verifyTool = createReviewerTestDataTools(manager, workspace.runId, store).find( + (candidate) => candidate.name === 'verify_test_data_cleanup', + ); + assert.ok(verifyTool); + const unread = (await verifyTool.execute( + 'verify-before-read', + { dataId, decision: 'confirm' } as never, + undefined, + undefined, + {} as never, + )) as AgentToolResult>; + assert.equal(unread.details.error, true); + const tool = createReviewerEvidenceTools(store).find( (candidate) => candidate.name === 'read_evidence_image', ); @@ -171,6 +205,15 @@ describe('Phase 4 browser and evidence boundaries', () => { createReviewerEvidenceTools(store).some((item) => item.name === 'run_fixture_command'), false, ); + const verified = (await verifyTool.execute( + 'verify-after-read', + { dataId, decision: 'confirm' } as never, + undefined, + undefined, + {} as never, + )) as AgentToolResult>; + assert.equal(verified.details.error, undefined); + assert.equal(manager.finalize(workspace.runId).ok, true); await store.cleanupLocal(); assert.deepEqual( (await store.list()).map((file) => file.name),