Skip to content

fix: verify test-data cleanup from independent evidence #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shenjiecode merged 1 commit into from
Sep 1, 2026
Merged

fix: verify test-data cleanup from independent evidence #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 60 additions & 3 deletions src/server/runs/evidence.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { AgentToolResult, ToolDefinition } from '@earendil-works/pi-coding-

import type { EvidenceReference } from '../../shared/types.js';
import { createTextResult } from './agent-session.js';
import type { TestDataVerificationReceipt } from './test-data.js';
import type { OssAdapter } from '../storage/oss.js';
import { contentTypeFor } from '../storage/oss.js';
import { RunWorkspace, type RunEvidenceFile } from './workspace.js';
Expand All @@ -29,7 +30,14 @@ export interface RunEvidenceStore {
readFailureCount?: () => number;
recordReadFailure?: () => void;
reviewReadCount?: () => number;
recordReviewRead?: () => void;
recordReviewRead?: (evidenceId?: string) => void;
captureCleanupQuery(
receipt: TestDataVerificationReceipt,
redactedContent: string,
): Promise<string>;
isCleanupClaimEvidence(evidenceId: string, runId: string, dataId: string): Promise<boolean>;
readCleanupTextEvidence(evidenceId: string, runId: string, dataId: string): Promise<string>;
isReviewedCleanupEvidence(evidenceId: string): boolean;
}

export interface EvidenceReadResult {
Expand All @@ -48,6 +56,9 @@ class DefaultRunEvidenceStore implements RunEvidenceStore {
private readonly references = new Map<string, EvidenceReference>();
private readFailures = 0;
private reviewReads = 0;
private cleanupSequence = 0;
private readonly cleanupEvidence = new Map<string, TestDataVerificationReceipt>();
private readonly reviewedEvidence = new Set<string>();

constructor(
private readonly workspace: RunWorkspace,
Expand Down Expand Up @@ -157,8 +168,54 @@ class DefaultRunEvidenceStore implements RunEvidenceStore {
return this.reviewReads;
}

recordReviewRead(): void {
recordReviewRead(evidenceId?: string): void {
this.reviewReads += 1;
if (evidenceId) this.reviewedEvidence.add(evidenceId);
}

async captureCleanupQuery(
receipt: TestDataVerificationReceipt,
redactedContent: string,
): Promise<string> {
this.cleanupSequence += 1;
const evidenceId = `cleanup-query-${receipt.sha256.slice(0, 16)}-${this.cleanupSequence}.json`;
await this.workspace.writeHarnessEvidence(
evidenceId,
`${JSON.stringify({ provenance: receipt, redactedContent }, null, 2)}\n`,
);
this.cleanupEvidence.set(evidenceId, { ...receipt });
return evidenceId;
}

async isCleanupClaimEvidence(
evidenceId: string,
runId: string,
dataId: string,
): Promise<boolean> {
const captured = this.cleanupEvidence.get(evidenceId);
if (captured) return captured.runId === runId && captured.dataId === dataId;
if (!/\.(?:png|jpe?g|webp)$/i.test(evidenceId)) return false;
return (await this.list()).some((file) => file.name === evidenceId);
}

async readCleanupTextEvidence(
evidenceId: string,
runId: string,
dataId: string,
): Promise<string> {
const captured = this.cleanupEvidence.get(evidenceId);
if (!captured || captured.runId !== runId || captured.dataId !== dataId) {
throw new Error('清理证据不属于当前 Run/data ID 或不是受控文本查询证据');
}
const evidence = await this.readUploaded(evidenceId);
if (evidence.body.byteLength > 256 * 1024) throw new Error('清理证据超过审核大小限制');
this.recordReviewRead(evidenceId);
return evidence.body.toString('utf8');
}

isReviewedCleanupEvidence(evidenceId: string): boolean {
const captured = this.cleanupEvidence.get(evidenceId);
return this.reviewedEvidence.has(evidenceId) && (captured ? captured.absent : true);
}
}

Expand Down Expand Up @@ -256,7 +313,7 @@ export function createReviewerEvidenceTools(store: RunEvidenceStore): ToolDefini
store.recordReadFailure?.();
return createTextResult('截图超过审核大小限制', { error: true });
}
store.recordReviewRead?.();
store.recordReviewRead?.(params.filename);
return {
content: [
{
Expand Down
60 changes: 46 additions & 14 deletions src/server/runs/orchestrator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,12 @@ import {
type RunEvidenceStore,
} from './evidence.js';
import { createProviderAdapter, type ProviderAdapter } from './provider.js';
import { createTestDataManager, createTestDataTools, type TestDataManager } from './test-data.js';
import {
createReviewerTestDataTools,
createTestDataManager,
createTestDataTools,
type TestDataManager,
} from './test-data.js';
import {
createRoleInstructionLoader,
RoleInstructionError,
Expand Down Expand Up @@ -402,6 +407,7 @@ class DefaultRunOrchestrator implements RunOrchestrator {
'Reviewer 未读取截图 evidence，无法完成独立视觉审核。',
]);
}
await this.finalizeTestData(workspace, context);
if (runnerCleanup.uploaded && !runnerCleanup.uploadFailed) {
await this.cleanupRetainedEvidence(workspace, context, evidenceStore);
}
Expand Down Expand Up @@ -743,7 +749,11 @@ class DefaultRunOrchestrator implements RunOrchestrator {
this.options.configuration.getRepository(),
this.options.secretStore,
),
...createTestDataTools(this.options.testData ?? createTestDataManager(), context.runId),
...createTestDataTools(
this.options.testData ?? createTestDataManager(),
context.runId,
evidenceStore,
),
...(evidenceStore ? createRunnerEvidenceTools(evidenceStore) : []),
createArtifactWriterTool(
'write_execution',
Expand Down Expand Up @@ -888,12 +898,7 @@ class DefaultRunOrchestrator implements RunOrchestrator {
notes.push('UI 场景没有产生可审核的 evidence。');
}

let cleanup: {
ok: boolean;
attempted: number;
failed: string[];
message: string;
};
let cleanup: Awaited<ReturnType<TestDataManager['cleanup']>>;
try {
cleanup = await this.options.testData!.cleanup(context.runId);
} catch {
Expand All @@ -902,12 +907,16 @@ class DefaultRunOrchestrator implements RunOrchestrator {
attempted: 0,
failed: [],
message: '测试数据清理适配器执行失败',
receipts: [],
};
}
if (!cleanup.ok) {
this.addBlockingReason(context, `测试数据清理失败：${cleanup.message}`);
}
notes.push(`测试数据清理：${cleanup.message}`);
notes.push(`测试数据清理适配器：${cleanup.message}`);
notes.push(
...cleanup.receipts.map(
(receipt) =>
`清理核验 receipt：${receipt.dataId} · ${receipt.sourceId} · ${receipt.queriedAt} · ${receipt.statusCode === undefined ? `exit ${receipt.exitCode ?? 'n/a'}` : `HTTP ${receipt.statusCode}`} · summary ${receipt.summary} · sha256 ${receipt.sha256}`,
),
);
if (cleanup.failed.length > 0) {
notes.push(`测试数据清理失败项数量：${cleanup.failed.length}`);
}
Expand All @@ -918,6 +927,24 @@ class DefaultRunOrchestrator implements RunOrchestrator {
return { uploaded, uploadFailed };
}

private async finalizeTestData(workspace: RunWorkspace, context: RunContext): Promise<void> {
const result = (this.options.testData ?? createTestDataManager()).finalize(context.runId);
const notes = [`测试数据最终核验：${result.message}`];
if (!result.ok) {
this.addBlockingReason(context, `测试数据清理失败：${result.message}`);
notes.push(
...result.pending.map(
(entry) =>
`测试数据残留：${entry.id}（${entry.status}${entry.rejectionReason ? `：${entry.rejectionReason}` : ''}）`,
),
);
}
if (context.blockingReasons.length > 0) {
notes.push(...context.blockingReasons.map((reason) => `Harness 阻塞：${reason}`));
}
await this.appendExecutionNotes(workspace, notes);
}

private async cleanupRetainedEvidence(
workspace: RunWorkspace,
context: RunContext,
Expand Down Expand Up @@ -976,6 +1003,11 @@ class DefaultRunOrchestrator implements RunOrchestrator {
]),
),
...(evidenceStore ? createReviewerEvidenceTools(evidenceStore) : []),
...createReviewerTestDataTools(
this.options.testData ?? createTestDataManager(),
context.runId,
evidenceStore,
),
createArtifactWriterTool(
'write_review',
'写入独立审核',
Expand Down Expand Up @@ -1703,7 +1735,7 @@ ${JSON.stringify(runnerContext(context), null, 2)}`;

function runnerOutputContract(): string {
return `先读取 plan.md，再按计划使用受控 target、工作场景、命令、环境、测试数据和 evidence 工具。UI 场景只能使用受控的 headless、isolated Playwright MCP，优先使用 accessibility snapshot/ref；截图使用相对文件名并通过 list_evidence_files 确认存在。
测试账号只用于当前操作，绝不能写入日志、命令输出、Markdown 或证据。使用 get_test_data_prefix 标记临时数据，登记后执行 cleanup_test_data。每个场景记录实际观察、命令退出码、决定性/辅助证据、偏差和清理结果。结束前分别通过 write_execution 和 write_draft_report 写完整工件；不可用条件记录为 blocked。`;
测试账号只用于当前操作，绝不能写入日志、命令输出、Markdown 或证据。使用 get_test_data_prefix 标记临时数据；创建后立即登记，删除后只能提交 Harness 捕获的受控查询证据或 Playwright 截图声明，并检查待核验列表。不能自填 evidence 正文、状态码、摘要或 hash。每个场景记录实际观察、命令退出码、决定性/辅助证据、偏差和清理结果。结束前分别通过 write_execution 和 write_draft_report 写完整工件；不可用条件记录为 blocked。`;
}

function reviewerUserMessage(context: RunContext): string {
Expand All @@ -1714,7 +1746,7 @@ ${JSON.stringify(reviewerContext(context), null, 2)}`;
}

function reviewerOutputContract(): string {
return `依次读取 plan.md、execution.md、draft-report.md 和存在的 scenario-changes.patch；原始证据先于 Runner 草稿。查看截图只能使用 list_evidence_files 和 read_evidence_image，不能执行命令、读取测试账号或任意路径。
return `依次读取 plan.md、execution.md、draft-report.md 和存在的 scenario-changes.patch；原始证据先于 Runner 草稿。清理声明必须先通过 read_test_data_cleanup_evidence 读取 Harness 捕获的受控文本证据，或通过 read_evidence_image 实际查看删除后截图，再调用 verify_test_data_cleanup 确认或拒绝；纯 Runner 声明不构成已清理。查看截图只能使用 list_evidence_files 和 read_evidence_image，不能执行命令、读取测试账号或任意路径。
截图不可访问、上传失败、视觉能力不足、清理未确认、场景缺失或影响不明时维持 blocked。零场景只有在 Main · 规划的计划确有依据时才能确认。结束前通过 write_review 写完整 review.md，并明确是否同意最终结果。`;
}

Expand Down
Loading
Loading