diff --git a/docs/operator/codex-appserver-terminal-event-recovery.md b/docs/operator/codex-appserver-terminal-event-recovery.md new file mode 100644 index 000000000000..583397415a69 --- /dev/null +++ b/docs/operator/codex-appserver-terminal-event-recovery.md @@ -0,0 +1,66 @@ +# Codex app-server terminal-event recovery + +Codex App Server 0.145.0 has a reported failure mode where a tool-enabled turn produces its final assistant output and valid artifact, accepts every tool result, and has no outstanding protocol work, but never emits `turn/completed`. A client that waits only for that event can block until timeout or replay work that already finished. + +## Snapshot contract + +Write an operation-scoped JSON snapshot after the final assistant output is received: + +```json +{ + "schema_version": 1, + "turn_id": "turn-id", + "turn_completed_seen": false, + "final_assistant_output_seen": true, + "artifact_required": true, + "artifact_verified": true, + "external_write_attempted": false, + "destination_verified": false, + "tool_requests_total": 3, + "tool_results_accepted": 3, + "outstanding_tool_requests": 0, + "outstanding_server_requests": 0, + "outstanding_approvals": 0, + "outstanding_protocol_items": 0, + "outstanding_subagents": 0, + "owned_background_processes": 0, + "seconds_since_last_event": 10 +} +``` + +Run the guard: + +```bash +bun scripts/operator/codex-appserver-turn-completion-guard.mjs \ + --input /absolute/path/turn-snapshot.json \ + --json +``` + +## Decisions + +- `protocol_complete`: Codex emitted `turn/completed`; continue normally. +- `verified_completion_without_terminal_event`: all obligations are settled, the artifact is verified, any external write is independently verified, and the quiet period elapsed. The client may record a local synthetic completion and continue without replaying the turn. +- `write_reconciliation_required`: an external write may have executed but its destination is not verified. Read the destination with the original operation identifier before replay. +- `terminal_state_not_proven`: keep the turn bounded and collect another snapshot. Do not create a replacement turn or retry external work. + +The guard always sets `automatic_retry_allowed` to `false` when the protocol terminal event is absent. + +## Integration rule + +Do not treat final text alone as completion. A synthetic local completion requires: + +1. all tool requests have accepted results; +2. no outstanding tool, server, approval, protocol, subagent, or owned background work; +3. the requested artifact is independently verified when required; +4. every external write is verified at its destination; +5. the configured quiet period has elapsed. + +The generated evidence receipt belongs in the durable operator ledger, not in the Codex transcript. + +## Validation + +```bash +bun scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs +``` + +The self-test covers normal protocol completion, safe synthetic completion, independently verified writes, uncertain writes, unfinished turns, evidence generation, and malformed input. diff --git a/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs new file mode 100644 index 000000000000..68be2b38e5fe --- /dev/null +++ b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs @@ -0,0 +1,105 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" + +const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-turn-completion-")) +const script = path.join(import.meta.dirname, "codex-appserver-turn-completion-guard.mjs") + +const base = { + schema_version: 1, + turn_id: "turn-test", + parent_turn_id: "turn-root", + expected_parent_turn_id: "turn-root", + turn_completed_seen: false, + final_assistant_output_seen: true, + artifact_required: true, + artifact_verified: true, + external_write_attempted: false, + destination_verified: false, + tool_requests_total: 3, + tool_results_accepted: 3, + outstanding_tool_requests: 0, + outstanding_server_requests: 0, + outstanding_approvals: 0, + outstanding_protocol_items: 0, + outstanding_subagents: 0, + owned_background_processes: 0, + seconds_since_last_event: 10, +} + +function run(name, snapshot) { + const directory = path.join(root, name) + const input = path.join(directory, "snapshot.json") + const evidence = path.join(directory, "evidence") + fs.mkdirSync(directory, { recursive: true }) + fs.writeFileSync(input, `${JSON.stringify(snapshot, null, 2)}\n`) + const result = spawnSync(process.execPath, [script, "--input", input, "--evidence-dir", evidence, "--json"], { + encoding: "utf8", + }) + return { ...result, json: result.stdout.trim() ? JSON.parse(result.stdout) : null } +} + +const protocol = run("protocol", { ...base, turn_completed_seen: true }) +assert.equal(protocol.status, 0, protocol.stderr) +assert.equal(protocol.json.status, "protocol_complete") +assert.equal(protocol.json.synthetic_local_completion, false) +assert.equal(protocol.json.parent_turn_id, "turn-root") + +const local = run("local", base) +assert.equal(local.status, 0, local.stderr) +assert.equal(local.json.status, "verified_completion_without_terminal_event") +assert.equal(local.json.synthetic_local_completion, true) +assert.equal(local.json.automatic_retry_allowed, false) +assert.equal(local.json.checks.parent_turn_lineage_matched, true) +assert.ok(fs.existsSync(local.json.evidence_file)) + +const legacyWithoutLineage = structuredClone(base) +delete legacyWithoutLineage.parent_turn_id +delete legacyWithoutLineage.expected_parent_turn_id +const legacyResult = run("legacy-without-lineage", legacyWithoutLineage) +assert.equal(legacyResult.status, 0, legacyResult.stderr) +assert.equal(legacyResult.json.status, "verified_completion_without_terminal_event") + +const lineageMismatch = run("lineage-mismatch", { + ...base, + parent_turn_id: "turn-other", +}) +assert.equal(lineageMismatch.status, 75, lineageMismatch.stderr) +assert.equal(lineageMismatch.json.status, "turn_lineage_mismatch") +assert.equal(lineageMismatch.json.checks.parent_turn_lineage_matched, false) +assert.equal(lineageMismatch.json.automatic_retry_allowed, false) + +const verifiedWrite = run("verified-write", { + ...base, + external_write_attempted: true, + destination_verified: true, +}) +assert.equal(verifiedWrite.status, 0, verifiedWrite.stderr) +assert.equal(verifiedWrite.json.status, "verified_completion_without_terminal_event") + +const uncertainWrite = run("uncertain-write", { + ...base, + external_write_attempted: true, + destination_verified: false, +}) +assert.equal(uncertainWrite.status, 75, uncertainWrite.stderr) +assert.equal(uncertainWrite.json.status, "write_reconciliation_required") +assert.equal(uncertainWrite.json.requires_destination_reconciliation, true) + +const unfinished = run("unfinished", { + ...base, + tool_results_accepted: 2, + outstanding_tool_requests: 1, + seconds_since_last_event: 1, +}) +assert.equal(unfinished.status, 75, unfinished.stderr) +assert.equal(unfinished.json.status, "terminal_state_not_proven") + +const malformed = run("malformed", { ...base, outstanding_tool_requests: -1 }) +assert.equal(malformed.status, 2) +assert.match(malformed.stderr, /non-negative integer/) + +fs.rmSync(root, { recursive: true, force: true }) +console.log("codex-appserver-turn-completion-guard self-test passed") diff --git a/scripts/operator/codex-appserver-turn-completion-guard.mjs b/scripts/operator/codex-appserver-turn-completion-guard.mjs new file mode 100644 index 000000000000..fc774b0900b9 --- /dev/null +++ b/scripts/operator/codex-appserver-turn-completion-guard.mjs @@ -0,0 +1,175 @@ +import fs from "node:fs" +import path from "node:path" +import { nowIso, parseArgs, sha256, stateRoot, writeJsonAtomic } from "./lib.mjs" + +const args = parseArgs(process.argv.slice(2)) +const inputFile = args.input +if (typeof inputFile !== "string" || !path.isAbsolute(inputFile)) { + console.error("Usage: bun scripts/operator/codex-appserver-turn-completion-guard.mjs --input /absolute/snapshot.json [--json]") + process.exit(2) +} + +let snapshot +try { + snapshot = JSON.parse(fs.readFileSync(inputFile, "utf8")) +} catch (error) { + console.error(`Unable to read completion snapshot: ${error.message}`) + process.exit(2) +} + +function boolean(name) { + if (typeof snapshot[name] !== "boolean") throw new Error(`${name} must be boolean`) + return snapshot[name] +} + +function count(name) { + const value = snapshot[name] + if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`) + return value +} + +function seconds(name) { + const value = snapshot[name] + if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`) + return value +} + +function optionalString(name) { + const value = snapshot[name] + if (value == null) return null + if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be null or a non-empty string`) + return value.trim() +} + +let state +try { + if (snapshot.schema_version !== 1) throw new Error("schema_version must equal 1") + if (typeof snapshot.turn_id !== "string" || snapshot.turn_id.trim() === "") throw new Error("turn_id is required") + + state = { + parent_turn_id: optionalString("parent_turn_id"), + expected_parent_turn_id: optionalString("expected_parent_turn_id"), + turn_completed_seen: boolean("turn_completed_seen"), + final_assistant_output_seen: boolean("final_assistant_output_seen"), + artifact_required: boolean("artifact_required"), + artifact_verified: boolean("artifact_verified"), + external_write_attempted: boolean("external_write_attempted"), + destination_verified: boolean("destination_verified"), + tool_requests_total: count("tool_requests_total"), + tool_results_accepted: count("tool_results_accepted"), + outstanding_tool_requests: count("outstanding_tool_requests"), + outstanding_server_requests: count("outstanding_server_requests"), + outstanding_approvals: count("outstanding_approvals"), + outstanding_protocol_items: count("outstanding_protocol_items"), + outstanding_subagents: count("outstanding_subagents"), + owned_background_processes: count("owned_background_processes"), + seconds_since_last_event: seconds("seconds_since_last_event"), + } +} catch (error) { + console.error(error.message) + process.exit(2) +} + +const minimumQuietSeconds = Number(args["minimum-quiet-seconds"] || process.env.OPERATOR_APP_SERVER_TERMINAL_QUIET_SECONDS || 5) +if (!Number.isFinite(minimumQuietSeconds) || minimumQuietSeconds < 0) { + console.error("minimum quiet seconds must be a non-negative number") + process.exit(2) +} + +const obligations = + state.outstanding_tool_requests + + state.outstanding_server_requests + + state.outstanding_approvals + + state.outstanding_protocol_items + + state.outstanding_subagents + + state.owned_background_processes +const toolResultsMatched = state.tool_requests_total === state.tool_results_accepted +const artifactSatisfied = !state.artifact_required || state.artifact_verified +const writeSatisfied = !state.external_write_attempted || state.destination_verified +const lineageMatched = + state.expected_parent_turn_id === null || state.parent_turn_id === state.expected_parent_turn_id +const semanticallySettled = + state.final_assistant_output_seen && + toolResultsMatched && + obligations === 0 && + artifactSatisfied && + lineageMatched && + state.seconds_since_last_event >= minimumQuietSeconds + +let status +let safeToContinue +let syntheticCompletion +let requiresReconciliation + +if (!lineageMatched) { + status = "turn_lineage_mismatch" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = state.external_write_attempted && !state.destination_verified +} else if (state.turn_completed_seen) { + status = "protocol_complete" + safeToContinue = true + syntheticCompletion = false + requiresReconciliation = false +} else if (semanticallySettled && writeSatisfied) { + status = "verified_completion_without_terminal_event" + safeToContinue = true + syntheticCompletion = true + requiresReconciliation = false +} else if (state.external_write_attempted && !state.destination_verified) { + status = "write_reconciliation_required" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = true +} else { + status = "terminal_state_not_proven" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = false +} + +const report = { + schema_version: 1, + observed_at: nowIso(), + turn_id: snapshot.turn_id, + parent_turn_id: state.parent_turn_id, + expected_parent_turn_id: state.expected_parent_turn_id, + status, + safe_to_continue: safeToContinue, + synthetic_local_completion: syntheticCompletion, + requires_destination_reconciliation: requiresReconciliation, + automatic_retry_allowed: false, + minimum_quiet_seconds: minimumQuietSeconds, + checks: { + parent_turn_lineage_matched: lineageMatched, + final_assistant_output_seen: state.final_assistant_output_seen, + tool_results_matched: toolResultsMatched, + outstanding_obligations: obligations, + artifact_satisfied: artifactSatisfied, + external_write_satisfied: writeSatisfied, + quiet_period_satisfied: state.seconds_since_last_event >= minimumQuietSeconds, + }, + snapshot_sha256: sha256(JSON.stringify(snapshot)), + remediation: + status === "turn_lineage_mismatch" + ? "Quarantine the nested result, preserve both turn identifiers, and do not use it to authorize completion or an external write. Reconcile any attempted write before replay." + : status === "write_reconciliation_required" + ? "Read the destination using the original operation identifier before any replay." + : status === "terminal_state_not_proven" + ? "Keep the turn bounded and collect another snapshot; do not create a replacement turn or retry external work." + : null, +} + +const directory = path.resolve(args["evidence-dir"] || path.join(stateRoot(args), "appserver-turn-completion")) +const evidenceFile = path.join(directory, `${report.observed_at.replace(/[:.]/g, "-")}-${snapshot.turn_id.replace(/[^A-Za-z0-9_.-]/g, "_")}.json`) +writeJsonAtomic(evidenceFile, report) + +const result = { ...report, evidence_file: evidenceFile } +if (args.json) console.log(JSON.stringify(result)) +else { + console.log(`Codex app-server turn ${snapshot.turn_id}: ${status}`) + console.log(`Evidence: ${evidenceFile}`) + if (!safeToContinue) console.error(report.remediation || "Completion is not proven") +} + +process.exit(safeToContinue ? 0 : 75)