From de2359301bfeddef581f95eadcd929cf0cfb52be Mon Sep 17 00:00:00 2001 From: urbanplanningpros Date: Tue, 28 Jul 2026 14:49:36 -0500 Subject: [PATCH 1/5] fix(operator): contain missing app-server completion events --- .../codex-appserver-turn-completion-guard.mjs | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 scripts/operator/codex-appserver-turn-completion-guard.mjs diff --git a/scripts/operator/codex-appserver-turn-completion-guard.mjs b/scripts/operator/codex-appserver-turn-completion-guard.mjs new file mode 100644 index 000000000000..f877c49c4d76 --- /dev/null +++ b/scripts/operator/codex-appserver-turn-completion-guard.mjs @@ -0,0 +1,153 @@ +import fs from "node:fs" +import path from "node:path" +import { nowIso, parseArgs, sha256, stateRoot, writeJsonAtomic } from "./lib.mjs" + +const args = parseArgs(process.argv.slice(2)) +const inputFile = args.input +if (typeof inputFile !== "string" || !path.isAbsolute(inputFile)) { + console.error("Usage: bun scripts/operator/codex-appserver-turn-completion-guard.mjs --input /absolute/snapshot.json [--json]") + process.exit(2) +} + +let snapshot +try { + snapshot = JSON.parse(fs.readFileSync(inputFile, "utf8")) +} catch (error) { + console.error(`Unable to read completion snapshot: ${error.message}`) + process.exit(2) +} + +function boolean(name) { + if (typeof snapshot[name] !== "boolean") throw new Error(`${name} must be boolean`) + return snapshot[name] +} + +function count(name) { + const value = snapshot[name] + if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`) + return value +} + +function seconds(name) { + const value = snapshot[name] + if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`) + return value +} + +let state +try { + if (snapshot.schema_version !== 1) throw new Error("schema_version must equal 1") + if (typeof snapshot.turn_id !== "string" || snapshot.turn_id.trim() === "") throw new Error("turn_id is required") + + state = { + turn_completed_seen: boolean("turn_completed_seen"), + final_assistant_output_seen: boolean("final_assistant_output_seen"), + artifact_required: boolean("artifact_required"), + artifact_verified: boolean("artifact_verified"), + external_write_attempted: boolean("external_write_attempted"), + destination_verified: boolean("destination_verified"), + tool_requests_total: count("tool_requests_total"), + tool_results_accepted: count("tool_results_accepted"), + outstanding_tool_requests: count("outstanding_tool_requests"), + outstanding_server_requests: count("outstanding_server_requests"), + outstanding_approvals: count("outstanding_approvals"), + outstanding_protocol_items: count("outstanding_protocol_items"), + outstanding_subagents: count("outstanding_subagents"), + owned_background_processes: count("owned_background_processes"), + seconds_since_last_event: seconds("seconds_since_last_event"), + } +} catch (error) { + console.error(error.message) + process.exit(2) +} + +const minimumQuietSeconds = Number(args["minimum-quiet-seconds"] || process.env.OPERATOR_APP_SERVER_TERMINAL_QUIET_SECONDS || 5) +if (!Number.isFinite(minimumQuietSeconds) || minimumQuietSeconds < 0) { + console.error("minimum quiet seconds must be a non-negative number") + process.exit(2) +} + +const obligations = + state.outstanding_tool_requests + + state.outstanding_server_requests + + state.outstanding_approvals + + state.outstanding_protocol_items + + state.outstanding_subagents + + state.owned_background_processes +const toolResultsMatched = state.tool_requests_total === state.tool_results_accepted +const artifactSatisfied = !state.artifact_required || state.artifact_verified +const writeSatisfied = !state.external_write_attempted || state.destination_verified +const semanticallySettled = + state.final_assistant_output_seen && + toolResultsMatched && + obligations === 0 && + artifactSatisfied && + state.seconds_since_last_event >= minimumQuietSeconds + +let status +let safeToContinue +let syntheticCompletion +let requiresReconciliation + +if (state.turn_completed_seen) { + status = "protocol_complete" + safeToContinue = true + syntheticCompletion = false + requiresReconciliation = false +} else if (semanticallySettled && writeSatisfied) { + status = "verified_completion_without_terminal_event" + safeToContinue = true + syntheticCompletion = true + requiresReconciliation = false +} else if (state.external_write_attempted && !state.destination_verified) { + status = "write_reconciliation_required" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = true +} else { + status = "terminal_state_not_proven" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = false +} + +const report = { + schema_version: 1, + observed_at: nowIso(), + turn_id: snapshot.turn_id, + status, + safe_to_continue: safeToContinue, + synthetic_local_completion: syntheticCompletion, + requires_destination_reconciliation: requiresReconciliation, + automatic_retry_allowed: false, + minimum_quiet_seconds: minimumQuietSeconds, + checks: { + final_assistant_output_seen: state.final_assistant_output_seen, + tool_results_matched: toolResultsMatched, + outstanding_obligations: obligations, + artifact_satisfied: artifactSatisfied, + external_write_satisfied: writeSatisfied, + quiet_period_satisfied: state.seconds_since_last_event >= minimumQuietSeconds, + }, + snapshot_sha256: sha256(JSON.stringify(snapshot)), + remediation: + status === "write_reconciliation_required" + ? "Read the destination using the original operation identifier before any replay." + : status === "terminal_state_not_proven" + ? "Keep the turn bounded and collect another snapshot; do not create a replacement turn or retry external work." + : null, +} + +const directory = path.resolve(args["evidence-dir"] || path.join(stateRoot(args), "appserver-turn-completion")) +const evidenceFile = path.join(directory, `${report.observed_at.replace(/[:.]/g, "-")}-${snapshot.turn_id.replace(/[^A-Za-z0-9_.-]/g, "_")}.json`) +writeJsonAtomic(evidenceFile, report) + +const result = { ...report, evidence_file: evidenceFile } +if (args.json) console.log(JSON.stringify(result)) +else { + console.log(`Codex app-server turn ${snapshot.turn_id}: ${status}`) + console.log(`Evidence: ${evidenceFile}`) + if (!safeToContinue) console.error(report.remediation || "Completion is not proven") +} + +process.exit(safeToContinue ? 0 : 75) From f39c9754943cd8198ff322ffcced556b44d15a5e Mon Sep 17 00:00:00 2001 From: urbanplanningpros Date: Tue, 28 Jul 2026 14:49:52 -0500 Subject: [PATCH 2/5] test(operator): cover app-server completion guard --- ...pserver-turn-completion-guard-selftest.mjs | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs diff --git a/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs new file mode 100644 index 000000000000..d77ffff781aa --- /dev/null +++ b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs @@ -0,0 +1,85 @@ +import assert from "node:assert/strict" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" + +const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-turn-completion-")) +const script = path.join(import.meta.dirname, "codex-appserver-turn-completion-guard.mjs") + +const base = { + schema_version: 1, + turn_id: "turn-test", + turn_completed_seen: false, + final_assistant_output_seen: true, + artifact_required: true, + artifact_verified: true, + external_write_attempted: false, + destination_verified: false, + tool_requests_total: 3, + tool_results_accepted: 3, + outstanding_tool_requests: 0, + outstanding_server_requests: 0, + outstanding_approvals: 0, + outstanding_protocol_items: 0, + outstanding_subagents: 0, + owned_background_processes: 0, + seconds_since_last_event: 10, +} + +function run(name, snapshot) { + const directory = path.join(root, name) + const input = path.join(directory, "snapshot.json") + const evidence = path.join(directory, "evidence") + fs.mkdirSync(directory, { recursive: true }) + fs.writeFileSync(input, `${JSON.stringify(snapshot, null, 2)}\n`) + const result = spawnSync(process.execPath, [script, "--input", input, "--evidence-dir", evidence, "--json"], { + encoding: "utf8", + }) + return { ...result, json: result.stdout.trim() ? JSON.parse(result.stdout) : null } +} + +const protocol = run("protocol", { ...base, turn_completed_seen: true }) +assert.equal(protocol.status, 0, protocol.stderr) +assert.equal(protocol.json.status, "protocol_complete") +assert.equal(protocol.json.synthetic_local_completion, false) + +const local = run("local", base) +assert.equal(local.status, 0, local.stderr) +assert.equal(local.json.status, "verified_completion_without_terminal_event") +assert.equal(local.json.synthetic_local_completion, true) +assert.equal(local.json.automatic_retry_allowed, false) +assert.ok(fs.existsSync(local.json.evidence_file)) + +const verifiedWrite = run("verified-write", { + ...base, + external_write_attempted: true, + destination_verified: true, +}) +assert.equal(verifiedWrite.status, 0, verifiedWrite.stderr) +assert.equal(verifiedWrite.json.status, "verified_completion_without_terminal_event") + +const uncertainWrite = run("uncertain-write", { + ...base, + external_write_attempted: true, + destination_verified: false, +}) +assert.equal(uncertainWrite.status, 75, uncertainWrite.stderr) +assert.equal(uncertainWrite.json.status, "write_reconciliation_required") +assert.equal(uncertainWrite.json.requires_destination_reconciliation, true) + +const unfinished = run("unfinished", { + ...base, + tool_results_accepted: 2, + outstanding_tool_requests: 1, + seconds_since_last_event: 1, +}) +assert.equal(unfinished.status, 75, unfinished.stderr) +assert.equal(unfinished.json.status, "terminal_state_not_proven") + +const malformed = run("malformed", { ...base, outstanding_tool_requests: -1 }) +assert.equal(malformed.status, 2) +assert.match(malformed.stderr, /non-negative integer/) + +fs.rmSync(root, { recursive: true, force: true }) +console.log("codex-appserver-turn-completion-guard self-test passed") From 6a590af4a98b84abe71ce20468a129d3c144ed3c Mon Sep 17 00:00:00 2001 From: urbanplanningpros Date: Tue, 28 Jul 2026 14:50:11 -0500 Subject: [PATCH 3/5] docs(operator): document missing terminal-event recovery --- ...codex-appserver-terminal-event-recovery.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/operator/codex-appserver-terminal-event-recovery.md diff --git a/docs/operator/codex-appserver-terminal-event-recovery.md b/docs/operator/codex-appserver-terminal-event-recovery.md new file mode 100644 index 000000000000..583397415a69 --- /dev/null +++ b/docs/operator/codex-appserver-terminal-event-recovery.md @@ -0,0 +1,66 @@ +# Codex app-server terminal-event recovery + +Codex App Server 0.145.0 has a reported failure mode where a tool-enabled turn produces its final assistant output and valid artifact, accepts every tool result, and has no outstanding protocol work, but never emits `turn/completed`. A client that waits only for that event can block until timeout or replay work that already finished. + +## Snapshot contract + +Write an operation-scoped JSON snapshot after the final assistant output is received: + +```json +{ + "schema_version": 1, + "turn_id": "turn-id", + "turn_completed_seen": false, + "final_assistant_output_seen": true, + "artifact_required": true, + "artifact_verified": true, + "external_write_attempted": false, + "destination_verified": false, + "tool_requests_total": 3, + "tool_results_accepted": 3, + "outstanding_tool_requests": 0, + "outstanding_server_requests": 0, + "outstanding_approvals": 0, + "outstanding_protocol_items": 0, + "outstanding_subagents": 0, + "owned_background_processes": 0, + "seconds_since_last_event": 10 +} +``` + +Run the guard: + +```bash +bun scripts/operator/codex-appserver-turn-completion-guard.mjs \ + --input /absolute/path/turn-snapshot.json \ + --json +``` + +## Decisions + +- `protocol_complete`: Codex emitted `turn/completed`; continue normally. +- `verified_completion_without_terminal_event`: all obligations are settled, the artifact is verified, any external write is independently verified, and the quiet period elapsed. The client may record a local synthetic completion and continue without replaying the turn. +- `write_reconciliation_required`: an external write may have executed but its destination is not verified. Read the destination with the original operation identifier before replay. +- `terminal_state_not_proven`: keep the turn bounded and collect another snapshot. Do not create a replacement turn or retry external work. + +The guard always sets `automatic_retry_allowed` to `false` when the protocol terminal event is absent. + +## Integration rule + +Do not treat final text alone as completion. A synthetic local completion requires: + +1. all tool requests have accepted results; +2. no outstanding tool, server, approval, protocol, subagent, or owned background work; +3. the requested artifact is independently verified when required; +4. every external write is verified at its destination; +5. the configured quiet period has elapsed. + +The generated evidence receipt belongs in the durable operator ledger, not in the Codex transcript. + +## Validation + +```bash +bun scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs +``` + +The self-test covers normal protocol completion, safe synthetic completion, independently verified writes, uncertain writes, unfinished turns, evidence generation, and malformed input. From 08beaec03536955659ce7a9e15292c007de2f492 Mon Sep 17 00:00:00 2001 From: urbanplanningpros Date: Tue, 28 Jul 2026 17:45:23 -0500 Subject: [PATCH 4/5] fix(operator): preserve nested Codex turn lineage --- .../codex-appserver-turn-completion-guard.mjs | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/operator/codex-appserver-turn-completion-guard.mjs b/scripts/operator/codex-appserver-turn-completion-guard.mjs index f877c49c4d76..fc774b0900b9 100644 --- a/scripts/operator/codex-appserver-turn-completion-guard.mjs +++ b/scripts/operator/codex-appserver-turn-completion-guard.mjs @@ -34,12 +34,21 @@ function seconds(name) { return value } +function optionalString(name) { + const value = snapshot[name] + if (value == null) return null + if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be null or a non-empty string`) + return value.trim() +} + let state try { if (snapshot.schema_version !== 1) throw new Error("schema_version must equal 1") if (typeof snapshot.turn_id !== "string" || snapshot.turn_id.trim() === "") throw new Error("turn_id is required") state = { + parent_turn_id: optionalString("parent_turn_id"), + expected_parent_turn_id: optionalString("expected_parent_turn_id"), turn_completed_seen: boolean("turn_completed_seen"), final_assistant_output_seen: boolean("final_assistant_output_seen"), artifact_required: boolean("artifact_required"), @@ -77,11 +86,14 @@ const obligations = const toolResultsMatched = state.tool_requests_total === state.tool_results_accepted const artifactSatisfied = !state.artifact_required || state.artifact_verified const writeSatisfied = !state.external_write_attempted || state.destination_verified +const lineageMatched = + state.expected_parent_turn_id === null || state.parent_turn_id === state.expected_parent_turn_id const semanticallySettled = state.final_assistant_output_seen && toolResultsMatched && obligations === 0 && artifactSatisfied && + lineageMatched && state.seconds_since_last_event >= minimumQuietSeconds let status @@ -89,7 +101,12 @@ let safeToContinue let syntheticCompletion let requiresReconciliation -if (state.turn_completed_seen) { +if (!lineageMatched) { + status = "turn_lineage_mismatch" + safeToContinue = false + syntheticCompletion = false + requiresReconciliation = state.external_write_attempted && !state.destination_verified +} else if (state.turn_completed_seen) { status = "protocol_complete" safeToContinue = true syntheticCompletion = false @@ -115,6 +132,8 @@ const report = { schema_version: 1, observed_at: nowIso(), turn_id: snapshot.turn_id, + parent_turn_id: state.parent_turn_id, + expected_parent_turn_id: state.expected_parent_turn_id, status, safe_to_continue: safeToContinue, synthetic_local_completion: syntheticCompletion, @@ -122,6 +141,7 @@ const report = { automatic_retry_allowed: false, minimum_quiet_seconds: minimumQuietSeconds, checks: { + parent_turn_lineage_matched: lineageMatched, final_assistant_output_seen: state.final_assistant_output_seen, tool_results_matched: toolResultsMatched, outstanding_obligations: obligations, @@ -131,11 +151,13 @@ const report = { }, snapshot_sha256: sha256(JSON.stringify(snapshot)), remediation: - status === "write_reconciliation_required" - ? "Read the destination using the original operation identifier before any replay." - : status === "terminal_state_not_proven" - ? "Keep the turn bounded and collect another snapshot; do not create a replacement turn or retry external work." - : null, + status === "turn_lineage_mismatch" + ? "Quarantine the nested result, preserve both turn identifiers, and do not use it to authorize completion or an external write. Reconcile any attempted write before replay." + : status === "write_reconciliation_required" + ? "Read the destination using the original operation identifier before any replay." + : status === "terminal_state_not_proven" + ? "Keep the turn bounded and collect another snapshot; do not create a replacement turn or retry external work." + : null, } const directory = path.resolve(args["evidence-dir"] || path.join(stateRoot(args), "appserver-turn-completion")) From ad1ce702ed2db4902f159a3e443b781976006293 Mon Sep 17 00:00:00 2001 From: urbanplanningpros Date: Tue, 28 Jul 2026 17:45:40 -0500 Subject: [PATCH 5/5] test(operator): cover nested Codex turn lineage --- ...pserver-turn-completion-guard-selftest.mjs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs index d77ffff781aa..68be2b38e5fe 100644 --- a/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs +++ b/scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs @@ -10,6 +10,8 @@ const script = path.join(import.meta.dirname, "codex-appserver-turn-completion-g const base = { schema_version: 1, turn_id: "turn-test", + parent_turn_id: "turn-root", + expected_parent_turn_id: "turn-root", turn_completed_seen: false, final_assistant_output_seen: true, artifact_required: true, @@ -43,14 +45,32 @@ const protocol = run("protocol", { ...base, turn_completed_seen: true }) assert.equal(protocol.status, 0, protocol.stderr) assert.equal(protocol.json.status, "protocol_complete") assert.equal(protocol.json.synthetic_local_completion, false) +assert.equal(protocol.json.parent_turn_id, "turn-root") const local = run("local", base) assert.equal(local.status, 0, local.stderr) assert.equal(local.json.status, "verified_completion_without_terminal_event") assert.equal(local.json.synthetic_local_completion, true) assert.equal(local.json.automatic_retry_allowed, false) +assert.equal(local.json.checks.parent_turn_lineage_matched, true) assert.ok(fs.existsSync(local.json.evidence_file)) +const legacyWithoutLineage = structuredClone(base) +delete legacyWithoutLineage.parent_turn_id +delete legacyWithoutLineage.expected_parent_turn_id +const legacyResult = run("legacy-without-lineage", legacyWithoutLineage) +assert.equal(legacyResult.status, 0, legacyResult.stderr) +assert.equal(legacyResult.json.status, "verified_completion_without_terminal_event") + +const lineageMismatch = run("lineage-mismatch", { + ...base, + parent_turn_id: "turn-other", +}) +assert.equal(lineageMismatch.status, 75, lineageMismatch.stderr) +assert.equal(lineageMismatch.json.status, "turn_lineage_mismatch") +assert.equal(lineageMismatch.json.checks.parent_turn_lineage_matched, false) +assert.equal(lineageMismatch.json.automatic_retry_allowed, false) + const verifiedWrite = run("verified-write", { ...base, external_write_attempted: true,