fix(operator): contain missing app-server completion events

docs/operator/codex-appserver-terminal-event-recovery.md

@@ -0,0 +1,66 @@
+# Codex app-server terminal-event recovery
+
+Codex App Server 0.145.0 has a reported failure mode where a tool-enabled turn produces its final assistant output and valid artifact, accepts every tool result, and has no outstanding protocol work, but never emits `turn/completed`. A client that waits only for that event can block until timeout or replay work that already finished.
+
+## Snapshot contract
+
+Write an operation-scoped JSON snapshot after the final assistant output is received:
+
+```json
+{
+  "schema_version": 1,
+  "turn_id": "turn-id",
+  "turn_completed_seen": false,
+  "final_assistant_output_seen": true,
+  "artifact_required": true,
+  "artifact_verified": true,
+  "external_write_attempted": false,
+  "destination_verified": false,
+  "tool_requests_total": 3,
+  "tool_results_accepted": 3,
+  "outstanding_tool_requests": 0,
+  "outstanding_server_requests": 0,
+  "outstanding_approvals": 0,
+  "outstanding_protocol_items": 0,
+  "outstanding_subagents": 0,
+  "owned_background_processes": 0,
+  "seconds_since_last_event": 10
+}
+```
+
+Run the guard:
+
+```bash
+bun scripts/operator/codex-appserver-turn-completion-guard.mjs \
+  --input /absolute/path/turn-snapshot.json \
+  --json
+```
+
+## Decisions
+
+- `protocol_complete`: Codex emitted `turn/completed`; continue normally.
+- `verified_completion_without_terminal_event`: all obligations are settled, the artifact is verified, any external write is independently verified, and the quiet period elapsed. The client may record a local synthetic completion and continue without replaying the turn.
+- `write_reconciliation_required`: an external write may have executed but its destination is not verified. Read the destination with the original operation identifier before replay.
+- `terminal_state_not_proven`: keep the turn bounded and collect another snapshot. Do not create a replacement turn or retry external work.
+
+The guard always sets `automatic_retry_allowed` to `false` when the protocol terminal event is absent.
+
+## Integration rule
+
+Do not treat final text alone as completion. A synthetic local completion requires:
+
+1. all tool requests have accepted results;
+2. no outstanding tool, server, approval, protocol, subagent, or owned background work;
+3. the requested artifact is independently verified when required;
+4. every external write is verified at its destination;
+5. the configured quiet period has elapsed.
+
+The generated evidence receipt belongs in the durable operator ledger, not in the Codex transcript.
+
+## Validation
+
+```bash
+bun scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs
+```
+
+The self-test covers normal protocol completion, safe synthetic completion, independently verified writes, uncertain writes, unfinished turns, evidence generation, and malformed input.

scripts/operator/codex-appserver-turn-completion-guard-selftest.mjs

@@ -0,0 +1,105 @@
+import assert from "node:assert/strict"
+import fs from "node:fs"
+import os from "node:os"
+import path from "node:path"
+import { spawnSync } from "node:child_process"
+
+const root = fs.mkdtempSync(path.join(os.tmpdir(), "codex-turn-completion-"))
+const script = path.join(import.meta.dirname, "codex-appserver-turn-completion-guard.mjs")
+
+const base = {
+  schema_version: 1,
+  turn_id: "turn-test",
+  parent_turn_id: "turn-root",
+  expected_parent_turn_id: "turn-root",
+  turn_completed_seen: false,
+  final_assistant_output_seen: true,
+  artifact_required: true,
+  artifact_verified: true,
+  external_write_attempted: false,
+  destination_verified: false,
+  tool_requests_total: 3,
+  tool_results_accepted: 3,
+  outstanding_tool_requests: 0,
+  outstanding_server_requests: 0,
+  outstanding_approvals: 0,
+  outstanding_protocol_items: 0,
+  outstanding_subagents: 0,
+  owned_background_processes: 0,
+  seconds_since_last_event: 10,
+}
+
+function run(name, snapshot) {
+  const directory = path.join(root, name)
+  const input = path.join(directory, "snapshot.json")
+  const evidence = path.join(directory, "evidence")
+  fs.mkdirSync(directory, { recursive: true })
+  fs.writeFileSync(input, `${JSON.stringify(snapshot, null, 2)}\n`)
+  const result = spawnSync(process.execPath, [script, "--input", input, "--evidence-dir", evidence, "--json"], {
+    encoding: "utf8",
+  })
+  return { ...result, json: result.stdout.trim() ? JSON.parse(result.stdout) : null }
+}
+
+const protocol = run("protocol", { ...base, turn_completed_seen: true })
+assert.equal(protocol.status, 0, protocol.stderr)
+assert.equal(protocol.json.status, "protocol_complete")
+assert.equal(protocol.json.synthetic_local_completion, false)
+assert.equal(protocol.json.parent_turn_id, "turn-root")
+
+const local = run("local", base)
+assert.equal(local.status, 0, local.stderr)
+assert.equal(local.json.status, "verified_completion_without_terminal_event")
+assert.equal(local.json.synthetic_local_completion, true)
+assert.equal(local.json.automatic_retry_allowed, false)
+assert.equal(local.json.checks.parent_turn_lineage_matched, true)
+assert.ok(fs.existsSync(local.json.evidence_file))
+
+const legacyWithoutLineage = structuredClone(base)
+delete legacyWithoutLineage.parent_turn_id
+delete legacyWithoutLineage.expected_parent_turn_id
+const legacyResult = run("legacy-without-lineage", legacyWithoutLineage)
+assert.equal(legacyResult.status, 0, legacyResult.stderr)
+assert.equal(legacyResult.json.status, "verified_completion_without_terminal_event")
+
+const lineageMismatch = run("lineage-mismatch", {
+  ...base,
+  parent_turn_id: "turn-other",
+})
+assert.equal(lineageMismatch.status, 75, lineageMismatch.stderr)
+assert.equal(lineageMismatch.json.status, "turn_lineage_mismatch")
+assert.equal(lineageMismatch.json.checks.parent_turn_lineage_matched, false)
+assert.equal(lineageMismatch.json.automatic_retry_allowed, false)
+
+const verifiedWrite = run("verified-write", {
+  ...base,
+  external_write_attempted: true,
+  destination_verified: true,
+})
+assert.equal(verifiedWrite.status, 0, verifiedWrite.stderr)
+assert.equal(verifiedWrite.json.status, "verified_completion_without_terminal_event")
+
+const uncertainWrite = run("uncertain-write", {
+  ...base,
+  external_write_attempted: true,
+  destination_verified: false,
+})
+assert.equal(uncertainWrite.status, 75, uncertainWrite.stderr)
+assert.equal(uncertainWrite.json.status, "write_reconciliation_required")
+assert.equal(uncertainWrite.json.requires_destination_reconciliation, true)
+
+const unfinished = run("unfinished", {
+  ...base,
+  tool_results_accepted: 2,
+  outstanding_tool_requests: 1,
+  seconds_since_last_event: 1,
+})
+assert.equal(unfinished.status, 75, unfinished.stderr)
+assert.equal(unfinished.json.status, "terminal_state_not_proven")
+
+const malformed = run("malformed", { ...base, outstanding_tool_requests: -1 })
+assert.equal(malformed.status, 2)
+assert.match(malformed.stderr, /non-negative integer/)
+
+fs.rmSync(root, { recursive: true, force: true })
+console.log("codex-appserver-turn-completion-guard self-test passed")

scripts/operator/codex-appserver-turn-completion-guard.mjs

@@ -0,0 +1,175 @@
+import fs from "node:fs"
+import path from "node:path"
+import { nowIso, parseArgs, sha256, stateRoot, writeJsonAtomic } from "./lib.mjs"
+
+const args = parseArgs(process.argv.slice(2))
+const inputFile = args.input
+if (typeof inputFile !== "string" || !path.isAbsolute(inputFile)) {
+  console.error("Usage: bun scripts/operator/codex-appserver-turn-completion-guard.mjs --input /absolute/snapshot.json [--json]")
+  process.exit(2)
+}
+
+let snapshot
+try {
+  snapshot = JSON.parse(fs.readFileSync(inputFile, "utf8"))
+} catch (error) {
+  console.error(`Unable to read completion snapshot: ${error.message}`)
+  process.exit(2)
+}
+
+function boolean(name) {
+  if (typeof snapshot[name] !== "boolean") throw new Error(`${name} must be boolean`)
+  return snapshot[name]
+}
+
+function count(name) {
+  const value = snapshot[name]
+  if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`)
+  return value
+}
+
+function seconds(name) {
+  const value = snapshot[name]
+  if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`)
+  return value
+}
+
+function optionalString(name) {
+  const value = snapshot[name]
+  if (value == null) return null
+  if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be null or a non-empty string`)
+  return value.trim()
+}
+
+let state
+try {
+  if (snapshot.schema_version !== 1) throw new Error("schema_version must equal 1")
+  if (typeof snapshot.turn_id !== "string" || snapshot.turn_id.trim() === "") throw new Error("turn_id is required")
+
+  state = {
+    parent_turn_id: optionalString("parent_turn_id"),
+    expected_parent_turn_id: optionalString("expected_parent_turn_id"),
+    turn_completed_seen: boolean("turn_completed_seen"),
+    final_assistant_output_seen: boolean("final_assistant_output_seen"),
+    artifact_required: boolean("artifact_required"),
+    artifact_verified: boolean("artifact_verified"),
+    external_write_attempted: boolean("external_write_attempted"),
+    destination_verified: boolean("destination_verified"),
+    tool_requests_total: count("tool_requests_total"),
+    tool_results_accepted: count("tool_results_accepted"),
+    outstanding_tool_requests: count("outstanding_tool_requests"),
+    outstanding_server_requests: count("outstanding_server_requests"),
+    outstanding_approvals: count("outstanding_approvals"),
+    outstanding_protocol_items: count("outstanding_protocol_items"),
+    outstanding_subagents: count("outstanding_subagents"),
+    owned_background_processes: count("owned_background_processes"),
+    seconds_since_last_event: seconds("seconds_since_last_event"),
+  }
+} catch (error) {
+  console.error(error.message)
+  process.exit(2)
+}
+
+const minimumQuietSeconds = Number(args["minimum-quiet-seconds"] || process.env.OPERATOR_APP_SERVER_TERMINAL_QUIET_SECONDS || 5)
+if (!Number.isFinite(minimumQuietSeconds) || minimumQuietSeconds < 0) {
+  console.error("minimum quiet seconds must be a non-negative number")
+  process.exit(2)
+}
+
+const obligations =
+  state.outstanding_tool_requests +
+  state.outstanding_server_requests +
+  state.outstanding_approvals +
+  state.outstanding_protocol_items +
+  state.outstanding_subagents +
+  state.owned_background_processes
+const toolResultsMatched = state.tool_requests_total === state.tool_results_accepted
+const artifactSatisfied = !state.artifact_required || state.artifact_verified
+const writeSatisfied = !state.external_write_attempted || state.destination_verified
+const lineageMatched =
+  state.expected_parent_turn_id === null || state.parent_turn_id === state.expected_parent_turn_id
+const semanticallySettled =
+  state.final_assistant_output_seen &&
+  toolResultsMatched &&
+  obligations === 0 &&
+  artifactSatisfied &&
+  lineageMatched &&
+  state.seconds_since_last_event >= minimumQuietSeconds
+
+let status
+let safeToContinue
+let syntheticCompletion
+let requiresReconciliation
+
+if (!lineageMatched) {
+  status = "turn_lineage_mismatch"
+  safeToContinue = false
+  syntheticCompletion = false
+  requiresReconciliation = state.external_write_attempted && !state.destination_verified
+} else if (state.turn_completed_seen) {
+  status = "protocol_complete"
+  safeToContinue = true
+  syntheticCompletion = false
+  requiresReconciliation = false
+} else if (semanticallySettled && writeSatisfied) {
+  status = "verified_completion_without_terminal_event"
+  safeToContinue = true
+  syntheticCompletion = true
+  requiresReconciliation = false
+} else if (state.external_write_attempted && !state.destination_verified) {
+  status = "write_reconciliation_required"
+  safeToContinue = false
+  syntheticCompletion = false
+  requiresReconciliation = true
+} else {
+  status = "terminal_state_not_proven"
+  safeToContinue = false
+  syntheticCompletion = false
+  requiresReconciliation = false
+}
+
+const report = {
+  schema_version: 1,
+  observed_at: nowIso(),
+  turn_id: snapshot.turn_id,
+  parent_turn_id: state.parent_turn_id,
+  expected_parent_turn_id: state.expected_parent_turn_id,
+  status,
+  safe_to_continue: safeToContinue,
+  synthetic_local_completion: syntheticCompletion,
+  requires_destination_reconciliation: requiresReconciliation,
+  automatic_retry_allowed: false,
+  minimum_quiet_seconds: minimumQuietSeconds,
+  checks: {
+    parent_turn_lineage_matched: lineageMatched,
+    final_assistant_output_seen: state.final_assistant_output_seen,
+    tool_results_matched: toolResultsMatched,
+    outstanding_obligations: obligations,
+    artifact_satisfied: artifactSatisfied,
+    external_write_satisfied: writeSatisfied,
+    quiet_period_satisfied: state.seconds_since_last_event >= minimumQuietSeconds,
+  },
+  snapshot_sha256: sha256(JSON.stringify(snapshot)),
+  remediation:
+    status === "turn_lineage_mismatch"
+      ? "Quarantine the nested result, preserve both turn identifiers, and do not use it to authorize completion or an external write. Reconcile any attempted write before replay."
+      : status === "write_reconciliation_required"
+        ? "Read the destination using the original operation identifier before any replay."
+        : status === "terminal_state_not_proven"
+          ? "Keep the turn bounded and collect another snapshot; do not create a replacement turn or retry external work."
+          : null,
+}
+
+const directory = path.resolve(args["evidence-dir"] || path.join(stateRoot(args), "appserver-turn-completion"))
+const evidenceFile = path.join(directory, `${report.observed_at.replace(/[:.]/g, "-")}-${snapshot.turn_id.replace(/[^A-Za-z0-9_.-]/g, "_")}.json`)
+writeJsonAtomic(evidenceFile, report)
+
+const result = { ...report, evidence_file: evidenceFile }
+if (args.json) console.log(JSON.stringify(result))
+else {
+  console.log(`Codex app-server turn ${snapshot.turn_id}: ${status}`)
+  console.log(`Evidence: ${evidenceFile}`)
+  if (!safeToContinue) console.error(report.remediation || "Completion is not proven")
+}
+
+process.exit(safeToContinue ? 0 : 75)