M5 W6: production security, recovery, and observability controls

Dockerfile

@@ -1,17 +1,28 @@
-# Deployed runtime image (P4). Builds the API + Twilio voice server.
-FROM python:3.12-slim
+# Pinned multi-arch Docker Official Image index for 3.12.10 slim-bookworm.
+FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
 
 WORKDIR /app
 COPY pyproject.toml ./
 COPY src ./src
 COPY data ./data
-RUN pip install --no-cache-dir -e ".[serve]"
+COPY config ./config
+COPY scripts ./scripts
+RUN pip install --no-cache-dir ".[serve,trace,voice,llm]" \
+    && groupadd --gid 10001 sku \
+    && useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin sku \
+    && mkdir -p /var/lib/sku-resolver \
+    && chown -R 10001:10001 /var/lib/sku-resolver
 
 # Runtime config via env (see src/runtime/config.py). Provide a real
 # SKU_SESSION_SECRET and SKU_WEBHOOK_SECRET in production; set
 # SKU_LLM_PROVIDER + the matching *_API_KEY to enable the LLM seams; set
 # TWILIO_* / ASSEMBLYAI_API_KEY for live voice.
-ENV PORT=8000
+ENV PORT=8000 \
+    PYTHONPATH=/app/src \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+VOLUME ["/var/lib/sku-resolver"]
 EXPOSE 8000
+USER 10001:10001
 
-CMD ["sh", "-c", "uvicorn runtime.app:create_app --factory --host 0.0.0.0 --port ${PORT}"]
+CMD ["uvicorn", "runtime.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]

data/m5_legacy_schema_lock.json

@@ -26,11 +26,13 @@
       }
     },
     "quote_store": {
-      "schema_sha256": "b78342c2f525a361338c7fb82102650af72229aeb2bff51e17c2ced7ec1fd4a8",
+      "schema_sha256": "2131cb30d2ce8d4b717457d8260408e0378059d894bb8f70e57d966824594244",
       "tables": [
         "artifact_guard_events",
         "artifact_jobs",
         "capability_access_events",
+        "data_legal_holds",
+        "deletion_audit",
         "delivery_jobs",
         "issuance_events",
         "provider_callback_events",
@@ -44,12 +46,15 @@
         "quote_schema",
         "quote_seq",
         "quotes",
+        "retention_context",
         "worker_heartbeats"
       ],
       "row_counts": {
         "artifact_guard_events": 0,
         "artifact_jobs": 0,
         "capability_access_events": 0,
+        "data_legal_holds": 0,
+        "deletion_audit": 0,
         "delivery_jobs": 0,
         "issuance_events": 0,
         "provider_callback_events": 0,
@@ -63,6 +68,7 @@
         "quote_schema": 1,
         "quote_seq": 1,
         "quotes": 1,
+        "retention_context": 1,
         "worker_heartbeats": 0
       },
       "known_gaps": [
@@ -72,5 +78,5 @@
       ]
     }
   },
-  "combined_sha256": "151432b988d1ace9cd6cd2af51039246f68d33f59a8977032cc678b808f592fb"
+  "combined_sha256": "a5b47bdf7a6faf6a95598d204def3997ad3ea63da82c1ab6e8f396fc93c6bc8b"
 }

deploy/processes.json

@@ -0,0 +1,15 @@
+{
+  "schema_version": 1,
+  "api": [
+    "uvicorn", "runtime.app:create_app", "--factory", "--host", "0.0.0.0",
+    "--port", "8000", "--no-access-log"
+  ],
+  "worker": [
+    "python", "scripts/quote_worker.py", "run", "--max-poll-seconds", "5",
+    "--lease-seconds", "30"
+  ],
+  "pre_stop": [
+    "POST", "/internal/drain"
+  ],
+  "state_mount": "/var/lib/sku-resolver"
+}

docs/DECISION_LOG.md

@@ -1610,3 +1610,57 @@ recipient allowlists, exact attachments/link-only SMS, signed callbacks,
 heartbeats, bounded polling, graceful drain, and operator inspect/retry/
 quarantine/revoke controls. **Status:** implemented and verified offline;
 credentialed staging sends and callbacks remain under P04.
+
+## 2026-07-11 — D22: production is one encrypted transactional state volume and two processes
+
+**Decision.** M5 production is one API process plus one worker process sharing a
+single local SQLite database on one encrypted durable volume. Quote, outbox,
+capability, provider, authoritative conversation cursor, and call-completion
+state therefore participate in one backup and one tenant-purge boundary. Network
+filesystems, in-memory stores, horizontal API/worker replicas, split quote/call
+databases, simulated providers, fixture sources, example origins, insecure
+secrets, stale schemas, and a missing worker heartbeat all fail startup.
+
+The container is digest-pinned, non-root, has explicit API and worker commands,
+disables access logs that would expose capability paths, and mounts all mutable
+state outside the image. The public health endpoint proves liveness only. The
+private readiness endpoint is token-gated and returns a generic state; detailed
+failure reasons stay local. A private drain endpoint rejects new work before a
+restart or migration.
+
+**Why.** SQLite is appropriate for the locked single-host topology only when its
+transaction and recovery boundary is not accidentally split across processes or
+volumes. Readiness is a safety gate, not a public diagnostic oracle.
+
+**Outcome.** WAL, foreign keys, busy timeout, full synchronization, integrity
+checks, schema writer locking, disk headroom, source freshness, worker heartbeat,
+provider, backup, encryption, retention, and PostHog/OTLP checks are executable.
+**Status:** implementation locked; production remains blocked on policy and
+credential evidence.
+
+## 2026-07-11 — D23: encrypted restore proof and PII-free PostHog are operational controls, not truth stores
+
+**Decision.** Online SQLite snapshots are encrypted with AES-256-GCM under a
+versioned key ID. The plaintext envelope contains table names, counts, and keyed
+aggregate digests—not quote, account, caller, provider, or capability
+identities. A backup is considered healthy only after a clean-host restore
+matches the encrypted database hash, integrity check, schemas, artifact set,
+conversation cursor set, and all transactional rows. Key rotation keeps old keys
+until their backup horizon expires.
+
+Tenant deletion is a single-database transaction. An active legal hold blocks
+it; an approved purge temporarily opens the immutable-row deletion gate, removes
+tenant quote/conversation/provider/capability state, closes the gate, and retains
+only a keyed tenant pseudonym plus deletion counts and approved actor/reason.
+
+OpenTelemetry remains the trace contract. PostHog is the sole production event/
+trace destination and receives only allowlisted scalar properties under a shared
+pseudonymous correlation ID. Transcript text, account/contact values, quote
+numbers/tokens, documents, and secrets are not exported. Event export is bounded,
+asynchronous, and fail-open, so telemetry cannot change a spoken or issued result.
+Phoenix remains an optional W7 local comparison, not a production dependency.
+
+**Outcome.** Backup/restore and retention CLIs, legal-hold tests, generic
+readiness, PII-free journey events, worker spans, alert thresholds, and the M5
+operations runbook are implemented. **Status:** code verified offline; P06/P07
+policy, clean-host container evidence, and live sink receipts remain external.

docs/M5_BUILD_JOURNAL.md

@@ -774,3 +774,51 @@ simulation refusal tests pass.
 Twilio/SendGrid credentials, staging recipient allowlists, one allowlisted email
 and SMS, provider IDs, signed delivered/undelivered callbacks, and downloaded
 artifact hash receipts. No production or Vercel deployment was attempted.
+
+## 2026-07-11 — Session 12: W6 production security, recovery, and observability controls
+
+Added a fail-closed production validator and private readiness surface. It rejects
+every development/example secret or origin, fixture/synthetic/ephemeral source,
+split or stale database, stale inventory/price source, missing live provider,
+simulated delivery, stale worker heartbeat, stale restore receipt, low disk,
+inconsistent ElevenLabs/application retention, missing encryption/key IDs, and
+missing PostHog/OTLP configuration. Existing P01–P16 policy blockers remain
+independent, so operationally healthy fixtures cannot turn production green.
+
+Hardened SQLite with WAL, foreign keys, five-second busy timeout, full sync,
+integrity checks, and a writer lock across migrations. Quote schema v5 adds legal
+holds, deletion audit, and a transaction-only immutable-row purge gate. The
+production topology requires quote and conversation schemas in one database so
+tenant offboarding removes both in one transaction.
+
+Added AES-256-GCM online backup, key-ID rotation support, clean-host restore and
+semantic verification, plus backup and retention CLIs. Backup envelopes expose no
+quote/caller/provider identities. Restore proves exact encrypted bytes, schemas,
+row sets, guarded artifact hashes, and conversation cursors before writing a
+fresh readiness marker.
+
+Added process drain, source-aware application rate limits, generic `/healthz`
+and private `/readyz`, a pinned non-root image, explicit API/worker process
+commands, durable state mount, and disabled capability-path access logs. The M5
+runbook covers parity, queue/provider, ambiguous lease, contact, invalid-reference,
+disk, backup/restore, key rotation, active-call restart, tenant offboarding, and
+rollback incidents.
+
+OpenTelemetry remains the tracing contract. A bounded asynchronous PostHog sink
+emits only allowlisted pseudonymous journey events, and worker stages emit duration
+and outcome spans. Alert evaluation pages immediately on parity or queue-SLO
+breach and covers provider outage, stuck lease, invalid-reference surge, disk,
+worker, and backup/restore signals.
+
+**Verification:** 932 tests collected and full pytest exits zero; ruff is clean;
+configured mypy reports no issues across 89 source files. Backup tamper/restore,
+legal hold/purge, SQLite posture, policy-vs-operational readiness, private drain,
+rate limiting, PostHog redaction/nonblocking behavior, alert thresholds, and
+static non-root image/process contracts pass.
+
+**External W6 proof still required:** approved P06 retention/encryption/key/
+ElevenLabs deletion policy; approved P07 destination/RPO/RTO/owner/drill policy;
+Docker-capable build and non-root restart receipt; encrypted-volume attestation;
+clean-host measured backup/restore drill; and live PostHog OTLP/event receipts.
+Docker is unavailable on this host. No production or Vercel deployment was
+attempted.

docs/M5_OPERATIONS_RUNBOOK.md

@@ -0,0 +1,79 @@
+# M5 production operations runbook
+
+This runbook is executable only after P02–P15 are locked for the tenant. Never
+paste a secret, destination, transcript, account number, quote capability token,
+or document content into a ticket, alert, terminal transcript, or PostHog.
+
+## Start, drain, and restart
+
+- Start the worker command from `deploy/processes.json` first and wait for a
+  fresh durable heartbeat.
+- Start the API command. `/healthz` proves only process liveness. Query `/readyz`
+  from the private network with `X-Internal-Ready-Token`; do not expose it at the
+  public edge.
+- Before restart or migration, call `POST /internal/drain` with the internal
+  token. Wait for `active=0`, stop the API, take an encrypted backup, then restart.
+  If an active call cannot drain, leave the version in service and escalate; do
+  not move its conversation cursor between processes.
+
+## Quote parity incident
+
+Page immediately. Confirm the artifact job is quarantined and no delivery job
+crossed the provider boundary. Preserve only the correlation ID, renderer/guard
+versions, and hashes. Reproduce with the frozen `QuoteDocument`; never deliver a
+manually edited PDF. Fix, run the parity battery, then use the operator retry.
+
+## Queue SLO, stuck lease, or provider outage
+
+- Inspect with `python scripts/quote_worker.py inspect`.
+- For an expired artifact or pre-send lease known not accepted, use the scoped
+  `retry` command. For `sending`, `manual_review`, or ambiguous network loss,
+  inspect the provider by stable delivery key before any retry.
+- Quarantine poison artifacts so later quotes continue. During provider outage,
+  preserve retry backoff and recipient allowlists; never switch destinations.
+
+## Missing/changed contact
+
+`needs_reauthorization` is terminal for the frozen assent. Correct the contact of
+record, reverify the caller, and obtain a new delivery assent (and quote revision
+when the frozen contract changes). Operators may not retarget a job.
+
+## Invalid-reference or link flood
+
+Keep refusals neutral. Confirm application and edge rate limits, inspect only
+pseudonymous source/tenant counters, rotate or revoke affected capability links,
+and avoid confirming whether a quote exists.
+
+## Disk pressure
+
+Drain API and worker before the SQLite volume reaches the configured floor.
+Preserve WAL/SHM files with the database. Expand the encrypted local volume or
+restore to a larger clean volume; never move live SQLite onto a network filesystem.
+
+## Backup, restore, and key rotation
+
+- Create: `python scripts/m5_backup.py create BACKUP_PATH`.
+- Restore drill: `python scripts/m5_backup.py restore-verify BACKUP_PATH EMPTY_DIR
+  --status-path STATUS_PATH`.
+- A backup is healthy only after clean-host restore verifies integrity, schemas,
+  rows, quote artifacts/hashes, job/provider state, capabilities, conversation
+  cursors, and completion events within the approved P07 RPO/RTO.
+- Add the new key ID and bytes, make it active, create and restore-verify a new
+  backup, then retire the old key only after its retention horizon. Never rewrite
+  an old envelope under a new key ID.
+
+## Tenant offboarding, legal hold, and deletion
+
+P06 supplies the approved actor, reason, retention horizon, and ElevenLabs
+audio/transcript deletion receipt. An active legal hold blocks deletion. With no
+hold, run the transactional tenant purge against the single state database; keep
+only its keyed tenant pseudonym and deletion counts. Delete provider/platform data
+under their approved APIs and attach only receipt IDs.
+
+## Rollback
+
+Rollback application and worker together to a schema-compatible image. Never
+downgrade across an unknown schema. For hosted voice, use the W4 pinned
+compensating rollback and verify the existing phone/SIP assignment. Re-run
+`/readyz`, a synthetic inbound call, a guarded quote, and a clean restore drill.
+

scripts/m5_backup.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Encrypted online backup and clean-host restore verification CLI."""
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO / 'src'))
+
+from runtime.backup import create_backup, restore_and_verify  # noqa: E402
+
+
+def _keys() -> dict[str, bytes]:
+    try:
+        raw = json.loads(os.environ.get('SKU_BACKUP_KEYS_JSON', '{}'))
+        keys = {str(name): base64.b64decode(value, validate=True)
+                for name, value in raw.items()}
+    except Exception as exc:
+        raise SystemExit('SKU_BACKUP_KEYS_JSON is invalid') from exc
+    if not keys or any(len(value) != 32 for value in keys.values()):
+        raise SystemExit('at least one 32-byte backup key is required')
+    return keys
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest='command', required=True)
+    create = sub.add_parser('create')
+    create.add_argument('output')
+    restore = sub.add_parser('restore-verify')
+    restore.add_argument('backup')
+    restore.add_argument('destination')
+    restore.add_argument('--status-path', required=True)
+    args = parser.parse_args()
+    keys = _keys()
+    if args.command == 'create':
+        active = os.environ.get('SKU_BACKUP_ACTIVE_KEY_ID', '')
+        if active not in keys:
+            raise SystemExit('SKU_BACKUP_ACTIVE_KEY_ID is unavailable')
+        quote = os.environ.get('SKU_QUOTE_STORE_DB', '')
+        conversation = os.environ.get('SKU_CONVERSATION_STORE_DB', '')
+        if not quote or not conversation:
+            raise SystemExit('durable database paths are required')
+        sources = {'state': quote}
+        if Path(quote).resolve() != Path(conversation).resolve():
+            sources['conversations'] = conversation
+        receipt = create_backup(
+            sources=sources, output=args.output, key=keys[active], key_id=active)
+    else:
+        receipt = restore_and_verify(
+            backup=args.backup, destination=args.destination, keys=keys,
+            status_path=args.status_path)
+    print(json.dumps(receipt, sort_keys=True))
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())