From a4ddb5ed3f681b914a61367593187bc5fa018c55 Mon Sep 17 00:00:00 2001 From: Owie Schon Date: Sat, 11 Jul 2026 09:33:33 -0400 Subject: [PATCH 1/3] fix: key document parity and repair quote outbox --- data/m5_baseline_defects.json | 2 -- src/gateway/orchestrator.py | 1 + src/quoting/document_guard.py | 24 +++++++++++++++++++++--- src/quoting/store.py | 6 ++++++ tests/test_m5_baseline_defects.py | 16 ++++++---------- tests/test_quote_renderers_golden.py | 2 ++ 6 files changed, 36 insertions(+), 15 deletions(-) diff --git a/data/m5_baseline_defects.json b/data/m5_baseline_defects.json index e4a86b1..188596a 100644 --- a/data/m5_baseline_defects.json +++ b/data/m5_baseline_defects.json @@ -3,9 +3,7 @@ "baseline_commit": "8001643fe503254ed9f12f0996894a49efe52342", "contract": "A probe passes only when its positive control works and the named baseline defect is behaviorally reproduced.", "defects": [ - {"id": "M5-DEF-07", "owner_wave": "W3", "test": "tests/test_m5_baseline_defects.py::test_defect_retry_does_not_repair_missing_queue_row", "expected_reason": "idempotent_retry_leaves_orphan_quote"}, {"id": "M5-DEF-08", "owner_wave": "W5", "test": "tests/test_m5_baseline_defects.py::test_defect_queue_claim_is_not_exclusive", "expected_reason": "same_pending_job_claimed_twice"}, - {"id": "M5-DEF-09", "owner_wave": "W3", "test": "tests/test_m5_baseline_defects.py::test_defect_document_guard_accepts_cross_line_quantity_swap", "expected_reason": "flat_fact_set_loses_line_association"}, {"id": "M5-DEF-11", "owner_wave": "W7", "test": "tests/test_m5_baseline_defects.py::test_defect_latency_harness_rejects_chunked_readback", "expected_reason": "latency_harness_expects_immediate_quote_number"} ] } diff --git a/src/gateway/orchestrator.py b/src/gateway/orchestrator.py index 08aebe3..9b097a3 100644 --- a/src/gateway/orchestrator.py +++ b/src/gateway/orchestrator.py @@ -1180,6 +1180,7 @@ def _issue_quote(self, sid, token, order) -> QuoteDocument: if not is_new: existing = self.quote_store.get(quote_number, revision) if existing is not None: + self.quote_store.repair_outbox(existing) return existing else: quote_number, prior_revision = revision_context diff --git a/src/quoting/document_guard.py b/src/quoting/document_guard.py index 51dd0fd..7693ab9 100644 --- a/src/quoting/document_guard.py +++ b/src/quoting/document_guard.py @@ -32,6 +32,13 @@ Fact = tuple[str, str] +def _line_fact(line_no, sku, qty, unit_price, extended_price, + ship_date) -> Fact: + return ('line', '|'.join((str(line_no), str(sku), str(qty), + _money(unit_price), _money(extended_price), + str(ship_date or 'TBD')[:10]))) + + class DocumentParityViolation(ValueError): """A rendered document's binding facts do not exactly match the QuoteDocument that produced it. Fail closed: never deliver.""" @@ -52,6 +59,10 @@ def document_facts(document: QuoteDocument) -> set[Fact]: ('freight_status', document.totals.freight_status), } for line in document.lines: + facts.add(_line_fact( + line.line_no, line.sku, line.requested_qty, line.unit_price, + line.extended_price, + line.ship_date.date().isoformat() if line.ship_date else 'TBD')) facts.add(('sku', line.sku)) facts.add(('qty', str(line.requested_qty))) facts.add(('unit_price', _money(line.unit_price))) @@ -111,13 +122,15 @@ def extract_pdf_facts(pdf_bytes: bytes) -> set[Fact]: row = lines[j:j + n] if not row[0].isdigit(): break - _, sku, _desc, qty, unit_price, ext_price, ship_date = row + line_no, sku, _desc, qty, unit_price, ext_price, ship_date = row + facts.add(_line_fact(line_no, sku, qty, + um.group(1) if (um := _MONEY_RE.match(unit_price)) else '', + em.group(1) if (em := _MONEY_RE.match(ext_price)) else '', + ship_date)) facts.add(('sku', sku)) facts.add(('qty', qty)) - um = _MONEY_RE.match(unit_price) if um: facts.add(('unit_price', um.group(1))) - em = _MONEY_RE.match(ext_price) if em: facts.add(('extended_price', em.group(1))) if _DATE_RE.match(ship_date): @@ -166,6 +179,11 @@ def extract_xlsx_facts(xlsx_bytes: bytes) -> set[Fact]: facts.add(('unit_price', _money(ws.cell(row=r, column=6).value))) facts.add(('extended_price', _money(ws.cell(row=r, column=7).value))) ship_date = ws.cell(row=r, column=8).value + facts.add(_line_fact( + line_no, ws.cell(row=r, column=2).value, + int(ws.cell(row=r, column=4).value), + ws.cell(row=r, column=6).value, + ws.cell(row=r, column=7).value, ship_date or 'TBD')) if ship_date: facts.add(('ship_date', str(ship_date)[:10])) r += 1 diff --git a/src/quoting/store.py b/src/quoting/store.py index 263f983..2574e39 100644 --- a/src/quoting/store.py +++ b/src/quoting/store.py @@ -90,6 +90,8 @@ def status_of(self, quote_number: str, revision: int) -> str | None: ... def enqueue(self, quote_number: str, revision: int) -> None: ... + def repair_outbox(self, document: QuoteDocument) -> None: ... + def claim_pending(self) -> QueueItem | None: ... def mark_delivered(self, quote_number: str, revision: int) -> None: ... @@ -254,6 +256,10 @@ def enqueue(self, quote_number: str, revision: int) -> None: (quote_number, revision)) self._conn.commit() + def repair_outbox(self, document: QuoteDocument) -> None: + """Idempotently restore the required artifact queue row for a quote.""" + self.enqueue(document.header.quote_number, document.header.revision) + def claim_pending(self) -> QueueItem | None: """Claim the oldest pending row for the single M5 worker.""" row = self._conn.execute( diff --git a/tests/test_m5_baseline_defects.py b/tests/test_m5_baseline_defects.py index f7e1afd..39723f7 100644 --- a/tests/test_m5_baseline_defects.py +++ b/tests/test_m5_baseline_defects.py @@ -179,7 +179,7 @@ def test_agent_turn_priced_order_line_carries_structural_provenance(tmp_path, mo assert payload["surfaced_values"]["order_lines"][0]["unit_price"] is not None -def test_defect_retry_does_not_repair_missing_queue_row(tmp_path): +def test_idempotent_retry_repairs_missing_outbox_row(tmp_path): gateway, _, _, caller, token, issued = _issue_one(tmp_path, "orphan-retry") number = issued.meta["quote_number"] assert gateway.quote_store.queue_item(number, 1) is not None @@ -190,10 +190,7 @@ def test_defect_retry_does_not_repair_missing_queue_row(tmp_path): gateway.converse(caller, token, "place the order") retried = gateway.converse(caller, token, "confirm the order") assert retried.meta["quote_number"] == number - assert gateway.quote_store.queue_item(number, 1) is None, ( - "idempotent_retry_leaves_orphan_quote" - ) - _observed("idempotent_retry_leaves_orphan_quote") + assert gateway.quote_store.queue_item(number, 1) is not None def test_defect_queue_claim_is_not_exclusive(tmp_path): @@ -209,7 +206,7 @@ def test_defect_queue_claim_is_not_exclusive(tmp_path): _observed("same_pending_job_claimed_twice") -def test_defect_document_guard_accepts_cross_line_quantity_swap(tmp_path): +def test_document_guard_rejects_cross_line_quantity_swap(tmp_path): from quoting.document_guard import DocumentParityViolation, verify_rendered_document from quoting.render_pdf import render_pdf from quoting.render_xlsx import render_xlsx @@ -228,9 +225,9 @@ def test_defect_document_guard_accepts_cross_line_quantity_swap(tmp_path): object.__setattr__(bad, 'lines', ( dataclasses.replace(document.lines[0], requested_qty=2), dataclasses.replace(document.lines[1], requested_qty=10))) - verify_rendered_document( - document, pdf_bytes=render_pdf(bad), xlsx_bytes=render_xlsx(bad) - ) + with pytest.raises(DocumentParityViolation): + verify_rendered_document( + document, pdf_bytes=render_pdf(bad), xlsx_bytes=render_xlsx(bad)) obvious = object.__new__(type(document)) for field in dataclasses.fields(document): object.__setattr__(obvious, field.name, getattr(document, field.name)) @@ -241,7 +238,6 @@ def test_defect_document_guard_accepts_cross_line_quantity_swap(tmp_path): verify_rendered_document( document, pdf_bytes=render_pdf(obvious), xlsx_bytes=render_xlsx(obvious) ) - _observed("flat_fact_set_loses_line_association") def test_revision_lookup_is_tenant_and_account_scoped_with_neutral_denial(tmp_path): diff --git a/tests/test_quote_renderers_golden.py b/tests/test_quote_renderers_golden.py index b2d19e3..5ff6184 100644 --- a/tests/test_quote_renderers_golden.py +++ b/tests/test_quote_renderers_golden.py @@ -109,6 +109,8 @@ def test_xlsx_structural_golden_matches_fixed_expected_fact_set(): ('extended_price', '299.38'), ('ship_date', '2026-07-12'), ('subtotal', '549.28'), ('tax_status', 'not_included'), ('freight_status', 'quoted_separately'), + ('line', '1|K5-24SBC|10|24.99|249.90|2026-07-11'), + ('line', '2|BH6-36SBC|2|149.69|299.38|2026-07-12'), } assert extract_xlsx_facts(xlsx_bytes) == expected From 24cd4b95cb2f7a5b256c8f7a96bf3baf0a23f931 Mon Sep 17 00:00:00 2001 From: Owie Schon Date: Sat, 11 Jul 2026 10:22:00 -0400 Subject: [PATCH 2/3] feat(m5): make quote issuance and artifacts atomic --- data/m5_legacy_schema_lock.json | 30 +- docs/M5_PRODUCTION_BUILD_E2E_EVAL_PLAN.md | 24 + scripts/m5_legacy_inventory.py | 6 +- src/gateway/conversation_snapshot.py | 27 +- src/gateway/conversation_store.py | 4 + src/gateway/orchestrator.py | 150 ++-- src/gateway/order.py | 5 +- src/quoting/__init__.py | 5 +- src/quoting/capabilities.py | 174 +++++ src/quoting/delivery.py | 37 - src/quoting/display.py | 27 + src/quoting/document_guard.py | 404 ++++++----- src/quoting/models.py | 13 + src/quoting/projection.py | 27 +- src/quoting/render_pdf.py | 163 +++-- src/quoting/render_xlsx.py | 172 +++-- src/quoting/store.py | 822 +++++++++++++++++++++- src/quoting/worker.py | 67 +- src/runtime/app.py | 35 +- src/runtime/config.py | 28 +- tests/gateway_fixtures.py | 18 +- tests/test_quote_artifact_layout.py | 120 ++++ tests/test_quote_artifacts.py | 175 +++++ tests/test_quote_authoritative_lineage.py | 66 ++ tests/test_quote_delivery.py | 42 +- tests/test_quote_migration.py | 121 ++++ tests/test_quote_renderers_golden.py | 121 +++- tests/test_quote_retrieval_route.py | 48 ++ tests/test_quote_store.py | 306 +++++++- tests/test_quote_worker.py | 73 +- tests/test_quoting_purity.py | 21 +- 31 files changed, 2788 insertions(+), 543 deletions(-) create mode 100644 src/quoting/capabilities.py create mode 100644 src/quoting/display.py create mode 100644 tests/test_quote_artifact_layout.py create mode 100644 tests/test_quote_artifacts.py create mode 100644 tests/test_quote_authoritative_lineage.py create mode 100644 tests/test_quote_migration.py create mode 100644 tests/test_quote_retrieval_route.py diff --git a/data/m5_legacy_schema_lock.json b/data/m5_legacy_schema_lock.json index ff5dcf4..1dd8f8b 100644 --- a/data/m5_legacy_schema_lock.json +++ b/data/m5_legacy_schema_lock.json @@ -26,27 +26,47 @@ } }, "quote_store": { - "schema_sha256": "6d2175921c806746d4f4232e99b05cccd1aceac385656c25af3cc83f1d3d3b14", + "schema_sha256": "d26de5eaaf158fd47094f886b9b94a94c21578b7f5bdf80a0cdeb22f3410a6d8", "tables": [ + "artifact_guard_events", + "artifact_jobs", + "capability_access_events", + "delivery_jobs", + "issuance_events", + "quote_artifacts", + "quote_capabilities", "quote_idempotency", + "quote_issue_keys", + "quote_number_allocations", "quote_queue", "quote_revision_seq", + "quote_schema", "quote_seq", "quotes" ], "row_counts": { + "artifact_guard_events": 0, + "artifact_jobs": 0, + "capability_access_events": 0, + "delivery_jobs": 0, + "issuance_events": 0, + "quote_artifacts": 0, + "quote_capabilities": 0, "quote_idempotency": 1, + "quote_issue_keys": 0, + "quote_number_allocations": 0, "quote_queue": 1, "quote_revision_seq": 0, + "quote_schema": 1, "quote_seq": 1, "quotes": 1 }, "known_gaps": [ - "idempotency is not tenant-bound", - "reserve/save/enqueue are separate commits", - "queue rows have no lease or timestamps" + "pre-W3 compatibility tables remain until migration evidence is retained", + "delivery job leases and authenticated call completion land in W5", + "legacy rows without assent or artifact proof remain explicitly held" ] } }, - "combined_sha256": "6471bcd46b9fc33be420adf67e7183cb997364d36d8d94ec87113c6a32612de0" + "combined_sha256": "1a6d06e44585df041fe2fdb6b4869c4446f1309eaa9ae512dfeb2bf90efc55db" } diff --git a/docs/M5_PRODUCTION_BUILD_E2E_EVAL_PLAN.md b/docs/M5_PRODUCTION_BUILD_E2E_EVAL_PLAN.md index 8e92791..30895a0 100644 --- a/docs/M5_PRODUCTION_BUILD_E2E_EVAL_PLAN.md +++ b/docs/M5_PRODUCTION_BUILD_E2E_EVAL_PLAN.md @@ -721,6 +721,17 @@ Build: depth/oldest age/lease/retry, render/guard/provider duration, parity/contact/ provider failures, hangup-to-accept, terminal callback status, link refusal, DB busy/disk, worker heartbeat, backup age/restore result, and rate-limit events. +- Use OpenTelemetry as the vendor-neutral trace contract and PostHog as the + single production destination for tracing/AI observability plus pseudonymous + product/voice journey events and funnels (call connected, clarification, + readback, assent, hangup, provider acceptance, download). PostHog is not the + source of truth for quote facts, deterministic safety evals, or transactional + state. A shared pseudonymous correlation ID must join events, traces, eval + runs, and the latency report without exporting transcript text, + account/contact values, quote capability tokens, or document contents. Arize + Phoenix is an optional local W7 bake-off only; it does not become a second + required production sink unless measured workflow value justifies the extra + retention, privacy, correlation, and operational surface. - Add alerts/runbooks for parity incident, queue SLO, provider outage, stuck lease, missing contact, invalid-reference attack, disk pressure, failed backup/restore, key rotation/revocation, active-call restart, tenant offboarding, and rollback. @@ -763,6 +774,19 @@ DoD: audit, independent resolution eval, and runtime readiness remain green. - The latency harness drives the actual multi-turn readback/assent protocol and reports workload, sample count, p50/p95/p99/max, commit, schema, and environment. +- The same run exports stage spans and corresponding pseudonymous journey events + to a local PostHog-compatible capture sink. Its evidence bundle proves + trace/event/eval/report correlation and includes + STT endpointing/finalization, network/tool, resolver, optional query rewrite + and retrieval, LLM first-token, TTS first-audio, and player/telephony timing + when each stage exists. Instrumentation overhead and export failure are tested; + telemetry remains non-blocking and cannot alter the spoken or issued result. +- Query-rewrite model racing is evaluated only on a non-binding retrieval lane, + against a single-model and raw-query fallback baseline. The report includes + answer/retrieval quality, fallback rate, provider-outage behavior, cost, and + p50/p75/p95/p99—not latency alone. No fastest-response-wins policy may bind a + SKU, requested quantity, account, price, availability, quote number, or + document field. - Twin per-turn order operations meet p95 <=800 ms; transactional issuance meets p95 <=300 ms; no renderer/provider call appears in the synchronous trace. - At P12 steady plus 2× burst, hangup-to-provider-acceptance p95 <=60 s, no orphan diff --git a/scripts/m5_legacy_inventory.py b/scripts/m5_legacy_inventory.py index 9db5dad..05db8e1 100755 --- a/scripts/m5_legacy_inventory.py +++ b/scripts/m5_legacy_inventory.py @@ -145,9 +145,9 @@ def compute() -> dict: "tables": quote_tables, "row_counts": _row_counts(store._conn, quote_tables), "known_gaps": [ - "idempotency is not tenant-bound", - "reserve/save/enqueue are separate commits", - "queue rows have no lease or timestamps", + "pre-W3 compatibility tables remain until migration evidence is retained", + "delivery job leases and authenticated call completion land in W5", + "legacy rows without assent or artifact proof remain explicitly held", ], }, } diff --git a/src/gateway/conversation_snapshot.py b/src/gateway/conversation_snapshot.py index 2ec93f4..2a321af 100644 --- a/src/gateway/conversation_snapshot.py +++ b/src/gateway/conversation_snapshot.py @@ -5,8 +5,6 @@ """ from __future__ import annotations -import hashlib -import json from dataclasses import asdict from typing import Any @@ -24,6 +22,7 @@ from gateway.order_parser import OrderClause from gateway.session import Session from quoting.models import DeliveryPreflight +from quoting.store import order_version_digest from sku_translator.quantity import RequestedQty SNAPSHOT_VERSION = 1 @@ -74,7 +73,9 @@ def dump_call(gateway, caller_id: str) -> dict[str, Any]: 'order': None if order is None else { 'lines': {key: _line(line) for key, line in order.state.lines.items()}, 'sequence': list(order.state.sequence), 'assented': order.state.assented, - 'assent_turn': order.state.assent_turn, 'next_id': order._next_id}, + 'assent_turn': order.state.assent_turn, + 'assent_occurred_at': order.state.assent_occurred_at, + 'next_id': order._next_id}, 'session': None if session is None else { 'session_id': session.session_id, 'channel_id': session.channel_id, 'created_at': session.created_at, 'last_active': session.last_active, @@ -133,6 +134,7 @@ def load_call(gateway, caller_id: str, value: dict[str, Any]) -> str | None: order.state.sequence = list(raw_order['sequence']) order.state.assented = raw_order['assented'] order.state.assent_turn = raw_order['assent_turn'] + order.state.assent_occurred_at = raw_order.get('assent_occurred_at') order._next_id = raw_order['next_id'] gateway._orders[caller_id] = order raw_session = value.get('session') @@ -204,11 +206,10 @@ def authoritative_events(before: dict[str, Any], after: dict[str, Any], else 'line_resolution') events.append((kind, {'line_id': line_id, 'state': state, 'sku': line.get('sku')})) - order_digest = hashlib.sha256(json.dumps( - [(line_id, new_lines[line_id].get('sku'), - new_lines[line_id].get('requested_qty')) - for line_id in new_order['sequence']], separators=(',', ':') - ).encode()).hexdigest() + order_digest = order_version_digest(tuple( + (line_id, new_lines[line_id].get('sku'), + new_lines[line_id].get('requested_qty')) + for line_id in new_order['sequence'])) readback_receipt_id = f'readback:{order_digest[:24]}' readback_completed = (not before.get('completed_order_readback') and after.get('completed_order_readback')) @@ -230,14 +231,20 @@ def authoritative_events(before: dict[str, Any], after: dict[str, Any], 'order_digest': order_digest, 'line_versions': list(new_order['sequence']), 'chunk_coverage': chunks, 'running_counts': running, - 'occurred_at': (after.get('session') or {}).get('last_active')})) + 'occurred_at': ( + new_order.get('assent_occurred_at') + if assent_granted and new_order.get('assent_occurred_at') is not None + else (after.get('session') or {}).get('last_active'))})) if assent_granted: events.append(('assent_receipt', { 'turn_id': turn_id, 'order_digest': order_digest, 'readback_receipt_id': readback_receipt_id, 'utterance_digest': text_digest, 'line_versions': list(new_order['sequence']), - 'occurred_at': (after.get('session') or {}).get('last_active')})) + 'occurred_at': ( + new_order.get('assent_occurred_at') + if new_order.get('assent_occurred_at') is not None + else (after.get('session') or {}).get('last_active'))})) return events diff --git a/src/gateway/conversation_store.py b/src/gateway/conversation_store.py index 78b866e..420140f 100644 --- a/src/gateway/conversation_store.py +++ b/src/gateway/conversation_store.py @@ -16,6 +16,10 @@ }) _LEGACY_QUOTE_TABLES = frozenset({ 'quote_idempotency', 'quote_queue', 'quote_revision_seq', 'quote_seq', 'quotes', + 'quote_number_allocations', 'quote_issue_keys', 'issuance_events', + 'artifact_jobs', 'delivery_jobs', 'quote_artifacts', + 'quote_capabilities', 'artifact_guard_events', 'quote_schema', + 'capability_access_events', }) diff --git a/src/gateway/orchestrator.py b/src/gateway/orchestrator.py index 9b097a3..02e5d49 100644 --- a/src/gateway/orchestrator.py +++ b/src/gateway/orchestrator.py @@ -72,7 +72,7 @@ QuoteTerms, default_valid_until, ) -from quoting.store import QuoteStore, order_content_hash +from quoting.store import QuoteStore, order_version_digest from quoting.totals import build_totals, extended_price from resolution import ResolutionService from sku_translator.quantity import RequestedQty, extract_requested_qty @@ -245,7 +245,8 @@ def _fault_escalation(self, session_id, token, exc) -> TurnResponse: # -- orchestration-backed turn (CONVERSATION_STATE_SPEC) ------------------- def converse(self, caller_id: str, token: str, text: str, *, - channel: Channel = Channel.TYPED) -> TurnResponse: + channel: Channel = Channel.TYPED, + turn_id: str | None = None) -> TurnResponse: """Caller-led, multi-part, closure-aware turn. REPLACES the fixed-sequence `turn()` on the /agent/turn path. Reuses the answer builders (so `surfaced` provenance and the authorization/lockout gates are preserved by @@ -259,6 +260,8 @@ def converse(self, caller_id: str, token: str, text: str, *, durable state. Fail-closed: an internal fault becomes a coherent escalation, never a 500 and NEVER a silent revert to legacy determinism.""" conv = self._conversations.get(caller_id) + authoritative_turn = turn_id or caller_id + utterance_digest = hashlib.sha256(text.encode()).hexdigest() if conv is None: conv = Conversation(horizons=self.disclosure_horizons) self._conversations[caller_id] = conv @@ -266,7 +269,9 @@ def converse(self, caller_id: str, token: str, text: str, *, order = self._orders.get(caller_id) if order is not None and order.active_lines and not order.is_assented: return self._with_order_disclosure(caller_id, self._decided( - self._converse_order_assent(caller_id, token, conv), conv, + self._converse_order_assent( + caller_id, token, conv, authoritative_turn, + utterance_digest), conv, move='order_completion_readback')) conv.note_completion_signal() # only the caller closes (inv 7) return self._with_order_disclosure(caller_id, self._decided(TurnResponse( @@ -275,7 +280,8 @@ def converse(self, caller_id: str, token: str, text: str, *, conv, move='close')) try: response = self._converse_dispatch( - caller_id, token, text, channel, conv) + caller_id, token, text, channel, conv, authoritative_turn, + utterance_digest) return self._with_order_disclosure(caller_id, response) except Exception as e: return self._fault_escalation(caller_id, token, e) @@ -324,7 +330,8 @@ def _order_for(self, caller_id: str) -> Order: self._orders[caller_id] = order return order - def _converse_dispatch(self, caller_id, token, text, channel, conv): + def _converse_dispatch(self, caller_id, token, text, channel, conv, + authoritative_turn, utterance_digest): pending_quote = self._pending_quote_reference.get(caller_id) if pending_quote is not None: suffix = pending_quote.rsplit('-', 1)[-1] @@ -400,7 +407,9 @@ def _converse_dispatch(self, caller_id, token, text, channel, conv): if caller_id in self._revision_context: self._pending_revision_assent.add(caller_id) return self._decided( - self._converse_order_assent(caller_id, token, conv), + self._converse_order_assent( + caller_id, token, conv, authoritative_turn, + utterance_digest), conv, move='order_assent') st = self.sessions.state_of(caller_id, token).value return self._decided(TurnResponse( @@ -409,7 +418,9 @@ def _converse_dispatch(self, caller_id, token, text, channel, conv): 'when it is correct.'), conv, move='order_readback_wait') if _is_order_assent_signal(text): return self._decided( - self._converse_order_assent(caller_id, token, conv), + self._converse_order_assent( + caller_id, token, conv, authoritative_turn, + utterance_digest), conv, move='order_assent') order = self._orders.get(caller_id) if order is not None: @@ -1049,7 +1060,9 @@ def _converse_withdraw_line(self, sid, token, line_id) -> TurnResponse: return TurnResponse(kind='order', session_state=st, text=f"Dropped the {line.sku} — anything else?") - def _converse_order_assent(self, sid, token, conv) -> TurnResponse: + def _converse_order_assent(self, sid, token, conv, + authoritative_turn=None, + utterance_digest=None) -> TurnResponse: """The last line of defense before a quote can exist (§1.5): a full-order readback and explicit caller assent. Attempts to price any still-CONFIRMED lines now (an account may have verified since they @@ -1128,7 +1141,9 @@ def _converse_order_assent(self, sid, token, conv) -> TurnResponse: text=f'Reading back the revised order: {lines_say}. ' 'Say confirm the quote when that is correct.') try: - order.grant_assent(readback_turn=sid) + order.grant_assent( + readback_turn=authoritative_turn or sid, + occurred_at=self.sessions._sessions[sid].last_active) except OrderNotIssuable: if own is None: return TurnResponse( @@ -1141,7 +1156,19 @@ def _converse_order_assent(self, sid, token, conv) -> TurnResponse: kind='order', session_state=st, text="A couple of lines still need to be sorted out before I " "can prepare the quote — let's finish those first.") - document = self._issue_quote(sid, token, order) + line_versions = [line.line_id for line in order.active_lines] + order_digest = order_version_digest(tuple( + (line.line_id, line.sku, line.requested_qty) + for line in order.active_lines)) + document = self._issue_quote(sid, token, order, assent_receipt={ + 'session_id': sid, + 'turn_id': authoritative_turn or sid, + 'order_digest': order_digest, + 'readback_receipt_id': f'readback:{order_digest[:24]}', + 'utterance_digest': utterance_digest or hashlib.sha256(b'').hexdigest(), + 'line_versions': line_versions, + 'occurred_at': order.state.assent_occurred_at, + }) self.journal.record(EventType.ORDER_ASSENT, sid, line_ids=[ln.line_id for ln in order.active_lines], quote_number=document.header.quote_number, @@ -1155,16 +1182,18 @@ def _converse_order_assent(self, sid, token, conv) -> TurnResponse: f"together and sent over. No sales order has been placed.", meta={'quote_number': document.header.quote_number}) - def _issue_quote(self, sid, token, order) -> QuoteDocument: + def _issue_quote(self, sid, token, order, *, assent_receipt) -> QuoteDocument: """Freeze the Order into a QuoteDocument (spec §3.1: "the caller never waits on a renderer" — freeze + number + journal is the ENTIRE in-call cost here; rendering/delivery, M5c/d, do not exist yet). Prices are read FRESH at this moment, never reused from an earlier confirm-time read — the same "read perishable facts at the moment of the binding action" discipline the disclosure gate already holds to. - Idempotent on (session_id, order content hash) via `quote_store. - reserve`: a retried assent for the SAME order content returns the - ALREADY-issued document rather than minting a second quote_number.""" + Idempotent on the tenant/account/session/order/action key: a retried + assent for the same order returns the already-issued immutable + document. Initial issuance and its lineage/outbox rows commit in one + store transaction; a failed transaction may consume a quote number + but can never reuse it.""" own = _verified_account(self.sessions, sid, token) if own is None: raise OrderNotIssuable( @@ -1172,25 +1201,19 @@ def _issue_quote(self, sid, token, order) -> QuoteDocument: now = self.now_fn() tenant_id = self.catalog.tenant_id() revision_context = self._revision_context.get(sid) + content_hash = order_version_digest(tuple( + (line.line_id, line.sku, line.requested_qty) + for line in order.active_lines)) if revision_context is None: - content_hash = order_content_hash( - tuple((ln.sku, ln.requested_qty) for ln in order.active_lines)) - quote_number, revision, is_new = self.quote_store.reserve( - tenant_id=tenant_id, session_id=sid, content_hash=content_hash) - if not is_new: - existing = self.quote_store.get(quote_number, revision) - if existing is not None: - self.quote_store.repair_outbox(existing) - return existing + quote_number = '' else: quote_number, prior_revision = revision_context prior = self.quote_store.get(quote_number, prior_revision) if prior is None or prior.header.account_id != own: raise OrderNotIssuable( 'revision account authorization changed before assent') - # Reserve only after every fresh price/availability read below. + # Commit only after every fresh price/availability read below. # Readback and non-assent turns therefore remain store-read-only. - revision = 0 lines = [] price_versions: set[str] = set() inventory_versions: set[str] = set() @@ -1234,42 +1257,59 @@ def _issue_quote(self, sid, token, order) -> QuoteDocument: ship_date=ship_dt, ship_date_as_of=inventory_as_of, resolution_source=dl.resolution_source, resolution_confidence=dl.resolution_confidence)) - if revision_context is not None: - revision = self.quote_store.reserve_revision(quote_number) - header = QuoteHeader( - tenant_id=tenant_id, quote_number=quote_number, revision=revision, - account_id=own, account_tier=self.account_tier_of(own), - created_at=now, valid_until=default_valid_until(now), - prepared_by='agent/gateway-v1', catalog_version=self.catalog_version) terms = QuoteTerms( validity_window_days=30, pricing_disclaimer='Prices subject to change; valid for the ' 'stated window.', tenant_terms_text='') customer = self.sessions.customer_db.by_number(own) - document = QuoteDocument(header=header, lines=tuple(lines), - totals=build_totals(tuple(lines)), terms=terms, - presentation=QuotePresentation( - seller_name='SKU Resolver Demo Seller', - seller_address='', branding_ref='default', - customer_display_name=(customer.name - if customer else own), - customer_account_id=own), - provenance=QuoteProvenance( - gateway_build_sha=getattr( - self, 'gateway_build_sha', 'test-build'), - renderer_schema_version='pdf-xlsx-v1', - guard_schema_version='document-guard-v1', - hosted_agent_version=getattr( - self, 'hosted_agent_version', 'local-test'), - catalog_version=self.catalog_version, - price_source_versions=tuple(sorted(price_versions)), - inventory_source_versions=tuple( - sorted(inventory_versions))), - delivery_preflight=self._delivery_preflight_by_call[sid]) - self.quote_store.save(document) - self.quote_store.enqueue(document.header.quote_number, - document.header.revision) + frozen_lines = tuple(lines) + preflight = self._delivery_preflight_by_call[sid] + + def build_document(number: str, document_revision: int) -> QuoteDocument: + header = QuoteHeader( + tenant_id=tenant_id, quote_number=number, + revision=document_revision, account_id=own, + account_tier=self.account_tier_of(own), created_at=now, + valid_until=default_valid_until(now), + prepared_by='agent/gateway-v1', + catalog_version=self.catalog_version) + return QuoteDocument( + header=header, lines=frozen_lines, + totals=build_totals(frozen_lines), terms=terms, + presentation=QuotePresentation( + seller_name='SKU Resolver Demo Seller', seller_address='', + branding_ref='default', + customer_display_name=(customer.name if customer else own), + customer_account_id=own), + provenance=QuoteProvenance( + gateway_build_sha=getattr( + self, 'gateway_build_sha', 'test-build'), + renderer_schema_version='pdf-xlsx-v1', + guard_schema_version='document-guard-v1', + hosted_agent_version=getattr( + self, 'hosted_agent_version', 'local-test'), + catalog_version=self.catalog_version, + price_source_versions=tuple(sorted(price_versions)), + inventory_source_versions=tuple( + sorted(inventory_versions))), + delivery_preflight=preflight) + + if revision_context is None: + document = self.quote_store.issue_atomic( + tenant_id=tenant_id, account_id=own, session_id=sid, + order_digest=content_hash, action='issue_quote', + assent_receipt=assent_receipt, + delivery_channel=preflight.channel, + document_factory=build_document) + else: + document = self.quote_store.revise_atomic( + tenant_id=tenant_id, account_id=own, + quote_number=quote_number, + expected_revision=prior_revision, session_id=sid, + order_digest=content_hash, assent_receipt=assent_receipt, + delivery_channel=preflight.channel, + document_factory=build_document) if revision_context is not None: self._revision_context.pop(sid, None) self._pending_revision_assent.discard(sid) diff --git a/src/gateway/order.py b/src/gateway/order.py index dd16ce0..f5ec907 100644 --- a/src/gateway/order.py +++ b/src/gateway/order.py @@ -128,6 +128,7 @@ class OrderState: sequence: list[str] = field(default_factory=list) # CURRENT line_ids, caller-facing order assented: bool = False assent_turn: str = '' + assent_occurred_at: float | None = None class Order: @@ -232,7 +233,8 @@ def transition(self, line_id: str, new_state: LineState) -> DraftLine: # -- full-order assent gate (§1.5) --------------------------------------- - def grant_assent(self, *, readback_turn: str = '') -> None: + def grant_assent(self, *, readback_turn: str = '', + occurred_at: float | None = None) -> None: """The last line of defense before a quote can exist: every active line must be PRICED (or already QUOTED, on a revision), and there must be at least one line. Fails loudly rather than let issuance @@ -245,6 +247,7 @@ def grant_assent(self, *, readback_turn: str = '') -> None: raise OrderNotIssuable(f'lines not ready for assent: {blockers}') self.state.assented = True self.state.assent_turn = readback_turn + self.state.assent_occurred_at = occurred_at @property def is_assented(self) -> bool: diff --git a/src/quoting/__init__.py b/src/quoting/__init__.py index e3053af..99adea6 100644 --- a/src/quoting/__init__.py +++ b/src/quoting/__init__.py @@ -11,8 +11,6 @@ Deliverer, DeliveryPayload, SimulatedDeliverer, - mint_link, - verify_link, ) from quoting.document_guard import ( DocumentParityViolation, @@ -41,6 +39,5 @@ 'render_pdf', 'render_xlsx', 'DocumentParityViolation', 'assert_document_parity', 'document_facts', 'extract_pdf_facts', 'extract_xlsx_facts', 'verify_rendered_document', - 'Deliverer', 'DeliveryPayload', 'SimulatedDeliverer', 'mint_link', - 'verify_link', 'QueueItem', + 'Deliverer', 'DeliveryPayload', 'SimulatedDeliverer', 'QueueItem', ] diff --git a/src/quoting/capabilities.py b/src/quoting/capabilities.py new file mode 100644 index 0000000..8822db2 --- /dev/null +++ b/src/quoting/capabilities.py @@ -0,0 +1,174 @@ +"""Opaque, tenant-bound capabilities for immutable guarded quote artifacts.""" +from __future__ import annotations + +import hashlib +import hmac +import re +import secrets +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime + +from quoting.models import QuoteDocument +from quoting.store import ArtifactRecord, CapabilityRecord, SqliteQuoteStore + +_TOKEN = re.compile(r'q1\.([A-Za-z0-9_-]{1,32})\.([A-Za-z0-9_-]{43})') +_FORMATS = frozenset({'pdf', 'xlsx'}) + + +@dataclass +class CapabilityKeyring: + keys: dict[str, dict[str, bytes]] + active_key_ids: dict[str, str] + + def __post_init__(self) -> None: + for tenant_id, key_id in self.active_key_ids.items(): + if not tenant_id or key_id not in self.keys.get(tenant_id, {}): + raise ValueError('every tenant requires a configured active key') + for tenant_keys in self.keys.values(): + for key_id, secret in tenant_keys.items(): + if (not re.fullmatch(r'[A-Za-z0-9_-]{1,32}', key_id) + or len(secret) < 32): + raise ValueError( + 'capability keys require canonical id and 32+ byte secret') + + def active(self, tenant_id: str) -> tuple[str, bytes]: + key_id = self.active_key_ids[tenant_id] + return key_id, self.keys[tenant_id][key_id] + + def key(self, tenant_id: str, key_id: str) -> bytes | None: + return self.keys.get(tenant_id, {}).get(key_id) + + def rotate(self, tenant_id: str, key_id: str, secret: bytes) -> None: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,32}', key_id) or len(secret) < 32: + raise ValueError('capability keys require canonical id and 32+ byte secret') + self.keys.setdefault(tenant_id, {})[key_id] = secret + self.active_key_ids[tenant_id] = key_id + + def retire(self, tenant_id: str, key_id: str) -> None: + if self.active_key_ids.get(tenant_id) == key_id: + raise ValueError('cannot retire the active capability key') + self.keys.get(tenant_id, {}).pop(key_id, None) + + +@dataclass(frozen=True) +class RetrievalResponse: + body: bytes + media_type: str + filename: str + headers: dict[str, str] + artifact_sha256: str + token_fingerprint: str + + +@dataclass +class CapabilityService: + store: SqliteQuoteStore + keyring: CapabilityKeyring + token_factory: Callable[[], str] = field( + default=lambda: secrets.token_urlsafe(32)) + + @staticmethod + def _digest(token: str, secret: bytes) -> str: + return hmac.new(secret, token.encode(), hashlib.sha256).hexdigest() + + @staticmethod + def fingerprint(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest()[:16] + + def mint(self, document: QuoteDocument, format_name: str) -> str: + if format_name not in _FORMATS: + raise ValueError('unsupported quote artifact format') + h = document.header + if self.store.artifact( + h.tenant_id, h.quote_number, h.revision, format_name) is None: + raise ValueError('cannot mint a capability before verified persistence') + key_id, secret = self.keyring.active(h.tenant_id) + random_part = self.token_factory() + if not re.fullmatch(r'[A-Za-z0-9_-]{43}', random_part): + raise ValueError('token factory returned noncanonical entropy') + token = f'q1.{key_id}.{random_part}' + self.store.save_capability(CapabilityRecord( + token_hash=self._digest(token, secret), tenant_id=h.tenant_id, + quote_number=h.quote_number, revision=h.revision, + format=format_name, key_id=key_id, + expires_at=h.valid_until, revoked_at=None)) + return token + + def revoke(self, tenant_id: str, token: str, *, at: datetime) -> bool: + parsed = self._parse(tenant_id, token) + if parsed is None: + return False + _key_id, token_hash = parsed + return self.store.revoke_capability(token_hash, at=at) + + def retrieve(self, tenant_id: str, token: str, format_name: str, *, + now: datetime) -> RetrievalResponse | None: + fingerprint = self.fingerprint(token) if isinstance(token, str) else 'invalid' + + def refuse() -> None: + if now.tzinfo is not None and now.utcoffset() is not None: + self.store.record_capability_access( + tenant_id=tenant_id, token_fingerprint=fingerprint, + format_name=str(format_name)[:16], outcome='refused', + occurred_at=now) + + if (format_name not in _FORMATS or now.tzinfo is None + or now.utcoffset() is None): + refuse() + return None + parsed = self._parse(tenant_id, token) + if parsed is None: + refuse() + return None + key_id, token_hash = parsed + record = self.store.capability(token_hash) + if (record is None or record.tenant_id != tenant_id + or record.key_id != key_id or record.format != format_name + or record.revoked_at is not None or now >= record.expires_at): + refuse() + return None + document = self.store.get(record.quote_number, record.revision) + if (document is None or document.header.tenant_id != tenant_id + or record.expires_at > document.header.valid_until + or now >= document.header.valid_until): + refuse() + return None + artifact = self.store.artifact( + tenant_id, record.quote_number, record.revision, format_name) + if artifact is None or not self._valid_artifact(artifact): + refuse() + return None + filename = f'quote-{record.quote_number}-r{record.revision}.{format_name}' + self.store.record_capability_access( + tenant_id=tenant_id, token_fingerprint=fingerprint, + format_name=format_name, outcome='served', occurred_at=now) + return RetrievalResponse( + body=artifact.content, media_type=artifact.media_type, + filename=filename, + headers={ + 'Cache-Control': 'private, no-store', + 'X-Content-Type-Options': 'nosniff', + 'Content-Disposition': f'attachment; filename="{filename}"', + }, + artifact_sha256=artifact.sha256, + token_fingerprint=fingerprint) + + def _parse(self, tenant_id: str, token: str) -> tuple[str, str] | None: + if not isinstance(token, str) or len(token) > 96: + return None + match = _TOKEN.fullmatch(token) + if match is None: + return None + key_id = match.group(1) + secret = self.keyring.key(tenant_id, key_id) + if secret is None: + return None + return key_id, self._digest(token, secret) + + @staticmethod + def _valid_artifact(artifact: ArtifactRecord) -> bool: + return (artifact.size == len(artifact.content) + and hmac.compare_digest( + artifact.sha256, + hashlib.sha256(artifact.content).hexdigest())) diff --git a/src/quoting/delivery.py b/src/quoting/delivery.py index 54d6a69..fb3d65d 100644 --- a/src/quoting/delivery.py +++ b/src/quoting/delivery.py @@ -1,11 +1,7 @@ """M5d offline delivery seam and HMAC-signed quote links.""" from __future__ import annotations -import base64 -import hashlib -import hmac from dataclasses import dataclass, field -from datetime import datetime from typing import Protocol @@ -49,36 +45,3 @@ def deliver(self, *, account: AccountContact, quote_number: str, quote_number=quote_number, revision=revision, summary=summary, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, signed_link=signed_link)) - - -def _sign(payload: bytes, secret: bytes) -> str: - return hmac.new(secret, payload, hashlib.sha256).hexdigest() - - -def mint_link(quote_number: str, revision: int, valid_until: datetime, - secret: bytes) -> str: - """Mint a token over quote identity and the quote's own validity horizon.""" - payload = f'{quote_number}|{revision}|{valid_until.isoformat()}'.encode() - encoded = base64.urlsafe_b64encode(payload).rstrip(b'=').decode() - return f'{encoded}.{_sign(payload, secret)}' - - -def verify_link(token: str, now: datetime, - secret: bytes) -> tuple[str, int] | None: - """Return quote identity only for a valid, unexpired token.""" - try: - encoded, supplied_sig = token.split('.', 1) - padding = '=' * (-len(encoded) % 4) - payload = base64.b64decode(encoded + padding, altchars=b'-_', validate=True) - if not hmac.compare_digest(_sign(payload, secret), supplied_sig): - return None - quote_number, revision_text, expiry_text = payload.decode().split('|', 2) - revision = int(revision_text) - expiry = datetime.fromisoformat(expiry_text) - if now.tzinfo is None or expiry.tzinfo is None or now >= expiry: - return None - if not quote_number or revision < 1: - return None - return quote_number, revision - except (UnicodeDecodeError, ValueError): - return None diff --git a/src/quoting/display.py b/src/quoting/display.py new file mode 100644 index 0000000..630ffc5 --- /dev/null +++ b/src/quoting/display.py @@ -0,0 +1,27 @@ +"""Canonical customer-display text and spreadsheet injection hardening.""" +from __future__ import annotations + +import re + +_CONTROL = re.compile(r'[\x00-\x1f\x7f]') +_FORMULA = re.compile(r'^\s*[=+\-@]') + + +def display_text(value: object) -> str: + """Preserve content while making forbidden controls visible and inert.""" + text = '' if value is None else str(value) + return _CONTROL.sub( + lambda match: f'\\u{ord(match.group()):04x}', text).rstrip(' ') + + +def xlsx_text(value: object) -> str: + """Return a literal Excel string; a leading apostrophe is not displayed.""" + text = display_text(value) + return f"'{text}" if _FORMULA.match(text) else text + + +def unquote_xlsx_text(value: object) -> str: + text = '' if value is None else str(value) + if text.startswith("'") and _FORMULA.match(text[1:]): + return text[1:] + return text diff --git a/src/quoting/document_guard.py b/src/quoting/document_guard.py index 7693ab9..4f7257d 100644 --- a/src/quoting/document_guard.py +++ b/src/quoting/document_guard.py @@ -1,214 +1,279 @@ -"""M5 Stage C, spec §3.3 — the document-level never-invent gate: the same -discipline `say_guard`/`provenance.assert_complete` apply to a spoken -sentence, applied to a rendered PAGE. Runs post-render, pre-delivery: -extracts every binding fact from the rendered PDF (text extraction) and -xlsx (cell reads), and asserts EXACT set-parity against the frozen -`QuoteDocument` that produced them. Anything on the page not in the -object — or in the object but missing from the page — fails closed; the -delivery step (M5d, not built) must never proceed past a raised -`DocumentParityViolation`. - -Extraction is STRUCTURAL, not generic text-mining: both renderers produce a -FIXED, known layout, so this module parses by position/pattern against -that known shape — the same discipline `gateway.provenance.surfaced()` -uses (read structurally from a known response shape, never by loosely -parsing prose). Binding facts, per spec §3.3: SKUs, quantities, unit and -extended prices, dates, quote number — NOT line numbers, catalog -descriptions, or availability notes, which are descriptive, not binding. -""" +"""Full customer-projection parity gate for rendered PDF and XLSX bytes.""" from __future__ import annotations import io +import json import re +from collections.abc import Mapping from decimal import Decimal from openpyxl import load_workbook from pypdf import PdfReader +from quoting.display import unquote_xlsx_text from quoting.models import QuoteDocument -from quoting.render_pdf import LINE_HEADER -from quoting.render_xlsx import HEADER_ROW +from quoting.projection import customer_projection +from quoting.render_pdf import ( + HEADER_LABELS, + LINE_LABELS, + TERM_LABELS, + TOTAL_LABELS, +) +from quoting.render_xlsx import HEADER_FIELDS, HEADER_ROW, TERM_FIELDS, TOTAL_FIELDS Fact = tuple[str, str] - - -def _line_fact(line_no, sku, qty, unit_price, extended_price, - ship_date) -> Fact: - return ('line', '|'.join((str(line_no), str(sku), str(qty), - _money(unit_price), _money(extended_price), - str(ship_date or 'TBD')[:10]))) +_FORBIDDEN_NAMES = ( + 'resolution_source', 'resolution_confidence', 'resolution_candidates', + 'resolution_open_questions', 'quantity_on_hand', 'destination_fingerprint', + 'contact_version', 'contact_id', 'journal', 'event_id', 'assent_json', + 'quantity on hand', 'on-hand quantity', 'on hand quantity', +) class DocumentParityViolation(ValueError): - """A rendered document's binding facts do not exactly match the - QuoteDocument that produced it. Fail closed: never deliver.""" + """Rendered customer bytes are incomplete, altered, or internally leaky.""" -def _money(value: Decimal | float | int) -> str: +def _money(value: object) -> str: return f'{Decimal(str(value)):.2f}' -def document_facts(document: QuoteDocument) -> set[Fact]: - """The (kind, value) fact set a rendered document must contain - EXACTLY — the canonical side of the parity check.""" - facts: set[Fact] = { - ('quote_number', document.header.quote_number), - ('revision', str(document.header.revision)), - ('subtotal', _money(document.totals.subtotal)), - ('tax_status', document.totals.tax_status), - ('freight_status', document.totals.freight_status), +def _value(value: object) -> str: + return '' if value is None else str(value) + + +def _line_fact(values: Mapping[str, object]) -> Fact: + canonical = { + key: _value(values[key]) for _, key in LINE_LABELS } - for line in document.lines: - facts.add(_line_fact( - line.line_no, line.sku, line.requested_qty, line.unit_price, - line.extended_price, - line.ship_date.date().isoformat() if line.ship_date else 'TBD')) - facts.add(('sku', line.sku)) - facts.add(('qty', str(line.requested_qty))) - facts.add(('unit_price', _money(line.unit_price))) - facts.add(('extended_price', _money(line.extended_price))) - if line.ship_date is not None: - facts.add(('ship_date', line.ship_date.date().isoformat())) + canonical['line_no'] = str(int(canonical['line_no'])) + canonical['requested_qty'] = str(int(canonical['requested_qty'])) + canonical['unit_price'] = _money(canonical['unit_price']) + canonical['extended_price'] = _money(canonical['extended_price']) + canonical['ship_date'] = canonical['ship_date'] or 'TBD' + return ('line', json.dumps(canonical, sort_keys=True, + ensure_ascii=False, separators=(',', ':'))) + + +def document_facts(document: QuoteDocument) -> set[Fact]: + projection = customer_projection(document) + facts: set[Fact] = set() + for _label, section, key in HEADER_LABELS: + facts.add((f'{section}.{key}', _value(getattr(projection, section)[key]))) + for line in projection.lines: + facts.add(_line_fact(dict(line))) + for _label, key in TOTAL_LABELS: + value = projection.totals[key] + facts.add((f'totals.{key}', _money(value) if key == 'subtotal' + else _value(value))) + for _label, key in TERM_LABELS: + facts.add((f'terms.{key}', _value(projection.terms[key]))) return facts -# -- PDF extraction (pypdf text) --------------------------------------------- +def _label_value(text: str, labels: dict[str, str]) -> tuple[str, str] | None: + for label, key in labels.items(): + prefix = f'{label}:' + if text.startswith(prefix): + return key, text[len(prefix):].strip() + return None -_QUOTE_LINE_RE = re.compile(r'Quote (\S+) \(Revision (\d+)\)') -_SUBTOTAL_RE = re.compile(r'Subtotal: \$(\d+\.\d{2})') -_TAX_RE = re.compile(r'Tax: (\S+)') -_FREIGHT_RE = re.compile(r'Freight: (\S+)') -_MONEY_RE = re.compile(r'^\$(\d+\.\d{2})$') -_DATE_RE = re.compile(r'^\d{4}-\d{2}-\d{2}$') +def _strip_money(value: str) -> str: + match = re.fullmatch(r'[A-Z]{3}\s+(-?\d+(?:\.\d+)?)', value) + return _money(match.group(1) if match else value) -def extract_pdf_facts(pdf_bytes: bytes) -> set[Fact]: - """Structural extraction from render_pdf's known layout: the header - line, the totals lines, and the line-item table. reportlab's Platypus - Table extracts as one cell value per text line in reading order, so - once the known `LINE_HEADER` row is located, each following group of - `len(LINE_HEADER)` lines is one row.""" - reader = PdfReader(io.BytesIO(pdf_bytes)) - lines: list[str] = [] - for page in reader.pages: - lines.extend((page.extract_text() or '').splitlines()) +def extract_pdf_facts(pdf_bytes: bytes) -> set[Fact]: + lines = [line.strip() for page in PdfReader(io.BytesIO(pdf_bytes)).pages + for line in (page.extract_text() or '').splitlines()] + header_labels = {label: f'{section}.{key}' + for label, section, key in HEADER_LABELS} + line_labels = dict(LINE_LABELS) + total_labels = dict(TOTAL_LABELS) + term_labels = dict(TERM_LABELS) facts: set[Fact] = set() - for text_line in lines: - m = _QUOTE_LINE_RE.search(text_line) - if m: - facts.add(('quote_number', m.group(1))) - facts.add(('revision', m.group(2))) - continue - m = _SUBTOTAL_RE.search(text_line) - if m: - facts.add(('subtotal', m.group(1))) + current_line: dict[str, str] | None = None + header_values: dict[str, str] = {} + total_values: dict[str, str] = {} + term_values: dict[str, str] = {} + last_target: tuple[dict[str, str], str] | None = None + section = 'header' + + def finish_line() -> None: + nonlocal current_line + if current_line is not None: + if set(current_line) == {key for _, key in LINE_LABELS}: + facts.add(_line_fact(current_line)) + current_line = None + + for text in lines: + if text.startswith('Line Item ') and text.removeprefix('Line Item ').isdigit(): + finish_line() + section = 'line' + current_line = {} + last_target = None continue - m = _TAX_RE.search(text_line) - if m: - facts.add(('tax_status', m.group(1))) + if text == 'Totals': + finish_line() + section = 'totals' + last_target = None continue - m = _FREIGHT_RE.search(text_line) - if m: - facts.add(('freight_status', m.group(1))) - - header = list(LINE_HEADER) - n = len(header) - for i in range(len(lines) - n + 1): - if lines[i:i + n] != header: + if text == 'Terms': + finish_line() + section = 'terms' + last_target = None continue - j = i + n - while j + n <= len(lines): - row = lines[j:j + n] - if not row[0].isdigit(): - break - line_no, sku, _desc, qty, unit_price, ext_price, ship_date = row - facts.add(_line_fact(line_no, sku, qty, - um.group(1) if (um := _MONEY_RE.match(unit_price)) else '', - em.group(1) if (em := _MONEY_RE.match(ext_price)) else '', - ship_date)) - facts.add(('sku', sku)) - facts.add(('qty', qty)) - if um: - facts.add(('unit_price', um.group(1))) - if em: - facts.add(('extended_price', em.group(1))) - if _DATE_RE.match(ship_date): - facts.add(('ship_date', ship_date)) - j += n - break + parsed = None + if section == 'header': + parsed = _label_value(text, header_labels) + if parsed: + key, value = parsed + header_values[key] = value + last_target = (header_values, key) + elif section == 'line' and current_line is not None: + parsed = _label_value(text, line_labels) + if parsed: + key, value = parsed + if key in ('unit_price', 'extended_price'): + value = _strip_money(value) + current_line[key] = value + last_target = (current_line, key) + elif section == 'totals': + parsed = _label_value(text, total_labels) + if parsed: + key, value = parsed + total_values[key] = (_strip_money(value) + if key == 'subtotal' else value) + last_target = (total_values, key) + elif section == 'terms': + parsed = _label_value(text, term_labels) + if parsed: + key, value = parsed + term_values[key] = value + last_target = (term_values, key) + if (parsed is None and text and text != 'QUOTE' + and last_target is not None): + target, key = last_target + target[key] = f'{target[key]} {text}' + elif parsed is None and text and text != 'QUOTE' and last_target is None: + facts.add(('unexpected_text', text)) + finish_line() + facts.update((key, value) for key, value in header_values.items()) + facts.update((f'totals.{key}', value) + for key, value in total_values.items()) + facts.update((f'terms.{key}', value) for key, value in term_values.items()) return facts -# -- XLSX extraction (openpyxl cell reads) ----------------------------------- - def extract_xlsx_facts(xlsx_bytes: bytes) -> set[Fact]: - """Structural extraction from render_xlsx's known cell layout: the - named header block and the line-item table — cell reads, never text - parsing. The table start row is located by matching the known - `HEADER_ROW` tuple rather than a hardcoded offset, so this stays - correct if render_xlsx's header-block length ever changes.""" - wb = load_workbook(io.BytesIO(xlsx_bytes), data_only=True) - ws = wb.active + wb = load_workbook(io.BytesIO(xlsx_bytes), data_only=False, keep_links=False) + ws = wb['Quote'] facts: set[Fact] = set() - - for row in ws.iter_rows(): + header_by_label = {label: (section, key) + for label, section, key in HEADER_FIELDS} + for row in ws.iter_rows(min_col=1, max_col=2): label = row[0].value - if label == 'Quote Number': - facts.add(('quote_number', str(row[1].value))) - elif label == 'Revision': - facts.add(('revision', str(row[1].value))) - - header_row_idx = None - n = len(HEADER_ROW) - for r in range(1, ws.max_row + 1): - values = tuple(ws.cell(row=r, column=c).value for c in range(1, n + 1)) - if values == HEADER_ROW: - header_row_idx = r - break - if header_row_idx is None: - return facts - - r = header_row_idx + 1 - while True: - line_no = ws.cell(row=r, column=1).value - if line_no is None: - break - facts.add(('sku', str(ws.cell(row=r, column=2).value))) - facts.add(('qty', str(int(ws.cell(row=r, column=4).value)))) - facts.add(('unit_price', _money(ws.cell(row=r, column=6).value))) - facts.add(('extended_price', _money(ws.cell(row=r, column=7).value))) - ship_date = ws.cell(row=r, column=8).value - facts.add(_line_fact( - line_no, ws.cell(row=r, column=2).value, - int(ws.cell(row=r, column=4).value), - ws.cell(row=r, column=6).value, - ws.cell(row=r, column=7).value, ship_date or 'TBD')) - if ship_date: - facts.add(('ship_date', str(ship_date)[:10])) - r += 1 - - r += 1 # blank separator row before totals - for _ in range(3): - label = ws.cell(row=r, column=6).value - value = ws.cell(row=r, column=7).value - if label == 'Subtotal': - facts.add(('subtotal', _money(value))) - elif label == 'Tax': - facts.add(('tax_status', value)) - elif label == 'Freight': - facts.add(('freight_status', value)) - r += 1 + if label in header_by_label: + section, key = header_by_label[label] + facts.add((f'{section}.{key}', unquote_xlsx_text(row[1].value))) + + table = ws.tables['QuoteLines'] + start, end = table.ref.split(':') + start_row = ws[start].row + end_row = ws[end].row + headers = [ws.cell(start_row, column).value + for column in range(1, len(HEADER_ROW) + 1)] + if tuple(headers) == HEADER_ROW: + keys = [key for _, key in LINE_LABELS] + for row in range(start_row + 1, end_row + 1): + raw = [ws.cell(row, column).value + for column in range(1, len(HEADER_ROW) + 1)] + values = {key: unquote_xlsx_text(value) + for key, value in zip(keys, raw, strict=True)} + facts.add(_line_fact(values)) + + total_by_label = dict(TOTAL_FIELDS) + term_by_label = dict(TERM_FIELDS) + for row in ws.iter_rows(): + for label_column, value_column, labels, prefix in ( + (6, 7, total_by_label, 'totals'), + (1, 2, term_by_label, 'terms')): + label = row[label_column - 1].value + if label in labels: + key = labels[label] + value = unquote_xlsx_text(row[value_column - 1].value) + facts.add((f'{prefix}.{key}', _money(value) + if key == 'subtotal' else value)) + allowed = { + ws.cell(row, column).coordinate + for row in range(1, len(HEADER_FIELDS) + 1) for column in (1, 2)} + allowed.update( + ws.cell(row, column).coordinate + for row in range(start_row, end_row + 1) + for column in range(1, len(HEADER_ROW) + 1)) + for row in range(end_row + 2, end_row + 6): + allowed.update((ws.cell(row, 6).coordinate, ws.cell(row, 7).coordinate)) + for row in range(end_row + 7, end_row + 10): + allowed.update((ws.cell(row, 1).coordinate, ws.cell(row, 2).coordinate)) + for row in ws.iter_rows(): + for cell in row: + if cell.value is not None and cell.coordinate not in allowed: + facts.add(('unexpected_cell', + f'{cell.coordinate}:{unquote_xlsx_text(cell.value)}')) return facts -# -- the gate ----------------------------------------------------------------- +def _validate_xlsx_contract(document: QuoteDocument, xlsx_bytes: bytes) -> None: + wb = load_workbook(io.BytesIO(xlsx_bytes), data_only=False, keep_links=False) + if wb.sheetnames != ['Quote']: + raise DocumentParityViolation('unexpected workbook sheet contract') + ws = wb['Quote'] + expected_ref = f'A{len(HEADER_FIELDS) + 2}:L{len(HEADER_FIELDS) + 2 + len(document.lines)}' + if set(ws.tables) != {'QuoteLines'} or ws.tables['QuoteLines'].ref != expected_ref: + raise DocumentParityViolation('unexpected quote table contract') + expected_names = { + 'Quote' + ''.join(part.title() for part in label.split()) + for label, *_rest in (*HEADER_FIELDS, *TOTAL_FIELDS, *TERM_FIELDS)} + if set(wb.defined_names) != expected_names: + raise DocumentParityViolation('unexpected workbook defined-name contract') + + +def _scan_forbidden(document: QuoteDocument, pdf_bytes: bytes, + xlsx_bytes: bytes) -> None: + pdf_text = '\n'.join(page.extract_text() or '' + for page in PdfReader(io.BytesIO(pdf_bytes)).pages) + wb = load_workbook(io.BytesIO(xlsx_bytes), data_only=False, keep_links=False) + if wb.sheetnames != ['Quote'] or getattr(wb, '_external_links', ()): + raise DocumentParityViolation('unexpected workbook sheet or external link') + for ws in wb.worksheets: + for row in ws.iter_rows(): + for cell in row: + if cell.data_type == 'f' or cell.hyperlink is not None: + raise DocumentParityViolation('active spreadsheet content rendered') + xlsx_values = [str(cell.value) for ws in wb.worksheets + for row in ws.iter_rows() for cell in row + if cell.value is not None] + haystack = '\n'.join((pdf_text, *xlsx_values)).casefold() + forbidden = list(_FORBIDDEN_NAMES) + forbidden.extend(( + document.presentation.branding_ref, + document.delivery_preflight.contact_id, + document.delivery_preflight.contact_version, + document.delivery_preflight.destination_fingerprint, + document.delivery_preflight.masked_destination, + )) + for line in document.lines: + forbidden.extend((line.resolution_source, line.resolution_confidence)) + for value in forbidden: + if value not in _FORBIDDEN_NAMES and len(str(value)) < 6: + continue + if value and str(value).casefold() in haystack: + raise DocumentParityViolation('forbidden internal quote data rendered') + def assert_document_parity(document: QuoteDocument, rendered_facts: set[Fact]) -> None: - """The gate: `rendered_facts` must be EXACTLY `document_facts(document)` - — no more (an invented/altered fact reached the page), no less (a real - fact silently dropped). Fails closed rather than allowing delivery.""" expected = document_facts(document) missing = expected - rendered_facts extra = rendered_facts - expected @@ -221,8 +286,7 @@ def assert_document_parity(document: QuoteDocument, def verify_rendered_document(document: QuoteDocument, *, pdf_bytes: bytes, xlsx_bytes: bytes) -> None: - """Runs the parity gate against BOTH rendered formats (spec §3.3) — a - document is only safe to deliver if NEITHER format diverges from the - QuoteDocument that produced it.""" + _scan_forbidden(document, pdf_bytes, xlsx_bytes) + _validate_xlsx_contract(document, xlsx_bytes) assert_document_parity(document, extract_pdf_facts(pdf_bytes)) assert_document_parity(document, extract_xlsx_facts(xlsx_bytes)) diff --git a/src/quoting/models.py b/src/quoting/models.py index dce355d..dbbfbdf 100644 --- a/src/quoting/models.py +++ b/src/quoting/models.py @@ -12,6 +12,8 @@ """ from __future__ import annotations +import base64 +import binascii import re from dataclasses import dataclass from datetime import datetime, timedelta @@ -90,6 +92,7 @@ class QuotePresentation: branding_ref: str customer_display_name: str customer_account_id: str + seller_logo_png_base64: str = '' @dataclass(frozen=True) @@ -174,6 +177,16 @@ def _validate_document(document: QuoteDocument) -> None: document.presentation.branding_ref, document.presentation.customer_display_name)): raise ValueError('visible seller/customer presentation fields are required') + logo = document.presentation.seller_logo_png_base64 + if logo: + if len(logo) > 700_000: + raise ValueError('frozen seller logo exceeds the artifact limit') + try: + decoded_logo = base64.b64decode(logo, validate=True) + except (binascii.Error, ValueError) as exc: + raise ValueError('frozen seller logo must be canonical base64') from exc + if not decoded_logo.startswith(b'\x89PNG\r\n\x1a\n'): + raise ValueError('frozen seller logo must be PNG') if not header.tenant_id or not re.fullmatch( rf'Q-{re.escape(header.tenant_id)}-\d{{6}}', header.quote_number): raise ValueError('quote number must be canonical for its tenant') diff --git a/src/quoting/projection.py b/src/quoting/projection.py index 55db268..54ef275 100644 --- a/src/quoting/projection.py +++ b/src/quoting/projection.py @@ -4,6 +4,7 @@ from dataclasses import asdict, dataclass from typing import Any +from quoting.display import display_text from quoting.models import QuoteDocument @@ -20,22 +21,38 @@ def customer_projection(document: QuoteDocument) -> CustomerQuoteProjection: """Project approved fields; audit, source, score, and contact data stay out.""" return CustomerQuoteProjection( header={ + 'tenant_id': display_text(document.header.tenant_id), 'quote_number': document.header.quote_number, 'revision': document.header.revision, + 'account_id': display_text(document.header.account_id), + 'account_tier': display_text(document.header.account_tier), 'currency': document.header.currency, 'created_at': document.header.created_at.isoformat(), 'valid_until': document.header.valid_until.isoformat(), + 'catalog_version': display_text(document.header.catalog_version), + }, + presentation={ + 'seller_name': display_text(document.presentation.seller_name), + 'seller_address': display_text(document.presentation.seller_address), + 'customer_display_name': display_text( + document.presentation.customer_display_name), + 'customer_account_id': display_text( + document.presentation.customer_account_id), }, - presentation=asdict(document.presentation), lines=tuple({ - 'line_no': line.line_no, 'sku': line.sku, - 'catalog_description': line.catalog_description, + 'line_no': line.line_no, 'sku': display_text(line.sku), + 'catalog_description': display_text(line.catalog_description), 'requested_qty': line.requested_qty, 'uom': line.uom, 'unit_price': str(line.unit_price), + 'unit_price_as_of': line.unit_price_as_of.isoformat(), 'extended_price': str(line.extended_price), + 'availability_note': display_text(line.availability_note), 'availability_status': line.availability_status, 'ship_date': line.ship_date.isoformat() if line.ship_date else None, + 'ship_date_as_of': line.ship_date_as_of.isoformat(), } for line in document.lines), - totals={key: str(value) if key == 'subtotal' else value + totals={key: (str(value) if key == 'subtotal' + else display_text(value).replace('_', ' ')) for key, value in asdict(document.totals).items()}, - terms=asdict(document.terms)) + terms={key: (display_text(value) if isinstance(value, str) else value) + for key, value in asdict(document.terms).items()}) diff --git a/src/quoting/render_pdf.py b/src/quoting/render_pdf.py index 43c3f74..9a97705 100644 --- a/src/quoting/render_pdf.py +++ b/src/quoting/render_pdf.py @@ -1,94 +1,117 @@ -"""M5 Stage C, spec §3.2/§3.4 — the PDF renderer. A deterministic function of -a frozen `QuoteDocument` -> bytes: no lookups, no math beyond formatting, no -model calls. Built on `reportlab`, not WeasyPrint (docs/DECISION_LOG.md D15: -a direct import spike showed WeasyPrint's system deps — Pango/Cairo/ -GDK-Pixbuf — do not resolve via pip alone in this environment). Real text -via `reportlab.platypus` (not an image render), so `document_guard`'s -post-render text extraction (pypdf) can actually read the binding facts -back off the page. - -One tenant-brandable template (spec §3.4): `tenant_terms_text`/ -`pricing_disclaimer` come from the frozen `QuoteTerms`, not hardcoded here; -multi-template theming is out of scope for M5 regardless of renderer. -""" +"""Deterministic, markup-safe PDF projection of one frozen quote.""" from __future__ import annotations +import base64 import io +from xml.sax.saxutils import escape -from reportlab.lib import colors from reportlab.lib.pagesizes import letter from reportlab.lib.styles import getSampleStyleSheet from reportlab.platypus import ( + Image, + KeepTogether, Paragraph, SimpleDocTemplate, Spacer, - Table, - TableStyle, ) +from quoting.display import display_text from quoting.models import QuoteDocument +from quoting.projection import customer_projection -# Public (not `_`-prefixed): document_guard.extract_pdf_facts locates the -# line-item table by matching this exact header, the same structural- -# extraction discipline gateway.provenance.surfaced() uses. -LINE_HEADER = ('Line', 'SKU', 'Description', 'Qty', 'Unit Price', - 'Extended Price', 'Ship Date') +LINE_HEADER = ( + 'Line', 'SKU', 'Description', 'Qty', 'UOM', 'Unit Price', + 'Price As Of', 'Extended Price', 'Availability', 'Availability Note', + 'Ship Date', 'Availability As Of', +) + +HEADER_LABELS = ( + ('Seller', 'presentation', 'seller_name'), + ('Seller Address', 'presentation', 'seller_address'), + ('Customer', 'presentation', 'customer_display_name'), + ('Customer Account', 'presentation', 'customer_account_id'), + ('Tenant', 'header', 'tenant_id'), + ('Quote Number', 'header', 'quote_number'), + ('Revision', 'header', 'revision'), + ('Account', 'header', 'account_id'), + ('Account Tier', 'header', 'account_tier'), + ('Currency', 'header', 'currency'), + ('Created', 'header', 'created_at'), + ('Valid Until', 'header', 'valid_until'), + ('Catalog Version', 'header', 'catalog_version'), +) + +LINE_LABELS = ( + ('Line', 'line_no'), ('SKU', 'sku'), + ('Description', 'catalog_description'), ('Qty', 'requested_qty'), + ('UOM', 'uom'), ('Unit Price', 'unit_price'), + ('Price As Of', 'unit_price_as_of'), + ('Extended Price', 'extended_price'), + ('Availability', 'availability_status'), + ('Availability Note', 'availability_note'), + ('Ship Date', 'ship_date'), ('Availability As Of', 'ship_date_as_of'), +) + +TOTAL_LABELS = ( + ('Subtotal', 'subtotal'), ('Tax', 'tax_status'), + ('Freight', 'freight_status'), ('Total Status', 'total_status'), +) + +TERM_LABELS = ( + ('Validity Window Days', 'validity_window_days'), + ('Pricing Disclaimer', 'pricing_disclaimer'), + ('Tenant Terms', 'tenant_terms_text'), +) + + +def _paragraph(label: str, value: object, style) -> Paragraph: + safe_label = escape(label) + safe_value = escape(display_text(value)) + return Paragraph(f'{safe_label}: {safe_value}', style) def render_pdf(document: QuoteDocument) -> bytes: + projection = customer_projection(document) buf = io.BytesIO() - # invariant=1: the determinism spike (docs/DECISION_LOG.md D15 outcome) - # found reportlab embeds a wall-clock /CreationDate by default, so two - # renders of the SAME document differ byte-for-byte; this flag pins that - # metadata to a fixed sentinel instead, making PDF output byte- - # deterministic (proven by tests/test_quote_renderers_golden.py's byte - # golden). openpyxl's xlsx output could not be made byte-deterministic - # this cheaply (the entropy is in the ZIP container's per-entry - # timestamp, not the workbook content) — its golden asserts on - # extracted structure instead, exactly as the spec anticipated. - doc = SimpleDocTemplate(buf, pagesize=letter, invariant=1) + doc = SimpleDocTemplate( + buf, pagesize=letter, invariant=1, + title=f'Quote {document.header.quote_number}') styles = getSampleStyleSheet() - h = document.header - story = [ - Paragraph(document.presentation.seller_name, styles['Heading2']), - Paragraph(document.presentation.seller_address, styles['Normal']), - Paragraph(f'Quote {h.quote_number} (Revision {h.revision})', - styles['Title']), - Paragraph(f'Customer: {document.presentation.customer_display_name}', - styles['Normal']), - Paragraph(f'Account: {h.account_id} ({h.account_tier})', - styles['Normal']), - Paragraph(f'Created: {h.created_at.isoformat()}', styles['Normal']), - Paragraph(f'Valid until: {h.valid_until.isoformat()}', styles['Normal']), - Paragraph(f'Catalog version: {h.catalog_version}', styles['Normal']), - Spacer(1, 12), - ] - - table_data = [list(LINE_HEADER)] - for line in document.lines: - table_data.append([ - str(line.line_no), line.sku, line.catalog_description, - str(line.requested_qty), f'${line.unit_price:.2f}', - f'${line.extended_price:.2f}', - line.ship_date.date().isoformat() if line.ship_date else 'TBD', - ]) - table = Table(table_data, repeatRows=1) - table.setStyle(TableStyle([ - ('BACKGROUND', (0, 0), (-1, 0), colors.lightgrey), - ('GRID', (0, 0), (-1, -1), 0.5, colors.grey), - ('FONTSIZE', (0, 0), (-1, -1), 9), - ])) - story.append(table) - story.append(Spacer(1, 12)) + story = [Paragraph('QUOTE', styles['Title'])] + if document.presentation.seller_logo_png_base64: + logo = Image(io.BytesIO(base64.b64decode( + document.presentation.seller_logo_png_base64, validate=True))) + logo._restrictSize(144, 48) + story.insert(0, logo) - t = document.totals - story.append(Paragraph(f'Subtotal: ${t.subtotal:.2f}', styles['Normal'])) - story.append(Paragraph(f'Tax: {t.tax_status}', styles['Normal'])) - story.append(Paragraph(f'Freight: {t.freight_status}', styles['Normal'])) + for label, section, key in HEADER_LABELS: + story.append(_paragraph(label, getattr(projection, section)[key], + styles['Normal'])) story.append(Spacer(1, 12)) - story.append(Paragraph(document.terms.pricing_disclaimer, styles['Normal'])) - if document.terms.tenant_terms_text: - story.append(Paragraph(document.terms.tenant_terms_text, styles['Normal'])) + + for line in projection.lines: + block = [Paragraph( + f"Line Item {int(line['line_no'])}", styles['Heading3'])] + for label, key in LINE_LABELS: + value = line[key] + if key in ('unit_price', 'extended_price'): + value = f"{document.header.currency} {value}" + elif key == 'ship_date' and value is None: + value = 'TBD' + block.append(_paragraph(label, value, styles['Normal'])) + block.append(Spacer(1, 8)) + story.append(KeepTogether(block)) + + story.append(Paragraph('Totals', styles['Heading3'])) + for label, key in TOTAL_LABELS: + value = projection.totals[key] + if key == 'subtotal': + value = f"{document.header.currency} {value}" + story.append(_paragraph(label, value, styles['Normal'])) + story.append(Spacer(1, 8)) + story.append(Paragraph('Terms', styles['Heading3'])) + for label, key in TERM_LABELS: + story.append(_paragraph(label, projection.terms[key], styles['Normal'])) doc.build(story) return buf.getvalue() diff --git a/src/quoting/render_xlsx.py b/src/quoting/render_xlsx.py index b80a9ec..e2e4423 100644 --- a/src/quoting/render_xlsx.py +++ b/src/quoting/render_xlsx.py @@ -1,74 +1,134 @@ -"""M5 Stage C, spec §3.2 — the xlsx renderer. A deterministic function of a -frozen `QuoteDocument` -> bytes: no lookups, no math beyond formatting, no -model calls (the same "pure" the spec means — not the strict zero-import -purity `quoting.totals`/`quoting.models` claim; this module legitimately -imports `openpyxl` to do the rendering, so it is NOT covered by -tests/test_quoting_purity.py). - -The xlsx is not a picture of the PDF (spec §3.2): quote metadata lands in a -named header block, lines in a proper table with TYPED cells (numbers as -numbers), so a tenant can ingest it programmatically. Money cells hold -`decimal.Decimal` directly (openpyxl accepts it as a numeric cell, exact in -the file's XML) rather than a `float()` cast — the M5 D11 discipline applied -to the one artifact that gets handed to someone else's system. -""" +"""Pure XLSX projection of one frozen customer quote.""" from __future__ import annotations +import base64 import io from openpyxl import Workbook +from openpyxl.drawing.image import Image +from openpyxl.styles import Font +from openpyxl.utils import get_column_letter +from openpyxl.workbook.defined_name import DefinedName +from openpyxl.worksheet.table import Table, TableStyleInfo +from quoting.display import xlsx_text from quoting.models import QuoteDocument +from quoting.projection import customer_projection -HEADER_ROW = ('Line', 'SKU', 'Description', 'Qty', 'UOM', 'Unit Price', - 'Extended Price', 'Ship Date', 'Availability') +HEADER_ROW = ( + 'Line', 'SKU', 'Description', 'Qty', 'UOM', 'Unit Price', + 'Price As Of', 'Extended Price', 'Availability', 'Availability Note', + 'Ship Date', 'Availability As Of', +) + +HEADER_FIELDS = ( + ('Seller', 'presentation', 'seller_name'), + ('Seller Address', 'presentation', 'seller_address'), + ('Customer', 'presentation', 'customer_display_name'), + ('Customer Account', 'presentation', 'customer_account_id'), + ('Tenant', 'header', 'tenant_id'), + ('Quote Number', 'header', 'quote_number'), + ('Revision', 'header', 'revision'), + ('Account', 'header', 'account_id'), + ('Account Tier', 'header', 'account_tier'), + ('Currency', 'header', 'currency'), + ('Created', 'header', 'created_at'), + ('Valid Until', 'header', 'valid_until'), + ('Catalog Version', 'header', 'catalog_version'), +) + +TOTAL_FIELDS = ( + ('Subtotal', 'subtotal'), ('Tax', 'tax_status'), + ('Freight', 'freight_status'), ('Total Status', 'total_status'), +) + +TERM_FIELDS = ( + ('Validity Window Days', 'validity_window_days'), + ('Pricing Disclaimer', 'pricing_disclaimer'), + ('Tenant Terms', 'tenant_terms_text'), +) + + +def _name(value: str) -> str: + return ''.join(part.title() for part in value.split()) + + +def _literal(value): + return xlsx_text(value) if isinstance(value, str) else value def render_xlsx(document: QuoteDocument) -> bytes: + projection = customer_projection(document) wb = Workbook() ws = wb.active ws.title = 'Quote' + ws.freeze_panes = 'A15' + if document.presentation.seller_logo_png_base64: + logo = Image(io.BytesIO(base64.b64decode( + document.presentation.seller_logo_png_base64, validate=True))) + scale = min(1, 180 / logo.width, 60 / logo.height) + logo.width *= scale + logo.height *= scale + ws.add_image(logo, 'D1') + + for row, (label, section, key) in enumerate(HEADER_FIELDS, start=1): + value = getattr(projection, section)[key] + ws.cell(row=row, column=1, value=label).font = Font(bold=True) + ws.cell(row=row, column=2, value=_literal(value)) + wb.defined_names.add(DefinedName( + f'Quote{_name(label)}', attr_text=f"'Quote'!$B${row}")) + + table_start = len(HEADER_FIELDS) + 2 + for column, label in enumerate(HEADER_ROW, start=1): + ws.cell(row=table_start, column=column, value=label) + for row, line in enumerate(projection.lines, start=table_start + 1): + values = ( + line['line_no'], line['sku'], line['catalog_description'], + line['requested_qty'], line['uom'], document.lines[ + row - table_start - 1].unit_price, + line['unit_price_as_of'], document.lines[ + row - table_start - 1].extended_price, + line['availability_status'], line['availability_note'], + line['ship_date'] or '', line['ship_date_as_of'], + ) + for column, value in enumerate(values, start=1): + ws.cell(row=row, column=column, value=_literal(value)) + + table_end = table_start + len(projection.lines) + table = Table( + displayName='QuoteLines', + ref=f'A{table_start}:{get_column_letter(len(HEADER_ROW))}{table_end}') + table.tableStyleInfo = TableStyleInfo( + name='TableStyleMedium2', showFirstColumn=False, + showLastColumn=False, showRowStripes=True, showColumnStripes=False) + ws.add_table(table) + + row = table_end + 2 + for label, key in TOTAL_FIELDS: + value = (document.totals.subtotal if key == 'subtotal' + else projection.totals[key]) + ws.cell(row=row, column=6, value=label).font = Font(bold=True) + ws.cell(row=row, column=7, value=_literal(value)) + wb.defined_names.add(DefinedName( + f'Quote{_name(label)}', attr_text=f"'Quote'!$G${row}")) + row += 1 + row += 1 + for label, key in TERM_FIELDS: + ws.cell(row=row, column=1, value=label).font = Font(bold=True) + ws.cell(row=row, column=2, value=_literal(projection.terms[key])) + wb.defined_names.add(DefinedName( + f'Quote{_name(label)}', attr_text=f"'Quote'!$B${row}")) + row += 1 - # Named header block (rows 1-7) — one fact per row, label in column A. - header_fields = ( - ('Seller', document.presentation.seller_name), - ('Seller Address', document.presentation.seller_address), - ('Customer', document.presentation.customer_display_name), - ('Quote Number', document.header.quote_number), - ('Revision', document.header.revision), - ('Account', document.header.account_id), - ('Account Tier', document.header.account_tier), - ('Created', document.header.created_at.isoformat()), - ('Valid Until', document.header.valid_until.isoformat()), - ('Catalog Version', document.header.catalog_version), - ) - for i, (label, value) in enumerate(header_fields, start=1): - ws.cell(row=i, column=1, value=label) - ws.cell(row=i, column=2, value=value) - - # Line table, typed cells. - table_start = len(header_fields) + 2 - for col, label in enumerate(HEADER_ROW, start=1): - ws.cell(row=table_start, column=col, value=label) - for i, line in enumerate(document.lines, start=table_start + 1): - ws.cell(row=i, column=1, value=line.line_no) - ws.cell(row=i, column=2, value=line.sku) - ws.cell(row=i, column=3, value=line.catalog_description) - ws.cell(row=i, column=4, value=line.requested_qty) - ws.cell(row=i, column=5, value=line.uom) - ws.cell(row=i, column=6, value=line.unit_price) - ws.cell(row=i, column=7, value=line.extended_price) - ws.cell(row=i, column=8, - value=line.ship_date.isoformat() if line.ship_date else '') - ws.cell(row=i, column=9, value=line.availability_note) - - totals_row = table_start + len(document.lines) + 2 - ws.cell(row=totals_row, column=6, value='Subtotal') - ws.cell(row=totals_row, column=7, value=document.totals.subtotal) - ws.cell(row=totals_row + 1, column=6, value='Tax') - ws.cell(row=totals_row + 1, column=7, value=document.totals.tax_status) - ws.cell(row=totals_row + 2, column=6, value='Freight') - ws.cell(row=totals_row + 2, column=7, value=document.totals.freight_status) + for column in (6, 8): + for cell in ws.iter_cols(min_col=column, max_col=column, + min_row=table_start + 1, max_row=table_end): + cell[0].number_format = '$0.00' + ws.column_dimensions['A'].width = 22 + ws.column_dimensions['B'].width = 28 + ws.column_dimensions['C'].width = 44 + for column in range(4, len(HEADER_ROW) + 1): + ws.column_dimensions[get_column_letter(column)].width = 20 buf = io.BytesIO() wb.save(buf) diff --git a/src/quoting/store.py b/src/quoting/store.py index 2574e39..e7857f0 100644 --- a/src/quoting/store.py +++ b/src/quoting/store.py @@ -17,12 +17,15 @@ import dataclasses import json +import re import sqlite3 +import threading from dataclasses import dataclass from datetime import datetime from decimal import Decimal from hashlib import sha256 -from typing import Protocol +from pathlib import Path +from typing import Callable, Protocol from quoting.models import ( DeliveryPreflight, @@ -35,6 +38,21 @@ QuoteTotals, ) +_PRE_W3_TABLES = frozenset({ + 'quote_seq', 'quote_idempotency', 'quotes', 'quote_revision_seq', + 'quote_queue', +}) +_W3_TABLES = frozenset({ + 'quote_schema', 'quote_number_allocations', 'quote_issue_keys', + 'issuance_events', 'artifact_jobs', 'delivery_jobs', 'quote_artifacts', + 'artifact_guard_events', 'quote_capabilities', + 'capability_access_events', +}) +_SHARED_CONVERSATION_TABLES = frozenset({ + 'conversation_schema', 'conversations', 'processed_turns', + 'conversation_events', +}) + @dataclass(frozen=True) class QueueItem: @@ -45,6 +63,37 @@ class QueueItem: detail: str +class RevisionConflict(RuntimeError): + """The expected active revision changed before the revision committed.""" + + +@dataclass(frozen=True) +class ArtifactRecord: + tenant_id: str + quote_number: str + revision: int + format: str + content: bytes + sha256: str + media_type: str + size: int + renderer_version: str + guard_version: str + verified_at: datetime + + +@dataclass(frozen=True) +class CapabilityRecord: + token_hash: str + tenant_id: str + quote_number: str + revision: int + format: str + key_id: str + expires_at: datetime + revoked_at: datetime | None + + def order_content_hash(sku_qty_pairs: tuple[tuple[str, int | None], ...]) -> str: """A stable hash of an order's line CONTENT (sku, qty) pairs — the other half of the idempotency key alongside session_id. Deliberately excludes @@ -55,6 +104,38 @@ def order_content_hash(sku_qty_pairs: tuple[tuple[str, int | None], ...]) -> str return sha256(payload.encode()).hexdigest()[:16] +def order_version_digest( + line_versions: tuple[tuple[str, str | None, int | None], ...]) -> str: + """Digest the exact ordered line versions covered by readback/assent.""" + payload = json.dumps(line_versions, separators=(',', ':'), ensure_ascii=False) + return sha256(payload.encode()).hexdigest() + + +def _canonical_assent_receipt(receipt: dict, *, session_id: str, + order_digest: str) -> str: + required = { + 'session_id', 'turn_id', 'order_digest', 'readback_receipt_id', + 'utterance_digest', 'line_versions', 'occurred_at', + } + if set(receipt) != required: + raise ValueError('assent receipt has missing or unknown fields') + line_versions = receipt['line_versions'] + if (receipt['session_id'] != session_id + or receipt['order_digest'] != order_digest + or receipt['readback_receipt_id'] != f'readback:{order_digest[:24]}' + or not isinstance(receipt['turn_id'], str) + or not receipt['turn_id'] or len(receipt['turn_id']) > 128 + or not isinstance(line_versions, list) or not line_versions + or len(set(line_versions)) != len(line_versions) + or any(not re.fullmatch(r'line:\d+', item) + for item in line_versions) + or not re.fullmatch(r'[0-9a-f]{64}', + str(receipt['utterance_digest'])) + or not isinstance(receipt['occurred_at'], (int, float))): + raise ValueError('assent receipt is not bound to exact readback/order state') + return json.dumps(receipt, sort_keys=True, separators=(',', ':')) + + def _connect(source) -> sqlite3.Connection: if isinstance(source, sqlite3.Connection): return source @@ -63,7 +144,59 @@ def _connect(source) -> sqlite3.Connection: return conn +def migrate_quote_database(path: str | Path, *, + plant_failure: bool = False) -> dict[str, object]: + """Upgrade/backfill a file DB, restoring original bytes on any failure.""" + target = Path(path) + existed = target.exists() + original = target.read_bytes() if existed else b'' + store: SqliteQuoteStore | None = None + try: + store = SqliteQuoteStore(target) + if plant_failure: + raise RuntimeError('planted quote migration failure') + foreign_key_errors = store._conn.execute( + 'PRAGMA foreign_key_check').fetchall() + if foreign_key_errors: + raise RuntimeError('quote migration produced foreign-key violations') + counts = {table: store._conn.execute( + f'SELECT COUNT(*) FROM {table}').fetchone()[0] for table in ( + 'quotes', 'quote_issue_keys', 'issuance_events', + 'artifact_jobs', 'delivery_jobs')} + store._conn.close() + store = None + upgraded = target.read_bytes() + return { + 'before_sha256': sha256(original).hexdigest(), + 'after_sha256': sha256(upgraded).hexdigest(), + 'counts': counts, + 'schema_version': 3, + } + except Exception: + if store is not None: + store._conn.close() + if existed: + target.write_bytes(original) + elif target.exists(): + target.unlink() + raise + + class QuoteStore(Protocol): + def issue_atomic( + self, *, tenant_id: str, account_id: str, session_id: str, + order_digest: str, action: str, assent_receipt: dict, + delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: ... + + def revise_atomic( + self, *, tenant_id: str, account_id: str, quote_number: str, + expected_revision: int, session_id: str, order_digest: str, + assent_receipt: dict, delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: ... + def reserve(self, *, tenant_id: str, session_id: str, content_hash: str) -> tuple[str, int, bool]: """Return (quote_number, revision, is_new). Idempotent: the SAME @@ -92,6 +225,17 @@ def enqueue(self, quote_number: str, revision: int) -> None: ... def repair_outbox(self, document: QuoteDocument) -> None: ... + def persist_artifacts_atomic( + self, document: QuoteDocument, *, pdf_bytes: bytes, + xlsx_bytes: bytes, verified_at: datetime, + fault_after: int | None = None) -> tuple[ArtifactRecord, ArtifactRecord]: ... + + def artifact(self, tenant_id: str, quote_number: str, revision: int, + format_name: str) -> ArtifactRecord | None: ... + + def record_artifact_failure(self, document: QuoteDocument, *, + reason: str, detail: str) -> None: ... + def claim_pending(self) -> QueueItem | None: ... def mark_delivered(self, quote_number: str, revision: int) -> None: ... @@ -103,7 +247,24 @@ def mark_blocked(self, quote_number: str, revision: int, class SqliteQuoteStore: def __init__(self, source) -> None: self._conn = _connect(source) + self._lock = threading.RLock() self._conn.row_factory = sqlite3.Row + self._conn.execute('PRAGMA foreign_keys=ON') + pre_tables = {row[0] for row in self._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' " + "AND name NOT LIKE 'sqlite_%'")} + unknown = pre_tables - ( + _PRE_W3_TABLES | _W3_TABLES | _SHARED_CONVERSATION_TABLES) + legacy = pre_tables & _PRE_W3_TABLES + if unknown: + raise RuntimeError( + f'unknown nonempty quote database schema: {sorted(unknown)}') + if legacy and legacy not in (_PRE_W3_TABLES, frozenset({'quotes'})): + raise RuntimeError('partial pre-W3 quote schema is not recognized') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS quote_schema ' + '(singleton INTEGER PRIMARY KEY CHECK(singleton=1), ' + 'version INTEGER NOT NULL)') self._conn.execute( 'CREATE TABLE IF NOT EXISTS quote_seq ' '(tenant_id TEXT PRIMARY KEY, next_seq INTEGER NOT NULL)') @@ -116,6 +277,9 @@ def __init__(self, source) -> None: '(quote_number TEXT, revision INTEGER, tenant_id TEXT NOT NULL, ' 'payload_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT "active", ' 'PRIMARY KEY (quote_number, revision))') + self._conn.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS quote_tenant_identity_idx ON ' + 'quotes(tenant_id,quote_number,revision)') try: self._conn.execute( 'ALTER TABLE quotes ADD COLUMN status TEXT NOT NULL ' @@ -123,6 +287,14 @@ def __init__(self, source) -> None: except sqlite3.OperationalError as exc: if 'duplicate column name' not in str(exc).lower(): raise + self._conn.executescript(""" + CREATE TRIGGER IF NOT EXISTS quote_payload_immutable + BEFORE UPDATE OF quote_number,revision,tenant_id,payload_json ON quotes + BEGIN SELECT RAISE(ABORT,'quote revisions are immutable'); END; + CREATE TRIGGER IF NOT EXISTS quote_row_delete_forbidden + BEFORE DELETE ON quotes + BEGIN SELECT RAISE(ABORT,'quote revisions are insert-only'); END; + """) self._conn.execute( 'CREATE TABLE IF NOT EXISTS quote_revision_seq ' '(quote_number TEXT PRIMARY KEY, next_revision INTEGER NOT NULL)') @@ -131,8 +303,636 @@ def __init__(self, source) -> None: '(quote_number TEXT, revision INTEGER, status TEXT NOT NULL, ' 'attempts INTEGER NOT NULL DEFAULT 0, detail TEXT NOT NULL DEFAULT "", ' 'PRIMARY KEY (quote_number, revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS quote_number_allocations ' + '(allocation_id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id TEXT NOT NULL)') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS quote_issue_keys ' + '(tenant_id TEXT NOT NULL, account_id TEXT NOT NULL, ' + 'session_id TEXT NOT NULL, order_digest TEXT NOT NULL, action TEXT NOT NULL, ' + 'quote_number TEXT NOT NULL, revision INTEGER NOT NULL, ' + 'PRIMARY KEY(tenant_id,account_id,session_id,order_digest,action), ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS issuance_events ' + '(event_id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id TEXT NOT NULL, ' + 'quote_number TEXT NOT NULL, revision INTEGER NOT NULL, session_id TEXT NOT NULL, ' + 'order_digest TEXT NOT NULL, assent_json TEXT NOT NULL, ' + 'UNIQUE(tenant_id,quote_number,revision), ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS artifact_jobs ' + '(tenant_id TEXT NOT NULL, quote_number TEXT NOT NULL, revision INTEGER NOT NULL, ' + 'status TEXT NOT NULL DEFAULT "pending", detail TEXT NOT NULL DEFAULT "", ' + 'PRIMARY KEY(tenant_id,quote_number,revision), ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS delivery_jobs ' + '(tenant_id TEXT NOT NULL, quote_number TEXT NOT NULL, revision INTEGER NOT NULL, ' + 'channel TEXT NOT NULL, status TEXT NOT NULL DEFAULT "held", ' + 'completion_event_id TEXT, detail TEXT NOT NULL DEFAULT "", ' + 'PRIMARY KEY(tenant_id,quote_number,revision,channel), ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS quote_artifacts ' + '(tenant_id TEXT NOT NULL, quote_number TEXT NOT NULL, ' + 'revision INTEGER NOT NULL, format TEXT NOT NULL, content BLOB NOT NULL, ' + 'sha256 TEXT NOT NULL, media_type TEXT NOT NULL, size INTEGER NOT NULL, ' + 'renderer_version TEXT NOT NULL, guard_version TEXT NOT NULL, ' + 'verified_at TEXT NOT NULL, ' + 'PRIMARY KEY(tenant_id,quote_number,revision,format), ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.executescript(""" + CREATE TRIGGER IF NOT EXISTS quote_artifact_update_forbidden + BEFORE UPDATE ON quote_artifacts + BEGIN SELECT RAISE(ABORT,'quote artifacts are immutable'); END; + CREATE TRIGGER IF NOT EXISTS quote_artifact_delete_forbidden + BEFORE DELETE ON quote_artifacts + BEGIN SELECT RAISE(ABORT,'quote artifacts are insert-only'); END; + """) + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS artifact_guard_events ' + '(event_id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id TEXT NOT NULL, ' + 'quote_number TEXT NOT NULL, revision INTEGER NOT NULL, ' + 'reason TEXT NOT NULL, detail TEXT NOT NULL, created_at TEXT NOT NULL, ' + 'FOREIGN KEY(tenant_id,quote_number,revision) REFERENCES ' + 'quotes(tenant_id,quote_number,revision))') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS quote_capabilities ' + '(token_hash TEXT PRIMARY KEY, tenant_id TEXT NOT NULL, ' + 'quote_number TEXT NOT NULL, revision INTEGER NOT NULL, ' + 'format TEXT NOT NULL, key_id TEXT NOT NULL, expires_at TEXT NOT NULL, ' + 'revoked_at TEXT, FOREIGN KEY(tenant_id,quote_number,revision,format) ' + 'REFERENCES quote_artifacts(tenant_id,quote_number,revision,format))') + self._conn.execute( + 'CREATE INDEX IF NOT EXISTS quote_capability_scope_idx ON ' + 'quote_capabilities(tenant_id,quote_number,revision,format)') + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS capability_access_events ' + '(event_id INTEGER PRIMARY KEY AUTOINCREMENT, tenant_id TEXT NOT NULL, ' + 'token_fingerprint TEXT NOT NULL, format TEXT NOT NULL, ' + 'outcome TEXT NOT NULL, occurred_at TEXT NOT NULL)') + if 'quotes' in pre_tables and 'quote_issue_keys' not in pre_tables: + self._backfill_pre_w3() + self._conn.execute( + 'INSERT INTO quote_schema(singleton,version) VALUES(1,3) ' + 'ON CONFLICT(singleton) DO UPDATE SET version=excluded.version') + self._conn.commit() + + def _backfill_pre_w3(self) -> None: + """Preserve recognized quote rows without inventing missing proof.""" + idempotency = { + (row['quote_number'], row['revision']): row + for row in self._conn.execute( + 'SELECT session_id,content_hash,quote_number,revision ' + 'FROM quote_idempotency')} + queues = { + (row['quote_number'], row['revision']): row + for row in self._conn.execute('SELECT * FROM quote_queue')} + allocated: set[str] = set() + rows = self._conn.execute( + 'SELECT quote_number,revision,tenant_id,payload_json FROM quotes ' + 'ORDER BY quote_number,revision').fetchall() + for row in rows: + document = _document_from_dict(json.loads(row['payload_json'])) + h = document.header + legacy_key = idempotency.get((h.quote_number, h.revision)) + session_id = (legacy_key['session_id'] if legacy_key is not None + else f'legacy:{h.quote_number}:r{h.revision}') + digest = (legacy_key['content_hash'] if legacy_key is not None + else sha256(row['payload_json'].encode()).hexdigest()[:16]) + action = ('issue_quote' if legacy_key is not None + else f'legacy_import:r{h.revision}') + assent = json.dumps({ + 'migration': 'pre_w3_preserved', + 'verified': False, + 'reason': 'legacy row has no authoritative assent receipt', + }, sort_keys=True, separators=(',', ':')) + self._conn.execute( + 'INSERT OR IGNORE INTO quote_issue_keys VALUES(?,?,?,?,?,?,?)', + (h.tenant_id, h.account_id, session_id, digest, action, + h.quote_number, h.revision)) + self._conn.execute( + 'INSERT OR IGNORE INTO issuance_events ' + '(tenant_id,quote_number,revision,session_id,order_digest,assent_json) ' + 'VALUES(?,?,?,?,?,?)', + (h.tenant_id, h.quote_number, h.revision, session_id, + digest, assent)) + queue = queues.get((h.quote_number, h.revision)) + artifact_status = ('pending' if queue is not None + and queue['status'] == 'pending' + else 'legacy_review') + detail = ('' if artifact_status == 'pending' + else 'legacy artifact bytes were not persisted') + self._conn.execute( + 'INSERT OR IGNORE INTO artifact_jobs ' + '(tenant_id,quote_number,revision,status,detail) VALUES(?,?,?,?,?)', + (h.tenant_id, h.quote_number, h.revision, + artifact_status, detail)) + self._conn.execute( + 'INSERT OR IGNORE INTO delivery_jobs ' + '(tenant_id,quote_number,revision,channel,status,detail) ' + 'VALUES(?,?,?,? ,"held",?)', + (h.tenant_id, h.quote_number, h.revision, + document.delivery_preflight.channel, + 'legacy import requires artifact and call-completion review')) + if h.quote_number not in allocated: + self._conn.execute( + 'INSERT INTO quote_number_allocations(tenant_id) VALUES(?)', + (h.tenant_id,)) + allocated.add(h.quote_number) + + def issue_atomic( + self, *, tenant_id: str, account_id: str, session_id: str, + order_digest: str, action: str, assent_receipt: dict, + delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: + with self._lock: + return self._issue_atomic( + tenant_id=tenant_id, account_id=account_id, + session_id=session_id, order_digest=order_digest, + action=action, assent_receipt=assent_receipt, + delivery_channel=delivery_channel, + document_factory=document_factory, fault_after=fault_after) + + def _issue_atomic( + self, *, tenant_id: str, account_id: str, session_id: str, + order_digest: str, action: str, assent_receipt: dict, + delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: + """Allocate once, then atomically insert immutable quote + lineage/outboxes.""" + key = (tenant_id, account_id, session_id, order_digest, action) + existing = self._conn.execute( + 'SELECT quote_number,revision FROM quote_issue_keys WHERE ' + 'tenant_id=? AND account_id=? AND session_id=? AND order_digest=? ' + 'AND action=?', key).fetchone() + if existing is not None: + document = self.get(existing['quote_number'], existing['revision']) + if document is None: + raise RuntimeError('idempotency key points to missing quote') + self._repair_atomic_outboxes(document, delivery_channel) + return document + + # Committed separately by design: a failed issuance leaves a legal gap + # and the identifier can never be reused. + quote_number = self._allocate_quote_number(tenant_id) + revision = 1 + document = document_factory(quote_number, revision) + if (document.header.tenant_id != tenant_id + or document.header.account_id != account_id + or document.header.quote_number != quote_number + or document.header.revision != revision): + raise ValueError('document factory returned mismatched quote identity') + payload = json.dumps(dataclasses.asdict(document), default=str, + sort_keys=True, separators=(',', ':')) + assent_json = _canonical_assent_receipt( + assent_receipt, session_id=session_id, order_digest=order_digest) + step = 0 + + def executed() -> None: + nonlocal step + step += 1 + if fault_after == step: + raise RuntimeError(f'planted issue fault after statement {step}') + + self._conn.execute('BEGIN IMMEDIATE') + try: + winner = self._conn.execute( + 'SELECT quote_number,revision FROM quote_issue_keys WHERE ' + 'tenant_id=? AND account_id=? AND session_id=? AND order_digest=? ' + 'AND action=?', key).fetchone() + if winner is not None: + self._conn.rollback() + existing_doc = self.get(winner['quote_number'], winner['revision']) + if existing_doc is None: + raise RuntimeError('winning issue key has no quote') + self._repair_atomic_outboxes(existing_doc, delivery_channel) + return existing_doc + self._conn.execute( + 'INSERT INTO quotes(quote_number,revision,tenant_id,payload_json,status) ' + 'VALUES(?,?,?,? ,"active")', + (quote_number, revision, tenant_id, payload)) + executed() + self._conn.execute( + 'INSERT INTO issuance_events ' + '(tenant_id,quote_number,revision,session_id,order_digest,assent_json) ' + 'VALUES(?,?,?,?,?,?)', + (tenant_id, quote_number, revision, session_id, + order_digest, assent_json)) + executed() + self._conn.execute( + 'INSERT INTO artifact_jobs(tenant_id,quote_number,revision) ' + 'VALUES(?,?,?)', (tenant_id, quote_number, revision)) + executed() + self._conn.execute( + 'INSERT INTO delivery_jobs ' + '(tenant_id,quote_number,revision,channel) VALUES(?,?,?,?)', + (tenant_id, quote_number, revision, delivery_channel)) + executed() + self._conn.execute( + 'INSERT INTO quote_issue_keys VALUES(?,?,?,?,?,?,?)', + (*key, quote_number, revision)) + executed() + # Legacy worker compatibility projection; W5 consumes delivery_jobs. + self._conn.execute( + 'INSERT INTO quote_queue VALUES(?,?,"pending",0,"") ' + 'ON CONFLICT(quote_number,revision) DO NOTHING', + (quote_number, revision)) + executed() + self._conn.commit() + return document + except Exception: + self._conn.rollback() + raise + + def _allocate_quote_number(self, tenant_id: str) -> str: + """Commit a per-tenant allocation independently from issuance. + + Gaps are intentional: once returned, a number survives a later issue + rollback and can never be handed out again. + """ + self._conn.execute('BEGIN IMMEDIATE') + try: + row = self._conn.execute( + 'SELECT next_seq FROM quote_seq WHERE tenant_id=?', + (tenant_id,)).fetchone() + sequence = int(row['next_seq']) if row is not None else 1 + self._conn.execute( + 'INSERT INTO quote_seq(tenant_id,next_seq) VALUES(?,?) ' + 'ON CONFLICT(tenant_id) DO UPDATE SET next_seq=excluded.next_seq', + (tenant_id, sequence + 1)) + self._conn.execute( + 'INSERT INTO quote_number_allocations(tenant_id) VALUES(?)', + (tenant_id,)) + self._conn.commit() + except Exception: + self._conn.rollback() + raise + return f'Q-{tenant_id}-{sequence:06d}' + + def revise_atomic( + self, *, tenant_id: str, account_id: str, quote_number: str, + expected_revision: int, session_id: str, order_digest: str, + assent_receipt: dict, delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: + with self._lock: + return self._revise_atomic( + tenant_id=tenant_id, account_id=account_id, + quote_number=quote_number, expected_revision=expected_revision, + session_id=session_id, order_digest=order_digest, + assent_receipt=assent_receipt, + delivery_channel=delivery_channel, + document_factory=document_factory, fault_after=fault_after) + + def _revise_atomic( + self, *, tenant_id: str, account_id: str, quote_number: str, + expected_revision: int, session_id: str, order_digest: str, + assent_receipt: dict, delivery_channel: str, + document_factory: Callable[[str, int], QuoteDocument], + fault_after: int | None = None) -> QuoteDocument: + """CAS one immutable revision, supersession, lineage, and outboxes.""" + action = f'revise:{quote_number}:{expected_revision}' + key = (tenant_id, account_id, session_id, order_digest, action) + existing = self._conn.execute( + 'SELECT quote_number,revision FROM quote_issue_keys WHERE ' + 'tenant_id=? AND account_id=? AND session_id=? AND order_digest=? ' + 'AND action=?', key).fetchone() + if existing is not None: + document = self.get(existing['quote_number'], existing['revision']) + if document is None: + raise RuntimeError('revision idempotency key points to missing quote') + self._repair_atomic_outboxes(document, delivery_channel) + return document + + new_revision = expected_revision + 1 + document = document_factory(quote_number, new_revision) + if (document.header.tenant_id != tenant_id + or document.header.account_id != account_id + or document.header.quote_number != quote_number + or document.header.revision != new_revision): + raise ValueError('document factory returned mismatched revision identity') + payload = json.dumps(dataclasses.asdict(document), default=str, + sort_keys=True, separators=(',', ':')) + assent_json = _canonical_assent_receipt( + assent_receipt, session_id=session_id, order_digest=order_digest) + step = 0 + + def executed() -> None: + nonlocal step + step += 1 + if fault_after == step: + raise RuntimeError(f'planted revision fault after statement {step}') + + self._conn.execute('BEGIN IMMEDIATE') + try: + winner = self._conn.execute( + 'SELECT quote_number,revision FROM quote_issue_keys WHERE ' + 'tenant_id=? AND account_id=? AND session_id=? AND order_digest=? ' + 'AND action=?', key).fetchone() + if winner is not None: + self._conn.rollback() + existing_doc = self.get(winner['quote_number'], winner['revision']) + if existing_doc is None: + raise RuntimeError('winning revision key has no quote') + self._repair_atomic_outboxes(existing_doc, delivery_channel) + return existing_doc + active = self._conn.execute( + 'SELECT payload_json FROM quotes WHERE tenant_id=? AND ' + 'quote_number=? AND revision=? AND status="active"', + (tenant_id, quote_number, expected_revision)).fetchone() + if active is None: + raise RevisionConflict( + f'expected active revision {expected_revision} is no longer current') + prior = _document_from_dict(json.loads(active['payload_json'])) + if prior.header.account_id != account_id: + raise RevisionConflict('revision ownership changed') + + self._conn.execute( + 'INSERT INTO quotes(quote_number,revision,tenant_id,payload_json,status) ' + 'VALUES(?,?,?,? ,"active")', + (quote_number, new_revision, tenant_id, payload)) + executed() + self._conn.execute( + 'INSERT INTO issuance_events ' + '(tenant_id,quote_number,revision,session_id,order_digest,assent_json) ' + 'VALUES(?,?,?,?,?,?)', + (tenant_id, quote_number, new_revision, session_id, + order_digest, assent_json)) + executed() + self._conn.execute( + 'INSERT INTO artifact_jobs(tenant_id,quote_number,revision) ' + 'VALUES(?,?,?)', (tenant_id, quote_number, new_revision)) + executed() + self._conn.execute( + 'INSERT INTO delivery_jobs ' + '(tenant_id,quote_number,revision,channel) VALUES(?,?,?,?)', + (tenant_id, quote_number, new_revision, delivery_channel)) + executed() + changed = self._conn.execute( + 'UPDATE quotes SET status="superseded" WHERE tenant_id=? AND ' + 'quote_number=? AND revision=? AND status="active"', + (tenant_id, quote_number, expected_revision)) + if changed.rowcount != 1: + raise RevisionConflict('active revision lost compare-and-swap') + executed() + self._conn.execute( + 'INSERT INTO quote_issue_keys VALUES(?,?,?,?,?,?,?)', + (*key, quote_number, new_revision)) + executed() + self._conn.execute( + 'INSERT INTO quote_revision_seq(quote_number,next_revision) ' + 'VALUES(?,?) ON CONFLICT(quote_number) DO UPDATE SET ' + 'next_revision=MAX(next_revision,excluded.next_revision)', + (quote_number, new_revision + 1)) + executed() + self._conn.execute( + 'INSERT INTO quote_queue VALUES(?,? ,"pending",0,"")', + (quote_number, new_revision)) + executed() + self._conn.commit() + return document + except Exception: + self._conn.rollback() + raise + + def _repair_atomic_outboxes(self, document: QuoteDocument, + channel: str) -> None: + h = document.header + self._conn.execute('BEGIN IMMEDIATE') + try: + self._conn.execute( + 'INSERT INTO artifact_jobs(tenant_id,quote_number,revision) ' + 'VALUES(?,?,?) ON CONFLICT DO NOTHING', + (h.tenant_id, h.quote_number, h.revision)) + self._conn.execute( + 'INSERT INTO delivery_jobs(tenant_id,quote_number,revision,channel) ' + 'VALUES(?,?,?,?) ON CONFLICT DO NOTHING', + (h.tenant_id, h.quote_number, h.revision, channel)) + self._conn.execute( + 'INSERT INTO quote_queue VALUES(?,?,"pending",0,"") ' + 'ON CONFLICT DO NOTHING', (h.quote_number, h.revision)) + self._conn.commit() + except Exception: + self._conn.rollback() + raise + + def persist_artifacts_atomic( + self, document: QuoteDocument, *, pdf_bytes: bytes, + xlsx_bytes: bytes, verified_at: datetime, + fault_after: int | None = None) -> tuple[ArtifactRecord, ArtifactRecord]: + with self._lock: + return self._persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, + verified_at=verified_at, fault_after=fault_after) + + def _persist_artifacts_atomic( + self, document: QuoteDocument, *, pdf_bytes: bytes, + xlsx_bytes: bytes, verified_at: datetime, + fault_after: int | None = None) -> tuple[ArtifactRecord, ArtifactRecord]: + """Insert both verified formats once and complete the artifact job.""" + if verified_at.tzinfo is None or verified_at.utcoffset() is None: + raise ValueError('verified_at must be timezone-aware') + h = document.header + formats = ( + ('pdf', pdf_bytes, 'application/pdf'), + ('xlsx', xlsx_bytes, + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'), + ) + record_list = [ArtifactRecord( + tenant_id=h.tenant_id, quote_number=h.quote_number, + revision=h.revision, format=format_name, content=content, + sha256=sha256(content).hexdigest(), media_type=media_type, + size=len(content), + renderer_version=document.provenance.renderer_schema_version, + guard_version=document.provenance.guard_schema_version, + verified_at=verified_at) + for format_name, content, media_type in formats] + records = (record_list[0], record_list[1]) + existing = tuple(self.artifact( + h.tenant_id, h.quote_number, h.revision, record.format) + for record in records) + if any(existing): + if existing == records: + return records + raise ValueError('verified quote artifacts are immutable') + + self._conn.execute('BEGIN IMMEDIATE') + step = 0 + try: + quote = self._conn.execute( + 'SELECT 1 FROM quotes WHERE tenant_id=? AND quote_number=? ' + 'AND revision=?', + (h.tenant_id, h.quote_number, h.revision)).fetchone() + job = self._conn.execute( + 'SELECT status FROM artifact_jobs WHERE tenant_id=? AND ' + 'quote_number=? AND revision=?', + (h.tenant_id, h.quote_number, h.revision)).fetchone() + if quote is None: + raise KeyError('artifact quote identity does not exist') + if job is None: + # Compatibility backfill for a recognized pre-W3 queue row. + self._conn.execute( + 'INSERT INTO artifact_jobs(tenant_id,quote_number,revision) ' + 'VALUES(?,?,?)', + (h.tenant_id, h.quote_number, h.revision)) + for record in records: + self._conn.execute( + 'INSERT INTO quote_artifacts VALUES(?,?,?,?,?,?,?,?,?,?,?)', + (record.tenant_id, record.quote_number, record.revision, + record.format, record.content, record.sha256, + record.media_type, record.size, record.renderer_version, + record.guard_version, record.verified_at.isoformat())) + step += 1 + if fault_after == step: + raise RuntimeError( + f'planted artifact fault after statement {step}') + self._conn.execute( + 'UPDATE artifact_jobs SET status="verified",detail="" WHERE ' + 'tenant_id=? AND quote_number=? AND revision=?', + (h.tenant_id, h.quote_number, h.revision)) + step += 1 + if fault_after == step: + raise RuntimeError( + f'planted artifact fault after statement {step}') + self._conn.commit() + return records + except Exception: + self._conn.rollback() + raise + + def artifact(self, tenant_id: str, quote_number: str, revision: int, + format_name: str) -> ArtifactRecord | None: + row = self._conn.execute( + 'SELECT * FROM quote_artifacts WHERE tenant_id=? AND quote_number=? ' + 'AND revision=? AND format=?', + (tenant_id, quote_number, revision, format_name)).fetchone() + if row is None: + return None + return ArtifactRecord( + tenant_id=row['tenant_id'], quote_number=row['quote_number'], + revision=row['revision'], format=row['format'], + content=bytes(row['content']), sha256=row['sha256'], + media_type=row['media_type'], size=row['size'], + renderer_version=row['renderer_version'], + guard_version=row['guard_version'], + verified_at=datetime.fromisoformat(row['verified_at'])) + + def record_artifact_failure(self, document: QuoteDocument, *, + reason: str, detail: str) -> None: + h = document.header + now = datetime.now().astimezone().isoformat() + self._conn.execute('BEGIN IMMEDIATE') + try: + self._conn.execute( + 'INSERT INTO artifact_jobs ' + '(tenant_id,quote_number,revision,status,detail) ' + 'VALUES(?,?,?,"quarantined",?) ON CONFLICT DO UPDATE SET ' + 'status="quarantined",detail=excluded.detail', + (h.tenant_id, h.quote_number, h.revision, detail)) + self._conn.execute( + 'INSERT INTO artifact_guard_events ' + '(tenant_id,quote_number,revision,reason,detail,created_at) ' + 'VALUES(?,?,?,?,?,?)', + (h.tenant_id, h.quote_number, h.revision, reason, detail, now)) + self._conn.commit() + except Exception: + self._conn.rollback() + raise + + def save_capability(self, record: CapabilityRecord) -> None: + self._conn.execute( + 'INSERT INTO quote_capabilities VALUES(?,?,?,?,?,?,?,?)', + (record.token_hash, record.tenant_id, record.quote_number, + record.revision, record.format, record.key_id, + record.expires_at.isoformat(), + record.revoked_at.isoformat() if record.revoked_at else None)) self._conn.commit() + def capability(self, token_hash: str) -> CapabilityRecord | None: + row = self._conn.execute( + 'SELECT * FROM quote_capabilities WHERE token_hash=?', + (token_hash,)).fetchone() + if row is None: + return None + return CapabilityRecord( + token_hash=row['token_hash'], tenant_id=row['tenant_id'], + quote_number=row['quote_number'], revision=row['revision'], + format=row['format'], key_id=row['key_id'], + expires_at=datetime.fromisoformat(row['expires_at']), + revoked_at=(datetime.fromisoformat(row['revoked_at']) + if row['revoked_at'] else None)) + + def revoke_capability(self, token_hash: str, *, at: datetime) -> bool: + changed = self._conn.execute( + 'UPDATE quote_capabilities SET revoked_at=? WHERE token_hash=? ' + 'AND revoked_at IS NULL', (at.isoformat(), token_hash)) + self._conn.commit() + return changed.rowcount == 1 + + def record_capability_access(self, *, tenant_id: str, + token_fingerprint: str, format_name: str, + outcome: str, occurred_at: datetime) -> None: + self._conn.execute( + 'INSERT INTO capability_access_events ' + '(tenant_id,token_fingerprint,format,outcome,occurred_at) ' + 'VALUES(?,?,?,?,?)', + (tenant_id, token_fingerprint, format_name, outcome, + occurred_at.isoformat())) + self._conn.commit() + + def export_lineage(self, tenant_id: str, quote_number: str, + revision: int) -> dict[str, object] | None: + quote = self._conn.execute( + 'SELECT payload_json,status FROM quotes WHERE tenant_id=? AND ' + 'quote_number=? AND revision=?', + (tenant_id, quote_number, revision)).fetchone() + if quote is None: + return None + sections: dict[str, object] = { + 'identity': { + 'tenant_id': tenant_id, 'quote_number': quote_number, + 'revision': revision, 'status': quote['status'], + 'payload_sha256': sha256( + quote['payload_json'].encode()).hexdigest(), + }, + 'issue_key': [dict(row) for row in self._conn.execute( + 'SELECT tenant_id,account_id,session_id,order_digest,action,' + 'quote_number,revision FROM quote_issue_keys WHERE tenant_id=? ' + 'AND quote_number=? AND revision=? ORDER BY action', + (tenant_id, quote_number, revision))], + 'issuance_event': [dict(row) for row in self._conn.execute( + 'SELECT tenant_id,quote_number,revision,session_id,order_digest,' + 'assent_json FROM issuance_events WHERE tenant_id=? AND ' + 'quote_number=? AND revision=?', + (tenant_id, quote_number, revision))], + 'artifact_job': [dict(row) for row in self._conn.execute( + 'SELECT tenant_id,quote_number,revision,status,detail FROM ' + 'artifact_jobs WHERE tenant_id=? AND quote_number=? AND revision=?', + (tenant_id, quote_number, revision))], + 'delivery_jobs': [dict(row) for row in self._conn.execute( + 'SELECT tenant_id,quote_number,revision,channel,status,' + 'completion_event_id,detail FROM delivery_jobs WHERE tenant_id=? ' + 'AND quote_number=? AND revision=? ORDER BY channel', + (tenant_id, quote_number, revision))], + 'artifacts': [dict(row) for row in self._conn.execute( + 'SELECT tenant_id,quote_number,revision,format,sha256,media_type,' + 'size,renderer_version,guard_version,verified_at FROM ' + 'quote_artifacts WHERE tenant_id=? AND quote_number=? AND ' + 'revision=? ORDER BY format', + (tenant_id, quote_number, revision))], + } + canonical = json.dumps( + sections, sort_keys=True, separators=(',', ':'), default=str) + return {**sections, 'lineage_sha256': sha256(canonical.encode()).hexdigest()} + def reserve(self, *, tenant_id: str, session_id: str, content_hash: str) -> tuple[str, int, bool]: row = self._conn.execute( @@ -159,11 +959,23 @@ def reserve(self, *, tenant_id: str, session_id: str, return quote_number, 1, True def save(self, document: QuoteDocument) -> None: - payload = json.dumps(dataclasses.asdict(document), default=str) + payload = json.dumps(dataclasses.asdict(document), default=str, + sort_keys=True, separators=(',', ':')) + existing = self._conn.execute( + 'SELECT tenant_id,payload_json FROM quotes WHERE quote_number=? ' + 'AND revision=?', (document.header.quote_number, + document.header.revision)).fetchone() + if existing is not None: + existing_canonical = json.dumps( + json.loads(existing['payload_json']), sort_keys=True, + separators=(',', ':')) + if (existing['tenant_id'] != document.header.tenant_id + or existing_canonical != payload): + raise ValueError('quote revisions are immutable and insert-only') + return self._conn.execute( - 'INSERT INTO quotes (quote_number, revision, tenant_id, payload_json) ' - 'VALUES (?, ?, ?, ?) ON CONFLICT(quote_number, revision) ' - 'DO UPDATE SET payload_json = excluded.payload_json', + 'INSERT INTO quotes(quote_number,revision,tenant_id,payload_json) ' + 'VALUES(?,?,?,?)', (document.header.quote_number, document.header.revision, document.header.tenant_id, payload)) self._conn.commit() diff --git a/src/quoting/worker.py b/src/quoting/worker.py index 14f8bb3..b00b621 100644 --- a/src/quoting/worker.py +++ b/src/quoting/worker.py @@ -1,7 +1,9 @@ """M5d off-call quote worker with fail-closed document parity.""" from __future__ import annotations +import hashlib from collections.abc import Callable +from datetime import datetime, timezone from gateway.journal import ConversationJournal, EventType from quoting.delivery import AccountContact, Deliverer @@ -17,17 +19,27 @@ def _record_block(store: QuoteStore, journal: ConversationJournal, *, quote_number: str, revision: int, reason: str, - detail: str) -> None: + detail: str, document: QuoteDocument | None = None, + escalation_hook: Callable[[str, dict], None] = lambda *_: None + ) -> None: + if document is not None: + store.record_artifact_failure(document, reason=reason, detail=detail) store.mark_blocked(quote_number, revision, detail) journal.record( EventType.QUOTE_DELIVERY_BLOCKED, quote_number, quote_number=quote_number, revision=revision, reason=reason, detail=detail) + escalation_hook(reason, { + 'quote_number': quote_number, 'revision': revision, + 'detail': detail, + }) def run_once(*, store: QuoteStore, deliverer: Deliverer, account_lookup: AccountLookup, journal: ConversationJournal, - link_factory: LinkFactory) -> bool: + link_factory: LinkFactory, + now_fn: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + escalation_hook: Callable[[str, dict], None] = lambda *_: None) -> bool: """Process at most one pending quote revision after parity verification.""" item = store.claim_pending() if item is None: @@ -37,27 +49,58 @@ def run_once(*, store: QuoteStore, deliverer: Deliverer, _record_block( store, journal, quote_number=item.quote_number, revision=item.revision, reason='quote_missing', - detail='quote_missing') + detail='quote_missing', escalation_hook=escalation_hook) return True account = account_lookup(document.header.account_id) if account is None or account.email is None: _record_block( store, journal, quote_number=item.quote_number, revision=item.revision, reason='contact_missing', - detail='contact_missing') + detail='contact_missing', escalation_hook=escalation_hook) return True - pdf_bytes = render_pdf(document) - xlsx_bytes = render_xlsx(document) - try: - verify_rendered_document( - document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) - except DocumentParityViolation as exc: + h = document.header + persisted_pdf = store.artifact( + h.tenant_id, h.quote_number, h.revision, 'pdf') + persisted_xlsx = store.artifact( + h.tenant_id, h.quote_number, h.revision, 'xlsx') + if (persisted_pdf is None) != (persisted_xlsx is None): _record_block( store, journal, quote_number=item.quote_number, - revision=item.revision, reason='document_parity', - detail=str(exc)) + revision=item.revision, reason='partial_artifact_set', + detail='partial_artifact_set', document=document, + escalation_hook=escalation_hook) return True + if persisted_pdf is None: + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + try: + verify_rendered_document( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + except DocumentParityViolation as exc: + _record_block( + store, journal, quote_number=item.quote_number, + revision=item.revision, reason='document_parity', + detail=str(exc), document=document, + escalation_hook=escalation_hook) + return True + persisted_pdf, persisted_xlsx = store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, + verified_at=now_fn()) + else: + assert persisted_xlsx is not None + for artifact in (persisted_pdf, persisted_xlsx): + if (artifact.size != len(artifact.content) + or artifact.sha256 != hashlib.sha256( + artifact.content).hexdigest()): + _record_block( + store, journal, quote_number=item.quote_number, + revision=item.revision, reason='artifact_hash_mismatch', + detail='artifact_hash_mismatch', document=document, + escalation_hook=escalation_hook) + return True + pdf_bytes = persisted_pdf.content + xlsx_bytes = persisted_xlsx.content deliverer.deliver( account=account, quote_number=item.quote_number, diff --git a/src/runtime/app.py b/src/runtime/app.py index 93a6586..966b31d 100644 --- a/src/runtime/app.py +++ b/src/runtime/app.py @@ -35,7 +35,8 @@ ) -def create_app(*, streaming_asr=None, tts=None, persona=None, improvement=None): +def create_app(*, streaming_asr=None, tts=None, persona=None, improvement=None, + gateway_bundle=None): from fastapi import FastAPI, Request, Response, WebSocket, WebSocketDisconnect from fastapi.responses import JSONResponse @@ -44,7 +45,8 @@ def create_app(*, streaming_asr=None, tts=None, persona=None, improvement=None): init_tracing() # OTel/Phoenix spans iff SKU_OBS_TRACING is set + otel installed; else no-op app = FastAPI(title='SKU Resolution Gateway', version='1.0.0') - gateway, sessions = build_gateway() + gateway, sessions = gateway_bundle if gateway_bundle is not None else build_gateway() + app.state.gateway = gateway # Voice persona (name/accent/voice/greeting) + ASR/TTS (injectable for tests). persona = persona if persona is not None else build_persona() asr = streaming_asr if streaming_asr is not None else build_streaming_asr() @@ -81,23 +83,17 @@ async def verify(session_id: str, request: Request): def tools(): return tools_manifest() - @app.get('/quote/{token}') - def retrieve_quote(token: str): - """Serve a frozen PDF only while its HMAC link remains valid.""" - from quoting.delivery import verify_link - from quoting.render_pdf import render_pdf - identity = verify_link(token, gateway.now_fn(), - gateway.quote_link_secret) - if identity is None: - return JSONResponse({'error': 'quote_link_invalid'}, status_code=404) - quote_number, revision = identity - document = gateway.quote_store.get(quote_number, revision) - if document is None: - return JSONResponse({'error': 'quote_not_found'}, status_code=404) + @app.get('/quote/{token}/{format_name}') + def retrieve_quote(token: str, format_name: str): + """Serve only persisted guarded bytes through an opaque capability.""" + tenant_id = gateway.catalog.tenant_id() + artifact = gateway.quote_capability_service.retrieve( + tenant_id, token, format_name, now=gateway.now_fn()) + if artifact is None: + return Response(status_code=404) return Response( - content=render_pdf(document), media_type='application/pdf', - headers={'Content-Disposition': - f'inline; filename="{quote_number}-r{revision}.pdf"'}) + content=artifact.body, media_type=artifact.media_type, + headers=artifact.headers) # -- voice-agent tool: the deterministic gateway as a callable tool -------- # A hosted voice agent (e.g. ElevenLabs Agents) handles speech, small talk, @@ -158,7 +154,8 @@ async def agent_turn(request: Request): # legacy fixed-sequence turn(). Legacy stays only on the other channels # (/voice, /v1/turns); it is NOT a fallback here — converse fails closed to # a coherent escalation, never reverts to legacy determinism. - resp = gateway.converse(sid, tok, text, channel=Channel.TYPED) + resp = gateway.converse( + sid, tok, text, channel=Channel.TYPED, turn_id=turn_id) # Capture the same self-improvement data as the voice path (off unless # SKU_IMPROVEMENT is configured): the agent path feeds the loop too. if improvement is not None: diff --git a/src/runtime/config.py b/src/runtime/config.py index 8317d16..507a58b 100644 --- a/src/runtime/config.py +++ b/src/runtime/config.py @@ -139,8 +139,10 @@ def build_gateway(): gateway.corrections = corrections # shared with the improvement loop gateway.deliverer = build_quote_deliverer() gateway.quote_link_secret = quote_link_secret() + gateway.quote_capability_service = build_quote_capability_service( + quote_store, catalog.tenant_id(), gateway.quote_link_secret) gateway.quote_link_factory = build_quote_link_factory( - gateway.quote_link_secret, quote_public_base_url()) + gateway.quote_capability_service, quote_public_base_url()) return gateway, sessions @@ -193,15 +195,25 @@ def quote_public_base_url() -> str: return base_url -def build_quote_link_factory(secret: bytes, base_url: str): - """Build complete customer URLs targeting the runtime retrieval route.""" - from quoting.delivery import mint_link +def build_quote_capability_service(store, tenant_id: str, secret: bytes): + """Build the single-tenant opaque artifact capability service.""" + import hashlib + + from quoting.capabilities import CapabilityKeyring, CapabilityService + key_id = os.environ.get('SKU_QUOTE_CAPABILITY_KEY_ID', 'local-v1') + derived = hashlib.sha256(secret).digest() + return CapabilityService( + store, CapabilityKeyring( + keys={tenant_id: {key_id: derived}}, + active_key_ids={tenant_id: key_id})) + + +def build_quote_link_factory(capability_service, base_url: str): + """Mint an opaque PDF capability only after guarded bytes are persisted.""" def link_for(document): - token = mint_link( - document.header.quote_number, document.header.revision, - document.header.valid_until, secret) - return f'{base_url.rstrip("/")}/quote/{token}' + token = capability_service.mint(document, 'pdf') + return f'{base_url.rstrip("/")}/quote/{token}/pdf' return link_for diff --git a/tests/gateway_fixtures.py b/tests/gateway_fixtures.py index 417a8c3..b4e06d7 100644 --- a/tests/gateway_fixtures.py +++ b/tests/gateway_fixtures.py @@ -78,17 +78,25 @@ def in_stock_sku(gw) -> str: def worker_pass(gw, journal): """Run one deterministic off-call worker pass without changing fixture shape.""" - from quoting.delivery import SimulatedDeliverer, mint_link + import hashlib + + from quoting.capabilities import CapabilityKeyring, CapabilityService + from quoting.delivery import SimulatedDeliverer from quoting.worker import run_once accounts = {account.account_id: account for account in ACCOUNTS} deliverer = SimulatedDeliverer() + tenant_id = gw.catalog.tenant_id() + capabilities = CapabilityService( + gw.quote_store, CapabilityKeyring( + keys={tenant_id: {'fixture-v1': hashlib.sha256( + QUOTE_LINK_SECRET).digest()}}, + active_key_ids={tenant_id: 'fixture-v1'})) + gw.quote_capability_service = capabilities def link_factory(document): - token = mint_link( - document.header.quote_number, document.header.revision, - document.header.valid_until, QUOTE_LINK_SECRET) - return f'https://quotes.example.test/quote/{token}' + token = capabilities.mint(document, 'pdf') + return f'https://quotes.example.test/quote/{token}/pdf' run_once(store=gw.quote_store, deliverer=deliverer, account_lookup=accounts.get, journal=journal, diff --git a/tests/test_quote_artifact_layout.py b/tests/test_quote_artifact_layout.py new file mode 100644 index 0000000..fd64ad1 --- /dev/null +++ b/tests/test_quote_artifact_layout.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import base64 +import dataclasses +import io +import shutil +import subprocess + +import pytest +from openpyxl import load_workbook +from PIL import Image +from pypdf import PdfReader +from test_quote_renderers_golden import _fixture_document + +from quoting.document_guard import verify_rendered_document +from quoting.render_pdf import render_pdf +from quoting.render_xlsx import render_xlsx +from quoting.totals import build_totals + + +def _maximum_layout_document(): + original = _fixture_document() + description = ( + 'Long Unicode-safe chrome exhaust assembly - fitment, finish, and ' + 'customer-visible handling notes. ' * 2) + lines = tuple(dataclasses.replace( + original.lines[0], line_no=index, sku=f'SKU-{index:03d}', + catalog_description=description, requested_qty=index, + extended_price=(original.lines[0].unit_price * index).quantize( + original.lines[0].unit_price)) for index in range(1, 26)) + return dataclasses.replace( + original, lines=lines, totals=build_totals(lines), + terms=dataclasses.replace( + original.terms, + tenant_terms_text=( + 'Net 30. Returns require authorization. Freight quoted ' + 'separately. ' * 20))) + + +def test_maximum_layout_has_full_parity_and_printable_page_bounds(tmp_path): + document = _maximum_layout_document() + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + verify_rendered_document( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + + reader = PdfReader(io.BytesIO(pdf_bytes)) + assert 2 <= len(reader.pages) <= len(document.lines) + 2 + + pdf_path = tmp_path / 'maximum.pdf' + pdf_path.write_bytes(pdf_bytes) + prefix = tmp_path / 'maximum-page' + subprocess.run( + ['pdftoppm', '-png', '-r', '96', str(pdf_path), str(prefix)], + check=True, capture_output=True, timeout=30) + images = sorted(tmp_path.glob('maximum-page-*.png')) + assert len(images) == len(reader.pages) + for path in images: + image = Image.open(path).convert('RGB') + pixels = list(image.getdata()) + nonwhite = [pixel for pixel in pixels if pixel != (255, 255, 255)] + assert nonwhite + assert len(nonwhite) / len(pixels) < 0.20 + bbox = Image.eval(image, lambda value: 255 - value).getbbox() + assert bbox is not None + left, top, right, bottom = bbox + assert left >= 25 and top >= 25 + assert right <= image.width - 25 and bottom <= image.height - 25 +@pytest.mark.skipif(shutil.which('soffice') is None, + reason='pinned headless office runtime unavailable') +def test_xlsx_opens_and_round_trips_in_headless_office(tmp_path): + source = tmp_path / 'input' + output = tmp_path / 'output' + source.mkdir() + output.mkdir() + path = source / 'quote.xlsx' + path.write_bytes(render_xlsx(_maximum_layout_document())) + result = subprocess.run( + [shutil.which('soffice'), '--headless', '--convert-to', 'xlsx', + '--outdir', str(output), str(path)], + capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr + round_tripped = output / 'quote.xlsx' + assert round_tripped.exists() + workbook = load_workbook(round_tripped, data_only=False, keep_links=False) + assert workbook.sheetnames == ['Quote'] + assert workbook['Quote'].tables['QuoteLines'].ref == 'A15:L40' + assert workbook['Quote']['D16'].value == 1 + assert workbook['Quote']['D40'].value == 25 + + +def test_frozen_tenant_branding_and_terms_render_without_default_residue(): + logo_buffer = io.BytesIO() + logo = Image.new('RGB', (180, 60), color=(19, 67, 112)) + logo.save(logo_buffer, format='PNG') + original = _fixture_document() + document = dataclasses.replace( + original, + presentation=dataclasses.replace( + original.presentation, seller_name='Acme Exhaust Supply', + seller_address='42 Fleet Way', branding_ref='acme-2026', + seller_logo_png_base64=base64.b64encode( + logo_buffer.getvalue()).decode()), + terms=dataclasses.replace( + original.terms, tenant_terms_text='Acme approved Net 30 terms.')) + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + verify_rendered_document( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + + pdf = PdfReader(io.BytesIO(pdf_bytes)) + pdf_text = '\n'.join(page.extract_text() or '' for page in pdf.pages) + assert 'Seller: Acme Exhaust Supply' in pdf_text + assert 'Seller Address: 42 Fleet Way' in pdf_text + assert 'Tenant Terms: Acme approved Net 30 terms.' in pdf_text + assert 'Seller: Seller' not in pdf_text and 'Net 30.' not in pdf_text + assert any(page.images for page in pdf.pages) + workbook = load_workbook(io.BytesIO(xlsx_bytes), data_only=False) + assert workbook['Quote']['B1'].value == 'Acme Exhaust Supply' + assert workbook['Quote']._images diff --git a/tests/test_quote_artifacts.py b/tests/test_quote_artifacts.py new file mode 100644 index 0000000..a379158 --- /dev/null +++ b/tests/test_quote_artifacts.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import timedelta + +import pytest +from test_quote_store import NOW, _issue + +from quoting.capabilities import CapabilityKeyring, CapabilityService +from quoting.document_guard import verify_rendered_document +from quoting.render_pdf import render_pdf +from quoting.render_xlsx import render_xlsx +from quoting.store import SqliteQuoteStore + + +def _persisted(tmp_path): + store = SqliteQuoteStore(tmp_path / 'artifacts.sqlite') + document = _issue(store) + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + verify_rendered_document( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + records = store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, + verified_at=NOW + timedelta(seconds=1)) + return store, document, pdf_bytes, xlsx_bytes, records + + +def test_guarded_artifacts_are_persisted_once_with_exact_metadata(tmp_path): + store, document, pdf_bytes, xlsx_bytes, records = _persisted(tmp_path) + assert [record.format for record in records] == ['pdf', 'xlsx'] + for expected, content in zip(records, (pdf_bytes, xlsx_bytes), strict=True): + fetched = store.artifact( + document.header.tenant_id, document.header.quote_number, + document.header.revision, expected.format) + assert fetched == expected + assert fetched.content == content + assert fetched.size == len(content) + assert fetched.sha256 == hashlib.sha256(content).hexdigest() + assert fetched.renderer_version == document.provenance.renderer_schema_version + assert fetched.guard_version == document.provenance.guard_schema_version + status = store._conn.execute('SELECT status FROM artifact_jobs').fetchone()[0] + assert status == 'verified' + assert store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, + verified_at=NOW + timedelta(seconds=1)) == records + with pytest.raises(ValueError, match='immutable'): + store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes + b'tamper', xlsx_bytes=xlsx_bytes, + verified_at=NOW + timedelta(seconds=1)) + + +@pytest.mark.parametrize('fault_after', (1, 2, 3)) +def test_artifact_transaction_fault_has_no_partial_format(tmp_path, fault_after): + store = SqliteQuoteStore(tmp_path / f'artifact-fault-{fault_after}.sqlite') + document = _issue(store) + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + with pytest.raises(RuntimeError, match=f'statement {fault_after}'): + store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, + verified_at=NOW, fault_after=fault_after) + assert store._conn.execute( + 'SELECT COUNT(*) FROM quote_artifacts').fetchone()[0] == 0 + assert store._conn.execute( + 'SELECT status FROM artifact_jobs').fetchone()[0] == 'pending' + + +def _service(store, tokens=None): + keyring = CapabilityKeyring( + keys={'t1': {'k1': b'a' * 32}, 't2': {'k1': b'b' * 32}}, + active_key_ids={'t1': 'k1', 't2': 'k1'}) + values = iter(tokens or ('A' * 43, 'B' * 43, 'C' * 43, 'D' * 43)) + return CapabilityService(store, keyring, token_factory=lambda: next(values)) + + +def test_opaque_capability_serves_exact_persisted_bytes_and_safe_headers(tmp_path): + store, document, pdf_bytes, _xlsx_bytes, _ = _persisted(tmp_path) + service = _service(store) + token = service.mint(document, 'pdf') + assert document.header.quote_number not in token and 't1' not in token + assert store._conn.execute( + 'SELECT COUNT(*) FROM quote_capabilities WHERE token_hash=?', + (token,)).fetchone()[0] == 0 + + response = service.retrieve('t1', token, 'pdf', now=NOW) + assert response is not None + assert response.body == pdf_bytes + assert response.artifact_sha256 == hashlib.sha256(pdf_bytes).hexdigest() + assert response.media_type == 'application/pdf' + assert response.filename == 'quote-Q-t1-000001-r1.pdf' + assert response.headers == { + 'Cache-Control': 'private, no-store', + 'X-Content-Type-Options': 'nosniff', + 'Content-Disposition': + 'attachment; filename="quote-Q-t1-000001-r1.pdf"', + } + assert token not in response.token_fingerprint + audits = store._conn.execute( + 'SELECT token_fingerprint,outcome FROM capability_access_events').fetchall() + assert [(row['token_fingerprint'], row['outcome']) for row in audits] == [ + (response.token_fingerprint, 'served')] + assert token not in json.dumps([dict(row) for row in audits]) + + +def test_capability_refusal_matrix_is_neutral(tmp_path): + store, document, _pdf, _xlsx, _ = _persisted(tmp_path) + service = _service(store) + token = service.mint(document, 'pdf') + cases = ( + ('t2', token, 'pdf', NOW), + ('t1', token, 'xlsx', NOW), + ('t1', '', 'pdf', NOW), + ('t1', 'q1.k1.short', 'pdf', NOW), + ('t1', token + 'x' * 100, 'pdf', NOW), + ('t1', token[:-1] + 'Z', 'pdf', NOW), + ('t1', token, 'pdf', document.header.valid_until), + ('t1', token, 'pdf', document.header.valid_until + timedelta(seconds=1)), + ) + assert all(service.retrieve(tenant, candidate, format_name, now=now) is None + for tenant, candidate, format_name, now in cases) + events = store._conn.execute( + 'SELECT token_fingerprint,outcome FROM capability_access_events').fetchall() + assert len(events) == len(cases) + assert {row['outcome'] for row in events} == {'refused'} + assert all(len(row['token_fingerprint']) == 16 for row in events) + + +def test_revocation_rotation_retirement_and_corrupt_artifact_fail_closed(tmp_path): + store, document, _pdf, _xlsx, _ = _persisted(tmp_path) + service = _service(store) + old_token = service.mint(document, 'pdf') + service.keyring.rotate('t1', 'k2', b'c' * 32) + new_token = service.mint(document, 'xlsx') + assert service.retrieve('t1', old_token, 'pdf', now=NOW) is not None + assert service.retrieve('t1', new_token, 'xlsx', now=NOW) is not None + service.keyring.retire('t1', 'k1') + assert service.retrieve('t1', old_token, 'pdf', now=NOW) is None + with pytest.raises(ValueError, match='active'): + service.keyring.retire('t1', 'k2') + assert service.revoke('t1', new_token, at=NOW) + assert service.retrieve('t1', new_token, 'xlsx', now=NOW) is None + + corrupt_token = service.mint(document, 'pdf') + store._conn.execute('DROP TRIGGER quote_artifact_update_forbidden') + store._conn.execute( + 'UPDATE quote_artifacts SET content=? WHERE format="pdf"', (b'bad',)) + store._conn.commit() + assert service.retrieve('t1', corrupt_token, 'pdf', now=NOW) is None + + +def test_artifact_rows_are_sql_immutable_and_lineage_export_is_checksummed(tmp_path): + store, document, _pdf, _xlsx, records = _persisted(tmp_path) + with pytest.raises(Exception, match='immutable'): + store._conn.execute( + 'UPDATE quote_artifacts SET content=? WHERE format="pdf"', (b'bad',)) + with pytest.raises(Exception, match='insert-only'): + store._conn.execute('DELETE FROM quote_artifacts WHERE format="pdf"') + + lineage = store.export_lineage( + document.header.tenant_id, document.header.quote_number, + document.header.revision) + assert lineage is not None + assert len(lineage['lineage_sha256']) == 64 + assert {item['sha256'] for item in lineage['artifacts']} == { + record.sha256 for record in records} + assert len(lineage['issuance_event']) == len(lineage['issue_key']) == 1 + assert len(lineage['artifact_job']) == len(lineage['delivery_jobs']) == 1 + assert store.export_lineage( + document.header.tenant_id, document.header.quote_number, + document.header.revision) == lineage + assert store.export_lineage( + 'other-tenant', document.header.quote_number, + document.header.revision) is None diff --git a/tests/test_quote_authoritative_lineage.py b/tests/test_quote_authoritative_lineage.py new file mode 100644 index 0000000..7b8a7cd --- /dev/null +++ b/tests/test_quote_authoritative_lineage.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json + +from fastapi.testclient import TestClient + +from gateway import SimulatedStreamingASR, SimulatedTTS, VoicePersona +from runtime.app import create_app + + +def test_hosted_assent_quote_and_lineage_share_exact_authoritative_receipt( + tmp_path, monkeypatch): + customers = tmp_path / 'customers.json' + customers.write_text(json.dumps([{ + 'account_id': '1001', 'name': 'TENANT CUSTOMER', + 'phone': '+15550100100', 'email': 'quotes@example.test', + 'contact_id': 'contact-1001', 'contact_version': 'v1', + 'phone_verified': True, 'email_verified': True, + }])) + monkeypatch.setenv('SKU_CUSTOMER_DB', str(customers)) + monkeypatch.setenv('SKU_QUOTE_STORE_DB', str(tmp_path / 'quotes.sqlite')) + monkeypatch.setenv( + 'SKU_CONVERSATION_STORE_DB', str(tmp_path / 'conversations.sqlite')) + monkeypatch.setenv('SKU_SESSION_SECRET', 's' * 32) + monkeypatch.setenv('SKU_QUOTE_LINK_SECRET', 'q' * 32) + monkeypatch.setenv( + 'SKU_QUOTE_PUBLIC_BASE_URL', 'https://quotes.example.test') + app = create_app( + streaming_asr=SimulatedStreamingASR(), tts=SimulatedTTS(), + persona=VoicePersona(), improvement=False) + client = TestClient(app) + caller = 'hosted-lineage-call' + headers = {'X-Agent-Id': 'agent-version-1'} + + def turn(turn_id, text): + response = client.post('/agent/turn', headers=headers, json={ + 'caller_id': caller, 'turn_id': turn_id, 'text': text}) + assert response.status_code == 200, response.text + return response.json() + + turn('turn-verify', 'my account number is 1001') + turn('turn-add', '10 of the K5-24SBC') + readback = turn('turn-readback', 'place the order') + assert readback['needs_confirmation'] is True + issued = turn('turn-assent', 'confirm the order') + assert issued['kind'] == 'order' and 'quote number' in issued['say'].lower() + + gateway = app.state.gateway + quote_row = gateway.quote_store._conn.execute( + 'SELECT quote_number,revision,assent_json FROM issuance_events').fetchone() + receipt = json.loads(quote_row['assent_json']) + events = gateway.conversation_store.events('tenant_001', caller) + by_type = {event['event_type']: event['payload'] for event in events} + assent = by_type['assent_receipt'] + readback_receipt = by_type['readback_receipt'] + + assert receipt == {'session_id': caller, **assent} + assert receipt['turn_id'] == 'turn-assent' + assert receipt['readback_receipt_id'] == readback_receipt['receipt_id'] + assert receipt['order_digest'] == readback_receipt['order_digest'] + assert receipt['line_versions'] == readback_receipt['line_versions'] + origins = {event['payload']['line_id'] for event in events + if event['event_type'] in ('line_origin', 'line_amendment')} + assert set(receipt['line_versions']) <= origins + assert gateway.quote_store._conn.execute( + 'SELECT COUNT(*) FROM quotes').fetchone()[0] == 1 diff --git a/tests/test_quote_delivery.py b/tests/test_quote_delivery.py index dbfbbba..989d2fb 100644 --- a/tests/test_quote_delivery.py +++ b/tests/test_quote_delivery.py @@ -1,48 +1,8 @@ """M5d delivery seam and signed-link behavior.""" from __future__ import annotations -from datetime import datetime, timedelta, timezone - from gateway import Account -from quoting.delivery import SimulatedDeliverer, mint_link, verify_link - -NOW = datetime(2026, 7, 10, 10, 0, tzinfo=timezone.utc) -SECRET = b'quote-link-test-secret' - - -def test_link_round_trip_before_valid_until(): - valid_until = NOW + timedelta(days=30) - token = mint_link('Q-tenant_001-000001', 1, valid_until, SECRET) - assert verify_link(token, NOW, SECRET) == ('Q-tenant_001-000001', 1) - - -def test_link_refuses_tampered_signature_without_raising(): - token = mint_link('Q-tenant_001-000001', 1, NOW + timedelta(days=30), SECRET) - replacement = '0' if token[-1] != '0' else '1' - assert verify_link(token[:-1] + replacement, NOW, SECRET) is None - - -def test_link_refuses_wrong_secret(): - token = mint_link('Q-tenant_001-000001', 1, NOW + timedelta(days=30), SECRET) - assert verify_link(token, NOW, b'wrong-secret') is None - - -def test_link_expires_at_valid_until_boundary(): - valid_until = NOW + timedelta(days=30) - token = mint_link('Q-tenant_001-000001', 1, valid_until, SECRET) - assert verify_link(token, valid_until - timedelta(microseconds=1), SECRET) - assert verify_link(token, valid_until, SECRET) is None - - -def test_link_refuses_malformed_tokens(): - for token in ('', 'not-a-token', 'a.b.c', '%%%%.deadbeef'): - assert verify_link(token, NOW, SECRET) is None - - -def test_link_payload_contains_no_account_contact_data(): - token = mint_link('Q-tenant_001-000001', 1, NOW + timedelta(days=30), SECRET) - assert '1001' not in token - assert 'example.test' not in token +from quoting.delivery import SimulatedDeliverer def test_simulated_deliverer_captures_both_formats_and_link(): diff --git a/tests/test_quote_migration.py b/tests/test_quote_migration.py new file mode 100644 index 0000000..4f7dba6 --- /dev/null +++ b/tests/test_quote_migration.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import dataclasses +import hashlib +import json +import sqlite3 + +import pytest +from test_quote_store import _document + +from quoting.store import SqliteQuoteStore, migrate_quote_database + + +def _legacy_database(path): + connection = sqlite3.connect(path) + connection.executescript(""" + CREATE TABLE quote_seq( + tenant_id TEXT PRIMARY KEY, next_seq INTEGER NOT NULL); + CREATE TABLE quote_idempotency( + session_id TEXT, content_hash TEXT, quote_number TEXT NOT NULL, + revision INTEGER NOT NULL, PRIMARY KEY(session_id,content_hash)); + CREATE TABLE quotes( + quote_number TEXT, revision INTEGER, tenant_id TEXT NOT NULL, + payload_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', + PRIMARY KEY(quote_number,revision)); + CREATE TABLE quote_revision_seq( + quote_number TEXT PRIMARY KEY, next_revision INTEGER NOT NULL); + CREATE TABLE quote_queue( + quote_number TEXT, revision INTEGER, status TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, detail TEXT NOT NULL DEFAULT '', + PRIMARY KEY(quote_number,revision)); + """) + document = _document('Q-t1-000001', 1) + payload = json.dumps(dataclasses.asdict(document), default=str) + connection.execute('INSERT INTO quote_seq VALUES(?,?)', ('t1', 2)) + connection.execute( + 'INSERT INTO quote_idempotency VALUES(?,?,?,?)', + ('legacy-session', 'legacy-digest', 'Q-t1-000001', 1)) + connection.execute( + 'INSERT INTO quotes VALUES(?,?,?,?,?)', + ('Q-t1-000001', 1, 't1', payload, 'active')) + connection.execute( + 'INSERT INTO quote_queue VALUES(?,?,?,?,?)', + ('Q-t1-000001', 1, 'pending', 0, '')) + connection.commit() + connection.close() + + +def _legacy_checksum(path): + connection = sqlite3.connect(path) + payload = [] + for table in ('quote_seq', 'quote_idempotency', 'quotes', + 'quote_revision_seq', 'quote_queue'): + payload.append((table, connection.execute( + f'SELECT * FROM {table} ORDER BY rowid').fetchall())) + connection.close() + return hashlib.sha256(repr(payload).encode()).hexdigest() + + +def test_p16_upgrade_preserves_legacy_rows_and_backfills_explicit_held_lineage( + tmp_path): + path = tmp_path / 'legacy.sqlite' + _legacy_database(path) + before = _legacy_checksum(path) + + report = migrate_quote_database(path) + + assert _legacy_checksum(path) == before + assert report['schema_version'] == 3 + assert report['counts'] == { + 'quotes': 1, 'quote_issue_keys': 1, 'issuance_events': 1, + 'artifact_jobs': 1, 'delivery_jobs': 1, + } + store = SqliteQuoteStore(path) + assert store._conn.execute( + 'SELECT version FROM quote_schema').fetchone()[0] == 3 + key = store._conn.execute('SELECT * FROM quote_issue_keys').fetchone() + assert (key['tenant_id'], key['account_id'], key['session_id'], + key['order_digest'], key['action']) == ( + 't1', '1001', 'legacy-session', 'legacy-digest', 'issue_quote') + assent = json.loads(store._conn.execute( + 'SELECT assent_json FROM issuance_events').fetchone()[0]) + assert assent['verified'] is False and assent['migration'] == 'pre_w3_preserved' + assert store._conn.execute( + 'SELECT status FROM artifact_jobs').fetchone()[0] == 'pending' + delivery = store._conn.execute( + 'SELECT status,detail FROM delivery_jobs').fetchone() + assert delivery['status'] == 'held' and 'review' in delivery['detail'] + assert store._conn.execute( + 'SELECT COUNT(*) FROM quote_artifacts').fetchone()[0] == 0 + assert store._conn.execute('PRAGMA foreign_key_check').fetchall() == [] + for table in ('quote_issue_keys', 'issuance_events', 'artifact_jobs', + 'delivery_jobs', 'quote_artifacts', 'quote_capabilities'): + assert store._conn.execute( + f'PRAGMA foreign_key_list({table})').fetchall() + + # Re-running the migration is a no-op over every preserved/backfilled row. + second = migrate_quote_database(path) + assert second['counts'] == report['counts'] + + +def test_p16_planted_failure_restores_legacy_database_byte_for_byte(tmp_path): + path = tmp_path / 'rollback.sqlite' + _legacy_database(path) + original = path.read_bytes() + with pytest.raises(RuntimeError, match='planted quote migration failure'): + migrate_quote_database(path, plant_failure=True) + assert path.read_bytes() == original + + +def test_unknown_database_is_rejected_without_silent_adoption(tmp_path): + path = tmp_path / 'unknown.sqlite' + connection = sqlite3.connect(path) + connection.execute('CREATE TABLE unrelated(secret TEXT)') + connection.execute('INSERT INTO unrelated VALUES("preserve")') + connection.commit() + connection.close() + original = path.read_bytes() + with pytest.raises(RuntimeError, match='unknown nonempty'): + migrate_quote_database(path) + assert path.read_bytes() == original diff --git a/tests/test_quote_renderers_golden.py b/tests/test_quote_renderers_golden.py index 5ff6184..4b9b48f 100644 --- a/tests/test_quote_renderers_golden.py +++ b/tests/test_quote_renderers_golden.py @@ -10,10 +10,14 @@ from __future__ import annotations import dataclasses +import io from datetime import datetime, timedelta, timezone from decimal import Decimal import pytest +from openpyxl import load_workbook +from pypdf import PdfReader, PdfWriter +from reportlab.pdfgen import canvas from quoting.document_guard import ( DocumentParityViolation, @@ -101,18 +105,7 @@ def test_pdf_byte_golden(): def test_xlsx_structural_golden_matches_fixed_expected_fact_set(): doc = _fixture_document() xlsx_bytes = render_xlsx(doc) - expected = { - ('quote_number', 'Q-tenant_001-000042'), ('revision', '1'), - ('sku', 'K5-24SBC'), ('qty', '10'), ('unit_price', '24.99'), - ('extended_price', '249.90'), ('ship_date', '2026-07-11'), - ('sku', 'BH6-36SBC'), ('qty', '2'), ('unit_price', '149.69'), - ('extended_price', '299.38'), ('ship_date', '2026-07-12'), - ('subtotal', '549.28'), ('tax_status', 'not_included'), - ('freight_status', 'quoted_separately'), - ('line', '1|K5-24SBC|10|24.99|249.90|2026-07-11'), - ('line', '2|BH6-36SBC|2|149.69|299.38|2026-07-12'), - } - assert extract_xlsx_facts(xlsx_bytes) == expected + assert extract_xlsx_facts(xlsx_bytes) == document_facts(doc) def test_pdf_structural_facts_match_the_same_expected_set(): @@ -177,8 +170,8 @@ def test_guard_catches_a_fabricated_quote_number_on_the_page(): # render, or a tampered file swapped in before delivery). doc = _fixture_document() forged_facts = extract_pdf_facts(render_pdf(doc)) - { - ('quote_number', doc.header.quote_number)} | { - ('quote_number', 'Q-tenant_001-999999')} + ('header.quote_number', doc.header.quote_number)} | { + ('header.quote_number', 'Q-tenant_001-999999')} with pytest.raises(DocumentParityViolation) as exc: assert_document_parity(doc, forged_facts) assert 'Q-tenant_001-999999' in str(exc.value) @@ -189,8 +182,104 @@ def test_guard_catches_a_missing_binding_fact(): # the rendered page — a real fact silently dropped, not just an extra # invented one. doc = _fixture_document() - incomplete_facts = extract_pdf_facts(render_pdf(doc)) - { - ('extended_price', '249.90')} + line_fact = next(fact for fact in document_facts(doc) + if fact[0] == 'line' and 'K5-24SBC' in fact[1]) + incomplete_facts = extract_pdf_facts(render_pdf(doc)) - {line_fact} with pytest.raises(DocumentParityViolation) as exc: assert_document_parity(doc, incomplete_facts) assert '249.90' in str(exc.value) + + +def test_every_customer_visible_fact_is_individually_guarded_in_both_formats(): + document = _fixture_document() + rendered = ( + extract_pdf_facts(render_pdf(document)), + extract_xlsx_facts(render_xlsx(document)), + ) + for facts in rendered: + for fact in document_facts(document): + altered = set(facts) + altered.remove(fact) + altered.add((fact[0], f'{fact[1]}::tampered')) + with pytest.raises(DocumentParityViolation): + assert_document_parity(document, altered) + + +def test_xlsx_has_stable_table_defined_names_and_typed_numeric_cells(): + document = _fixture_document() + workbook = load_workbook(io.BytesIO(render_xlsx(document)), data_only=False) + sheet = workbook['Quote'] + table = sheet.tables['QuoteLines'] + assert table.ref == 'A15:L17' + assert set(workbook.defined_names) == { + 'QuoteSeller', 'QuoteSellerAddress', 'QuoteCustomer', + 'QuoteCustomerAccount', 'QuoteTenant', 'QuoteQuoteNumber', + 'QuoteRevision', 'QuoteAccount', 'QuoteAccountTier', 'QuoteCurrency', + 'QuoteCreated', 'QuoteValidUntil', 'QuoteCatalogVersion', + 'QuoteSubtotal', 'QuoteTax', 'QuoteFreight', 'QuoteTotalStatus', + 'QuoteValidityWindowDays', 'QuotePricingDisclaimer', 'QuoteTenantTerms', + } + assert sheet['D16'].data_type == 'n' and sheet['D16'].value == 10 + assert sheet['F16'].data_type == 'n' + assert sheet['H16'].data_type == 'n' + + +def test_dynamic_markup_formulas_links_and_controls_remain_inert_customer_text(): + original = _fixture_document() + line = dataclasses.replace( + original.lines[0], + catalog_description='not bold & literal\x01', + availability_note='=HYPERLINK("https://evil.invalid","click")') + document = dataclasses.replace( + original, lines=(line, original.lines[1]), + totals=build_totals((line, original.lines[1])), + presentation=dataclasses.replace( + original.presentation, seller_name='@SUM(1+1)'), + terms=dataclasses.replace( + original.terms, tenant_terms_text='+cmd|inert')) + pdf_bytes = render_pdf(document) + xlsx_bytes = render_xlsx(document) + + verify_rendered_document(document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + pdf_text = '\n'.join(page.extract_text() or '' + for page in PdfReader(io.BytesIO(pdf_bytes)).pages) + assert 'not bold & literal\\u0001' in pdf_text + workbook = load_workbook(io.BytesIO(xlsx_bytes), data_only=False, + keep_links=False) + assert not getattr(workbook, '_external_links', ()) + for row in workbook['Quote'].iter_rows(): + for cell in row: + assert cell.data_type != 'f' + assert cell.hyperlink is None + + +def test_guard_rejects_any_populated_xlsx_cell_outside_projection(): + document = _fixture_document() + workbook = load_workbook(io.BytesIO(render_xlsx(document)), data_only=False) + workbook['Quote']['Z99'] = 'Quantity On Hand: 27' + output = io.BytesIO() + workbook.save(output) + with pytest.raises(DocumentParityViolation): + verify_rendered_document( + document, pdf_bytes=render_pdf(document), + xlsx_bytes=output.getvalue()) + + +def test_guard_rejects_unmodeled_pdf_overlay_text(): + document = _fixture_document() + reader = PdfReader(io.BytesIO(render_pdf(document))) + overlay_buffer = io.BytesIO() + overlay = canvas.Canvas(overlay_buffer, invariant=1) + overlay.drawString(50, 770, 'Quantity On Hand: 27') + overlay.save() + overlay_page = PdfReader(io.BytesIO(overlay_buffer.getvalue())).pages[0] + reader.pages[0].merge_page(overlay_page) + writer = PdfWriter() + for page in reader.pages: + writer.add_page(page) + tampered = io.BytesIO() + writer.write(tampered) + with pytest.raises(DocumentParityViolation): + verify_rendered_document( + document, pdf_bytes=tampered.getvalue(), + xlsx_bytes=render_xlsx(document)) diff --git a/tests/test_quote_retrieval_route.py b/tests/test_quote_retrieval_route.py new file mode 100644 index 0000000..e6e064d --- /dev/null +++ b/tests/test_quote_retrieval_route.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import importlib +from types import SimpleNamespace + +from fastapi.testclient import TestClient +from test_quote_artifacts import _persisted +from test_quote_store import NOW + +from gateway import SimulatedStreamingASR, SimulatedTTS, VoicePersona +from quoting.capabilities import CapabilityKeyring, CapabilityService +from runtime.app import create_app + + +def test_http_retrieval_is_neutral_and_serves_only_persisted_guarded_bytes( + tmp_path, monkeypatch): + store, document, pdf_bytes, _xlsx_bytes, _records = _persisted(tmp_path) + service = CapabilityService( + store, CapabilityKeyring( + keys={'t1': {'route-v1': b'r' * 32}}, + active_key_ids={'t1': 'route-v1'}), + token_factory=lambda: 'R' * 43) + token = service.mint(document, 'pdf') + gateway = SimpleNamespace( + catalog=SimpleNamespace(tenant_id=lambda: 't1'), + quote_capability_service=service, now_fn=lambda: NOW) + monkeypatch.setattr( + importlib.import_module('quoting.render_pdf'), 'render_pdf', + lambda _document: (_ for _ in ()).throw( + AssertionError('retrieval must never re-render'))) + client = TestClient(create_app( + gateway_bundle=(gateway, SimpleNamespace()), + streaming_asr=SimulatedStreamingASR(), tts=SimulatedTTS(), + persona=VoicePersona(), improvement=False)) + + response = client.get(f'/quote/{token}/pdf') + assert response.status_code == 200 and response.content == pdf_bytes + assert response.headers['cache-control'] == 'private, no-store' + assert response.headers['x-content-type-options'] == 'nosniff' + assert response.headers['content-type'].startswith('application/pdf') + + refusals = ( + client.get(f'/quote/{token}/xlsx'), + client.get(f'/quote/{token[:-1]}Z/pdf'), + client.get('/quote/q1.route-v1.short/pdf'), + ) + assert {(item.status_code, item.content) for item in refusals} == { + (404, b'')} diff --git a/tests/test_quote_store.py b/tests/test_quote_store.py index b464081..c0265ba 100644 --- a/tests/test_quote_store.py +++ b/tests/test_quote_store.py @@ -2,9 +2,14 @@ save/get round-trip (M5 Stage C, spec §4.1).""" from __future__ import annotations +import dataclasses +import hashlib +import threading from datetime import datetime, timedelta, timezone from decimal import Decimal +import pytest + from quoting.models import ( DeliveryPreflight, QuoteDocument, @@ -14,7 +19,7 @@ QuoteProvenance, QuoteTerms, ) -from quoting.store import SqliteQuoteStore, order_content_hash +from quoting.store import RevisionConflict, SqliteQuoteStore, order_content_hash from quoting.totals import build_totals, extended_price NOW = datetime(2026, 7, 10, 10, 0, tzinfo=timezone.utc) @@ -111,3 +116,302 @@ def test_order_content_hash_is_stable_and_content_sensitive(): h3 = order_content_hash((('K5-24SBC', 11), ('BH6-36SBC', 2))) assert h1 == h2 assert h1 != h3 + + +def _issue(store, *, session='s1', digest='digest', fault_after=None): + return store.issue_atomic( + tenant_id='t1', account_id='1001', session_id=session, + order_digest=digest, action='issue_quote', + assent_receipt=_receipt(session, digest), + delivery_channel='email', + document_factory=lambda number, revision: _document(number, revision), + fault_after=fault_after) + + +def _receipt(session, digest, line_versions=None): + return { + 'session_id': session, 'turn_id': f'turn:{session}', + 'order_digest': digest, + 'readback_receipt_id': f'readback:{digest[:24]}', + 'utterance_digest': hashlib.sha256(b'confirm').hexdigest(), + 'line_versions': line_versions or ['line:1'], 'occurred_at': 1.0, + } + + +def _count(store, table): + return store._conn.execute(f'SELECT COUNT(*) FROM {table}').fetchone()[0] + + +def test_atomic_issue_commits_quote_lineage_and_both_outboxes_once(tmp_path): + store = SqliteQuoteStore(tmp_path / 'atomic.sqlite') + document = _issue(store) + + assert document.header.quote_number == 'Q-t1-000001' + assert _count(store, 'quotes') == 1 + assert _count(store, 'issuance_events') == 1 + assert _count(store, 'artifact_jobs') == 1 + assert _count(store, 'delivery_jobs') == 1 + assert _count(store, 'quote_issue_keys') == 1 + assert _count(store, 'quote_queue') == 1 + + +def test_atomic_issue_rejects_unbound_or_incomplete_assent_receipt(tmp_path): + store = SqliteQuoteStore(tmp_path / 'invalid-assent.sqlite') + receipt = _receipt('s1', 'digest') + for mutation in ( + {**receipt, 'order_digest': 'other'}, + {key: value for key, value in receipt.items() if key != 'turn_id'}, + {**receipt, 'line_versions': []}, + {**receipt, 'utterance_digest': 'not-a-digest'}): + with pytest.raises(ValueError, match='assent receipt'): + store.issue_atomic( + tenant_id='t1', account_id='1001', session_id='s1', + order_digest='digest', action='issue_quote', + assent_receipt=mutation, delivery_channel='email', + document_factory=lambda number, revision: _document( + number, revision)) + assert _count(store, 'quotes') == _count(store, 'issuance_events') == 0 + + +def test_atomic_issue_faults_roll_back_every_binding_row_and_never_reuse_number( + tmp_path): + binding_tables = ( + 'quotes', 'issuance_events', 'artifact_jobs', 'delivery_jobs', + 'quote_issue_keys', 'quote_queue') + for statement in range(1, 7): + store = SqliteQuoteStore(tmp_path / f'fault-{statement}.sqlite') + try: + _issue(store, fault_after=statement) + except RuntimeError as exc: + assert f'statement {statement}' in str(exc) + else: # pragma: no cover - makes a missing planted boundary explicit + raise AssertionError(f'fault boundary {statement} did not fire') + assert all(_count(store, table) == 0 for table in binding_tables) + + recovered = _issue(store) + assert recovered.header.quote_number == 'Q-t1-000002' + + +def test_atomic_issue_retry_returns_winner_and_repairs_missing_outboxes(tmp_path): + store = SqliteQuoteStore(tmp_path / 'retry.sqlite') + first = _issue(store) + h = first.header + store._conn.execute('DELETE FROM artifact_jobs') + store._conn.execute('DELETE FROM delivery_jobs') + store._conn.execute('DELETE FROM quote_queue') + store._conn.commit() + + retried = _issue(store) + + assert retried == first + assert _count(store, 'quotes') == _count(store, 'issuance_events') == 1 + assert _count(store, 'artifact_jobs') == 1 + assert _count(store, 'delivery_jobs') == 1 + assert store.queue_item(h.quote_number, h.revision) is not None + + +def test_quote_revision_rows_are_insert_only(): + store = SqliteQuoteStore(':memory:') + document = _document('Q-t1-000001', 1) + store.save(document) + store.save(document) + changed = dataclasses.replace( + document, terms=dataclasses.replace( + document.terms, pricing_disclaimer='silently changed')) + + try: + store.save(changed) + except ValueError as exc: + assert 'immutable and insert-only' in str(exc) + else: # pragma: no cover + raise AssertionError('an existing quote revision was mutated') + with pytest.raises(Exception, match='immutable'): + store._conn.execute( + 'UPDATE quotes SET payload_json="{}" WHERE quote_number=?', + (document.header.quote_number,)) + with pytest.raises(Exception, match='insert-only'): + store._conn.execute( + 'DELETE FROM quotes WHERE quote_number=?', + (document.header.quote_number,)) + + +def test_concurrent_identical_issue_has_one_winner_and_one_binding_set(tmp_path): + path = tmp_path / 'concurrent.sqlite' + first_store = SqliteQuoteStore(path) + second_store = SqliteQuoteStore(path) + barrier = threading.Barrier(2) + results = [] + errors = [] + + def issue(store): + def build(number, revision): + barrier.wait() + return _document(number, revision) + + try: + results.append(store.issue_atomic( + tenant_id='t1', account_id='1001', session_id='same-session', + order_digest='same-digest', action='issue_quote', + assent_receipt=_receipt('same-session', 'same-digest'), + delivery_channel='email', + document_factory=build)) + except Exception as exc: # captured so the main test owns the assertion + errors.append(exc) + + threads = [threading.Thread(target=issue, args=(store,)) + for store in (first_store, second_store)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert not errors + assert len(results) == 2 + assert results[0] == results[1] + assert _count(first_store, 'quotes') == 1 + assert _count(first_store, 'issuance_events') == 1 + assert _count(first_store, 'artifact_jobs') == 1 + assert _count(first_store, 'delivery_jobs') == 1 + assert _count(first_store, 'quote_issue_keys') == 1 + # Both contenders allocated before the winner was known; the losing + # number is a permanent legal gap, never a second quote. + assert _count(first_store, 'quote_number_allocations') == 2 + + +def test_shared_store_instance_serializes_simultaneous_initial_issuance(tmp_path): + store = SqliteQuoteStore(tmp_path / 'shared-instance.sqlite') + start = threading.Barrier(3) + results = [] + errors = [] + + def issue(): + start.wait() + try: + results.append(_issue( + store, session='shared-session', digest='shared-digest')) + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=issue) for _ in range(2)] + for thread in threads: + thread.start() + start.wait() + for thread in threads: + thread.join(timeout=10) + + assert not errors and len(results) == 2 + assert results[0] == results[1] + assert _count(store, 'quotes') == _count(store, 'issuance_events') == 1 + assert _count(store, 'quote_number_allocations') == 1 + + +def _revise(store, original, *, session='revision-session', digest='revised', + qty=15, fault_after=None): + return store.revise_atomic( + tenant_id='t1', account_id='1001', + quote_number=original.header.quote_number, + expected_revision=original.header.revision, session_id=session, + order_digest=digest, + assent_receipt=_receipt(session, digest), + delivery_channel='email', + document_factory=lambda number, revision: _document( + number, revision, qty=qty), fault_after=fault_after) + + +def test_atomic_revision_commits_new_row_and_supersedes_prior_together(tmp_path): + store = SqliteQuoteStore(tmp_path / 'revision.sqlite') + original = _issue(store) + revised = _revise(store, original) + + assert revised.header.quote_number == original.header.quote_number + assert revised.header.revision == 2 + assert store.status_of(original.header.quote_number, 1) == 'superseded' + assert store.status_of(original.header.quote_number, 2) == 'active' + assert _count(store, 'quotes') == 2 + assert _count(store, 'issuance_events') == 2 + assert _count(store, 'artifact_jobs') == 2 + assert _count(store, 'delivery_jobs') == 2 + + +def test_atomic_revision_faults_leave_original_active_and_no_descendant(tmp_path): + for statement in range(1, 9): + store = SqliteQuoteStore(tmp_path / f'revision-fault-{statement}.sqlite') + original = _issue(store) + try: + _revise(store, original, fault_after=statement) + except RuntimeError as exc: + assert f'statement {statement}' in str(exc) + else: # pragma: no cover + raise AssertionError(f'revision fault boundary {statement} did not fire') + + assert store.get(original.header.quote_number).header.revision == 1 + assert store.status_of(original.header.quote_number, 1) == 'active' + assert _count(store, 'quotes') == 1 + assert _count(store, 'issuance_events') == 1 + assert _count(store, 'artifact_jobs') == 1 + assert _count(store, 'delivery_jobs') == 1 + assert _count(store, 'quote_queue') == 1 + + +def test_atomic_revision_retry_is_idempotent_and_repairs_outboxes(tmp_path): + store = SqliteQuoteStore(tmp_path / 'revision-retry.sqlite') + original = _issue(store) + revised = _revise(store, original) + store._conn.execute('DELETE FROM artifact_jobs WHERE revision=2') + store._conn.execute('DELETE FROM delivery_jobs WHERE revision=2') + store._conn.execute('DELETE FROM quote_queue WHERE revision=2') + store._conn.commit() + + retried = _revise(store, original) + + assert retried == revised + assert _count(store, 'quotes') == _count(store, 'issuance_events') == 2 + assert _count(store, 'artifact_jobs') == 2 + assert _count(store, 'delivery_jobs') == 2 + assert _count(store, 'quote_queue') == 2 + + +def test_concurrent_revision_cas_allows_one_descendant(tmp_path): + path = tmp_path / 'revision-race.sqlite' + seed_store = SqliteQuoteStore(path) + original = _issue(seed_store) + stores = (SqliteQuoteStore(path), SqliteQuoteStore(path)) + barrier = threading.Barrier(2) + results = [] + errors = [] + + def revise(store, session, digest, qty): + def build(number, revision): + barrier.wait() + return _document(number, revision, qty=qty) + + try: + results.append(store.revise_atomic( + tenant_id='t1', account_id='1001', + quote_number=original.header.quote_number, + expected_revision=1, session_id=session, + order_digest=digest, + assent_receipt=_receipt(session, digest), + delivery_channel='email', document_factory=build)) + except Exception as exc: + errors.append(exc) + + threads = [ + threading.Thread(target=revise, + args=(stores[0], 'revision-a', 'digest-a', 11)), + threading.Thread(target=revise, + args=(stores[1], 'revision-b', 'digest-b', 12)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert len(results) == 1 + assert len(errors) == 1 and isinstance(errors[0], RevisionConflict) + assert _count(seed_store, 'quotes') == 2 + assert _count(seed_store, 'issuance_events') == 2 + assert _count(seed_store, 'artifact_jobs') == 2 + assert _count(seed_store, 'delivery_jobs') == 2 + assert seed_store.status_of(original.header.quote_number, 1) == 'superseded' + assert seed_store.status_of(original.header.quote_number, 2) == 'active' diff --git a/tests/test_quote_worker.py b/tests/test_quote_worker.py index d206143..d0bfd87 100644 --- a/tests/test_quote_worker.py +++ b/tests/test_quote_worker.py @@ -2,6 +2,7 @@ from __future__ import annotations import dataclasses +import hashlib from datetime import datetime, timedelta, timezone from decimal import Decimal from urllib.parse import urlparse @@ -9,7 +10,9 @@ import pytest from gateway import Account, ConversationJournal, EventType -from quoting.delivery import SimulatedDeliverer, mint_link, verify_link +from quoting.capabilities import CapabilityKeyring, CapabilityService +from quoting.delivery import SimulatedDeliverer +from quoting.document_guard import verify_rendered_document from quoting.models import ( DeliveryPreflight, QuoteDocument, @@ -72,9 +75,12 @@ def _account(_account_id): return Account('1001', 'DEMO TRUCK CENTER', email='quotes@example.test') -def _link(document): - return mint_link(document.header.quote_number, document.header.revision, - document.header.valid_until, SECRET) +def _link_factory(store): + service = CapabilityService( + store, CapabilityKeyring( + keys={'tenant_001': {'test-v1': hashlib.sha256(SECRET).digest()}}, + active_key_ids={'tenant_001': 'test-v1'})) + return lambda document: service.mint(document, 'pdf') def _unsafe_document(document, lines): @@ -100,6 +106,7 @@ def test_fail_closed_guard_called_once_blocks_and_never_delivers( from quoting.render_pdf import render_pdf from quoting.render_xlsx import render_xlsx guard_calls = 0 + escalations = [] def counted_guard(doc, *, pdf_bytes, xlsx_bytes): nonlocal guard_calls @@ -112,12 +119,20 @@ def counted_guard(doc, *, pdf_bytes, xlsx_bytes): assert run_once(store=store, deliverer=deliverer, account_lookup=_account, journal=journal, - link_factory=_link) is True + link_factory=_link_factory(store), now_fn=lambda: NOW, + escalation_hook=lambda reason, context: escalations.append( + (reason, context))) is True assert guard_calls == 1 assert deliverer.payloads == [] row = store.queue_item(document.header.quote_number, 1) assert row.status == 'blocked' + artifact_job = store._conn.execute( + 'SELECT status FROM artifact_jobs').fetchone() + assert artifact_job['status'] == 'quarantined' + assert store._conn.execute( + 'SELECT COUNT(*) FROM artifact_guard_events').fetchone()[0] == 1 + assert len(escalations) == 1 and escalations[0][0] == 'document_parity' blocked = journal.events(EventType.QUOTE_DELIVERY_BLOCKED) assert len(blocked) == 1 assert blocked[0]['quote_number'] == document.header.quote_number @@ -129,7 +144,7 @@ def test_untampered_positive_control_delivers_exactly_once(tmp_path): assert run_once(store=store, deliverer=deliverer, account_lookup=_account, journal=journal, - link_factory=_link) is True + link_factory=_link_factory(store), now_fn=lambda: NOW) is True assert len(deliverer.payloads) == 1 payload = deliverer.payloads[0] @@ -149,7 +164,8 @@ def test_missing_contact_of_record_blocks_without_delivery(tmp_path): assert run_once( store=store, deliverer=deliverer, account_lookup=lambda _aid: Account('1001', 'DEMO TRUCK CENTER'), - journal=journal, link_factory=_link) is True + journal=journal, link_factory=_link_factory(store), + now_fn=lambda: NOW) is True assert deliverer.payloads == [] row = store.queue_item(document.header.quote_number, 1) @@ -166,13 +182,13 @@ def test_no_pending_work_is_a_noop(tmp_path): now_fn=lambda: NOW.isoformat()) assert run_once(store=store, deliverer=deliverer, account_lookup=_account, journal=journal, - link_factory=_link) is False + link_factory=_link_factory(store), + now_fn=lambda: NOW) is False assert deliverer.payloads == [] def test_assent_enqueues_worker_delivers_working_link(tmp_path): from gateway_fixtures import ( - QUOTE_LINK_SECRET, build_gateway, worker_pass, ) @@ -201,26 +217,45 @@ def test_assent_enqueues_worker_delivers_working_link(tmp_path): parsed = urlparse(payload.signed_link) assert parsed.scheme == 'https' assert parsed.netloc == 'quotes.example.test' - assert parsed.path.startswith('/quote/') - signed_token = parsed.path.removeprefix('/quote/') - assert verify_link( - signed_token, document.header.created_at, - QUOTE_LINK_SECRET) == (quote_number, 1) + prefix = '/quote/' + assert parsed.path.startswith(prefix) and parsed.path.endswith('/pdf') + opaque_token = parsed.path.removeprefix(prefix).removesuffix('/pdf') + retrieved = gw.quote_capability_service.retrieve( + document.header.tenant_id, opaque_token, 'pdf', + now=document.header.created_at) + assert retrieved is not None and retrieved.body == payload.pdf_bytes def test_runtime_link_factory_targets_https_quote_route(monkeypatch): - from runtime.config import build_quote_link_factory, quote_public_base_url + from quoting.render_pdf import render_pdf + from quoting.render_xlsx import render_xlsx + from runtime.config import ( + build_quote_capability_service, + build_quote_link_factory, + quote_public_base_url, + ) monkeypatch.setenv('SKU_QUOTE_PUBLIC_BASE_URL', 'https://quotes.tenant.example/') base_url = quote_public_base_url() - link = build_quote_link_factory(SECRET, base_url)(_document()) + store = SqliteQuoteStore(':memory:') + document = _document() + store.save(document) + store.enqueue(document.header.quote_number, document.header.revision) + pdf_bytes, xlsx_bytes = render_pdf(document), render_xlsx(document) + verify_rendered_document( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes) + store.persist_artifacts_atomic( + document, pdf_bytes=pdf_bytes, xlsx_bytes=xlsx_bytes, verified_at=NOW) + service = build_quote_capability_service( + store, document.header.tenant_id, SECRET) + link = build_quote_link_factory(service, base_url)(document) parsed = urlparse(link) assert parsed.scheme == 'https' assert parsed.netloc == 'quotes.tenant.example' - signed_token = parsed.path.removeprefix('/quote/') - assert verify_link(signed_token, NOW, SECRET) == ( - _document().header.quote_number, 1) + token = parsed.path.removeprefix('/quote/').removesuffix('/pdf') + retrieved = service.retrieve('tenant_001', token, 'pdf', now=NOW) + assert retrieved is not None and retrieved.body == pdf_bytes monkeypatch.setenv('SKU_QUOTE_PUBLIC_BASE_URL', 'http://quotes.example.test') with pytest.raises(ValueError, match='must be an HTTPS origin'): diff --git a/tests/test_quoting_purity.py b/tests/test_quoting_purity.py index 7b6b34c..0cd5a5a 100644 --- a/tests/test_quoting_purity.py +++ b/tests/test_quoting_purity.py @@ -13,8 +13,27 @@ # Whitelist per PURE module. Anything imported outside this table fails the # test — including indirect creep like `sqlite3`, `gateway`, or a model SDK. ALLOWED = { - 'models.py': {'__future__', 'dataclasses', 'datetime', 'decimal', 're'}, + 'models.py': { + '__future__', 'base64', 'binascii', 'dataclasses', 'datetime', + 'decimal', 're', + }, 'totals.py': {'__future__', 'decimal', 'quoting.models'}, + 'display.py': {'__future__', 're'}, + 'projection.py': { + '__future__', 'dataclasses', 'quoting.display', 'quoting.models', + 'typing', + }, + 'render_pdf.py': { + '__future__', 'base64', 'io', 'quoting.display', 'quoting.models', + 'quoting.projection', 'reportlab.lib.pagesizes', + 'reportlab.lib.styles', 'reportlab.platypus', 'xml.sax.saxutils', + }, + 'render_xlsx.py': { + '__future__', 'base64', 'io', 'openpyxl', + 'openpyxl.drawing.image', 'openpyxl.styles', 'openpyxl.utils', + 'openpyxl.workbook.defined_name', 'openpyxl.worksheet.table', + 'quoting.display', 'quoting.models', 'quoting.projection', + }, } From 13450df0b067f573d520f0a47e0c23bea780532e Mon Sep 17 00:00:00 2001 From: Owie Schon Date: Sat, 11 Jul 2026 14:35:00 -0400 Subject: [PATCH 3/3] test: gate quote layout on Poppler runtime --- tests/test_quote_artifact_layout.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_quote_artifact_layout.py b/tests/test_quote_artifact_layout.py index fd64ad1..e66af59 100644 --- a/tests/test_quote_artifact_layout.py +++ b/tests/test_quote_artifact_layout.py @@ -37,6 +37,8 @@ def _maximum_layout_document(): 'separately. ' * 20))) +@pytest.mark.skipif(shutil.which('pdftoppm') is None, + reason='pinned Poppler visual runtime unavailable') def test_maximum_layout_has_full_parity_and_printable_page_bounds(tmp_path): document = _maximum_layout_document() pdf_bytes = render_pdf(document)