From e880e5a36375a3785ad33a93ff100063aec26235 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Tue, 25 Aug 2026 21:42:53 -0700 Subject: [PATCH 1/3] feat(catalyst): preserve complete session context evidence --- catalyst-gateway/src/catalyst/query_engine.py | 277 +++++++++++++++++- catalyst-gateway/src/catalyst/service.py | 102 ++++++- .../src/catalyst/session_context.py | 139 ++------- catalyst-gateway/src/catalyst/storage.py | 40 +-- catalyst-gateway/src/catalyst/workbench.py | 77 ++--- catalyst-gateway/tests/test_local_hub.py | 2 +- catalyst-gateway/tests/test_query_engine.py | 163 +++++++++++ catalyst-gateway/tests/test_query_notebook.py | 123 +++++++- .../tests/test_session_context.py | 202 ++++++------- .../tests/test_workbench_guidance.py | 27 +- .../tests/test_workbench_routes.py | 203 ++++++++++++- .../catalyst-query-request-v2.schema.json | 2 +- ...lyst-query-revision-context-v1.schema.json | 25 +- docs/contracts/catalyst-query-v1.schema.json | 130 ++++++++ ...rkbench-generation-evidence-v1.schema.json | 9 +- ...-workbench-guidance-request-v1.schema.json | 4 +- ...catalyst-workbench-guidance-v1.schema.json | 10 +- .../catalyst-workbench-turn-v1.schema.json | 12 +- docs/med-agent-hub.md | 7 + docs/specification.md | 25 +- 20 files changed, 1188 insertions(+), 391 deletions(-) diff --git a/catalyst-gateway/src/catalyst/query_engine.py b/catalyst-gateway/src/catalyst/query_engine.py index 57ba7aaa..146cabf8 100644 --- a/catalyst-gateway/src/catalyst/query_engine.py +++ b/catalyst-gateway/src/catalyst/query_engine.py @@ -81,6 +81,9 @@ "CATALYST_HUB_QUERY_PROFILE_URL", "http://med-agent-hub:8080/v1/hub/query-profiles" ) _HUB_TIMEOUT_SECONDS = float(os.getenv("CATALYST_HUB_TIMEOUT_SECONDS", "1800")) +_ROLE_REQUEST_EVIDENCE_CONTRACT = "med-agent-hub.catalyst-role-request-evidence.v1" +_REQUEST_EVIDENCE_ERROR_ATTRIBUTE = "_catalyst_request_evidence" +_HUB_ERROR_ATTRIBUTE = "_catalyst_hub_error" @dataclass(frozen=True) @@ -154,7 +157,11 @@ async def _backend_chat( temperature: float, dry_multiplier: float, max_tokens: Optional[int], -) -> tuple[str, Optional[Mapping[str, Any]]]: +) -> tuple[ + str, + Optional[Mapping[str, Any]], + Optional[Mapping[str, Any]], +]: """Call a named Hub query role; caller-provided model settings are ignored.""" payload: Dict[str, Any] = {"messages": messages} @@ -165,7 +172,50 @@ async def _backend_chat( json=payload, timeout=_HUB_TIMEOUT_SECONDS, ) - resp.raise_for_status() + if not resp.is_success: + detail: Any = None + try: + error_document = resp.json() + if isinstance(error_document, Mapping): + detail = error_document.get("detail") + except (TypeError, ValueError): + pass + request_evidence = None + hub_error = None + if isinstance(detail, Mapping): + if "request_evidence" in detail: + request_evidence = _validated_role_request_evidence( + detail.get("request_evidence"), + profile_id=profile_id, + role=role, + model=model, + caller_messages=messages, + response_format=response_format, + temperature=temperature, + dry_multiplier=dry_multiplier, + max_tokens=max_tokens, + ) + code = detail.get("code") + if isinstance(code, str) and code: + hub_error = { + "code": code, + "httpStatus": resp.status_code, + } + message = detail.get("message") + if isinstance(message, str) and message: + hub_error["message"] = message + try: + resp.raise_for_status() + except httpx.HTTPStatusError as error: + if request_evidence is not None: + setattr( + error, + _REQUEST_EVIDENCE_ERROR_ATTRIBUTE, + request_evidence, + ) + if hub_error is not None: + setattr(error, _HUB_ERROR_ATTRIBUTE, hub_error) + raise message = resp.json() content = message.get("content") if isinstance(message, Mapping) else None if not isinstance(content, str) or not content.strip(): @@ -173,7 +223,178 @@ async def _backend_chat( accounting = ( message.get("token_accounting") if isinstance(message, Mapping) else None ) - return content.strip(), accounting if isinstance(accounting, Mapping) else None + request_evidence = ( + _validated_role_request_evidence( + message.get("request_evidence"), + profile_id=profile_id, + role=role, + model=model, + caller_messages=messages, + response_format=response_format, + temperature=temperature, + dry_multiplier=dry_multiplier, + max_tokens=max_tokens, + ) + if isinstance(message, Mapping) and "request_evidence" in message + else None + ) + return ( + content.strip(), + accounting if isinstance(accounting, Mapping) else None, + request_evidence, + ) + + +def _validated_role_request_evidence( + value: Any, + *, + profile_id: str, + role: str, + model: str, + caller_messages: list[dict[str, str]], + response_format: Mapping[str, Any], + temperature: float, + dry_multiplier: float, + max_tokens: Optional[int], +) -> dict[str, Any]: + """Validate the exact Hub-owned role request before retaining it as evidence.""" + + def invalid(message: str) -> QueryContractError: + return QueryContractError(f"Hub request evidence {message}") + + if not isinstance(value, Mapping): + raise invalid("must be an object") + evidence = deepcopy(dict(value)) + if set(evidence) != { + "contractVersion", + "request", + "requestDigest", + "prompt", + "tokens", + }: + raise invalid("has unexpected or missing fields") + if evidence["contractVersion"] != _ROLE_REQUEST_EVIDENCE_CONTRACT: + raise invalid("uses an unsupported contract version") + + exact_request = evidence["request"] + if not isinstance(exact_request, Mapping) or set(exact_request) != { + "profileId", + "role", + "model", + "messages", + "responseFormat", + "config", + }: + raise invalid("request has unexpected or missing fields") + exact_messages = exact_request["messages"] + if ( + exact_request["profileId"] != profile_id + or exact_request["role"] != role + or exact_request["model"] != model + or not isinstance(exact_messages, list) + or len(exact_messages) != len(caller_messages) + 1 + or not isinstance(exact_messages[0], Mapping) + or exact_messages[0].get("role") != "system" + or not isinstance(exact_messages[0].get("content"), str) + or exact_messages[1:] != caller_messages + or exact_request["responseFormat"] != dict(response_format) + ): + raise invalid("does not match the configured role call") + config = exact_request["config"] + expected_config = { + "temperature": temperature, + "dryMultiplier": dry_multiplier, + "maxTokens": max_tokens, + } + if not isinstance(config, Mapping) or dict(config) != expected_config: + raise invalid("configuration does not match the configured role call") + request_digest = evidence["requestDigest"] + if not isinstance( + request_digest, str + ) or request_digest != _canonical_evidence_digest(exact_request): + raise invalid("requestDigest does not match the exact request") + + prompt = evidence["prompt"] + if not isinstance(prompt, Mapping) or not { + "renderedPrompt", + "renderedPromptDigest", + }.issubset(prompt): + raise invalid("prompt measurement is incomplete") + if not set(prompt).issubset( + {"renderedPrompt", "renderedPromptDigest", "unavailableReason"} + ): + raise invalid("prompt measurement has unexpected fields") + rendered_prompt = prompt["renderedPrompt"] + rendered_digest = prompt["renderedPromptDigest"] + if rendered_prompt is None: + if rendered_digest is not None or not isinstance( + prompt.get("unavailableReason"), str + ): + raise invalid("prompt absence is not explained") + elif ( + not isinstance(rendered_prompt, str) + or rendered_digest != _evidence_digest(rendered_prompt) + or "unavailableReason" in prompt + ): + raise invalid("renderedPromptDigest does not match the exact prompt") + + tokens = evidence["tokens"] + required_token_fields = { + "tokenizer", + "contextWindow", + "outputReserve", + "promptTokens", + "requiredTokens", + "fits", + } + if not isinstance(tokens, Mapping) or not required_token_fields.issubset(tokens): + raise invalid("token measurement is incomplete") + if not set(tokens).issubset( + required_token_fields + | {"contextWindowUnavailableReason", "promptTokensUnavailableReason"} + ): + raise invalid("token measurement has unexpected fields") + context_window = tokens["contextWindow"] + output_reserve = tokens["outputReserve"] + prompt_tokens = tokens["promptTokens"] + required_tokens = tokens["requiredTokens"] + fits = tokens["fits"] + if tokens["tokenizer"] != model: + raise invalid("tokenizer does not match the configured model") + if type(output_reserve) is not int or output_reserve < 0: + raise invalid("outputReserve must be a non-negative integer") + if max_tokens is not None and output_reserve != max_tokens: + raise invalid("outputReserve does not match the configured role call") + if context_window is None: + if not isinstance(tokens.get("contextWindowUnavailableReason"), str): + raise invalid("missing context window is not explained") + elif ( + type(context_window) is not int + or context_window < 1 + or "contextWindowUnavailableReason" in tokens + ): + raise invalid("contextWindow is invalid") + if prompt_tokens is None: + if ( + required_tokens is not None + or fits is not None + or not isinstance(tokens.get("promptTokensUnavailableReason"), str) + ): + raise invalid("missing prompt token count is not explained") + else: + if ( + type(prompt_tokens) is not int + or prompt_tokens < 0 + or "promptTokensUnavailableReason" in tokens + or required_tokens != prompt_tokens + output_reserve + ): + raise invalid("prompt token count is inconsistent") + expected_fit = ( + required_tokens <= context_window if context_window is not None else None + ) + if fits is not expected_fit: + raise invalid("fit result is inconsistent with the exact token counts") + return evidence def _evidence_digest(value: Any) -> str: @@ -361,20 +582,60 @@ async def _invoke_backend( dry_multiplier=dry_multiplier, max_tokens=max_tokens, ) - # Test doubles may still answer with a bare string; production answers - # (content, accounting). Either way each invocation keeps its own count. - content, accounting = ( - answered if isinstance(answered, tuple) else (answered, None) - ) + # Older test doubles may still answer with a bare string or the former + # (content, accounting) pair. The Hub now returns the exact configured + # role request as a third value. + if isinstance(answered, tuple) and len(answered) == 3: + content, accounting, request_evidence = answered + elif isinstance(answered, tuple) and len(answered) == 2: + content, accounting = answered + request_evidence = None + else: + content, accounting, request_evidence = answered, None, None invocation["tokenAccounting"] = ( dict(accounting) if isinstance(accounting, Mapping) else None ) + if isinstance(request_evidence, Mapping): + invocation["requestEvidence"] = deepcopy(dict(request_evidence)) + invocation["requestDigest"] = str(request_evidence["requestDigest"]) except asyncio.CancelledError as exc: _finish_invocation(invocation, outcome="cancelled", failure=repr(exc)) raise except (TimeoutError, httpx.TimeoutException) as exc: _finish_invocation(invocation, outcome="timed_out", failure=repr(exc)) raise + except httpx.HTTPStatusError as exc: + request_evidence = getattr( + exc, + _REQUEST_EVIDENCE_ERROR_ATTRIBUTE, + None, + ) + if isinstance(request_evidence, Mapping): + invocation["requestEvidence"] = deepcopy(dict(request_evidence)) + invocation["requestDigest"] = str(request_evidence["requestDigest"]) + hub_error = getattr(exc, _HUB_ERROR_ATTRIBUTE, None) + if isinstance(hub_error, Mapping): + invocation["hubError"] = deepcopy(dict(hub_error)) + outcome = ( + "pre_dispatch_rejected" + if isinstance(hub_error, Mapping) + and hub_error.get("code") == "context_window_exceeded" + else "transport_failed" + ) + _finish_invocation( + invocation, + outcome=outcome, + failure=( + dict(hub_error) + if isinstance(hub_error, Mapping) + else { + "type": type(exc).__name__, + "status": exc.response.status_code, + "message": str(exc), + } + ), + ) + raise except QueryContractError as exc: invocation["responseDigest"] = _evidence_digest("") _finish_invocation(invocation, outcome="contract_failed", failure=str(exc)) diff --git a/catalyst-gateway/src/catalyst/service.py b/catalyst-gateway/src/catalyst/service.py index dd88c94d..e3059f98 100644 --- a/catalyst-gateway/src/catalyst/service.py +++ b/catalyst-gateway/src/catalyst/service.py @@ -46,6 +46,7 @@ from .workbench import ( build_advisory_validation, build_revision_context, + finalize_revision_context_digest, normalize_findings, workbench_query_digest, ) @@ -1427,7 +1428,7 @@ def prepare_request( ): revision["sessionContext"] = build_session_context( guidance=store.active_guidance(session_id), - omitted_guidance=store.guidance_omitted_by_cap(session_id), + omitted_guidance=[], verified_examples=select_verified_examples( self._verified_examples(session, prior_turns), instruction=instruction, @@ -1437,6 +1438,7 @@ def prepare_request( ), relevant_failure=self._relevant_prior_failure(prior_turns), ) + finalize_revision_context_digest(revision) request = build_revision_query_request( instruction, runtime_catalog, @@ -2635,6 +2637,12 @@ def _generation_invocations( configuration = item.get("configuration") if isinstance(configuration, dict): invocation["configuration"] = deepcopy(configuration) + request_evidence = item.get("requestEvidence") + if isinstance(request_evidence, dict): + invocation["requestEvidence"] = deepcopy(request_evidence) + hub_error = item.get("hubError") + if isinstance(hub_error, dict): + invocation["hubError"] = deepcopy(hub_error) projected.append(invocation) return projected # Model invocations are Hub-owned evidence. A Gateway-to-Hub request is @@ -2680,6 +2688,14 @@ def _model_failure_stage( return f"{role}_output_contract", f"{role}_output_contract_failed" if outcome == "validation_failed": return f"{role}_output_contract", f"{role}_output_contract_failed" + if outcome == "pre_dispatch_rejected": + hub_error = terminal.get("hubError") + code = ( + str(hub_error.get("code")) + if isinstance(hub_error, dict) and hub_error.get("code") + else f"{role}_request_rejected" + ) + return f"{role}_request", code if outcome == "timed_out": return f"{role}_transport", f"{role}_timeout" if outcome == "cancelled": @@ -2972,11 +2988,11 @@ def _failure_check_details(outcome: dict[str, Any] | None) -> list[dict[str, Any # The diagnostic contract's detail shape is {name, value}. details.append( { - "name": str(check.get("name") or "unnamed_check")[:100], - "value": value[:4000], + "name": str(check.get("name") or "unnamed_check"), + "value": value, } ) - return details[:32] + return details @classmethod def _failure_details(cls, outcome: dict[str, Any] | None) -> list[dict[str, Any]]: @@ -2987,7 +3003,7 @@ def _failure_details(cls, outcome: dict[str, Any] | None) -> list[dict[str, Any] """ details = [ { - "name": str(finding.get("code"))[:100], + "name": str(finding.get("code")), "value": " ".join( part for part in ( @@ -3001,11 +3017,11 @@ def _failure_details(cls, outcome: dict[str, Any] | None) -> list[dict[str, Any] str(finding.get("suggestedAction") or "").strip(), ) if part - )[:4000], + ), } for finding in cls._unresolved_findings(outcome) ] - return (details + cls._failure_check_details(outcome))[:32] + return details + cls._failure_check_details(outcome) @staticmethod def _response_hub_trace_id(outcome: dict[str, Any]) -> str | None: @@ -3227,7 +3243,7 @@ def _raw_workbench_draft_seed(raw_output: object) -> dict[str, Any] | None: def pin_workbench_guidance( self, session_id: str, payload: dict[str, Any] ) -> ServiceResponse: - """Pin one instruction to a session, exactly as written.""" + """Record one optional experimental instruction exactly as written.""" try: self.contracts.validate( "catalyst-workbench-guidance-request-v1.schema.json", payload @@ -3276,20 +3292,52 @@ def unpin_workbench_guidance( def _verified_examples( session: Mapping[str, Any], prior_turns: Sequence[Mapping[str, Any]] ) -> list[dict[str, Any]]: - """Queries this session already accepted, as example candidates. - - A verified example is a kept version: a turn selected it and it - survived. Successful queries are examples, never guidance. - """ + """Return kept queries that were validated and successfully executed.""" versions = { str(version["versionId"]): version for version in session.get("versions", []) } + validations: dict[str, list[dict[str, Any]]] = {} + for validation in session.get("validations", []): + if not isinstance(validation, Mapping): + continue + version_id = str(validation.get("versionId") or "") + version = versions.get(version_id) + if version is None or validation.get("queryDigest") != version.get( + "queryDigest" + ): + continue + validations.setdefault(version_id, []).append(dict(validation)) + + successful_executions: dict[str, list[dict[str, Any]]] = {} + session_id = str(session.get("sessionId") or "") + for execution in session.get("executions", []): + if ( + not isinstance(execution, Mapping) + or execution.get("status") != "succeeded" + ): + continue + version_id = str(execution.get("versionId") or "") + version = versions.get(version_id) + if ( + version is None + or execution.get("queryDigest") != version.get("queryDigest") + or ( + execution.get("sessionId") is not None + and str(execution.get("sessionId")) != session_id + ) + ): + continue + successful_executions.setdefault(version_id, []).append(dict(execution)) + examples: list[dict[str, Any]] = [] for turn in prior_turns: selected = turn.get("selectedVersionId") version = versions.get(str(selected)) if selected else None - if version is None: + version_id = str(selected or "") + version_validations = validations.get(version_id, []) + version_executions = successful_executions.get(version_id, []) + if version is None or not version_validations or not version_executions: continue examples.append( { @@ -3299,6 +3347,32 @@ def _verified_examples( "queryDigest": str(version["queryDigest"]), "sourceId": str(turn.get("dataSourceId") or ""), "catalogVersion": str(turn.get("catalogVersion") or ""), + "advisoryValidations": [ + { + key: deepcopy(validation.get(key)) + for key in ( + "validationId", + "status", + "validatorRevision", + "validatorDigest", + "findings", + "createdAt", + ) + } + for validation in version_validations + ], + "successfulExecutions": [ + { + key: deepcopy(execution.get(key)) + for key in ( + "executionId", + "status", + "completedAt", + "durationMs", + ) + } + for execution in version_executions + ], } ) return examples diff --git a/catalyst-gateway/src/catalyst/session_context.py b/catalyst-gateway/src/catalyst/session_context.py index 98fd6762..70cc3b93 100644 --- a/catalyst-gateway/src/catalyst/session_context.py +++ b/catalyst-gateway/src/catalyst/session_context.py @@ -1,44 +1,20 @@ -"""The layered context a session hands the writer. +"""The complete eligible session context supplied to the writer. -Three bounded layers ride on every generation request: what a person pinned, -what already worked in this session, and the one failure this attempt should -not repeat. Their order is fixed, because position reads as authority to a -model, and their precedence is stated rather than implied. - -Nothing here summarises. Guidance is delivered exactly as written -- the -wording is the instruction -- and anything the caps exclude is recorded as an -omission rather than dropped silently. +Nothing here ranks, summarises, or caps context items. The Hub records the +actual assembled model request and determines whether that complete request +fits the selected model. """ from __future__ import annotations -import re from typing import Any, Iterable, Mapping, Sequence SESSION_CONTEXT_CONTRACT = "catalyst.query.session-context.v1" """What a Hub must advertise before Catalyst sends the layered context.""" -LAYER_ORDER: tuple[str, ...] = ( - "guidance", - "verifiedExamples", - "editorSnapshot", - "instructionHistory", - "relevantFailure", - "currentValidation", - "currentInstruction", -) -"""Delivery order inside the request, after contract, catalog, and policy. - -Contract, catalog and policy outrank all of this; the current instruction -comes last because it outranks the guidance retained above it. -""" - -MAX_VERIFIED_EXAMPLES = 3 - -_GUIDANCE_PRECEDENCE = ( - "Standing instructions for this session, in the order they were pinned. " - "They outrank the retained history but not the current instruction; " - "where two conflict, the later one wins." +_GUIDANCE_ROLE = ( + "Additional session guidance supplied for this experiment, preserved " + "verbatim with its provenance." ) _EXAMPLE_ROLE = ( "Evidence: queries already accepted in this session, for reference. " @@ -51,12 +27,6 @@ _MODEL_FACING_GUIDANCE = ("text", "source", "originTurnId", "createdAt") -_WORD = re.compile(r"[a-z0-9_]+") - - -def _words(text: str) -> set[str]: - return set(_WORD.findall(text.casefold())) - def select_verified_examples( candidates: Sequence[Mapping[str, Any]], @@ -66,37 +36,16 @@ def select_verified_examples( catalog_version: str, exclude_turn_id: str | None, ) -> list[dict[str, Any]]: - """The kept queries worth showing, most similar first. + """Keep every eligible earlier example in recorded session order.""" - Eligible means: accepted earlier in this session, against the same source - and catalog, and not this turn's own answer. Ranking is normalised word - overlap with the request, then the newest turn, then the stable id -- so - the same session always produces the same three. - """ - wanted = _words(instruction) - eligible = [ - candidate + _ = instruction # Kept in the call shape; relevance ranking is intentionally absent. + return [ + dict(candidate) for candidate in candidates if candidate.get("sourceId") == source_id and candidate.get("catalogVersion") == catalog_version and candidate.get("turnId") != exclude_turn_id ] - ranked = sorted( - eligible, - key=lambda candidate: ( - -len(wanted & _words(str(candidate.get("instruction", "")))), - # Newest turn first, then the stable id, so ties never depend on - # the order the rows happened to arrive in. - _descending(str(candidate.get("turnId", ""))), - str(candidate.get("turnId", "")), - ), - ) - return [dict(candidate) for candidate in ranked[:MAX_VERIFIED_EXAMPLES]] - - -def _descending(value: str) -> tuple[int, ...]: - """Sort key that orders strings newest-first without reversing the whole sort.""" - return tuple(-ord(character) for character in value) def build_session_context( @@ -119,20 +68,20 @@ def build_session_context( if guidance: context["guidance"] = { - "precedence": _GUIDANCE_PRECEDENCE, + "role": _GUIDANCE_ROLE, "entries": [ {key: entry.get(key) for key in _MODEL_FACING_GUIDANCE} for entry in guidance ], } - if omitted: - omissions.append( - { - "layer": "guidance", - "itemIds": [str(entry["entryId"]) for entry in omitted], - "reason": "active_entry_cap", - } - ) + omissions.extend( + { + "layer": "guidance", + "itemIds": [str(entry["entryId"])], + "reason": str(entry.get("omissionReason") or "not_supplied"), + } + for entry in omitted + ) if examples: context["verifiedExamples"] = { "role": _EXAMPLE_ROLE, @@ -145,51 +94,3 @@ def build_session_context( } context["omissions"] = omissions return context - - -class TokenAccountingError(ValueError): - """The request cannot be counted, so it must not be sent.""" - - -def account_for_tokens( - *, - rendered: str, - profile: Mapping[str, Any], - included_item_ids: Sequence[str], - omissions: Sequence[Mapping[str, Any]], - count_tokens: Any, -) -> dict[str, Any]: - """Count the fully rendered messages against the profile's declared window. - - Counting happens before the model is called, so an overflow is a refusal - rather than a silent truncation -- the failure mode that would drop the - guidance a person pinned and leave the turn looking like it honoured it. - - A profile that names no exact tokenizer cannot be counted at all. A - character-count substitute is precisely what the roadmap forbids, because - it is wrong in the direction that matters: it under-counts the dense, - punctuation-heavy JSON this context is made of. - """ - tokenizer = profile.get("tokenizer") - if not isinstance(tokenizer, str) or not tokenizer: - raise TokenAccountingError( - "profile declares no exact tokenizer; the request cannot be counted" - ) - window = int(profile["contextWindow"]) - reserve = int(profile["outputReserve"]) - prompt_tokens = int(count_tokens(rendered)) - omitted_ids = [ - str(item_id) - for omission in omissions - for item_id in omission.get("itemIds", []) - ] - return { - "tokenizer": tokenizer, - "contextWindow": window, - "outputReserve": reserve, - "promptTokens": prompt_tokens, - "includedItemIds": list(included_item_ids), - "omittedItemIds": omitted_ids, - "omissions": [dict(omission) for omission in omissions], - "fits": prompt_tokens + reserve <= window, - } diff --git a/catalyst-gateway/src/catalyst/storage.py b/catalyst-gateway/src/catalyst/storage.py index f7c9b216..c2ab21de 100644 --- a/catalyst-gateway/src/catalyst/storage.py +++ b/catalyst-gateway/src/catalyst/storage.py @@ -886,13 +886,9 @@ def _initialize(self) -> None: # ---------------------------------------------------------- guidance # - # What a person pins so the writer stops having to be told twice. Entries - # are append-only: unpinning or replacing one appends a lifecycle event - # and leaves the text where it was, because a turn's evidence must keep - # meaning what it meant when it ran. - - GUIDANCE_DELIVERY_CAP = 20 - """Active entries delivered to the writer. History is not capped.""" + # Optional session guidance remains an experimental seam. Entries are + # append-only: deactivating or replacing one appends a lifecycle event and + # leaves the original text intact so old turn evidence keeps its meaning. def pin_guidance( self, @@ -904,7 +900,7 @@ def pin_guidance( supersedes: str | None = None, actor_id: str | None = None, ) -> dict[str, Any]: - """Pin one instruction to a session, exactly as written.""" + """Record one optional experimental instruction exactly as written.""" if not text.strip(): raise ValueError("guidance text must not be blank") if source not in {"human", "system"}: @@ -1044,7 +1040,7 @@ def guidance_entry(self, session_id: str, entry_id: str) -> dict[str, Any]: return self._guidance_row(row) def guidance_history(self, session_id: str) -> list[dict[str, Any]]: - """Every entry ever pinned to this session, in order.""" + """Every experimental guidance entry for this session, in order.""" rows = self._connection.execute( """ SELECT * FROM catalyst_workbench_guidance @@ -1055,24 +1051,7 @@ def guidance_history(self, session_id: str) -> list[dict[str, Any]]: return [self._guidance_row(row) for row in rows] def active_guidance(self, session_id: str) -> list[dict[str, Any]]: - """The entries delivered to the writer, oldest first. - - Past the cap the oldest active entries stop being delivered; the - omission is recorded where the request is assembled, and nothing is - deleted here. - """ - rows = self._connection.execute( - """ - SELECT * FROM catalyst_workbench_guidance - WHERE session_id = ? AND state = 'active' ORDER BY entry_order - """, - (session_id,), - ).fetchall() - entries = [self._guidance_row(row) for row in rows] - return entries[-self.GUIDANCE_DELIVERY_CAP :] - - def guidance_omitted_by_cap(self, session_id: str) -> list[dict[str, Any]]: - """Active entries the cap keeps out of the delivered set.""" + """Every active entry delivered to the writer in recorded order.""" rows = self._connection.execute( """ SELECT * FROM catalyst_workbench_guidance @@ -1080,10 +1059,7 @@ def guidance_omitted_by_cap(self, session_id: str) -> list[dict[str, Any]]: """, (session_id,), ).fetchall() - entries = [self._guidance_row(row) for row in rows] - if len(entries) <= self.GUIDANCE_DELIVERY_CAP: - return [] - return entries[: -self.GUIDANCE_DELIVERY_CAP] + return [self._guidance_row(row) for row in rows] @staticmethod def _guidance_row(row: sqlite3.Row) -> dict[str, Any]: @@ -1952,7 +1928,7 @@ def fail_turn( failure = { "stage": stage, "code": code, - "message": message[:4000], + "message": message, "evidenceAvailable": evidence_available, "rawEvidenceRef": raw_ref, "diagnostic": { diff --git a/catalyst-gateway/src/catalyst/workbench.py b/catalyst-gateway/src/catalyst/workbench.py index 18c20e33..718d1388 100644 --- a/catalyst-gateway/src/catalyst/workbench.py +++ b/catalyst-gateway/src/catalyst/workbench.py @@ -83,19 +83,13 @@ def build_revision_context( effective_base: Mapping[str, Any] | None, editor_snapshot: Mapping[str, Any] | None, ) -> dict[str, Any]: - """Build bounded, digest-bound context without rows or historical SQL copies.""" + """Build complete, digest-bound context without rows or historical SQL copies.""" ordered = sorted(prior_turns, key=lambda item: int(item["ordinal"])) session_id = str(session["sessionId"]) if any(str(turn.get("sessionId")) != session_id for turn in ordered): raise ValueError("Revision history contains an unrelated session turn.") - initial = next((turn for turn in ordered if turn["kind"] == "initial"), None) - followups = [turn for turn in ordered if turn["kind"] == "followup"][-5:] - included = ([initial] if initial is not None else []) + followups - included_ids = {str(turn["turnId"]) for turn in included} - omitted = [turn for turn in ordered if str(turn["turnId"]) not in included_ids] - if len(omitted) > 1000: - raise ValueError("Revision history exceeds the deterministic omission bound.") + included = ordered history = [ { @@ -107,15 +101,7 @@ def build_revision_context( } for turn in included ] - omitted_refs = [ - { - "turnId": str(turn["turnId"]), - "ordinal": int(turn["ordinal"]), - "kind": str(turn["kind"]), - "instructionDigest": str(turn["instructionDigest"]), - } - for turn in omitted - ] + omitted_refs: list[dict[str, Any]] = [] # A turn answering the writer's question revises nothing: there is no # editor content, so evidence can only be looked up under the base. editor_digest = ( @@ -135,7 +121,6 @@ def build_revision_context( ] validation_context = None validation_ref = None - validation_omitted = 0 if matching_validations: validation = matching_validations[-1] findings = [ @@ -150,9 +135,8 @@ def build_revision_context( "message", ) } - for finding in validation.get("findings", [])[:50] + for finding in validation.get("findings", []) ] - validation_omitted = max(0, len(validation.get("findings", [])) - 50) validation_context = { "validationId": validation["validationId"], "versionId": validation["versionId"], @@ -173,8 +157,6 @@ def build_revision_context( ] execution_context = None execution_ref = None - execution_columns_omitted = 0 - diagnostic_truncated = False if matching_executions: execution = matching_executions[-1] result = ( @@ -182,7 +164,6 @@ def build_revision_context( ) raw_columns = result.get("columns") if isinstance(result, dict) else [] raw_columns = raw_columns if isinstance(raw_columns, list) else [] - execution_columns_omitted = max(0, len(raw_columns) - 128) columns = [ { "ordinal": int(column["ordinal"]), @@ -190,7 +171,7 @@ def build_revision_context( "databaseType": str(column["databaseType"]), "logicalType": str(column["logicalType"]), } - for column in raw_columns[:128] + for column in raw_columns if isinstance(column, Mapping) and all( key in column @@ -223,8 +204,8 @@ def build_revision_context( ), ) execution_warnings = [ - _SENSITIVE_DIAGNOSTIC.sub("[redacted]", warning)[:2000] - for warning in raw_warnings[:8] + _SENSITIVE_DIAGNOSTIC.sub("[redacted]", warning) + for warning in raw_warnings if isinstance(warning, str) and warning.strip() ] diagnostic = execution.get("databaseDiagnostic") @@ -241,9 +222,6 @@ def build_revision_context( value = diagnostic.get(key) if key in {"message", "detail", "hint"} and isinstance(value, str): value = _SENSITIVE_DIAGNOSTIC.sub("[redacted]", value) - if len(value) > 4000: - diagnostic_truncated = True - value = value[:4000] bounded_diagnostic[key] = value diagnostic = bounded_diagnostic else: @@ -270,9 +248,9 @@ def build_revision_context( "executionRef": execution_ref, "omissions": { "historyInstructionsOmitted": len(omitted_refs), - "validationFindingsOmitted": validation_omitted, - "executionColumnsOmitted": execution_columns_omitted, - "diagnosticTextTruncated": diagnostic_truncated, + "validationFindingsOmitted": 0, + "executionColumnsOmitted": 0, + "diagnosticTextTruncated": False, "prohibitedClasses": [ "database_credentials", "database_connection_details", @@ -309,10 +287,18 @@ def build_revision_context( "selection": selection, "contextDigest": "0" * 64, } - context["contextDigest"] = canonical_sha256( + finalize_revision_context_digest(context) + return context + + +def finalize_revision_context_digest(context: dict[str, Any]) -> str: + """Bind every supplied revision-context field, including session context.""" + + digest = canonical_sha256( {key: value for key, value in context.items() if key != "contextDigest"} ) - return context + context["contextDigest"] = digest + return digest def normalize_findings( @@ -347,9 +333,9 @@ def normalize_findings( if suggested_action is None: suggested_action = source.get("suggested_action") repairability = _normalize_repairability(source.get("repairability")) - evidence = _bounded_json(source.get("evidence")) - ast_unit = _bounded_json(source.get("astUnit", source.get("ast_unit"))) - span = _bounded_json(source.get("span")) + evidence = _evidence_json(source.get("evidence")) + ast_unit = _evidence_json(source.get("astUnit", source.get("ast_unit"))) + span = _evidence_json(source.get("span")) identity = { "queryDigest": query_digest, @@ -466,20 +452,15 @@ def _normalize_repairability(value: Any) -> str: return normalized if normalized in _REPAIRABILITY else "manual" -def _bounded_json(value: Any, *, depth: int = 0) -> Any: - """Keep evidence useful without allowing unbounded model/database payloads.""" +def _evidence_json(value: Any) -> Any: + """Preserve finding evidence in a stable JSON-compatible form.""" if value is None or isinstance(value, (bool, int, float)): return value if isinstance(value, str): - return value[:1000] - if depth >= 4: - return str(value)[:1000] + return value if isinstance(value, Mapping): - items = list(value.items())[:25] - return { - str(key)[:200]: _bounded_json(item, depth=depth + 1) for key, item in items - } + return {str(key): _evidence_json(item) for key, item in value.items()} if isinstance(value, (list, tuple)): - return [_bounded_json(item, depth=depth + 1) for item in value[:25]] - return str(value)[:1000] + return [_evidence_json(item) for item in value] + return str(value) diff --git a/catalyst-gateway/tests/test_local_hub.py b/catalyst-gateway/tests/test_local_hub.py index f05901f9..f6e47c15 100644 --- a/catalyst-gateway/tests/test_local_hub.py +++ b/catalyst-gateway/tests/test_local_hub.py @@ -272,7 +272,7 @@ async def test_the_local_hub_advertises_the_session_context_it_can_read( """The in-process engine reads the Phase 1 shape, so it says so. Without the advertisement Catalyst withholds the layer, and nothing would - ever receive the guidance a person pinned. + ever receive guidance supplied for an experiment. """ from src.catalyst.session_context import SESSION_CONTEXT_CONTRACT diff --git a/catalyst-gateway/tests/test_query_engine.py b/catalyst-gateway/tests/test_query_engine.py index 40a1b6ea..43e879fe 100644 --- a/catalyst-gateway/tests/test_query_engine.py +++ b/catalyst-gateway/tests/test_query_engine.py @@ -11,6 +11,7 @@ import json from unittest.mock import patch +import httpx import pytest from src.catalyst import query_engine @@ -148,6 +149,53 @@ def _collaborative_profile() -> EngineProfile: ) +def _role_request_evidence( + *, + profile_id: str, + role: str, + model: str, + caller_messages: list[dict], + response_format: dict, + context_window: int = 4096, + prompt_tokens: int = 100, + output_reserve: int = 512, +) -> dict: + exact_request = { + "profileId": profile_id, + "role": role, + "model": model, + "messages": [ + {"role": "system", "content": "Exact Hub-owned system prompt."}, + *copy.deepcopy(caller_messages), + ], + "responseFormat": copy.deepcopy(response_format), + "config": { + "temperature": 0.0, + "dryMultiplier": 0.0, + "maxTokens": output_reserve, + }, + } + rendered_prompt = "exact rendered role request" + required_tokens = prompt_tokens + output_reserve + return { + "contractVersion": "med-agent-hub.catalyst-role-request-evidence.v1", + "request": exact_request, + "requestDigest": query_engine._canonical_evidence_digest(exact_request), + "prompt": { + "renderedPrompt": rendered_prompt, + "renderedPromptDigest": query_engine._evidence_digest(rendered_prompt), + }, + "tokens": { + "tokenizer": model, + "contextWindow": context_window, + "outputReserve": output_reserve, + "promptTokens": prompt_tokens, + "requiredTokens": required_tokens, + "fits": required_tokens <= context_window, + }, + } + + def _queued_backend(responses: list, captured_messages: list | None = None): queue = [r if isinstance(r, str) else json.dumps(r) for r in responses] @@ -497,6 +545,121 @@ async def backend(client, profile_id, role, model, messages, **kwargs): assert result["_hubEvidence"]["tokenAccounting"] == accounting +@pytest.mark.asyncio +async def test_exact_hub_role_request_evidence_travels_with_a_successful_invocation(): + caller_messages = [{"role": "user", "content": "exact caller context"}] + response_format = {"type": "json_object"} + evidence = _role_request_evidence( + profile_id="profile-a", + role="query_generate", + model="model-a", + caller_messages=caller_messages, + response_format=response_format, + ) + accounting = { + "tokenizer": "model-a", + "contextWindow": 4096, + "outputReserve": 512, + "promptTokens": 100, + } + + async def respond(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "content": json.dumps(_ready_candidate()), + "token_accounting": accounting, + "request_evidence": evidence, + }, + ) + + invocations: list[dict] = [] + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + content = await query_engine._invoke_backend( + client, + "profile-a", + "query_generate", + "model-a", + caller_messages, + provider_id="med-agent-hub", + response_format=response_format, + temperature=0.0, + dry_multiplier=0.0, + max_tokens=512, + invocations=invocations, + role="writer", + stage="initial_generation", + attempt=1, + ) + + assert json.loads(content)["status"] == "ready" + assert invocations[0]["outcome"] == "succeeded" + assert invocations[0]["requestEvidence"] == evidence + assert invocations[0]["requestDigest"] == evidence["requestDigest"] + assert invocations[0]["tokenAccounting"] == accounting + + +@pytest.mark.asyncio +async def test_known_context_overflow_keeps_evidence_as_a_pre_dispatch_rejection(): + caller_messages = [{"role": "user", "content": "oversized caller context"}] + response_format = {"type": "json_object"} + evidence = _role_request_evidence( + profile_id="profile-a", + role="query_generate", + model="model-a", + caller_messages=caller_messages, + response_format=response_format, + context_window=600, + prompt_tokens=100, + ) + assert evidence["tokens"]["fits"] is False + message = "The exact rendered request exceeds the model context window." + + async def reject(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 422, + json={ + "detail": { + "code": "context_window_exceeded", + "message": message, + "request_evidence": evidence, + } + }, + ) + + invocations: list[dict] = [] + async with httpx.AsyncClient(transport=httpx.MockTransport(reject)) as client: + with pytest.raises(httpx.HTTPStatusError): + await query_engine._invoke_backend( + client, + "profile-a", + "query_generate", + "model-a", + caller_messages, + provider_id="med-agent-hub", + response_format=response_format, + temperature=0.0, + dry_multiplier=0.0, + max_tokens=512, + invocations=invocations, + role="writer", + stage="initial_generation", + attempt=1, + ) + + invocation = invocations[0] + assert invocation["outcome"] == "pre_dispatch_rejected" + assert invocation["requestEvidence"] == evidence + assert invocation["requestDigest"] == evidence["requestDigest"] + assert invocation["hubError"] == { + "code": "context_window_exceeded", + "httpStatus": 422, + "message": message, + } + assert invocation["responseDigest"] is None + assert invocation["failureDigest"] + + @pytest.mark.asyncio async def test_a_hub_that_counted_nothing_is_reported_as_nothing(): """No accounting is absence in the evidence, never an invented shape.""" diff --git a/catalyst-gateway/tests/test_query_notebook.py b/catalyst-gateway/tests/test_query_notebook.py index 87bd81dd..97c86543 100644 --- a/catalyst-gateway/tests/test_query_notebook.py +++ b/catalyst-gateway/tests/test_query_notebook.py @@ -11,7 +11,11 @@ ActiveTurnGenerationError, WorkbenchStore, ) -from src.catalyst.workbench import build_revision_context, workbench_query_digest +from src.catalyst.workbench import ( + build_revision_context, + finalize_revision_context_digest, + workbench_query_digest, +) CONTRACTS = Path(__file__).resolve().parents[2] / "docs" / "contracts" @@ -118,6 +122,102 @@ def test_editor_digest_uses_the_normative_golden_vector() -> None: ) +def test_revision_context_keeps_the_complete_instruction_history() -> None: + session_id = "00000000-0000-0000-0000-000000000001" + prior_turns = [ + { + "sessionId": session_id, + "turnId": f"00000000-0000-0000-0000-{index:012d}", + "ordinal": index, + "kind": "initial" if index == 1 else "followup", + "instruction": f"instruction {index}", + "instructionDigest": utf8_sha256(f"instruction {index}"), + } + for index in range(1, 10) + ] + + context = build_revision_context( + session={"sessionId": session_id, "validations": [], "executions": []}, + prior_turns=prior_turns, + turn_id="00000000-0000-0000-0000-000000000099", + instruction="next instruction", + base_classification="not_applicable", + observed_base=None, + effective_base=None, + editor_snapshot=None, + ) + + assert [item["instruction"] for item in context["instructionHistory"]] == [ + f"instruction {index}" for index in range(1, 10) + ] + assert context["selection"]["omissions"]["historyInstructionsOmitted"] == 0 + assert context["selection"]["omissions"]["omittedHistory"] == [] + ContractRegistry.load(CONTRACTS).validate( + "catalyst-query-revision-context-v1.schema.json", context + ) + + +@pytest.mark.parametrize( + "extra_context", + [ + { + "guidance": { + "role": "experimental", + "entries": [{"text": "Use CIEL."}], + } + }, + { + "verifiedExamples": { + "role": "evidence", + "examples": [{"turnId": "earlier", "sql": "SELECT 1"}], + } + }, + { + "relevantFailure": { + "role": "evidence", + "turnId": "earlier", + "code": "bad_query", + } + }, + ], +) +def test_revision_digest_binds_every_supplied_session_context_layer( + extra_context: dict, +) -> None: + session_id = "00000000-0000-0000-0000-000000000001" + prior = { + "sessionId": session_id, + "turnId": "00000000-0000-0000-0000-000000000002", + "ordinal": 1, + "kind": "initial", + "instruction": "Show one row", + "instructionDigest": utf8_sha256("Show one row"), + } + context = build_revision_context( + session={"sessionId": session_id, "validations": [], "executions": []}, + prior_turns=[prior], + turn_id="00000000-0000-0000-0000-000000000003", + instruction="Try again", + base_classification="not_applicable", + observed_base=None, + effective_base=None, + editor_snapshot=None, + ) + digest_without_session_context = context["contextDigest"] + context["sessionContext"] = { + "contractVersion": "catalyst.query.session-context.v1", + "omissions": [], + **extra_context, + } + + digest_with_session_context = finalize_revision_context_digest(context) + + assert digest_with_session_context != digest_without_session_context + assert digest_with_session_context == canonical_sha256( + {key: value for key, value in context.items() if key != "contextDigest"} + ) + + @pytest.mark.parametrize("execution_status", ["failed", "timed_out", "cancelled"]) def test_revision_context_sanitizes_terminal_execution_context( execution_status: str, @@ -855,3 +955,24 @@ def test_model_failure_stage_uses_terminal_invocation_role_for_reviewer_timeout( stage, code = CatalystService._model_failure_stage(evidence, reviewer=False) assert (stage, code) == ("reviewer_transport", "reviewer_timeout") + + +def test_model_failure_stage_keeps_known_context_overflow_out_of_transport() -> None: + from src.catalyst.service import CatalystService + + evidence = { + "modelInvocations": [ + { + "role": "writer", + "outcome": "pre_dispatch_rejected", + "hubError": { + "code": "context_window_exceeded", + "httpStatus": 422, + }, + } + ] + } + + stage, code = CatalystService._model_failure_stage(evidence, reviewer=False) + + assert (stage, code) == ("writer_request", "context_window_exceeded") diff --git a/catalyst-gateway/tests/test_session_context.py b/catalyst-gateway/tests/test_session_context.py index ca15a32b..642be5bd 100644 --- a/catalyst-gateway/tests/test_session_context.py +++ b/catalyst-gateway/tests/test_session_context.py @@ -1,25 +1,14 @@ -"""The layered context the writer receives, and the order it arrives in. - -Phase 1 adds three bounded layers to a request: session guidance, verified -examples from earlier in the session, and the one prior failure on this -revision line. The roadmap fixes their order and their precedence, because a -model reads position as authority: contract, catalog and policy outrank all -user context; the current instruction outranks retained guidance; later -guidance wins a guidance conflict; history, failures and examples are -evidence rather than commands. -""" +"""The complete eligible session context the writer receives.""" from __future__ import annotations from typing import Any -import pytest - from src.catalyst.session_context import ( - LAYER_ORDER, build_session_context, select_verified_examples, ) +from src.catalyst.service import CatalystService def _entry(order: int, text: str, **extra: Any) -> dict[str, Any]: @@ -37,18 +26,6 @@ def _entry(order: int, text: str, **extra: Any) -> dict[str, Any]: } -def test_the_layers_arrive_in_the_order_the_roadmap_fixes() -> None: - assert LAYER_ORDER == ( - "guidance", - "verifiedExamples", - "editorSnapshot", - "instructionHistory", - "relevantFailure", - "currentValidation", - "currentInstruction", - ) - - def test_guidance_is_delivered_verbatim_in_pin_order() -> None: context = build_session_context( guidance=[_entry(1, " Exclude do_not_perform. "), _entry(2, "Use CIEL.")], @@ -70,24 +47,23 @@ def test_guidance_is_delivered_verbatim_in_pin_order() -> None: } -def test_guidance_says_it_is_standing_instruction_not_evidence() -> None: - """Precedence is stated, because position alone is ambiguous.""" +def test_guidance_is_labelled_as_optional_experimental_context() -> None: context = build_session_context( guidance=[_entry(1, "Exclude do_not_perform.")], omitted_guidance=[], verified_examples=[], relevant_failure=None, ) - precedence = context["guidance"]["precedence"] - - assert "current instruction" in precedence - assert "later" in precedence.lower() + assert "experiment" in context["guidance"]["role"] + assert "provenance" in context["guidance"]["role"] -def test_an_entry_the_cap_pushed_out_is_recorded_as_omitted() -> None: +def test_an_explicitly_omitted_entry_keeps_its_reason() -> None: context = build_session_context( guidance=[_entry(2, "kept")], - omitted_guidance=[_entry(1, "pushed out")], + omitted_guidance=[ + _entry(1, "not supplied", omissionReason="operator_excluded") + ], verified_examples=[], relevant_failure=None, ) @@ -95,7 +71,7 @@ def test_an_entry_the_cap_pushed_out_is_recorded_as_omitted() -> None: omissions = context["omissions"] assert omissions[0]["layer"] == "guidance" assert omissions[0]["itemIds"] == ["guidance-1"] - assert omissions[0]["reason"] == "active_entry_cap" + assert omissions[0]["reason"] == "operator_excluded" def test_failures_and_examples_are_labelled_evidence_not_commands() -> None: @@ -150,7 +126,7 @@ def _kept(turn: str, instruction: str, digest: str = "a" * 64) -> dict[str, Any] } -def test_examples_rank_by_word_overlap_with_the_request() -> None: +def test_examples_remain_in_recorded_session_order() -> None: chosen = select_verified_examples( [ _kept("t1", "count medication requests by name"), @@ -163,10 +139,10 @@ def test_examples_rank_by_word_overlap_with_the_request() -> None: exclude_turn_id=None, ) - assert [item["turnId"] for item in chosen][:2] == ["t3", "t1"] + assert [item["turnId"] for item in chosen] == ["t1", "t2", "t3"] -def test_at_most_three_examples_travel() -> None: +def test_every_eligible_example_travels() -> None: chosen = select_verified_examples( [_kept(f"t{index}", "count medication requests") for index in range(6)], instruction="count medication requests", @@ -175,7 +151,7 @@ def test_at_most_three_examples_travel() -> None: exclude_turn_id=None, ) - assert len(chosen) == 3 + assert len(chosen) == 6 def test_a_turn_never_receives_its_own_answer_as_an_example() -> None: @@ -205,8 +181,7 @@ def test_examples_from_another_source_or_catalog_are_not_eligible() -> None: assert chosen == [] -def test_selection_is_deterministic_when_overlap_ties() -> None: - """Ties break on the newest turn, then the stable id -- never on chance.""" +def test_selection_does_not_invent_a_relevance_ranking() -> None: candidates = [ _kept("t1", "count medication requests"), _kept("t2", "count medication requests"), @@ -227,76 +202,93 @@ def test_selection_is_deterministic_when_overlap_ties() -> None: exclude_turn_id=None, ) - assert [item["turnId"] for item in first] == [item["turnId"] for item in second] - - -# --- token accounting ------------------------------------------------------ -# -# A profile declares its window, its output reserve, and the exact tokenizer. -# The fully rendered messages are counted against them before the model is -# called, so overflow is a refusal rather than a silent truncation that -# quietly drops the guidance a person pinned. - - -def test_accounting_reports_the_counted_prompt_against_the_declared_window() -> None: - from src.catalyst.session_context import account_for_tokens - - accounting = account_for_tokens( - rendered="a b c d", - profile={"contextWindow": 100, "outputReserve": 10, "tokenizer": "gemma-4"}, - included_item_ids=["guidance-1"], - omissions=[], - count_tokens=lambda text: len(text.split()), - ) - - assert accounting["promptTokens"] == 4 - assert accounting["contextWindow"] == 100 - assert accounting["outputReserve"] == 10 - assert accounting["tokenizer"] == "gemma-4" - assert accounting["includedItemIds"] == ["guidance-1"] - assert accounting["fits"] is True + assert [item["turnId"] for item in first] == ["t1", "t2", "t3"] + assert [item["turnId"] for item in second] == ["t3", "t2", "t1"] -def test_a_prompt_that_leaves_no_room_for_the_reply_does_not_fit() -> None: - from src.catalyst.session_context import account_for_tokens +def test_only_validated_and_successfully_executed_kept_queries_become_examples() -> ( + None +): + version = { + "versionId": "version-1", + "sql": "SELECT 1", + "queryDigest": "a" * 64, + } + turn = { + "turnId": "turn-1", + "instruction": "Show one row", + "selectedVersionId": version["versionId"], + "dataSourceId": "openmrs-hiv", + "catalogVersion": "runtime-catalog", + } + base_session = { + "sessionId": "session-1", + "versions": [version], + "validations": [], + "executions": [], + } + validation = { + "validationId": "validation-1", + "versionId": version["versionId"], + "queryDigest": version["queryDigest"], + "status": "invalid", + "validatorRevision": "validator-1", + "validatorDigest": "b" * 64, + "findings": [{"ruleCode": "advisory.warning", "severity": "warning"}], + "createdAt": "2026-08-25T00:00:00Z", + } + failed_execution = { + "executionId": "execution-failed", + "sessionId": base_session["sessionId"], + "versionId": version["versionId"], + "queryDigest": version["queryDigest"], + "status": "failed", + } + successful_execution = { + "executionId": "execution-succeeded", + "sessionId": base_session["sessionId"], + "versionId": version["versionId"], + "queryDigest": version["queryDigest"], + "status": "succeeded", + "completedAt": "2026-08-25T00:01:00Z", + "durationMs": 8, + "result": {"rows": [[{"type": "string", "value": "not context"}]]}, + } - accounting = account_for_tokens( - rendered="a b c d e f g h i j", - profile={"contextWindow": 12, "outputReserve": 5, "tokenizer": "gemma-4"}, - included_item_ids=[], - omissions=[], - count_tokens=lambda text: len(text.split()), + assert CatalystService._verified_examples(base_session, [turn]) == [] + assert ( + CatalystService._verified_examples( + base_session | {"validations": [validation]}, [turn] + ) + == [] ) - - assert accounting["fits"] is False - - -def test_a_profile_without_an_exact_tokenizer_cannot_be_counted() -> None: - """A character-count substitute is the thing the roadmap forbids.""" - from src.catalyst.session_context import TokenAccountingError, account_for_tokens - - with pytest.raises(TokenAccountingError, match="tokenizer"): - account_for_tokens( - rendered="a b", - profile={"contextWindow": 100, "outputReserve": 10}, - included_item_ids=[], - omissions=[], - count_tokens=lambda text: len(text.split()), + assert ( + CatalystService._verified_examples( + base_session + | {"validations": [validation], "executions": [failed_execution]}, + [turn], ) + == [] + ) - -def test_every_omission_travels_with_its_reason() -> None: - from src.catalyst.session_context import account_for_tokens - - accounting = account_for_tokens( - rendered="a", - profile={"contextWindow": 100, "outputReserve": 10, "tokenizer": "gemma-4"}, - included_item_ids=[], - omissions=[ - {"layer": "guidance", "itemIds": ["g1"], "reason": "active_entry_cap"} - ], - count_tokens=lambda text: len(text.split()), + examples = CatalystService._verified_examples( + base_session + | { + "validations": [validation], + "executions": [failed_execution, successful_execution], + }, + [turn], ) - assert accounting["omittedItemIds"] == ["g1"] - assert accounting["omissions"][0]["reason"] == "active_entry_cap" + assert len(examples) == 1 + assert examples[0]["advisoryValidations"][0]["status"] == "invalid" + assert examples[0]["advisoryValidations"][0]["findings"] == validation["findings"] + assert examples[0]["successfulExecutions"] == [ + { + "executionId": "execution-succeeded", + "status": "succeeded", + "completedAt": "2026-08-25T00:01:00Z", + "durationMs": 8, + } + ] + assert "result" not in examples[0] diff --git a/catalyst-gateway/tests/test_workbench_guidance.py b/catalyst-gateway/tests/test_workbench_guidance.py index c93f67b6..bf50c17f 100644 --- a/catalyst-gateway/tests/test_workbench_guidance.py +++ b/catalyst-gateway/tests/test_workbench_guidance.py @@ -1,9 +1,7 @@ -"""Session guidance: what a person pins so the writer stops being told once. +"""Optional session guidance retained verbatim for experiments. -An instruction that mattered on turn two mattered on turn five, and the only -thing that accumulated in a session was the list of instructions themselves -- -text with no standing, which the prompt explicitly ranks below the current -one. Guidance is durable, append-only, delivered verbatim, and capped. +Guidance is durable and append-only. Delivery does not impose a fixed item +limit or claim a precedence that the planned research has not established. """ from __future__ import annotations @@ -14,8 +12,6 @@ from src.catalyst.storage import WorkbenchStore -GUIDANCE_CAP = 20 - def _store(tmp_path: Path) -> WorkbenchStore: return WorkbenchStore(tmp_path / "gateway.sqlite3") @@ -53,8 +49,8 @@ def test_a_pin_is_stored_verbatim_with_its_provenance(tmp_path: Path) -> None: assert [event["action"] for event in entry["events"]] == ["pinned"] -def test_entries_keep_the_order_they_were_pinned_in(tmp_path: Path) -> None: - """Order is the tie-break when two entries conflict: later wins.""" +def test_entries_keep_the_order_they_were_recorded_in(tmp_path: Path) -> None: + """Recorded sequence is preserved without defining conflict precedence.""" store = _store(tmp_path) session_id = _session(store) for text in ("first", "second", "third"): @@ -103,22 +99,19 @@ def test_replacing_an_entry_supersedes_it_and_keeps_both(tmp_path: Path) -> None assert history[first["entryId"]]["supersededBy"] == second["entryId"] -def test_the_twenty_first_entry_pushes_the_oldest_out_of_delivery( +def test_more_than_twenty_active_entries_are_delivered_without_a_fixed_cap( tmp_path: Path, ) -> None: - """The cap bounds what is delivered, not what is remembered.""" store = _store(tmp_path) session_id = _session(store) - for index in range(GUIDANCE_CAP + 1): + for index in range(25): store.pin_guidance(session_id, text=f"entry {index}", source="human") active = store.active_guidance(session_id) - assert len(active) == GUIDANCE_CAP - assert active[0]["text"] == "entry 1", "the oldest left the delivered set" - assert active[-1]["text"] == f"entry {GUIDANCE_CAP}" - # Nothing was forgotten. - assert len(store.guidance_history(session_id)) == GUIDANCE_CAP + 1 + assert len(active) == 25 + assert active[0]["text"] == "entry 0" + assert active[-1]["text"] == "entry 24" def test_guidance_never_leaks_between_sessions(tmp_path: Path) -> None: diff --git a/catalyst-gateway/tests/test_workbench_routes.py b/catalyst-gateway/tests/test_workbench_routes.py index e164129c..2da89c93 100644 --- a/catalyst-gateway/tests/test_workbench_routes.py +++ b/catalyst-gateway/tests/test_workbench_routes.py @@ -406,6 +406,48 @@ async def aclose(self) -> None: return None +class RequestEvidenceHub(FakeHub): + async def generate_query(self, request: dict) -> dict: + query = await super().generate_query(request) + writer = query["_hubEvidence"]["modelInvocations"][0] + exact_request = { + "profileId": PROFILE_ID, + "role": "query_generate", + "model": writer["modelId"], + "messages": [ + {"role": "system", "content": "Exact Hub-owned writer prompt."}, + *deepcopy(request["messages"]), + ], + "responseFormat": None, + "config": { + "temperature": 0.0, + "dryMultiplier": 0.0, + "maxTokens": 1024, + }, + } + rendered = "exact rendered writer request" + request_evidence = { + "contractVersion": "med-agent-hub.catalyst-role-request-evidence.v1", + "request": exact_request, + "requestDigest": canonical_sha256(exact_request), + "prompt": { + "renderedPrompt": rendered, + "renderedPromptDigest": utf8_sha256(rendered), + }, + "tokens": { + "tokenizer": writer["modelId"], + "contextWindow": 24576, + "outputReserve": 1024, + "promptTokens": 2345, + "requiredTokens": 3369, + "fits": True, + }, + } + writer["requestEvidence"] = request_evidence + writer["requestDigest"] = request_evidence["requestDigest"] + return query + + class IncompleteProfileHub(FakeHub): async def list_query_profiles(self) -> list[dict]: profiles = await super().list_query_profiles() @@ -1884,6 +1926,69 @@ def test_failed_turn_names_its_failed_checks_in_the_failure_block( assert all(not detail["value"].startswith("passed") for detail in details) +def test_all_failure_context_reaches_the_next_model_request_without_truncation( + tmp_path: Path, +) -> None: + rejected = _rejected_query() + rejected["diagnosticCandidate"].pop("candidate") + rejected["diagnosticCandidate"]["rawOutput"] = '{"patches": []}' + findings = [ + { + "code": f"policy.{'specific_rule_' * 9}{index}", + "stage": "parameter_binding", + "severity": "error", + "path": f"$.sql[{index}]", + "message": f"Finding {index}: " + ("full diagnostic text " * 300), + "evidence": f"evidence-{index}-" + ("x" * 5000), + "suggestedAction": f"Correct finding {index} without dropping it.", + } + for index in range(40) + ] + attempt = rejected["diagnosticCandidate"]["attempts"][0] + attempt["findings"] = findings + attempt["finding_codes"] = [finding["code"] for finding in findings] + hub = FailingFollowupHub(_ready_query(), rejected) + client, _ = _client(tmp_path, _ready_query(), hub=hub) + session = _create_session(client) + base = session["currentVersion"] + snapshot = { + "contractVersion": "catalyst.workbench.editor-snapshot.v1", + "sql": base["sql"], + "parameters": base["parameters"], + "expectedColumns": base["expectedColumns"], + "editorDigest": base["queryDigest"], + } + + def follow_up(instruction: str): + return client.post( + f"/v1/catalyst/workbench/sessions/{session['sessionId']}/turns", + json={ + "contractVersion": "catalyst.workbench.turn.request.v1", + "instruction": instruction, + "profileId": PROFILE_ID, + "observedBase": { + "versionId": base["versionId"], + "queryDigest": base["queryDigest"], + }, + "editorSnapshot": snapshot, + }, + ) + + failed = follow_up("Try the requested change") + assert failed.status_code == 201, failed.text + assert failed.json()["status"] == "failed" + stored_details = failed.json()["failure"]["diagnostic"]["details"] + assert len(stored_details) == 41 # every finding plus the failed named check + assert stored_details[0]["name"] == findings[0]["code"] + assert findings[0]["evidence"] in stored_details[0]["value"] + + second = follow_up("Use the failure evidence and try again") + assert second.status_code == 201, second.text + relevant = hub.requests[-1]["catalystQuery"]["revision"]["sessionContext"] + supplied = relevant["relevantFailure"]["findings"] + assert supplied == stored_details + + def test_structured_raw_only_diagnostic_is_preserved_for_manual_recovery( tmp_path: Path, ) -> None: @@ -2468,6 +2573,55 @@ def test_initial_and_followup_turn_routes_preserve_exact_context_and_evidence( assert "hidden_reasoning" in detail["prohibitedClasses"] +def test_retained_instruction_history_survives_a_service_reload_without_guidance( + tmp_path: Path, +) -> None: + first_hub = FakeHub(_ready_query()) + first_client, _ = _client(tmp_path, _ready_query(), hub=first_hub) + session = _create_session(first_client) + + def follow_up(client: TestClient, base: dict, instruction: str) -> None: + response = client.post( + f"/v1/catalyst/workbench/sessions/{session['sessionId']}/turns", + json={ + "contractVersion": "catalyst.workbench.turn.request.v1", + "instruction": instruction, + "profileId": PROFILE_ID, + "observedBase": { + "versionId": base["versionId"], + "queryDigest": base["queryDigest"], + }, + "editorSnapshot": { + "contractVersion": "catalyst.workbench.editor-snapshot.v1", + "sql": base["sql"], + "parameters": base["parameters"], + "expectedColumns": base["expectedColumns"], + "editorDigest": workbench_query_digest( + base["sql"], base["parameters"], base["expectedColumns"] + ), + }, + }, + ) + assert response.status_code == 201, response.text + + follow_up(first_client, session["currentVersion"], "Only finalized observations") + + reloaded_hub = FakeHub(_ready_query()) + reloaded_client, _ = _client(tmp_path, _ready_query(), hub=reloaded_hub) + restored = reloaded_client.get( + f"/v1/catalyst/workbench/sessions/{session['sessionId']}" + ).json() + assert restored["guidance"] == [] + follow_up(reloaded_client, restored["currentVersion"], "Now group by test name") + + revision = reloaded_hub.requests[-1]["catalystQuery"]["revision"] + assert [item["instruction"] for item in revision["instructionHistory"]] == [ + QUESTION, + "Only finalized observations", + ] + assert "guidance" not in revision["sessionContext"] + + def test_followup_rejects_bad_snapshot_digest_without_events(tmp_path: Path) -> None: client, _ = _client(tmp_path, _ready_query()) session = _create_session(client) @@ -2815,10 +2969,10 @@ def test_blank_guidance_is_refused_with_a_clear_error(tmp_path: Path) -> None: assert response.json()["error"]["code"] == "invalid_request" -def test_pinned_guidance_reaches_the_writer_on_the_next_turn( +def test_optional_guidance_reaches_the_writer_on_the_next_turn( tmp_path: Path, ) -> None: - """A composer pin becomes active on the next turn, not retroactively.""" + """Experimental guidance becomes active on the next turn, not retroactively.""" hub = FailingFollowupHub(_ready_query(), _ready_query()) client, _ = _client(tmp_path, _ready_query(), hub=hub) session = _create_session(client) @@ -2858,6 +3012,9 @@ def test_pinned_guidance_reaches_the_writer_on_the_next_turn( assert [item["text"] for item in context["guidance"]["entries"]] == [ "Exclude do_not_perform rows." ] + assert revision["contextDigest"] == canonical_sha256( + {key: value for key, value in revision.items() if key != "contextDigest"} + ) # The initial turn ran before the pin existed and must not have carried it. initial = hub.requests[0]["catalystQuery"] assert "sessionContext" not in initial or not initial["sessionContext"].get( @@ -2865,12 +3022,14 @@ def test_pinned_guidance_reaches_the_writer_on_the_next_turn( ) -def test_the_request_records_what_the_caps_left_out(tmp_path: Path) -> None: +def test_more_than_twenty_guidance_entries_reach_the_request_without_a_cap( + tmp_path: Path, +) -> None: hub = FailingFollowupHub(_ready_query(), _ready_query()) client, _ = _client(tmp_path, _ready_query(), hub=hub) session = _create_session(client) base = session["currentVersion"] - for index in range(21): + for index in range(25): client.post( f"/v1/catalyst/workbench/sessions/{session['sessionId']}/guidance", json={ @@ -2902,9 +3061,10 @@ def test_the_request_records_what_the_caps_left_out(tmp_path: Path) -> None: ) context = hub.requests[-1]["catalystQuery"]["revision"]["sessionContext"] - assert len(context["guidance"]["entries"]) == 20 - assert context["omissions"][0]["reason"] == "active_entry_cap" - assert len(context["omissions"][0]["itemIds"]) == 1 + assert [item["text"] for item in context["guidance"]["entries"]] == [ + f"entry {index}" for index in range(25) + ] + assert context["omissions"] == [] class HubWithoutSessionContext(FailingFollowupHub): @@ -3178,6 +3338,35 @@ def test_the_turns_token_evidence_is_published_with_the_evidence( assert writer["tokenAccounting"] == accounting +def test_the_exact_hub_role_request_is_published_with_the_turn_evidence( + tmp_path: Path, +) -> None: + hub = RequestEvidenceHub(_ready_query()) + client, _ = _client(tmp_path, _ready_query(), hub=hub) + session = _create_session(client) + + turn = client.get( + f"/v1/catalyst/workbench/sessions/{session['sessionId']}/turns" + ).json()["turns"][0] + evidence = client.get( + f"/v1/catalyst/workbench/sessions/{session['sessionId']}/turns/" + f"{turn['turnId']}/generation-evidence" + ).json() + + writer = [item for item in evidence["invocations"] if item["role"] == "writer"][0] + exact = writer["requestEvidence"] + assert exact["contractVersion"] == ( + "med-agent-hub.catalyst-role-request-evidence.v1" + ) + assert writer["requestDigest"] == exact["requestDigest"] + assert exact["request"]["messages"][0]["role"] == "system" + assert exact["tokens"]["fits"] is True + ContractRegistry.load(CONTRACTS).validate( + "catalyst-workbench-generation-evidence-v1.schema.json", + evidence, + ) + + # --- session guidance over HTTP --------------------------------------------- diff --git a/docs/contracts/catalyst-query-request-v2.schema.json b/docs/contracts/catalyst-query-request-v2.schema.json index 352047f3..3240cd53 100644 --- a/docs/contracts/catalyst-query-request-v2.schema.json +++ b/docs/contracts/catalyst-query-request-v2.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://openelis-global.org/catalyst/contracts/catalyst-query-request-v2.schema.json", "title": "Catalyst query-profile revision request v2", - "description": "OpenAI-compatible Hub request for a follow-up query. The sole user message is the current instruction; bounded prior context is typed under catalystQuery.revision.", + "description": "OpenAI-compatible Hub request for a follow-up query. The sole user message is the current instruction; eligible prior context is typed under catalystQuery.revision.", "type": "object", "additionalProperties": false, "required": ["model", "stream", "messages", "catalystQuery"], diff --git a/docs/contracts/catalyst-query-revision-context-v1.schema.json b/docs/contracts/catalyst-query-revision-context-v1.schema.json index 3c36571c..1db6880f 100644 --- a/docs/contracts/catalyst-query-revision-context-v1.schema.json +++ b/docs/contracts/catalyst-query-revision-context-v1.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://openelis-global.org/contracts/catalyst-query-revision-context-v1.schema.json", "title": "Catalyst query revision context v1", - "description": "Bounded deterministic context supplied to both Hub model roles for one follow-up turn.", + "description": "Complete deterministic context assembled for one follow-up turn. Per-model physical requests are recorded separately by the Hub.", "type": "object", "additionalProperties": false, "required": [ @@ -74,7 +74,6 @@ "instructionHistory": { "type": "array", "minItems": 1, - "maxItems": 6, "items": { "$ref": "#/$defs/historyItem" } @@ -106,7 +105,7 @@ "$ref": "catalyst-workbench-editor-snapshot-v1.schema.json#/$defs/digest" }, "sessionContext": { - "description": "The session's layered context: pinned guidance, verified examples from earlier turns, and the one prior failure on this revision line. Optional, so a request built before Phase 1 stays valid; every omission a cap caused is recorded inside it.", + "description": "The session context actually supplied: optional guidance, verified examples from earlier turns, and the relevant prior failure. Optional for compatibility with Hubs that do not advertise this contract; any omission is recorded with its reason.", "type": "object", "additionalProperties": true, "required": [ @@ -226,8 +225,7 @@ }, "message": { "type": "string", - "minLength": 1, - "maxLength": 4000 + "minLength": 1 } } }, @@ -271,7 +269,6 @@ }, "findings": { "type": "array", - "maxItems": 50, "items": { "$ref": "#/$defs/finding" } @@ -358,22 +355,19 @@ "maxLength": 100 }, "message": { - "type": "string", - "maxLength": 4000 + "type": "string" }, "detail": { "type": [ "string", "null" - ], - "maxLength": 4000 + ] }, "hint": { "type": [ "string", "null" - ], - "maxLength": 4000 + ] }, "position": { "type": [ @@ -438,18 +432,15 @@ }, "columns": { "type": "array", - "maxItems": 128, "items": { "$ref": "#/$defs/executionColumn" } }, "warnings": { "type": "array", - "maxItems": 8, "items": { "type": "string", - "minLength": 1, - "maxLength": 2000 + "minLength": 1 } }, "databaseDiagnostic": { @@ -557,7 +548,6 @@ }, "omittedHistory": { "type": "array", - "maxItems": 1000, "items": { "$ref": "#/$defs/omittedHistoryRef" } @@ -609,7 +599,6 @@ "includedHistoryTurnIds": { "type": "array", "minItems": 1, - "maxItems": 6, "uniqueItems": true, "items": { "type": "string", diff --git a/docs/contracts/catalyst-query-v1.schema.json b/docs/contracts/catalyst-query-v1.schema.json index 92446300..e093e380 100644 --- a/docs/contracts/catalyst-query-v1.schema.json +++ b/docs/contracts/catalyst-query-v1.schema.json @@ -772,6 +772,133 @@ } ] }, + "roleRequestEvidence": { + "description": "The exact configured-role request and the Hub's direct prompt/token measurements. Catalyst verifies the canonical request digest and consistency before retaining it.", + "type": "object", + "additionalProperties": false, + "required": ["contractVersion", "request", "requestDigest", "prompt", "tokens"], + "properties": { + "contractVersion": { + "const": "med-agent-hub.catalyst-role-request-evidence.v1" + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": ["profileId", "role", "model", "messages", "responseFormat", "config"], + "properties": { + "profileId": {"type": "string", "minLength": 1}, + "role": {"enum": ["query_generate", "query_review"]}, + "model": {"type": "string", "minLength": 1}, + "messages": { + "type": "array", + "minItems": 2, + "items": {"type": "object"} + }, + "responseFormat": { + "oneOf": [ + {"type": "null"}, + {"type": "object"} + ] + }, + "config": { + "type": "object", + "additionalProperties": false, + "required": ["temperature", "dryMultiplier", "maxTokens"], + "properties": { + "temperature": {"type": "number"}, + "dryMultiplier": {"type": "number", "minimum": 0}, + "maxTokens": {"type": "integer", "minimum": 1} + } + } + } + }, + "requestDigest": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "prompt": { + "type": "object", + "additionalProperties": false, + "required": ["renderedPrompt", "renderedPromptDigest"], + "properties": { + "renderedPrompt": {"type": ["string", "null"]}, + "renderedPromptDigest": { + "type": ["string", "null"], + "pattern": "^[a-f0-9]{64}$" + }, + "unavailableReason": {"type": "string", "minLength": 1} + }, + "allOf": [ + { + "if": { + "properties": {"renderedPrompt": {"type": "null"}}, + "required": ["renderedPrompt"] + }, + "then": { + "properties": {"renderedPromptDigest": {"type": "null"}}, + "required": ["unavailableReason"] + }, + "else": { + "properties": { + "renderedPromptDigest": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + }, + "not": {"required": ["unavailableReason"]} + } + } + ] + }, + "tokens": { + "type": "object", + "additionalProperties": false, + "required": ["tokenizer", "contextWindow", "outputReserve", "promptTokens", "requiredTokens", "fits"], + "properties": { + "tokenizer": {"type": "string", "minLength": 1}, + "contextWindow": {"type": ["integer", "null"], "minimum": 1}, + "contextWindowUnavailableReason": {"type": "string", "minLength": 1}, + "outputReserve": {"type": "integer", "minimum": 0}, + "promptTokens": {"type": ["integer", "null"], "minimum": 0}, + "promptTokensUnavailableReason": {"type": "string", "minLength": 1}, + "requiredTokens": {"type": ["integer", "null"], "minimum": 0}, + "fits": {"type": ["boolean", "null"]} + }, + "allOf": [ + { + "if": { + "properties": {"contextWindow": {"type": "null"}}, + "required": ["contextWindow"] + }, + "then": {"required": ["contextWindowUnavailableReason"]}, + "else": {"not": {"required": ["contextWindowUnavailableReason"]}} + }, + { + "if": { + "properties": {"promptTokens": {"type": "null"}}, + "required": ["promptTokens"] + }, + "then": { + "properties": { + "requiredTokens": {"type": "null"}, + "fits": {"type": "null"} + }, + "required": ["promptTokensUnavailableReason"] + }, + "else": { + "properties": {"requiredTokens": {"type": "integer", "minimum": 0}}, + "not": {"required": ["promptTokensUnavailableReason"]} + } + } + ] + } + } + }, + "hubInvocationError": { + "description": "A structured error returned by the configured Hub role endpoint.", + "type": "object", + "additionalProperties": false, + "required": ["code", "httpStatus"], + "properties": { + "code": {"type": "string", "minLength": 1}, + "message": {"type": "string", "minLength": 1}, + "httpStatus": {"type": "integer", "minimum": 400, "maximum": 599} + } + }, "modelInvocation": { "type": "object", "additionalProperties": false, @@ -838,9 +965,12 @@ } ] }, + "requestEvidence": {"$ref": "#/$defs/roleRequestEvidence"}, + "hubError": {"$ref": "#/$defs/hubInvocationError"}, "outcome": { "enum": [ "succeeded", + "pre_dispatch_rejected", "transport_failed", "contract_failed", "validation_failed", diff --git a/docs/contracts/catalyst-workbench-generation-evidence-v1.schema.json b/docs/contracts/catalyst-workbench-generation-evidence-v1.schema.json index 24637e90..84299b3d 100644 --- a/docs/contracts/catalyst-workbench-generation-evidence-v1.schema.json +++ b/docs/contracts/catalyst-workbench-generation-evidence-v1.schema.json @@ -508,7 +508,6 @@ "properties": { "included": { "type": "array", - "maxItems": 6, "items": {"$ref": "#/$defs/historyReference"} }, "includedDigest": { @@ -516,7 +515,6 @@ }, "omitted": { "type": "array", - "maxItems": 1000, "items": {"$ref": "#/$defs/historyReference"} }, "omittedDigest": { @@ -689,10 +687,17 @@ } ] }, + "requestEvidence": { + "$ref": "https://openelis-global.org/catalyst/contracts/catalyst-query-v1.schema.json#/$defs/roleRequestEvidence" + }, + "hubError": { + "$ref": "https://openelis-global.org/catalyst/contracts/catalyst-query-v1.schema.json#/$defs/hubInvocationError" + }, "outcome": { "enum": [ "in_progress", "succeeded", + "pre_dispatch_rejected", "transport_failed", "contract_failed", "validation_failed", diff --git a/docs/contracts/catalyst-workbench-guidance-request-v1.schema.json b/docs/contracts/catalyst-workbench-guidance-request-v1.schema.json index 7d2ed127..42f22297 100644 --- a/docs/contracts/catalyst-workbench-guidance-request-v1.schema.json +++ b/docs/contracts/catalyst-workbench-guidance-request-v1.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "catalyst-workbench-guidance-request-v1.schema.json", - "title": "Pin one guidance entry to a session", + "title": "Submit one experimental guidance entry for a session", "type": "object", "additionalProperties": false, "required": ["contractVersion", "text"], @@ -9,7 +9,7 @@ "contractVersion": { "const": "catalyst.workbench.guidance.request.v1" }, "text": { "type": "string", "minLength": 1, "maxLength": 2000 }, "source": { - "description": "A composer pin, or a finding accepted from a failed turn.", + "description": "Guidance supplied directly for an experiment, or a finding accepted from a failed turn.", "enum": ["human", "system"] }, "originTurnId": { "type": ["string", "null"], "format": "uuid" }, diff --git a/docs/contracts/catalyst-workbench-guidance-v1.schema.json b/docs/contracts/catalyst-workbench-guidance-v1.schema.json index a97e1b71..3d61f36c 100644 --- a/docs/contracts/catalyst-workbench-guidance-v1.schema.json +++ b/docs/contracts/catalyst-workbench-guidance-v1.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "catalyst-workbench-guidance-v1.schema.json", "title": "Catalyst workbench session guidance entry", - "description": "One durable instruction a person pinned to a session. Delivered to the writer verbatim: never summarised, rewritten, or normalised. Entries are append-only; removing or replacing one appends a lifecycle event rather than editing history.", + "description": "One optional experimental instruction supplied for a session. Delivered to the writer verbatim: never summarised, rewritten, or normalised. Entries are append-only; deactivating or replacing one appends a lifecycle event rather than editing history. Historical wire values retain pin terminology for compatibility and do not imply a required product interface.", "type": "object", "additionalProperties": false, "required": [ @@ -24,7 +24,7 @@ "entryId": { "type": "string", "minLength": 1 }, "sessionId": { "type": "string", "format": "uuid" }, "order": { - "description": "Server-assigned position. Delivery order, and the tie-break when guidance conflicts: later wins.", + "description": "Server-assigned recorded sequence. It does not define conflict precedence.", "type": "integer", "minimum": 1 }, @@ -39,11 +39,11 @@ "pattern": "^[0-9a-f]{64}$" }, "source": { - "description": "Pinned in the composer, or accepted from a finding on a failed turn.", + "description": "Supplied directly through the experimental API, or accepted from a finding on a failed turn.", "enum": ["human", "system"] }, "originTurnId": { - "description": "The turn whose finding was accepted; null for a composer pin.", + "description": "The turn whose finding was accepted; null for guidance supplied directly through the experimental API.", "type": ["string", "null"], "format": "uuid" }, @@ -65,7 +65,7 @@ }, "createdAt": { "type": "string", "format": "date-time" }, "events": { - "description": "Append-only lifecycle: pinned, then any unpin or supersede.", + "description": "Append-only lifecycle. The pinned and unpinned action names are retained wire identifiers, not a required user-interface design.", "type": "array", "minItems": 1, "items": { diff --git a/docs/contracts/catalyst-workbench-turn-v1.schema.json b/docs/contracts/catalyst-workbench-turn-v1.schema.json index b8f9dcda..28b61217 100644 --- a/docs/contracts/catalyst-workbench-turn-v1.schema.json +++ b/docs/contracts/catalyst-workbench-turn-v1.schema.json @@ -1125,11 +1125,13 @@ "properties": { "stage": { "enum": [ + "writer_request", "writer_transport", "writer_output_contract", "writer_validation", "writer_findings", "writer_decision", + "reviewer_request", "reviewer_transport", "reviewer_output_contract", "reviewer_validation", @@ -1146,8 +1148,7 @@ }, "message": { "type": "string", - "minLength": 1, - "maxLength": 4000 + "minLength": 1 }, "evidenceAvailable": { "type": "boolean" @@ -1229,7 +1230,6 @@ }, "details": { "type": "array", - "maxItems": 32, "items": { "type": "object", "additionalProperties": false, @@ -1240,12 +1240,10 @@ "properties": { "name": { "type": "string", - "minLength": 1, - "maxLength": 100 + "minLength": 1 }, "value": { - "type": "string", - "maxLength": 4000 + "type": "string" } } } diff --git a/docs/med-agent-hub.md b/docs/med-agent-hub.md index bdef3479..cceb2dd3 100644 --- a/docs/med-agent-hub.md +++ b/docs/med-agent-hub.md @@ -134,6 +134,13 @@ The engine must: correlation evidence. 8. Never send database credentials or result rows to Hub. +For each configured-role call, Hub returns the exact final model request, +including its system prompt, caller messages, response format, effective +configuration, canonical digest, and direct router measurements when +available. Catalyst retains this evidence on success and on structured Hub +errors. A proven context-window overflow is recorded as rejection before model +dispatch, not as a transport failure or a bad model answer. + Model review is a generation-quality control. Gateway still parses and validates every candidate before execution. diff --git a/docs/specification.md b/docs/specification.md index 57c20b0b..9c2ed24c 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -215,6 +215,17 @@ generation may be active per session. A failed generation records raw typed evidence and leaves the preceding query editable. `New session` is the boundary for unrelated work. +Follow-up generation supplies every prior user instruction. Query versions, +verified model results, and failures are supplied through their structured +records rather than replaying raw model replies as trusted conversation. It +also supplies the relevant prior failure and every earlier kept query that has both advisory +validation evidence and a successful database execution against the same +source. Optional session guidance remains an experimental API seam. Catalyst +does not cap or rank those items. Hub records each physical model request and +rejects for context size only when its exact measured prompt and reply reserve exceed +the selected model's advertised context window; Catalyst records that result +without silently removing context and retrying. + **The thread is one sequence of cells.** Model generations are recorded as turns and hand edits as query versions, but the analyst sees a single numbered thread: versions are numbered in the order they were appended, which @@ -288,7 +299,7 @@ against. provenance. Later edits or successors mark those results stale rather than hiding them. 11. A follow-up uses the exact active SQL/parameters, current instruction, - bounded instruction history and only exact-digest validation/execution + complete retained user-instruction history and only exact-digest validation/execution summaries. The writer returns a complete successor; when the selected Hub query profile includes a reviewer, it may approve or return one complete correction, which Gateway re-lints. @@ -444,6 +455,12 @@ the actual model ID plus assistant content. Hub owns profile configuration, provider/auth/timeout transport, and the model-router connection; Catalyst owns query semantics, SQL policy, orchestration, execution, and lineage. +For each configured-role call, Hub returns versioned evidence for the exact +system-plus-caller messages, response format, effective configuration, +canonical request digest, rendered prompt when available, tokenizer count, +advertised context window, reply reserve, and fit result. Catalyst validates +and retains that evidence on both successful calls and structured Hub errors. + Planned narrative reports remain separate: `single-e4b-checked` and `team-med-checked` are Hub product profiles whose Catalyst integration is deferred to R4. @@ -457,7 +474,7 @@ Contextual revisions use with the versioned revision-context, editor-snapshot and workbench-turn contracts. They bind the current instruction, target/catalog, policy, correlation IDs, current version/digest, exact editor SQL/parameters/digest, -bounded instruction history and matching validation/execution summaries. They +complete retained user-instruction history and matching validation/execution summaries. They exclude result rows, credentials, raw traces, historical SQL copies and unrelated sessions. Gateway selects the role-specific model-backend `response_format` and passes it through Hub's configured-role endpoint. @@ -680,8 +697,8 @@ claimed by the demo MVP. - **CAT-FR-013:** Preserve a compact chronological turn timeline and restore it without invoking a model. - **CAT-FR-014:** Generate a complete successor from the exact active editor - buffer and current follow-up instruction, with bounded context and no result - rows. + buffer, current follow-up instruction, and complete eligible session context, + with no result rows or silent context removal. - **CAT-FR-015:** Keep prior results visible and label the exact query version that produced them; mark them stale after edits or successor generation. - **CAT-FR-016:** Promote only a successful query execution into a Dataset draft From 899fd8d068ac85764f416d302fa175fdb3268240 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Tue, 25 Aug 2026 23:53:55 -0700 Subject: [PATCH 2/3] docs(catalyst): remove stale follow-up cap --- docs/med-agent-hub.md | 4 ++-- docs/roadmap.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/med-agent-hub.md b/docs/med-agent-hub.md index cceb2dd3..db2b026f 100644 --- a/docs/med-agent-hub.md +++ b/docs/med-agent-hub.md @@ -159,8 +159,8 @@ complete Gateway engine request: - `catalystQuery`: analytics target, compact runtime catalog, non-secret query policy, correlation IDs, and `requiredOutputContract: catalyst.query.v1`; - for v2, the exact active editor SQL/parameters/digest, current stored version - and digest, initial instruction plus at most five prior follow-ups, and only - exact-base validation/execution summaries. + and digest, every prior user instruction in stored order, and only exact-base + validation/execution summaries. The demo request contains no production actor, facility, tenant, or authorization context. diff --git a/docs/roadmap.md b/docs/roadmap.md index 289867b3..fe25f8a7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -311,7 +311,7 @@ The linear notebook extends R3 without adding chat or branching: 3. Validate and Run the exact active version, preserving version-labelled stale results. 4. Generate one complete successor from the exact editor snapshot and current - instruction, using at most five prior follow-up instructions. + instruction, using every prior user instruction in the session. 5. When the selected profile declares a reviewer, invoke it after the writer and deterministic lint, re-lint its complete correction, and preserve writer/reviewer evidence. The recommended GPU lane uses a different-family From c978f9322d6b0ddca27e5483a3955cad63dfca64 Mon Sep 17 00:00:00 2001 From: Piotr Mankowski Date: Tue, 25 Aug 2026 23:56:57 -0700 Subject: [PATCH 3/3] chore(catalyst): pin request-evidence hub --- scripts/bootstrap-med-agent-hub.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bootstrap-med-agent-hub.sh b/scripts/bootstrap-med-agent-hub.sh index 5f0c870a..c65a8164 100755 --- a/scripts/bootstrap-med-agent-hub.sh +++ b/scripts/bootstrap-med-agent-hub.sh @@ -5,7 +5,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TARGET_DIR="${ROOT_DIR}/.med-agent-hub" HUB_REPO="${MED_AGENT_HUB_REPO:-https://github.com/pmanko/med-agent-hub.git}" -HUB_REF="${MED_AGENT_HUB_REF:-e6095f520d9b3be53069d7a4fff710c91f18c246}" +HUB_REF="${MED_AGENT_HUB_REF:-939720e4f52adec8f2fd573157a049afc1656654}" if [ -d "${TARGET_DIR}/.git" ]; then echo "med-agent-hub checkout already exists at ${TARGET_DIR}"