diff --git a/README.md b/README.md
index 2092a7d..161181a 100644
--- a/README.md
+++ b/README.md
@@ -105,6 +105,16 @@ Interaction rules preserve their source-assigned `severity`. `DRUG_SAFETY_MIN_IN
- `POST /v1/hub/query-profiles/{profile}/roles/{role}/generate`: execute one
configured query role. Callers provide non-system messages and an optional
response format; callers cannot override the role model, prompt, or knobs.
+ Every request that reaches the model returns versioned request evidence with
+ the exact system and caller messages, selected profile/role/model, response
+ format and model settings, and an RFC 8785 canonical SHA-256 digest. When the
+ router supports it, the same evidence includes the exact rendered prompt and
+ token count against the model's advertised context window and the role's
+ configured output allowance. Missing rendering, counting, or window
+ information is reported with a stable reason and is never estimated. For an
+ otherwise valid request, only a known overflow blocks the model call; it
+ returns `422` with the same evidence. The existing `token_accounting` field
+ keeps its compatible successful shape.
- `POST /v1/hub/generate`: raw single-model compatibility endpoint for generic
consumers that own their own profile configuration. Catalyst does not use it.
- `GET /health`: service health, uptime, and process memory.
diff --git a/server/generic_role.py b/server/generic_role.py
index 6b9fbd2..00c8701 100644
--- a/server/generic_role.py
+++ b/server/generic_role.py
@@ -17,21 +17,28 @@
from __future__ import annotations
-from typing import Any, Dict, List, Optional
+import hashlib
+from copy import deepcopy
+from typing import Any, Dict, List, Mapping, Optional
import httpx
+import rfc8785
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, ConfigDict, Field
from . import team
+from .config import llm_config
from .levels_loader import (
ModelNotFoundError,
- catalyst_query_profile_metadata,
catalyst_query_profile_ids,
+ catalyst_query_profile_metadata,
get_catalyst_query_profile,
)
-from .openai_compat import _backend_discovery_metadata, _served_backend_model_metadata
-from .config import llm_config
+from .openai_compat import (
+ ROUTER_PROBE_TIMEOUT_SECONDS,
+ _backend_discovery_metadata,
+ _served_backend_model_metadata,
+)
from .prompt_loader import load_prompt
router = APIRouter()
@@ -65,9 +72,11 @@ class ProfileGenerateRequest(BaseModel):
class ProfileGenerateResponse(GenerateResponse):
profile_id: str
role: str
+ request_evidence: Dict[str, Any]
# Exact token evidence for the fully rendered request, counted with the
- # model's own template and tokenizer before the call. None when the
- # router could not count -- absence is honest, a character estimate is not.
+ # model's own template and tokenizer before the call. The compatible field
+ # is None unless all four legacy values are known; request_evidence records
+ # each missing fact and reason without substituting an estimate.
token_accounting: Optional[Dict[str, Any]] = None
@@ -76,69 +85,186 @@ def _backend_models() -> set[str] | None:
return None if discovered is None else set(discovered)
-_CONTEXT_WINDOWS: Dict[str, int] = {}
-
-
def _context_window(model: str) -> Optional[int]:
"""The --ctx-size the router actually launched this model with."""
- cached = _CONTEXT_WINDOWS.get(model)
- if cached:
- return cached
- metadata = _served_backend_model_metadata() or {}
- entry = metadata.get(model) or {}
- args = list(((entry.get("status") or {}).get("args")) or [])
+ metadata = _served_backend_model_metadata()
+ if not isinstance(metadata, Mapping):
+ return None
+ entry = metadata.get(model)
+ if not isinstance(entry, Mapping):
+ return None
+ status = entry.get("status")
+ if not isinstance(status, Mapping):
+ return None
+ advertised_args = status.get("args")
+ if not isinstance(advertised_args, (list, tuple)):
+ return None
+ args = list(advertised_args)
for index, item in enumerate(args[:-1]):
if item in ("--ctx-size", "-c"):
try:
- _CONTEXT_WINDOWS[model] = int(args[index + 1])
- return _CONTEXT_WINDOWS[model]
+ value = int(args[index + 1])
+ if value <= 0:
+ return None
+ return value
except (TypeError, ValueError):
return None
return None
-async def _token_accounting(
- model: str, messages: List[Dict[str, Any]], max_tokens: Optional[int]
-) -> Optional[Dict[str, Any]]:
- """Count the rendered request with the model's own template and tokenizer.
+async def _prompt_measurement(
+ model: str, messages: List[Dict[str, Any]], output_reserve: int
+) -> Dict[str, Any]:
+ """Render and count the exact configured-role prompt when the router can.
/apply-template renders the exact prompt the model will consume --
special tokens included -- and /tokenize counts it with the model's own
- vocabulary, so the number is the one the context window will see.
+ vocabulary. Missing router capabilities remain explicit; they are never
+ replaced with character or approximate token counts.
"""
+
window = _context_window(model)
- if window is None or max_tokens is None:
- return None
+ prompt: Optional[str] = None
+ prompt_tokens: Optional[int] = None
+ prompt_unavailable_reason: Optional[str] = None
+ count_unavailable_reason: Optional[str] = None
base = llm_config.base_url.rstrip("/")
headers = {}
if llm_config.api_key:
headers["Authorization"] = f"Bearer {llm_config.api_key}"
try:
- async with httpx.AsyncClient(headers=headers, timeout=30.0) as client:
- rendered = await client.post(
- f"{base}/apply-template",
- json={"model": model, "messages": messages},
- )
- rendered.raise_for_status()
- prompt = rendered.json().get("prompt")
- if not isinstance(prompt, str):
- return None
- tokenized = await client.post(
- f"{base}/tokenize",
- json={"model": model, "content": prompt},
- )
- tokenized.raise_for_status()
- tokens = tokenized.json().get("tokens")
- if not isinstance(tokens, list):
- return None
+ async with httpx.AsyncClient(
+ headers=headers,
+ timeout=ROUTER_PROBE_TIMEOUT_SECONDS,
+ ) as client:
+ try:
+ rendered = await client.post(
+ f"{base}/apply-template",
+ json={"model": model, "messages": messages},
+ )
+ rendered.raise_for_status()
+ candidate = rendered.json().get("prompt")
+ if isinstance(candidate, str):
+ prompt = candidate
+ else:
+ prompt_unavailable_reason = "prompt_rendering_unavailable"
+ except Exception:
+ prompt_unavailable_reason = "prompt_rendering_unavailable"
+
+ if prompt is None:
+ count_unavailable_reason = "rendered_prompt_unavailable"
+ else:
+ try:
+ tokenized = await client.post(
+ f"{base}/tokenize",
+ json={
+ "model": model,
+ "content": prompt,
+ # apply-template already emitted special tokens.
+ "add_special": False,
+ "parse_special": True,
+ },
+ )
+ tokenized.raise_for_status()
+ tokens = tokenized.json().get("tokens")
+ if isinstance(tokens, list):
+ prompt_tokens = len(tokens)
+ else:
+ count_unavailable_reason = "prompt_token_count_unavailable"
+ except Exception:
+ count_unavailable_reason = "prompt_token_count_unavailable"
except Exception:
- return None
- return {
+ if prompt is None:
+ prompt_unavailable_reason = "prompt_rendering_unavailable"
+ count_unavailable_reason = "rendered_prompt_unavailable"
+ elif prompt_tokens is None:
+ count_unavailable_reason = "prompt_token_count_unavailable"
+
+ prompt_evidence: Dict[str, Any] = {
+ "renderedPrompt": prompt,
+ "renderedPromptDigest": (
+ hashlib.sha256(prompt.encode("utf-8")).hexdigest()
+ if prompt is not None
+ else None
+ ),
+ }
+ if prompt is None:
+ prompt_evidence["unavailableReason"] = (
+ prompt_unavailable_reason or "prompt_rendering_unavailable"
+ )
+
+ required_tokens = (
+ prompt_tokens + output_reserve if prompt_tokens is not None else None
+ )
+ fits = (
+ required_tokens <= window
+ if required_tokens is not None and window is not None
+ else None
+ )
+ token_evidence: Dict[str, Any] = {
"tokenizer": model,
"contextWindow": window,
- "outputReserve": int(max_tokens),
- "promptTokens": len(tokens),
+ "outputReserve": output_reserve,
+ "promptTokens": prompt_tokens,
+ "requiredTokens": required_tokens,
+ "fits": fits,
+ }
+ if window is None:
+ token_evidence["contextWindowUnavailableReason"] = "context_window_unavailable"
+ if prompt_tokens is None:
+ token_evidence["promptTokensUnavailableReason"] = (
+ count_unavailable_reason or "prompt_token_count_unavailable"
+ )
+ return {"prompt": prompt_evidence, "tokens": token_evidence}
+
+
+def _request_evidence(
+ *,
+ profile_id: str,
+ role: str,
+ model: str,
+ messages: List[Dict[str, Any]],
+ response_format: Optional[Dict[str, Any]],
+ temperature: float,
+ dry_multiplier: float,
+ max_tokens: int,
+ measurement: Dict[str, Any],
+) -> Dict[str, Any]:
+ """Bind exact call inputs and router measurements with canonical JSON."""
+
+ request = {
+ "profileId": profile_id,
+ "role": role,
+ "model": model,
+ "messages": deepcopy(messages),
+ "responseFormat": deepcopy(response_format),
+ "config": {
+ "temperature": temperature,
+ "dryMultiplier": dry_multiplier,
+ "maxTokens": max_tokens,
+ },
}
+ return {
+ "contractVersion": "med-agent-hub.catalyst-role-request-evidence.v1",
+ "request": request,
+ "requestDigest": hashlib.sha256(rfc8785.dumps(request)).hexdigest(),
+ "prompt": deepcopy(measurement["prompt"]),
+ "tokens": deepcopy(measurement["tokens"]),
+ }
+
+
+def _compatible_token_accounting(
+ measurement: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+ """Keep the existing successful token_accounting response shape."""
+
+ tokens = measurement.get("tokens")
+ if not isinstance(tokens, dict):
+ return None
+ required = ("tokenizer", "contextWindow", "outputReserve", "promptTokens")
+ if any(tokens.get(key) is None for key in required):
+ return None
+ return {key: deepcopy(tokens[key]) for key in required}
def _profile_or_404(profile_id: str):
@@ -254,25 +380,74 @@ async def generate_query_role(
},
)
knobs = profile.knobs[role]
+ model = profile.models[role]
+ response_format = deepcopy(req.response_format)
+ temperature = float(knobs["temperature"])
+ dry_multiplier = float(knobs["dry"])
+ max_tokens = int(knobs["maxTokens"])
rendered_messages = [
{"role": "system", "content": load_prompt(profile.prompts[role])},
- *req.messages,
+ *deepcopy(req.messages),
]
- accounting = await _token_accounting(
- profile.models[role], rendered_messages, int(knobs["maxTokens"])
- )
- content = await _chat_or_bad_gateway(
- model=profile.models[role],
+ measurement = await _prompt_measurement(model, rendered_messages, max_tokens)
+ request_evidence = _request_evidence(
+ profile_id=profile_id,
+ role=role,
+ model=model,
messages=rendered_messages,
- response_format=req.response_format,
- temperature=float(knobs["temperature"]),
- dry_multiplier=float(knobs["dry"]),
- max_tokens=int(knobs["maxTokens"]),
+ response_format=response_format,
+ temperature=temperature,
+ dry_multiplier=dry_multiplier,
+ max_tokens=max_tokens,
+ measurement=measurement,
)
+ if measurement["tokens"].get("fits") is False:
+ raise HTTPException(
+ status_code=422,
+ detail={
+ "code": "context_window_exceeded",
+ "message": (
+ "The exact rendered role request plus its configured output "
+ "reserve exceeds the model context window."
+ ),
+ "request_evidence": request_evidence,
+ },
+ )
+ try:
+ content = await _chat_or_bad_gateway(
+ model=model,
+ messages=rendered_messages,
+ response_format=response_format,
+ temperature=temperature,
+ dry_multiplier=dry_multiplier,
+ max_tokens=max_tokens,
+ )
+ except HTTPException as error:
+ raise HTTPException(
+ status_code=error.status_code,
+ detail={
+ "code": "model_request_failed",
+ "message": str(error.detail),
+ "request_evidence": request_evidence,
+ },
+ headers=error.headers,
+ ) from error
+ except Exception as error:
+ raise HTTPException(
+ status_code=502,
+ detail={
+ "code": "model_request_failed",
+ "message": (
+ "The model backend did not return a usable assistant response."
+ ),
+ "request_evidence": request_evidence,
+ },
+ ) from error
return ProfileGenerateResponse(
profile_id=profile_id,
role=role,
- model=profile.models[role],
+ model=model,
content=content,
- token_accounting=accounting,
+ request_evidence=request_evidence,
+ token_accounting=_compatible_token_accounting(measurement),
)
diff --git a/server/openai_compat.py b/server/openai_compat.py
index 49a67af..310bc61 100644
--- a/server/openai_compat.py
+++ b/server/openai_compat.py
@@ -75,8 +75,7 @@ def _is_sensitive_metadata_key(normalized: str, parts: tuple[str, ...]) -> bool:
if normalized in _SENSITIVE_METADATA_KEYS:
return True
if any(
- part in {"credential", "credentials", "password", "secret"}
- for part in parts
+ part in {"credential", "credentials", "password", "secret"} for part in parts
):
return True
pairs = set(zip(parts, parts[1:]))
@@ -141,6 +140,9 @@ def _sanitize_backend_metadata(value: Any, *, key: str = "") -> Any:
return value
+ROUTER_PROBE_TIMEOUT_SECONDS = 3.0
+
+
def _served_backend_model_metadata() -> Optional[Dict[str, Dict[str, Any]]]:
"""Return the router catalog's per-model metadata, or ``None`` when discovery itself fails."""
headers = {}
@@ -150,7 +152,7 @@ def _served_backend_model_metadata() -> Optional[Dict[str, Dict[str, Any]]]:
response = httpx.get(
f"{llm_config.base_url.rstrip('/')}/v1/models",
headers=headers,
- timeout=3.0,
+ timeout=ROUTER_PROBE_TIMEOUT_SECONDS,
)
response.raise_for_status()
result: Dict[str, Dict[str, Any]] = {}
diff --git a/tests/test_generic_role.py b/tests/test_generic_role.py
index 798e976..88c8d6a 100644
--- a/tests/test_generic_role.py
+++ b/tests/test_generic_role.py
@@ -6,14 +6,16 @@
from __future__ import annotations
+import asyncio
+import hashlib
from typing import Any, Dict, List, Optional
from unittest.mock import patch
import httpx
+import rfc8785
from fastapi.testclient import TestClient
-from server import team
-from server import generic_role
+from server import generic_role, team
from server.main import app
@@ -126,11 +128,30 @@ def test_catalyst_query_profile_owns_model_prompt_and_knobs(monkeypatch):
},
)
+ async def fake_measurement(model, messages, output_reserve):
+ return {
+ "prompt": {
+ "renderedPrompt": "rendered",
+ "renderedPromptDigest": hashlib.sha256(b"rendered").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": model,
+ "contextWindow": 24576,
+ "outputReserve": output_reserve,
+ "promptTokens": 100,
+ "requiredTokens": 100 + output_reserve,
+ "fits": True,
+ },
+ }
+
async def fake_chat(client, model, messages, **kwargs):
captured.update(model=model, messages=messages, kwargs=kwargs)
return {"role": "assistant", "content": '{"status":"ready"}'}
- with patch.object(team, "_chat", side_effect=fake_chat):
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=fake_chat),
+ ):
response = TestClient(app).post(
"/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
json={
@@ -227,20 +248,28 @@ def test_a_role_generation_counts_its_rendered_request_first(monkeypatch):
)
counted: Dict[str, Any] = {}
- async def fake_accounting(model, messages, max_tokens):
- counted.update(model=model, turns=len(messages), reserve=max_tokens)
+ async def fake_measurement(model, messages, output_reserve):
+ counted.update(model=model, turns=len(messages), reserve=output_reserve)
return {
- "tokenizer": model,
- "contextWindow": 24576,
- "outputReserve": max_tokens,
- "promptTokens": 1234,
+ "prompt": {
+ "renderedPrompt": "rendered request",
+ "renderedPromptDigest": hashlib.sha256(b"rendered request").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": model,
+ "contextWindow": 24576,
+ "outputReserve": output_reserve,
+ "promptTokens": 1234,
+ "requiredTokens": 1234 + output_reserve,
+ "fits": True,
+ },
}
async def fake_chat(client, model, messages, **kwargs):
return {"role": "assistant", "content": '{"status":"ready"}'}
with (
- patch.object(generic_role, "_token_accounting", side_effect=fake_accounting),
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
patch.object(team, "_chat", side_effect=fake_chat),
):
response = TestClient(app).post(
@@ -260,6 +289,477 @@ async def fake_chat(client, model, messages, **kwargs):
assert counted == {"model": "gemma-e4b", "turns": 2, "reserve": 1024}
+def test_a_configured_role_records_the_exact_request_passed_to_chat(monkeypatch):
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
+ )
+ captured: Dict[str, Any] = {}
+ measurement = {
+ "prompt": {
+ "renderedPrompt": "exact rendered prompt",
+ "renderedPromptDigest": hashlib.sha256(
+ b"exact rendered prompt"
+ ).hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 24576,
+ "outputReserve": 1024,
+ "promptTokens": 1200,
+ "requiredTokens": 2224,
+ "fits": True,
+ },
+ }
+
+ async def fake_measurement(model, messages, output_reserve):
+ assert model == "gemma-e4b"
+ assert output_reserve == 1024
+ return measurement
+
+ async def fake_chat(client, model, messages, **kwargs):
+ captured.update(model=model, messages=messages, kwargs=kwargs)
+ return {"role": "assistant", "content": '{"status":"ready"}'}
+
+ caller_messages = [
+ {"role": "assistant", "content": "Earlier query"},
+ {"role": "user", "content": "Use the retained history"},
+ ]
+ response_format = {
+ "type": "json_schema",
+ "json_schema": {"name": "candidate", "strict": True, "schema": {}},
+ }
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=fake_chat),
+ ):
+ response = TestClient(app).post(
+ "/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
+ json={"messages": caller_messages, "response_format": response_format},
+ )
+
+ assert response.status_code == 200, response.text
+ evidence = response.json()["request_evidence"]
+ assert evidence["contractVersion"] == (
+ "med-agent-hub.catalyst-role-request-evidence.v1"
+ )
+ exact_request = evidence["request"]
+ assert exact_request == {
+ "profileId": "catalyst-query-e4b-qwen14b",
+ "role": "query_generate",
+ "model": "gemma-e4b",
+ "messages": captured["messages"],
+ "responseFormat": response_format,
+ "config": {
+ "temperature": 0.0,
+ "dryMultiplier": 0.0,
+ "maxTokens": 1024,
+ },
+ }
+ assert captured["messages"][1:] == caller_messages
+ assert captured["kwargs"] == {
+ "response_format": response_format,
+ "temperature": 0.0,
+ "dry_multiplier": 0.0,
+ "max_tokens": 1024,
+ }
+ assert (
+ evidence["requestDigest"]
+ == hashlib.sha256(rfc8785.dumps(exact_request)).hexdigest()
+ )
+ assert evidence["prompt"] == measurement["prompt"]
+ assert evidence["tokens"] == measurement["tokens"]
+
+
+def test_prompt_measurement_records_exact_rendering_count_and_fit(monkeypatch):
+ calls: list[tuple[str, Dict[str, Any]]] = []
+ client_options: Dict[str, Any] = {}
+
+ class Response:
+ def __init__(self, payload):
+ self.payload = payload
+
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return self.payload
+
+ class Client:
+ def __init__(self, *args, **kwargs):
+ client_options.update(kwargs)
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *args):
+ return None
+
+ async def post(self, url, json):
+ calls.append((url, json))
+ if url.endswith("/apply-template"):
+ return Response({"prompt": "exact prompt"})
+ return Response({"tokens": list(range(37))})
+
+ monkeypatch.setattr(generic_role, "_context_window", lambda model: 4096)
+ monkeypatch.setattr(generic_role.httpx, "AsyncClient", Client)
+ messages = [
+ {"role": "system", "content": "system"},
+ {"role": "user", "content": "caller"},
+ ]
+
+ measured = asyncio.run(generic_role._prompt_measurement("gemma-e4b", messages, 512))
+
+ prompt = "exact prompt"
+ assert measured == {
+ "prompt": {
+ "renderedPrompt": prompt,
+ "renderedPromptDigest": hashlib.sha256(prompt.encode()).hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 4096,
+ "outputReserve": 512,
+ "promptTokens": 37,
+ "requiredTokens": 549,
+ "fits": True,
+ },
+ }
+ assert calls[0][1] == {"model": "gemma-e4b", "messages": messages}
+ assert calls[1][1] == {
+ "model": "gemma-e4b",
+ "content": prompt,
+ "add_special": False,
+ "parse_special": True,
+ }
+ assert client_options["timeout"] == generic_role.ROUTER_PROBE_TIMEOUT_SECONDS
+
+
+def test_context_window_is_read_from_current_router_metadata(monkeypatch):
+ metadata = iter(
+ [
+ {"gemma-e4b": {"status": {"args": ["--ctx-size", "4096"]}}},
+ {"gemma-e4b": {"status": {"args": ["--ctx-size", "8192"]}}},
+ ]
+ )
+ monkeypatch.setattr(
+ generic_role, "_served_backend_model_metadata", lambda: next(metadata)
+ )
+
+ assert generic_role._context_window("gemma-e4b") == 4096
+ assert generic_role._context_window("gemma-e4b") == 8192
+
+
+def test_malformed_router_metadata_makes_context_window_unavailable(monkeypatch):
+ malformed_entries = [
+ "not-an-object",
+ {"status": "loaded"},
+ {"status": {"args": 7}},
+ ]
+
+ for entry in malformed_entries:
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda entry=entry: {"gemma-e4b": entry},
+ )
+ assert generic_role._context_window("gemma-e4b") is None
+
+
+def test_prompt_measurement_reports_why_render_count_and_window_are_unavailable(
+ monkeypatch,
+):
+ class Client:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *args):
+ return None
+
+ async def post(self, url, json):
+ request = httpx.Request("POST", url)
+ response = httpx.Response(503, request=request)
+ raise httpx.HTTPStatusError(
+ "unavailable", request=request, response=response
+ )
+
+ monkeypatch.setattr(generic_role, "_context_window", lambda model: None)
+ monkeypatch.setattr(generic_role.httpx, "AsyncClient", Client)
+
+ measured = asyncio.run(
+ generic_role._prompt_measurement(
+ "gemma-e4b", [{"role": "user", "content": "caller"}], 1024
+ )
+ )
+
+ assert measured == {
+ "prompt": {
+ "renderedPrompt": None,
+ "renderedPromptDigest": None,
+ "unavailableReason": "prompt_rendering_unavailable",
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": None,
+ "contextWindowUnavailableReason": "context_window_unavailable",
+ "outputReserve": 1024,
+ "promptTokens": None,
+ "promptTokensUnavailableReason": "rendered_prompt_unavailable",
+ "requiredTokens": None,
+ "fits": None,
+ },
+ }
+
+
+def test_prompt_measurement_keeps_rendering_when_only_token_count_fails(monkeypatch):
+ class Response:
+ def __init__(self, payload):
+ self.payload = payload
+
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return self.payload
+
+ class Client:
+ def __init__(self, *args, **kwargs):
+ self.calls = 0
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *args):
+ return None
+
+ async def post(self, url, json):
+ self.calls += 1
+ if self.calls == 1:
+ return Response({"prompt": "rendered"})
+ return Response({})
+
+ monkeypatch.setattr(generic_role, "_context_window", lambda model: 4096)
+ monkeypatch.setattr(generic_role.httpx, "AsyncClient", Client)
+
+ measured = asyncio.run(
+ generic_role._prompt_measurement(
+ "gemma-e4b", [{"role": "user", "content": "caller"}], 1024
+ )
+ )
+
+ assert measured["prompt"]["renderedPrompt"] == "rendered"
+ assert "unavailableReason" not in measured["prompt"]
+ assert measured["tokens"] == {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 4096,
+ "outputReserve": 1024,
+ "promptTokens": None,
+ "promptTokensUnavailableReason": "prompt_token_count_unavailable",
+ "requiredTokens": None,
+ "fits": None,
+ }
+
+
+def test_known_role_request_overflow_returns_evidence_without_calling_model(
+ monkeypatch,
+):
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
+ )
+ measurement = {
+ "prompt": {
+ "renderedPrompt": "too large",
+ "renderedPromptDigest": hashlib.sha256(b"too large").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 1500,
+ "outputReserve": 1024,
+ "promptTokens": 800,
+ "requiredTokens": 1824,
+ "fits": False,
+ },
+ }
+
+ async def fake_measurement(model, messages, output_reserve):
+ return measurement
+
+ async def must_not_chat(*args, **kwargs):
+ raise AssertionError("known overflow must not reach the model")
+
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=must_not_chat),
+ ):
+ response = TestClient(app).post(
+ "/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
+ json={"messages": [{"role": "user", "content": "catalog context"}]},
+ )
+
+ assert response.status_code == 422, response.text
+ detail = response.json()["detail"]
+ assert detail["code"] == "context_window_exceeded"
+ evidence = detail["request_evidence"]
+ assert evidence["tokens"] == measurement["tokens"]
+ assert (
+ evidence["requestDigest"]
+ == hashlib.sha256(rfc8785.dumps(evidence["request"])).hexdigest()
+ )
+
+
+def test_a_configured_role_backend_failure_still_returns_request_evidence(
+ monkeypatch,
+):
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
+ )
+ measurement = {
+ "prompt": {
+ "renderedPrompt": "rendered request",
+ "renderedPromptDigest": hashlib.sha256(b"rendered request").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 24576,
+ "outputReserve": 1024,
+ "promptTokens": 1200,
+ "requiredTokens": 2224,
+ "fits": True,
+ },
+ }
+
+ async def fake_measurement(model, messages, output_reserve):
+ return measurement
+
+ async def failed_chat(*args, **kwargs):
+ request = httpx.Request("POST", "http://router/v1/chat/completions")
+ response = httpx.Response(500, request=request)
+ raise httpx.HTTPStatusError("boom", request=request, response=response)
+
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=failed_chat),
+ ):
+ response = TestClient(app).post(
+ "/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
+ json={"messages": [{"role": "user", "content": "catalog context"}]},
+ )
+
+ assert response.status_code == 502, response.text
+ detail = response.json()["detail"]
+ assert detail["code"] == "model_request_failed"
+ assert "model backend returned 500" in detail["message"]
+ evidence = detail["request_evidence"]
+ assert evidence["contractVersion"] == (
+ "med-agent-hub.catalyst-role-request-evidence.v1"
+ )
+ assert evidence["tokens"] == measurement["tokens"]
+ assert (
+ evidence["requestDigest"]
+ == hashlib.sha256(rfc8785.dumps(evidence["request"])).hexdigest()
+ )
+
+
+def test_a_configured_role_empty_response_still_returns_request_evidence(
+ monkeypatch,
+):
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
+ )
+ measurement = {
+ "prompt": {
+ "renderedPrompt": "rendered request",
+ "renderedPromptDigest": hashlib.sha256(b"rendered request").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 24576,
+ "outputReserve": 1024,
+ "promptTokens": 1200,
+ "requiredTokens": 2224,
+ "fits": True,
+ },
+ }
+
+ async def fake_measurement(model, messages, output_reserve):
+ return measurement
+
+ async def empty_chat(*args, **kwargs):
+ return {"role": "assistant", "content": " "}
+
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=empty_chat),
+ ):
+ response = TestClient(app).post(
+ "/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
+ json={"messages": [{"role": "user", "content": "catalog context"}]},
+ )
+
+ assert response.status_code == 502, response.text
+ detail = response.json()["detail"]
+ assert detail["code"] == "model_request_failed"
+ assert "no assistant content" in detail["message"]
+ assert detail["request_evidence"]["request"]["model"] == "gemma-e4b"
+
+
+def test_a_configured_role_malformed_backend_response_keeps_request_evidence(
+ monkeypatch,
+):
+ monkeypatch.setattr(
+ generic_role,
+ "_served_backend_model_metadata",
+ lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
+ )
+ measurement = {
+ "prompt": {
+ "renderedPrompt": "rendered request",
+ "renderedPromptDigest": hashlib.sha256(b"rendered request").hexdigest(),
+ },
+ "tokens": {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 24576,
+ "outputReserve": 1024,
+ "promptTokens": 1200,
+ "requiredTokens": 2224,
+ "fits": True,
+ },
+ }
+
+ async def fake_measurement(model, messages, output_reserve):
+ return measurement
+
+ async def malformed_chat(*args, **kwargs):
+ raise KeyError("choices")
+
+ with (
+ patch.object(generic_role, "_prompt_measurement", side_effect=fake_measurement),
+ patch.object(team, "_chat", side_effect=malformed_chat),
+ ):
+ response = TestClient(app).post(
+ "/v1/hub/query-profiles/catalyst-query-e4b-qwen14b/roles/query_generate/generate",
+ json={"messages": [{"role": "user", "content": "catalog context"}]},
+ )
+
+ assert response.status_code == 502, response.text
+ detail = response.json()["detail"]
+ assert detail["code"] == "model_request_failed"
+ assert detail["message"] == (
+ "The model backend did not return a usable assistant response."
+ )
+ assert detail["request_evidence"]["request"]["model"] == "gemma-e4b"
+
+
def test_an_uncountable_request_is_answered_with_no_accounting_not_a_guess(
monkeypatch,
):
@@ -274,14 +774,31 @@ def test_an_uncountable_request_is_answered_with_no_accounting_not_a_guess(
lambda: {"gemma-e4b": {}, "qwen2.5-14b": {}},
)
- async def broken_accounting(model, messages, max_tokens):
- return None
+ async def broken_measurement(model, messages, output_reserve):
+ return {
+ "prompt": {
+ "renderedPrompt": None,
+ "renderedPromptDigest": None,
+ "unavailableReason": "prompt_rendering_unavailable",
+ },
+ "tokens": {
+ "tokenizer": model,
+ "contextWindow": 24576,
+ "outputReserve": output_reserve,
+ "promptTokens": None,
+ "promptTokensUnavailableReason": "rendered_prompt_unavailable",
+ "requiredTokens": None,
+ "fits": None,
+ },
+ }
async def fake_chat(client, model, messages, **kwargs):
return {"role": "assistant", "content": '{"status":"ready"}'}
with (
- patch.object(generic_role, "_token_accounting", side_effect=broken_accounting),
+ patch.object(
+ generic_role, "_prompt_measurement", side_effect=broken_measurement
+ ),
patch.object(team, "_chat", side_effect=fake_chat),
):
response = TestClient(app).post(
@@ -291,3 +808,12 @@ async def fake_chat(client, model, messages, **kwargs):
assert response.status_code == 200
assert response.json()["token_accounting"] is None
+ assert response.json()["request_evidence"]["tokens"] == {
+ "tokenizer": "gemma-e4b",
+ "contextWindow": 24576,
+ "outputReserve": 1024,
+ "promptTokens": None,
+ "promptTokensUnavailableReason": "rendered_prompt_unavailable",
+ "requiredTokens": None,
+ "fits": None,
+ }