diff --git a/.env.example b/.env.example index 4b5c96de..26746e0a 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,8 @@ REMEMBERSTACK_SELFHOST_API_PORT=8000 REMEMBERSTACK_OPENROUTER_API_KEY=replace-before-real-use # Optional exact OpenRouter embedding-provider slug; unset keeps automatic routing. # REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius +# 32k covers reasoning plus content; the provider account cap remains the monetary boundary. +REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 # Optional global reasoning effort for every chat-generation call. Allowed: # none, minimal, low, medium, high, xhigh, max. Unset uses the model default; # "none" reduces extraction latency but can reduce adjudication quality, and diff --git a/compose.yaml b/compose.yaml index 037a6f32..4ebba275 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,7 @@ x-app: &app REMEMBERSTACK_MINIO_SECRET_KEY: ${REMEMBERSTACK_MINIO_SECRET_KEY} REMEMBERSTACK_OPENROUTER_API_KEY: ${REMEMBERSTACK_OPENROUTER_API_KEY} REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER: ${REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER:-} + REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS: ${REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 2a382266..236860e0 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -162,6 +162,12 @@ reliably honour it. Two distinct violations were observed in real runs against appending a sentence period inside the `arguments_json` string, after the closing brace. The agent loop parses the first complete JSON object and records the trailing text on the trace row (§7). +- **2026-07-29 — `z-ai/glm-5.2` cap exhaustion and mitigation.** OpenRouter + requests sent no `max_tokens`, so production E1/E2 calls exhausted the + provider default on reasoning: `finish_reason='length'`, `content=null`, and + `reasoning_present=True`, or truncated JSON (`Unterminated string`), produced + dead-letters. The adapter now sends a 32,000-token reasoning-plus-content + budget by default; the provider account cap remains the monetary boundary. Neither is reproducible on demand: the same fact-label prompt succeeded on 24 consecutive retries after failing once, and three prompt variants each returned diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index cae61b2e..2cc76b38 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -38,6 +38,7 @@ _ALLOWED_REASONING_EFFORTS: Final[frozenset[str]] = frozenset( ("none", "minimal", "low", "medium", "high", "xhigh", "max") ) +_DEFAULT_MAX_COMPLETION_TOKENS: Final[int] = 32_000 class StrictSchemaError(ValueError): @@ -57,6 +58,15 @@ class OpenRouterSettings(BaseSettings): api_key: str = Field(min_length=1) base_url: str = Field(default="https://openrouter.ai/api/v1") timeout_s: float = Field(default=120.0, gt=0) + max_completion_tokens: int | None = Field( + default=_DEFAULT_MAX_COMPLETION_TOKENS, ge=1 + ) + """Combined reasoning-and-content budget for chat completions. + + The 32k default gives reasoning models deliberate generation headroom; + the provider account cap remains the deployment's monetary boundary. + Explicit ``None`` omits ``max_tokens`` from the provider payload. + """ embedding_provider: str | None = None reasoning_effort: ReasoningEffort | None = None reasoning_effort_map: dict[str, ReasoningEffort] | None = None @@ -76,6 +86,14 @@ def normalize_optional_string(cls, value: object) -> object: return value return value.strip() or None + @field_validator("max_completion_tokens", mode="before") + @classmethod + def default_empty_max_completion_tokens(cls, value: object) -> object: + """Treat an empty env value as the deliberate 32k default.""" + if isinstance(value, str) and not value.strip(): + return _DEFAULT_MAX_COMPLETION_TOKENS + return value + @field_validator("reasoning_effort_map", mode="before") @classmethod def parse_reasoning_effort_map(cls, value: object) -> object: @@ -158,6 +176,8 @@ def generate( } if request.temperature is not None: payload["temperature"] = request.temperature + if self._settings.max_completion_tokens is not None: + payload["max_tokens"] = self._settings.max_completion_tokens effort = self._reasoning_effort_for(model=request.model) if effort is not None: payload["reasoning"] = {"effort": effort} diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 9a2ef9b5..4d7e28c5 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -630,6 +630,11 @@ def _model_bindings() -> dict[str, str]: "p1_embedding": p1.embedding_model, "fact_label": p1.label_model, "openrouter_embedding_provider": openrouter.embedding_provider or "auto", + "openrouter_max_completion_tokens": ( + str(openrouter.max_completion_tokens) + if openrouter.max_completion_tokens is not None + else "unset" + ), "openrouter_reasoning_effort": openrouter.reasoning_effort or "auto", # Canonical (sorted-key) form so the effective per-model effort policy # is part of measurement provenance, not hidden behind the global pin. diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index 96744acb..bca6488a 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -66,6 +66,67 @@ def test_usage_fails_closed_when_required_accounting_is_unusable( _usage(body=body, requested_model="requested/model", latency_ms=1) +@pytest.mark.parametrize( + ("settings_override", "expected"), + (({}, 32_000), ({"max_completion_tokens": None}, None)), +) +def test_generation_forwards_configured_max_completion_tokens( + monkeypatch: pytest.MonkeyPatch, + settings_override: dict[str, object], + expected: int | None, +) -> None: + """The 32k default is sent, while explicit None leaves provider defaults.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings.model_validate( + {"api_key": "test-key", **settings_override} + ) + ) + observed: dict[str, object] = {} + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + observed.update(payload) + assert path == "/chat/completions" + return { + "model": "z-ai/glm-5.2", + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.2", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + assert ("max_tokens" in observed) is (expected is not None) + assert observed.get("max_tokens") == expected + + +def test_max_completion_tokens_empty_env_uses_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Compose's empty forwarded value keeps the deliberate 32k default.""" + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS", "") + + settings = OpenRouterSettings(api_key="test-key") + + assert settings.max_completion_tokens == 32_000 + + +@pytest.mark.parametrize("invalid", (0, -1, "not-an-integer")) +def test_max_completion_tokens_rejects_invalid_values(invalid: object) -> None: + """Zero, negative, and malformed caps cannot silently reach OpenRouter.""" + with pytest.raises(ValidationError) as raised: + OpenRouterSettings.model_validate( + {"api_key": "test-key", "max_completion_tokens": invalid} + ) + + assert raised.value.errors()[0]["loc"] == ("max_completion_tokens",) + + @pytest.mark.parametrize(("temperature", "present"), ((None, False), (0.0, True))) def test_generation_forwards_temperature_only_when_declared( monkeypatch: pytest.MonkeyPatch, temperature: float | None, present: bool diff --git a/src/tests/conftest.py b/src/tests/conftest.py index b651b7c1..ba48d792 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -24,6 +24,7 @@ #: Optional pins that alter provider request payloads when present. _LEAKY_OPTIONAL_PINS = ( "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER", + "REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS", "REMEMBERSTACK_OPENROUTER_REASONING_EFFORT", "REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP", ) diff --git a/src/tests/profiles/test_selfhost_profile.py b/src/tests/profiles/test_selfhost_profile.py index bf0b8c8e..37a2add7 100644 --- a/src/tests/profiles/test_selfhost_profile.py +++ b/src/tests/profiles/test_selfhost_profile.py @@ -87,6 +87,26 @@ def test_model_bindings_report_embedding_provider_without_secrets( assert "test-key" not in bindings.values() +@pytest.mark.parametrize( + ("configured", "reported"), (("64000", "64000"), ("", "32000")) +) +def test_compose_and_model_bindings_expose_max_completion_tokens( + monkeypatch: pytest.MonkeyPatch, configured: str, reported: str +) -> None: + """Compose forwards the cap and readiness fingerprints its effective value.""" + compose = (_ROOT / "compose.yaml").read_text(encoding="utf-8") + assert ( + "REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS:" + " ${REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS:-}" + ) in compose + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_API_KEY", "test-key") + monkeypatch.setenv("REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS", configured) + + bindings = _model_bindings() + + assert bindings["openrouter_max_completion_tokens"] == reported + + @pytest.mark.parametrize(("configured", "reported"), (("none", "none"), ("", "auto"))) def test_model_bindings_report_reasoning_effort( monkeypatch: pytest.MonkeyPatch, configured: str, reported: str