From 7c6fa4294e7fc1111f71dff18c8c4a33b091e3e0 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 31 Jul 2026 18:17:16 +0200 Subject: [PATCH] fix(benchmarks): ship LoCoMo v8 answer recovery Conv-47 scored 91/150 under v7: the six-word cap caused 7 invalid answers and constrained 19 longer gold answers, while 23/150 questions failed on first-step non-JSON completions. Raise the guarded answer limit to twenty words and extend the shared two-retry allowance to pre-tool invalid completions. --- benchmarks/locomo/README.md | 29 ++-- benchmarks/locomo/__init__.py | 2 +- benchmarks/locomo/cli.py | 2 +- benchmarks/locomo/dataset.py | 4 +- benchmarks/locomo/model.py | 14 +- benchmarks/locomo/protocol.py | 21 +-- benchmarks/locomo/runner.py | 35 +++- benchmarks/locomo/sharding/README.md | 2 +- decisions.md | 15 ++ design/benchmarks/next-steps.md | 11 ++ plan/designs/locomo_benchmark_design.md | 63 +++++-- src/tests/benchmarks/test_locomo_protocol.py | 27 +-- src/tests/benchmarks/test_locomo_runner.py | 169 +++++++++++++++++-- 13 files changed, 311 insertions(+), 83 deletions(-) diff --git a/benchmarks/locomo/README.md b/benchmarks/locomo/README.md index e029b575..9fbf4013 100644 --- a/benchmarks/locomo/README.md +++ b/benchmarks/locomo/README.md @@ -1,4 +1,4 @@ -# RS-LoCoMo-Full-v7 setup +# RS-LoCoMo-Full-v8 setup This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its @@ -16,13 +16,13 @@ The safe first command is local and makes no API or model call: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v7 \ + --protocol full-v8 \ --output .benchmark-runs/locomo-smoke ``` The harness validates the pinned bytes, renders session documents, and fingerprints the -eight-question smoke plan. `--protocol` is prepare-only: choose `full-v7` (the -default) or `full-v7-strong` there, and every later stage reads that immutable +eight-question smoke plan. `--protocol` is prepare-only: choose `full-v8` (the +default) or `full-v8-strong` there, and every later stage reads that immutable choice from `run.json`. Do not run remote stages until reviewing [`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md). @@ -40,6 +40,11 @@ containers so retrieved evidence does not get crowded out by audit metadata. Freshness, hydration-drop counts, and meaningful default-valued fields remain visible. +V8 requires the shortest phrase that fully names the requested entities or +values, permits up to twenty words, and forbids explanations or reasoning. Its +two-retry malformed-completion allowance is shared across the answer loop, +including a completion returned before the first tool call. + Build the image from the revision under test — Compose otherwise serves the published release image, and the harness refuses to run against an engine whose stamped revision does not match the prepared run: @@ -76,15 +81,17 @@ call can cross it, is recorded, and stops the run. Use the provider account cap monetary boundary. If that leaves later questions unanswered, they remain visible as zero-scored missing records; resuming them requires an explicitly higher threshold. -After at least one recipe result, an answer-agent completion that is not a valid -JSON answer step is retried at most twice. Those attempts consume the same -nine-call per-question and run-absolute call budgets; they are not extra calls -outside the cap. Each item records `reader_attempts`, and the summary records -`total_reader_retries`. Tool-selection failures before retrieval and judge -failures are not retried. +An answer-agent completion that is not a valid JSON step is retried at most +twice, whether it occurs before the first tool call or while reading retrieved +evidence. The two-retry allowance is shared across both positions. Every +attempt consumes the same nine-call per-question and run-absolute call budgets; +these are not extra calls outside the cap. Each item records reader-position +attempts in `reader_attempts` and pre-tool additional calls in +`first_step_retries`; the summary sums both signals separately. Plain provider +outages and judge failures are not retried. The strong protocol pins answer-agent reasoning effort to `none` on every answer -call. The default `full-v7` protocol sends no per-call effort field for its +call. The default `full-v8` protocol sends no per-call effort field for its non-reasoning `gpt-4o-mini` answer agent. Ambient OpenRouter effort-map settings therefore cannot change either prepared protocol's answer behavior. diff --git a/benchmarks/locomo/__init__.py b/benchmarks/locomo/__init__.py index a7c05d53..1b6b234e 100644 --- a/benchmarks/locomo/__init__.py +++ b/benchmarks/locomo/__init__.py @@ -1 +1 @@ -"""The pinned RS-LoCoMo-Full-v7 benchmark adapter.""" +"""The pinned RS-LoCoMo-Full-v8 benchmark adapter.""" diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index 44c5622d..37f50291 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -115,7 +115,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-Full-v7: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v8: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py index ce09241c..ea9405e9 100644 --- a/benchmarks/locomo/dataset.py +++ b/benchmarks/locomo/dataset.py @@ -139,9 +139,9 @@ def load_manifest(tier: str) -> QuestionManifest: f"manifest file {tier!r} declares tier {manifest.tier!r}" ) if manifest.dataset_commit != DATASET_COMMIT: - raise DatasetValidationError("manifest dataset commit is not RS-LoCoMo-Full-v7") + raise DatasetValidationError("manifest dataset commit is not RS-LoCoMo-Full-v8") if manifest.dataset_sha256 != DATASET_SHA256: - raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v7") + raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v8") actual = item_ids_hash(item_ids=manifest.item_ids) if actual != manifest.item_ids_sha256: raise DatasetValidationError( diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 85c34901..fa160c90 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v7 protocols.""" +"""Typed values for the full-system RS-LoCoMo-Full-v8 protocols.""" from __future__ import annotations @@ -26,8 +26,8 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] -ProtocolKey = Literal["full-v7", "full-v7-strong"] -ProtocolName = Literal["RS-LoCoMo-Full-v7", "RS-LoCoMo-Full-v7-strong"] +ProtocolKey = Literal["full-v8", "full-v8-strong"] +ProtocolName = Literal["RS-LoCoMo-Full-v8", "RS-LoCoMo-Full-v8-strong"] SourceTimezoneBasis = Literal["assumed_utc"] AnswerAgentModel = Literal["openai/gpt-4o-mini", "openai/gpt-5.6-luna"] JudgeModel = Literal["openai/gpt-5.6-luna"] @@ -118,7 +118,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v7" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v8" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -289,6 +289,7 @@ class AnswerRecord(FrozenModel): reader_called: bool agent_call_count: int = Field(default=0, ge=0) reader_attempts: int = Field(default=0, ge=0) + first_step_retries: int = Field(default=0, ge=0) reader_latency_ms: int | None = Field(default=None, ge=0) generated_answer: str | None = None reader_usage: ProviderCallUsage | None = None @@ -309,6 +310,8 @@ def require_answer_xor_failure(self) -> "AnswerRecord": raise ValueError("reader_called must match agent_call_count") if self.reader_attempts > self.agent_call_count: raise ValueError("reader attempts cannot exceed agent calls") + if self.first_step_retries > self.agent_call_count: + raise ValueError("first-step retries cannot exceed agent calls") if self.generated_answer is not None and self.reader_attempts < 1: raise ValueError("a generated answer requires a reader attempt") return self @@ -373,7 +376,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v7" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v8" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) @@ -385,6 +388,7 @@ class RunSummary(FrozenModel): failures: dict[str, int] answer_agent_calls: int = Field(ge=0) total_reader_retries: int = Field(ge=0) + total_first_step_retries: int = Field(default=0, ge=0) judge_calls: int = Field(ge=0) tokens_in: int = Field(ge=0) tokens_out: int = Field(ge=0) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index a2efa57f..d497c925 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -26,10 +26,10 @@ from benchmarks.locomo.model import ToolCallRecord from rememberstack.model import ToolDescriptor -PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v7" -STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v7-strong" -DEFAULT_PROTOCOL_KEY: Final = "full-v7" -ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07-hybrid-context" +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v8" +STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v8-strong" +DEFAULT_PROTOCOL_KEY: Final = "full-v8" +ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07-hybrid-context-v8answers" MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 ANSWER_READER_RETRY_BUDGET: Final = 2 @@ -72,7 +72,8 @@ automatically current fact. Use timestamps to resolve relative dates. Do not confuse people mentioned in a memory with the conversation speakers. Never use outside knowledge. If the deployment does not contain the answer, finish with -"Unknown". A final answer must be concise and at most six words. +"Unknown". The final answer must be the shortest phrase that fully names the +requested entities/values, at most twenty words, no explanations or reasoning. Loop discipline: never repeat a tool call with the same tool AND the same arguments. If a tool yields nothing useful, change the arguments meaningfully or switch tools rather than retrying @@ -127,8 +128,8 @@ class LoCoMoProtocol: answer_agent_reasoning_effort: str | None -_FULL_V7 = LoCoMoProtocol( - key="full-v7", +_FULL_V8 = LoCoMoProtocol( + key="full-v8", name=PROTOCOL_NAME, answer_agent_model=ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -145,8 +146,8 @@ class LoCoMoProtocol: answer_reader_retry_budget=ANSWER_READER_RETRY_BUDGET, answer_agent_reasoning_effort=None, ) -_FULL_V7_STRONG = LoCoMoProtocol( - key="full-v7-strong", +_FULL_V8_STRONG = LoCoMoProtocol( + key="full-v8-strong", name=STRONG_PROTOCOL_NAME, answer_agent_model=STRONG_ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -165,7 +166,7 @@ class LoCoMoProtocol: ) PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( - {_FULL_V7.key: _FULL_V7, _FULL_V7_STRONG.key: _FULL_V7_STRONG} + {_FULL_V8.key: _FULL_V8, _FULL_V8_STRONG.key: _FULL_V8_STRONG} ) diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index 793a688a..b3dbb17d 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -431,7 +431,7 @@ def answer_sample( ): raise ExecutionGuardError( "the deployment did not report the exact completed" - " RS-LoCoMo-Full-v7 pipeline and fresh P2/P3 projections" + " RS-LoCoMo-Full-v8 pipeline and fresh P2/P3 projections" ) _require_serving_revision(context=context, readiness=readiness) prior_readiness = context.state.readiness.get(sample_id) @@ -732,6 +732,9 @@ def summarize_run(*, run_dir: Path) -> RunSummary: max(record.reader_attempts - 1, 0) for record in context.state.answers.values() ), + total_first_step_retries=sum( + record.first_step_retries for record in context.state.answers.values() + ), judge_calls=sum( record.model_called for record in context.state.judges.values() ), @@ -967,7 +970,7 @@ def _validate_run( """Recompute immutable run identity before any local or remote stage.""" selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v7") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v8") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -977,7 +980,7 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v7") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v8") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if _models_hash(values=documents) != configuration.documents_sha256: @@ -1308,6 +1311,8 @@ def _answer_one( tool_latency_ms = 0 agent_call_count = 0 reader_attempts = 0 + first_step_retries = 0 + invalid_completion_attempts = 0 prior_calls = sum(record.agent_call_count for record in state.answers.values()) for _ in range(max_agent_calls_per_question): if prior_calls + agent_call_count >= max_agent_calls: @@ -1351,6 +1356,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts + int(bool(trace)), + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms + call_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1372,6 +1378,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms + call_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1385,6 +1392,7 @@ def _answer_one( if error.usage is not None: usages.append(error.usage) state.evaluator_cost_usd += error.usage.cost_usd + invalid_completion_attempts += 1 if trace: reader_attempts += 1 if agent_observation is not None: @@ -1394,13 +1402,14 @@ def _answer_one( outcome="provider_error", ) can_retry = ( - bool(trace) - and reader_attempts <= answer_reader_retry_budget + invalid_completion_attempts <= answer_reader_retry_budget and agent_call_count < max_agent_calls_per_question and prior_calls + agent_call_count < max_agent_calls and state.evaluator_cost_usd < max_evaluator_cost_usd ) if can_retry: + if not trace: + first_step_retries += 1 continue return _failed_answer( question=question, @@ -1410,6 +1419,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1436,6 +1446,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts + int(bool(trace)), + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms + call_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1472,6 +1483,7 @@ def _answer_one( reader_attempts=( reader_attempts + int(step.action == "answer" and bool(trace)) ), + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1495,12 +1507,13 @@ def _answer_one( retrieval_succeeded=False, agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, tool_calls=tuple(trace), usages=tuple(usages), ) answer = step.answer or "" - if len(answer.split()) > 6: + if len(answer.split()) > 20: if agent_observation is not None: agent_observation.finish( usage=response.usage, @@ -1510,11 +1523,12 @@ def _answer_one( return _failed_answer( question=question, kind="invalid_response", - message="answer agent exceeded the six-word answer limit", + message="answer agent exceeded the twenty-word answer limit", retrieval_latency_ms=tool_latency_ms, retrieval_succeeded=True, agent_call_count=agent_call_count, reader_attempts=reader_attempts + 1, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1547,6 +1561,7 @@ def _answer_one( reader_called=True, agent_call_count=agent_call_count, reader_attempts=reader_attempts + 1, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, generated_answer=answer, reader_usage=_aggregate_usage(usages=tuple(usages)), @@ -1560,6 +1575,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1576,6 +1592,7 @@ def _answer_one( retrieval_succeeded=True, agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1611,6 +1628,7 @@ def _answer_one( retrieval_succeeded=False, agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace( trace=tuple(trace), doc_sessions=doc_sessions @@ -1639,6 +1657,7 @@ def _answer_one( retrieval_succeeded=bool(trace), agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=agent_latency_ms, claims=_claims_from_trace(trace=tuple(trace), doc_sessions=doc_sessions), tool_calls=tuple(trace), @@ -1809,6 +1828,7 @@ def _failed_answer( retrieval_succeeded: bool, agent_call_count: int, reader_attempts: int = 0, + first_step_retries: int = 0, reader_latency_ms: int | None = None, claims: tuple[RetrievedClaim, ...] = (), tool_calls: tuple[ToolCallRecord, ...] = (), @@ -1832,6 +1852,7 @@ def _failed_answer( reader_called=agent_call_count > 0, agent_call_count=agent_call_count, reader_attempts=reader_attempts, + first_step_retries=first_step_retries, reader_latency_ms=reader_latency_ms, reader_usage=_aggregate_usage(usages=usages) if usages else None, failure=_failure(kind=kind, message=message), diff --git a/benchmarks/locomo/sharding/README.md b/benchmarks/locomo/sharding/README.md index 35c2b5fa..f29eb74a 100644 --- a/benchmarks/locomo/sharding/README.md +++ b/benchmarks/locomo/sharding/README.md @@ -122,7 +122,7 @@ The following environment variables tune the driver without changing its argumen | --- | ---: | --- | | `LOCOMO_PYTHON` | `.venv/bin/python` | repository virtual-environment Python | | `LOCOMO_TIER` | `publication` | prepared manifest tier | -| `LOCOMO_PROTOCOL` | `full-v7` | prepare-time protocol key | +| `LOCOMO_PROTOCOL` | `full-v8` | prepare-time protocol key | | `LOCOMO_MAX_DOCUMENTS` | `100` | per-sample ingest authorization | | `LOCOMO_MAX_QUESTIONS` | `1540` | run-absolute answer item authorization | | `LOCOMO_MAX_AGENT_CALLS` | `13860` | run-absolute answer-agent call ceiling | diff --git a/decisions.md b/decisions.md index 80d5aac5..30346d9f 100644 --- a/decisions.md +++ b/decisions.md @@ -2809,6 +2809,21 @@ may proceed to its first tagged artifact proof after CLA activation. > durable records retain the raw envelopes. The tool > catalog, prompt, adapter identity, and protocol fingerprints change, so v6 > and v7 results are not comparable. See the companion design §§2, 3, and 7. +> +> **Also amended 2026-07-31 (v8 — answer-stage correctness):** the conv-47 v7 +> re-score was 91/150. Its six-word answer cap caused 7 terminal invalid-answer +> failures and made 19 longer gold answers structurally impossible to reproduce +> completely. V8 requires the shortest complete entity/value phrase, permits +> twenty words, and forbids explanations or reasoning. The same run also lost +> 23/150 questions to malformed structured completions on the first agent call, +> before any tool result. The existing two-retry allowance now applies there as +> well as at the reader position and is shared across the answer loop; every +> attempt consumes the ordinary per-question, run-wide, and cost budgets. Plain +> provider outages remain non-retried. Reader-position attempts and additional +> first-step calls are reported separately. The prompt, adapter identity, +> runner behavior, and fingerprints change, while the tool catalog remains +> unchanged, so v7 and v8 results are not comparable. See the companion design +> §§2 and 7. **Decision.** The first competitive benchmark is **`RS-LoCoMo-Full-v1`** over the exact pinned LoCoMo ten-conversation file and categories 1–4. Each conversation is an isolated deployment; diff --git a/design/benchmarks/next-steps.md b/design/benchmarks/next-steps.md index 4dfbe0cd..59bb6e3e 100644 --- a/design/benchmarks/next-steps.md +++ b/design/benchmarks/next-steps.md @@ -28,6 +28,17 @@ and every extraction change forces re-ingestion of every store. ## Tier 2 — harness correctness/ergonomics +**Shipped 2026-07-31 in `RS-LoCoMo-Full-v8` — answer-stage correctness.** +The conv-47 v7 re-score was 91/150. The six-word answer cap produced 7 outright +`answer_invalid_response` failures, while 19 judged misses had gold answers +longer than six words; v8 now permits the shortest complete entity/value phrase +up to twenty words. Separately, 23/150 questions (about 15%) ended as +`answer_reader` with `reader_attempts: 0`, no tool calls, and a first-step +“completion content is not JSON” failure against prompts averaging about 22,800 +input tokens. V8 extends the existing shared two-retry allowance to that first +step, charging every attempt to the normal call and cost budgets while leaving +plain provider outages terminal. + 5. **Derive the protocol fingerprint from identity fields only** (drop `repository_revision` from the hash, keep it recorded). Evidence: the sharded run could not be officially merged with the sequential half diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 764af4dc..75fcb1a1 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -21,7 +21,7 @@ WP-8.2 remains in progress until an owner-authorized eight-question smoke finish ## 2. Fixed protocol ```text -protocol RS-LoCoMo-Full-v7 +protocol RS-LoCoMo-Full-v8 dataset commit 3eb6f2c585f5e1699204e3c3bdf7adc5c28cb376 dataset SHA-256 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4 categories 1, 2, 3, 4 @@ -29,7 +29,7 @@ answer-agent model openai/gpt-4o-mini answer temperature 0 max tool calls/question 8 max agent calls/question 9 -reader retries 2 additional attempts within the 9-call cap +invalid-completion retry 2 additional attempts within the 9-call cap answer reasoning effort adapter default (no field sent) judge model openai/gpt-5.6-luna judge temperature 0 @@ -67,10 +67,10 @@ are not directly comparable. changed only the answer agent to `openai/gpt-5.6-luna`. It exists because three smoke passes on a healthy store (coarse evidence-session recall 0.5, with the gold evidence at rank 1) scored only 1–2/8 with `openai/gpt-4o-mini`, which looped -past the tool-call limit or returned invalid responses. The v7 protocols retain -that seat distinction. Scores from `RS-LoCoMo-Full-v7` and -`RS-LoCoMo-Full-v7-strong` are never comparable. The weak-agent -`RS-LoCoMo-Full-v7` protocol remains the default measurement of what a harness +past the tool-call limit or returned invalid responses. The v8 protocols retain +that seat distinction. Scores from `RS-LoCoMo-Full-v8` and +`RS-LoCoMo-Full-v8-strong` are never comparable. The weak-agent +`RS-LoCoMo-Full-v8` protocol remains the default measurement of what a harness consumer experiences. **Reader retry and answer-effort pin (2026-07-29):** strong-agent smoke runs @@ -83,8 +83,8 @@ The answer reader now retries that same model call up to two times, for three attempts total. A retry uses the ordinary agent-call ledger: it consumes both the nine-call per-question allowance and the run's `max_agent_calls` ceiling. If either allowance is exhausted, the item keeps the same terminal failure it -would have recorded before. Tool-selection calls before any recipe result and -judge calls do not receive this retry. Each answer record stores +would have recorded before. In v5–v7, tool-selection calls before any recipe +result and judge calls did not receive this retry. Each answer record stores `reader_attempts`; the run summary adds `total_reader_retries` without changing the existing failure categories. @@ -94,7 +94,7 @@ raised the score from 1/8 to 3/8. Environment-only configuration was not reproducible because two runs with identical `run.json` files could behave differently. The strong protocol therefore pins `answer_agent_reasoning_effort` to `none` and sends it explicitly on every answer-agent call, including tool -selection and final reading. The default `full-v7` protocol pins `None`, which +selection and final reading. The default `full-v8` protocol pins `None`, which means no effort field is sent for its non-reasoning `gpt-4o-mini` answer agent. Engine worker seats still use the existing environment map; this override is only on benchmark answer requests. @@ -133,6 +133,30 @@ empty containers; it does not cap or discard retrieved evidence or freshness. The tool catalog, prompt, reader rendering, adapter version, and protocol fingerprints changed, so v6 and v7 results are not comparable. +**v7 → v8 (2026-07-31 — answer-stage correctness):** a fresh conv-47 re-score +under v7 scored 91/150 and exposed two protocol constraints that turned usable +model work into forced zeroes. First, the six-word final-answer cap caused 7 +terminal `answer_invalid_response` failures. Another 19 judged misses had gold +answers longer than six words, including multi-entity enumerations that could +not be named completely within the cap. The estimated cost was 13–26 questions +out of 150. V8 instead requires the shortest phrase that fully names the +requested entities or values, permits at most twenty words, and forbids +explanations or reasoning. The runner enforces the same twenty-word boundary. + +Second, 23/150 questions (about 15%) failed with `answer_reader` before any tool +call: `reader_attempts: 0`, an empty tool trace, and “completion content is not +JSON.” The larger v7 answer prompts averaged about 22,800 input tokens per +question, making this first-step failure common enough to bias the score. V8 +applies the existing two-retry malformed-completion allowance before the first +tool call as well as at the reader position. The allowance is shared across the +whole answer loop; every attempt consumes the normal per-question call count, +run-wide call ceiling, and evaluator-cost ceiling. Plain provider outages are +still terminal and are not retried. `reader_attempts` remains limited to calls +after tool results, while `first_step_retries` counts additional pre-tool calls; +the run summary totals both signals separately. The prompt, adapter identity, +runner behavior, and protocol fingerprints changed, while the public tool +catalog did not, so v7 and v8 results are not comparable. + ### 2.1 Why v2+ uses a stronger judge `RS-LoCoMo-Full-v1` used `openai/gpt-4o-mini` for both the answer agent and the judge. The @@ -405,18 +429,23 @@ For each question: recorded on the trace row, not discarded silently — see §2.4), and call `MemoryClient.run_recipe()`. 4. Append arguments, latency, and the complete envelope. -5. For `action="answer"`, require at least one tool call and at most six words. -6. After at least one tool result, retry a completion that cannot produce the - required JSON step up to two times. Every attempt counts toward the normal - call budgets; tool selection before retrieval is never retried. +5. For `action="answer"`, require at least one tool call and at most twenty + words. The prompt requires the shortest phrase that fully names the requested + entities or values and forbids explanations or reasoning. +6. Retry a completion that cannot produce the required JSON step up to two + times, including before the first tool call. The allowance is shared across + the loop; every attempt counts toward the normal per-question, run-wide, and + cost budgets. Plain provider outages are never retried. 7. Stop at eight tools or nine model calls; budget exhaustion is a visible wrong. -8. Checkpoint the terminal answer or failure, including `reader_attempts`. +8. Checkpoint the terminal answer or failure. `reader_attempts` counts only + reader-position attempts after tool results; `first_step_retries` counts + additional calls made before any tool result. The agent is instructed to orient, verify current facts, and audit evidence while respecting grain, validity, freshness, truncation, typed negatives, and hydration drops. It receives no gold answer, evidence IDs, summaries, or outside retrieval. -Loop guards in the frozen answer prompt (v7): never repeat a tool call with the +Loop guards in the frozen answer prompt (v8): never repeat a tool call with the same tool and the same arguments; if a tool yields nothing useful, switch tools rather than retrying it; use `question_context` first for ordinary recall and try it before answering "Unknown". These are prompt discipline, not harness @@ -440,11 +469,11 @@ Local preparation: uv run --extra benchmark python -m benchmarks.locomo prepare \ --dataset /absolute/path/locomo10.json \ --tier smoke \ - --protocol full-v7 \ + --protocol full-v8 \ --output .benchmark-runs/locomo-smoke ``` -`--protocol` exists only on `prepare`. Use `full-v7-strong` there to select the +`--protocol` exists only on `prepare`. Use `full-v8-strong` there to select the strong answer agent; ingest, answer, judge, and summarize read the pinned choice from the prepared run and expose no protocol override. diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 7dcfb65b..68203a6b 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -185,11 +185,18 @@ def test_frozen_tool_catalog_hash_matches_stock_full_system_recipes() -> None: ) -def test_protocol_is_v7_and_answer_prompt_has_loop_guards() -> None: - """The default v7 identity and answer-loop discipline remain unchanged.""" - assert PROTOCOL_NAME == "RS-LoCoMo-Full-v7" - assert DEFAULT_PROTOCOL_KEY == "full-v7" +def test_protocol_is_v8_and_answer_prompt_has_loop_guards() -> None: + """The default v8 identity and answer-loop discipline remain unchanged.""" + assert PROTOCOL_NAME == "RS-LoCoMo-Full-v8" + assert DEFAULT_PROTOCOL_KEY == "full-v8" prompt = ANSWER_AGENT_PROMPT_TEMPLATE + normalized_prompt = " ".join(prompt.split()) + assert ( + "The final answer must be the shortest phrase that fully names the requested " + "entities/values, at most twenty words, no explanations or reasoning." + in normalized_prompt + ) + assert "six words" not in normalized_prompt assert "never repeat a tool call with the same tool AND the same" in prompt assert "switch tools rather than retrying" in prompt assert "use question_context first" in prompt @@ -198,14 +205,14 @@ def test_protocol_is_v7_and_answer_prompt_has_loop_guards() -> None: def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None: - assert tuple(PROTOCOL_REGISTRY) == ("full-v7", "full-v7-strong") - default = PROTOCOL_REGISTRY["full-v7"] - strong = PROTOCOL_REGISTRY["full-v7-strong"] + assert tuple(PROTOCOL_REGISTRY) == ("full-v8", "full-v8-strong") + default = PROTOCOL_REGISTRY["full-v8"] + strong = PROTOCOL_REGISTRY["full-v8-strong"] - assert default.name == "RS-LoCoMo-Full-v7" + assert default.name == "RS-LoCoMo-Full-v8" assert default.answer_agent_model == "openai/gpt-4o-mini" assert default.answer_agent_reasoning_effort is None - assert strong.name == "RS-LoCoMo-Full-v7-strong" + assert strong.name == "RS-LoCoMo-Full-v8-strong" assert strong.answer_agent_model == "openai/gpt-5.6-luna" assert strong.answer_agent_reasoning_effort == "none" assert default.answer_reader_retry_budget == 2 @@ -231,7 +238,7 @@ def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None @pytest.mark.parametrize( ("extra_args", "expected"), - (((), "full-v7"), (("--protocol", "full-v7-strong"), "full-v7-strong")), + (((), "full-v8"), (("--protocol", "full-v8-strong"), "full-v8-strong")), ) def test_prepare_cli_selects_protocol_only_at_prepare( extra_args: tuple[str, ...], expected: str, monkeypatch: pytest.MonkeyPatch diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 076c34f8..06aea906 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -89,6 +89,7 @@ def test_agent_calls_public_recipe_then_answers() -> None: assert answer.retrieval_succeeded is True assert answer.generated_answer == "Prague" assert answer.agent_call_count == 2 + assert answer.first_step_retries == 0 assert [call.name for call in answer.tool_calls] == ["claims_verbatim"] assert len(provider.generated_prompts) == 2 @@ -172,9 +173,39 @@ def test_reader_retry_stops_when_run_agent_call_budget_is_exhausted() -> None: assert provider.answer_calls == 3 -def test_invalid_tool_selection_completion_is_not_retried() -> None: +def test_first_step_invalid_completion_retries_then_succeeds_within_budgets() -> None: client, raw_client = _memory_client() - provider = _CostProvider(cost=Decimal(0), invalid_tool_selection=True) + provider = _CostProvider(cost=Decimal("0.10"), invalid_first_step_completions=1) + state = _run_state() + try: + answer = _answer_one( + question=_question(), + client=client, + provider=provider, + tools=(_tool(),), + doc_sessions={}, + state=state, + max_agent_calls=3, + max_evaluator_cost_usd=Decimal("1"), + max_agent_calls_per_question=3, + ) + finally: + raw_client.close() + + assert answer.failure is None + assert answer.generated_answer == "Prague" + assert answer.reader_attempts == 1 + assert answer.first_step_retries == 1 + assert answer.agent_call_count == 3 + assert provider.answer_calls == 3 + assert state.evaluator_cost_usd == Decimal("0.30") + assert answer.reader_usage is not None + assert answer.reader_usage.tokens_in == 30 + + +def test_first_step_invalid_completion_fails_after_shared_retry_budget() -> None: + client, raw_client = _memory_client() + provider = _CostProvider(cost=Decimal(0), invalid_first_step_completions=3) try: answer = _answer_one( question=_question(), @@ -191,7 +222,35 @@ def test_invalid_tool_selection_completion_is_not_retried() -> None: assert answer.failure is not None assert answer.failure.kind == "reader" + assert answer.failure.message == ( + "AnswerAgentStep: completion content is not JSON (synthetic)" + ) assert answer.reader_attempts == 0 + assert answer.first_step_retries == 2 + assert answer.agent_call_count == 3 + assert provider.answer_calls == 3 + + +def test_first_step_provider_outage_is_not_retried() -> None: + client, raw_client = _memory_client() + provider = _CostProvider(cost=Decimal(0), first_step_provider_outage=True) + try: + answer = _answer_one( + question=_question(), + client=client, + provider=provider, + tools=(_tool(),), + doc_sessions={}, + state=_run_state(), + max_agent_calls=9, + max_evaluator_cost_usd=Decimal("1"), + ) + finally: + raw_client.close() + + assert answer.failure is not None + assert answer.failure.kind == "reader" + assert answer.first_step_retries == 0 assert answer.agent_call_count == 1 assert provider.answer_calls == 1 @@ -225,6 +284,47 @@ def test_answer_without_consulting_memory_is_rejected() -> None: assert answer.agent_call_count == 1 +@pytest.mark.parametrize( + ("word_count", "expected_failure"), ((20, None), (21, "invalid_response")) +) +def test_answer_word_limit_accepts_twenty_and_rejects_twenty_one( + word_count: int, expected_failure: str | None +) -> None: + answer_text = " ".join(f"word-{index}" for index in range(1, word_count + 1)) + + def tool_then_sized_answer(prompt: str, type_name: str) -> dict[str, object]: + step = _tool_then_answer(prompt, type_name) + if step["action"] == "answer": + step["answer"] = answer_text + return step + + client, raw_client = _memory_client() + provider = FakeModelProvider(generate_router=tool_then_sized_answer) + try: + answer = _answer_one( + question=_question(), + client=client, + provider=provider, + tools=(_tool(),), + doc_sessions={}, + state=_run_state(), + max_agent_calls=9, + max_evaluator_cost_usd=Decimal("1"), + ) + finally: + raw_client.close() + + if expected_failure is None: + assert answer.failure is None + assert answer.generated_answer == answer_text + else: + assert answer.failure is not None + assert answer.failure.kind == expected_failure + assert answer.failure.message == ( + "answer agent exceeded the twenty-word answer limit" + ) + + def test_agent_and_judge_share_one_run_absolute_cost_threshold() -> None: client, raw_client = _memory_client() provider = _CostProvider(cost=Decimal("0.30")) @@ -287,17 +387,19 @@ def test_a_call_that_crosses_the_cost_threshold_is_recorded_then_stops() -> None "protocol", "answer_agent_model", "reasoning_effort", + "invalid_first_step_completions", "invalid_reader_completions", ), ( - ("full-v7", "openai/gpt-4o-mini", None, 0), - ("full-v7-strong", "openai/gpt-5.6-luna", "none", 2), + ("full-v8", "openai/gpt-4o-mini", None, 1, 0), + ("full-v8-strong", "openai/gpt-5.6-luna", "none", 0, 2), ), ) def test_staged_mock_run_uses_prepared_protocol_and_resumes( protocol: ProtocolKey, answer_agent_model: str, reasoning_effort: str | None, + invalid_first_step_completions: int, invalid_reader_completions: int, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -316,7 +418,9 @@ def test_staged_mock_run_uses_prepared_protocol_and_resumes( client = MemoryClient(client=raw_client) preflight_provider = _PreflightProvider() provider = _CostProvider( - cost=Decimal(0), invalid_reader_completions=invalid_reader_completions + cost=Decimal(0), + invalid_first_step_completions=invalid_first_step_completions, + invalid_reader_completions=invalid_reader_completions, ) try: ingests = ingest_sample( @@ -371,7 +475,9 @@ def test_staged_mock_run_uses_prepared_protocol_and_resumes( assert first_answers == second_answers assert first_judges == second_judges assert preflight_provider.models == [answer_agent_model] - expected_answer_calls = 2 + invalid_reader_completions + expected_answer_calls = ( + 2 + invalid_first_step_completions + invalid_reader_completions + ) assert provider.models == [ *([answer_agent_model] * expected_answer_calls), JUDGE_MODEL, @@ -396,6 +502,7 @@ def test_staged_mock_run_uses_prepared_protocol_and_resumes( assert summary.official_f1 == 1 assert summary.answer_agent_calls == expected_answer_calls assert summary.total_reader_retries == invalid_reader_completions + assert summary.total_first_step_retries == invalid_first_step_completions class _FakeLangfuseObservation: @@ -776,8 +883,8 @@ def test_single_run_summary_json_is_unchanged( serialized = summarize_run(run_dir=run_dir).model_dump_json() assert serialized == ( - '{"protocol_name":"RS-LoCoMo-Full-v7","protocol_fingerprint":' - '"02c9ef2dde16bc3b4e2ce0c273fc5eaf5d61f3ba96598263014f90221437035a",' + '{"protocol_name":"RS-LoCoMo-Full-v8","protocol_fingerprint":' + '"dfcae6bbea8b0a0c65b10f6ed88f58071932ea2d06371bd6003ce5e448c618ac",' '"tier":"smoke","questions":1,"judge_correct":0,"judge_percent":0.0,' '"official_f1":0.0,"categories":[{"category":1,"questions":0,' '"judge_correct":0,"judge_percent":0.0,"official_f1":0.0},{"category":2,' @@ -789,7 +896,8 @@ def test_single_run_summary_json_is_unchanged( '"mean_session_recall":0.0,"complete_session_success":0.0,' '"warning":"session-grain diagnostic; not turn Recall@k"},' '"failures":{"missing_answer":1,"missing_judge":1},' - '"answer_agent_calls":0,"total_reader_retries":0,"judge_calls":0,' + '"answer_agent_calls":0,"total_reader_retries":0,' + '"total_first_step_retries":0,"judge_calls":0,' '"tokens_in":0,"tokens_out":0,"evaluator_cost_usd":"0",' '"ingestion_cost_source":"deployment cost ledger; not available through ' 'benchmark SDK"}' @@ -886,21 +994,27 @@ def test_prepared_protocol_pins_and_fingerprints_are_distinct( dataset_path=tmp_path / "synthetic.json", tier="smoke", output=strong_dir, - protocol="full-v7-strong", + protocol="full-v8-strong", ) - assert weak.protocol_name == "RS-LoCoMo-Full-v7" + assert weak.protocol_name == "RS-LoCoMo-Full-v8" assert weak.answer_agent_model == "openai/gpt-4o-mini" assert weak.answer_agent_reasoning_effort is None assert weak.answer_reader_retry_budget == 2 assert weak.protocol_fingerprint == ( + "dfcae6bbea8b0a0c65b10f6ed88f58071932ea2d06371bd6003ce5e448c618ac" + ) + assert weak.protocol_fingerprint != ( "02c9ef2dde16bc3b4e2ce0c273fc5eaf5d61f3ba96598263014f90221437035a" ) - assert strong.protocol_name == "RS-LoCoMo-Full-v7-strong" + assert strong.protocol_name == "RS-LoCoMo-Full-v8-strong" assert strong.answer_agent_model == "openai/gpt-5.6-luna" assert strong.answer_agent_reasoning_effort == "none" assert strong.answer_reader_retry_budget == 2 assert strong.protocol_fingerprint == ( + "ccf6b7b28397f4311a08403aa1c4639f209e90532d9430f54f12003fd017fe8b" + ) + assert strong.protocol_fingerprint != ( "70d765a8bd0c597f91b3c75171546e0aa7e954be22a09d41974b41cacbe1ae77" ) assert strong.protocol_fingerprint != weak.protocol_fingerprint @@ -1106,14 +1220,18 @@ def __init__( *, cost: Decimal, invalid_reader_completions: int = 0, - invalid_tool_selection: bool = False, + invalid_first_step_completions: int = 0, + first_step_provider_outage: bool = False, ) -> None: self.cost = cost self.invalid_reader_completions = invalid_reader_completions - self.invalid_tool_selection = invalid_tool_selection + self.invalid_first_step_completions = invalid_first_step_completions + self.first_step_provider_outage = first_step_provider_outage self.models: list[str] = [] self.requests: list[ModelRequest] = [] self.answer_calls = 0 + self.first_step_invalid_calls = 0 + self.reader_invalid_calls = 0 def generate( self, *, request: ModelRequest, response_type: type[ResponseT] @@ -1122,9 +1240,24 @@ def generate( self.requests.append(request) if response_type is AnswerAgentStep: self.answer_calls += 1 - if (self.invalid_tool_selection and self.answer_calls == 1) or ( - 1 < self.answer_calls <= self.invalid_reader_completions + 1 - ): + empty_trace = "TOOL TRACE SO FAR:\n[]" in request.prompt + if self.first_step_provider_outage and self.answer_calls == 1: + raise OpenRouterProviderError( + "OpenRouter /chat/completions returned 503 (synthetic)" + ) + invalid_first_step = ( + empty_trace + and self.first_step_invalid_calls < self.invalid_first_step_completions + ) + invalid_reader = ( + not empty_trace + and self.reader_invalid_calls < self.invalid_reader_completions + ) + if invalid_first_step or invalid_reader: + if invalid_first_step: + self.first_step_invalid_calls += 1 + else: + self.reader_invalid_calls += 1 raise OpenRouterInvalidResponseError( "AnswerAgentStep: completion content is not JSON (synthetic)", usage=ProviderCallUsage( @@ -1142,7 +1275,7 @@ def generate( "arguments_json": '{"query": "Where?"}', "answer": None, } - if self.answer_calls == 1 + if empty_trace else { "action": "answer", "tool_name": None,