fix(locomo): v8 — twenty-word answers + first-step invalid-completion retry

benchmarks/locomo/README.md

@@ -1,4 +1,4 @@
-# RS-LoCoMo-Full-v7 setup
+# RS-LoCoMo-Full-v8 setup
 
 This directory contains the unshipped full-system LoCoMo adapter. It does not vendor or
 auto-download LoCoMo. Supply the exact pinned `locomo10.json` only after confirming its
@@ -16,13 +16,13 @@ The safe first command is local and makes no API or model call:
 uv run --extra benchmark python -m benchmarks.locomo prepare \
   --dataset /absolute/path/locomo10.json \
   --tier smoke \
-  --protocol full-v7 \
+  --protocol full-v8 \
   --output .benchmark-runs/locomo-smoke
 ```
 
 The harness validates the pinned bytes, renders session documents, and fingerprints the
-eight-question smoke plan. `--protocol` is prepare-only: choose `full-v7` (the
-default) or `full-v7-strong` there, and every later stage reads that immutable
+eight-question smoke plan. `--protocol` is prepare-only: choose `full-v8` (the
+default) or `full-v8-strong` there, and every later stage reads that immutable
 choice from `run.json`. Do not run remote stages until reviewing
 [`locomo_benchmark_design.md`](../../plan/designs/locomo_benchmark_design.md).
 
@@ -40,6 +40,11 @@ containers so retrieved evidence does not get crowded out by audit metadata.
 Freshness, hydration-drop counts, and meaningful default-valued fields remain
 visible.
 
+V8 requires the shortest phrase that fully names the requested entities or
+values, permits up to twenty words, and forbids explanations or reasoning. Its
+two-retry malformed-completion allowance is shared across the answer loop,
+including a completion returned before the first tool call.
+
 Build the image from the revision under test — Compose otherwise serves the
 published release image, and the harness refuses to run against an engine whose
 stamped revision does not match the prepared run:
@@ -76,15 +81,17 @@ call can cross it, is recorded, and stops the run. Use the provider account cap
 monetary boundary. If that leaves later questions unanswered, they remain visible as zero-scored
 missing records; resuming them requires an explicitly higher threshold.
 
-After at least one recipe result, an answer-agent completion that is not a valid
-JSON answer step is retried at most twice. Those attempts consume the same
-nine-call per-question and run-absolute call budgets; they are not extra calls
-outside the cap. Each item records `reader_attempts`, and the summary records
-`total_reader_retries`. Tool-selection failures before retrieval and judge
-failures are not retried.
+An answer-agent completion that is not a valid JSON step is retried at most
+twice, whether it occurs before the first tool call or while reading retrieved
+evidence. The two-retry allowance is shared across both positions. Every
+attempt consumes the same nine-call per-question and run-absolute call budgets;
+these are not extra calls outside the cap. Each item records reader-position
+attempts in `reader_attempts` and pre-tool additional calls in
+`first_step_retries`; the summary sums both signals separately. Plain provider
+outages and judge failures are not retried.
 
 The strong protocol pins answer-agent reasoning effort to `none` on every answer
-call. The default `full-v7` protocol sends no per-call effort field for its
+call. The default `full-v8` protocol sends no per-call effort field for its
 non-reasoning `gpt-4o-mini` answer agent. Ambient OpenRouter effort-map settings
 therefore cannot change either prepared protocol's answer behavior.
 

benchmarks/locomo/__init__.py

@@ -1 +1 @@
-"""The pinned RS-LoCoMo-Full-v7 benchmark adapter."""
+"""The pinned RS-LoCoMo-Full-v8 benchmark adapter."""

benchmarks/locomo/cli.py

@@ -115,7 +115,7 @@ def _parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="python -m benchmarks.locomo",
         description=(
-            "RS-LoCoMo-Full-v7: prepare is local; ingest/answer/judge require "
+            "RS-LoCoMo-Full-v8: prepare is local; ingest/answer/judge require "
             "explicit execution acknowledgements"
         ),
     )

benchmarks/locomo/dataset.py

@@ -139,9 +139,9 @@ def load_manifest(tier: str) -> QuestionManifest:
             f"manifest file {tier!r} declares tier {manifest.tier!r}"
         )
     if manifest.dataset_commit != DATASET_COMMIT:
-        raise DatasetValidationError("manifest dataset commit is not RS-LoCoMo-Full-v7")
+        raise DatasetValidationError("manifest dataset commit is not RS-LoCoMo-Full-v8")
     if manifest.dataset_sha256 != DATASET_SHA256:
-        raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v7")
+        raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v8")
     actual = item_ids_hash(item_ids=manifest.item_ids)
     if actual != manifest.item_ids_sha256:
         raise DatasetValidationError(

benchmarks/locomo/model.py

@@ -1,4 +1,4 @@
-"""Typed values for the full-system RS-LoCoMo-Full-v7 protocols."""
+"""Typed values for the full-system RS-LoCoMo-Full-v8 protocols."""
 
 from __future__ import annotations
 
@@ -26,8 +26,8 @@
 Category = Literal[1, 2, 3, 4, 5]
 RetainedCategory = Literal[1, 2, 3, 4]
 Tier = Literal["smoke", "development", "publication"]
-ProtocolKey = Literal["full-v7", "full-v7-strong"]
-ProtocolName = Literal["RS-LoCoMo-Full-v7", "RS-LoCoMo-Full-v7-strong"]
+ProtocolKey = Literal["full-v8", "full-v8-strong"]
+ProtocolName = Literal["RS-LoCoMo-Full-v8", "RS-LoCoMo-Full-v8-strong"]
 SourceTimezoneBasis = Literal["assumed_utc"]
 AnswerAgentModel = Literal["openai/gpt-4o-mini", "openai/gpt-5.6-luna"]
 JudgeModel = Literal["openai/gpt-5.6-luna"]
@@ -118,7 +118,7 @@ class QuestionManifest(FrozenModel):
 class RunConfiguration(FrozenModel):
     """Immutable identity of one prepared benchmark run."""
 
-    protocol_name: ProtocolName = "RS-LoCoMo-Full-v7"
+    protocol_name: ProtocolName = "RS-LoCoMo-Full-v8"
     adapter_version: NonEmpty
     prepared_at: datetime
     repository_revision: NonEmpty
@@ -289,6 +289,7 @@ class AnswerRecord(FrozenModel):
     reader_called: bool
     agent_call_count: int = Field(default=0, ge=0)
     reader_attempts: int = Field(default=0, ge=0)
+    first_step_retries: int = Field(default=0, ge=0)
     reader_latency_ms: int | None = Field(default=None, ge=0)
     generated_answer: str | None = None
     reader_usage: ProviderCallUsage | None = None
@@ -309,6 +310,8 @@ def require_answer_xor_failure(self) -> "AnswerRecord":
             raise ValueError("reader_called must match agent_call_count")
         if self.reader_attempts > self.agent_call_count:
             raise ValueError("reader attempts cannot exceed agent calls")
+        if self.first_step_retries > self.agent_call_count:
+            raise ValueError("first-step retries cannot exceed agent calls")
         if self.generated_answer is not None and self.reader_attempts < 1:
             raise ValueError("a generated answer requires a reader attempt")
         return self
@@ -373,7 +376,7 @@ class SessionDiagnosticSummary(FrozenModel):
 class RunSummary(FrozenModel):
     """Publication-ready local aggregate with no hidden denominator."""
 
-    protocol_name: ProtocolName = "RS-LoCoMo-Full-v7"
+    protocol_name: ProtocolName = "RS-LoCoMo-Full-v8"
     protocol_fingerprint: NonEmpty
     tier: Tier
     questions: int = Field(ge=1)
@@ -385,6 +388,7 @@ class RunSummary(FrozenModel):
     failures: dict[str, int]
     answer_agent_calls: int = Field(ge=0)
     total_reader_retries: int = Field(ge=0)
+    total_first_step_retries: int = Field(default=0, ge=0)
     judge_calls: int = Field(ge=0)
     tokens_in: int = Field(ge=0)
     tokens_out: int = Field(ge=0)

benchmarks/locomo/protocol.py

@@ -26,10 +26,10 @@
 from benchmarks.locomo.model import ToolCallRecord
 from rememberstack.model import ToolDescriptor
 
-PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v7"
-STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v7-strong"
-DEFAULT_PROTOCOL_KEY: Final = "full-v7"
-ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07-hybrid-context"
+PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v8"
+STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v8-strong"
+DEFAULT_PROTOCOL_KEY: Final = "full-v8"
+ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07-hybrid-context-v8answers"
 MAX_TOOL_CALLS: Final = 8
 MAX_AGENT_CALLS: Final = 9
 ANSWER_READER_RETRY_BUDGET: Final = 2
@@ -72,7 +72,8 @@
 automatically current fact. Use timestamps to resolve relative dates. Do not
 confuse people mentioned in a memory with the conversation speakers. Never use
 outside knowledge. If the deployment does not contain the answer, finish with
-"Unknown". A final answer must be concise and at most six words.
+"Unknown". The final answer must be the shortest phrase that fully names the
+requested entities/values, at most twenty words, no explanations or reasoning.
 
 Loop discipline: never repeat a tool call with the same tool AND the same
 arguments. If a tool yields nothing useful, change the arguments meaningfully or switch tools rather than retrying
@@ -127,8 +128,8 @@ class LoCoMoProtocol:
     answer_agent_reasoning_effort: str | None
 
 
-_FULL_V7 = LoCoMoProtocol(
-    key="full-v7",
+_FULL_V8 = LoCoMoProtocol(
+    key="full-v8",
     name=PROTOCOL_NAME,
     answer_agent_model=ANSWER_AGENT_MODEL,
     judge_model=JUDGE_MODEL,
@@ -145,8 +146,8 @@ class LoCoMoProtocol:
     answer_reader_retry_budget=ANSWER_READER_RETRY_BUDGET,
     answer_agent_reasoning_effort=None,
 )
-_FULL_V7_STRONG = LoCoMoProtocol(
-    key="full-v7-strong",
+_FULL_V8_STRONG = LoCoMoProtocol(
+    key="full-v8-strong",
     name=STRONG_PROTOCOL_NAME,
     answer_agent_model=STRONG_ANSWER_AGENT_MODEL,
     judge_model=JUDGE_MODEL,
@@ -165,7 +166,7 @@ class LoCoMoProtocol:
 )
 
 PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType(
-    {_FULL_V7.key: _FULL_V7, _FULL_V7_STRONG.key: _FULL_V7_STRONG}
+    {_FULL_V8.key: _FULL_V8, _FULL_V8_STRONG.key: _FULL_V8_STRONG}
 )
 
 

benchmarks/locomo/runner.py

@@ -431,7 +431,7 @@ def answer_sample(
     ):
         raise ExecutionGuardError(
             "the deployment did not report the exact completed"
-            " RS-LoCoMo-Full-v7 pipeline and fresh P2/P3 projections"
+            " RS-LoCoMo-Full-v8 pipeline and fresh P2/P3 projections"
         )
     _require_serving_revision(context=context, readiness=readiness)
     prior_readiness = context.state.readiness.get(sample_id)
@@ -732,6 +732,9 @@ def summarize_run(*, run_dir: Path) -> RunSummary:
             max(record.reader_attempts - 1, 0)
             for record in context.state.answers.values()
         ),
+        total_first_step_retries=sum(
+            record.first_step_retries for record in context.state.answers.values()
+        ),
         judge_calls=sum(
             record.model_called for record in context.state.judges.values()
         ),
@@ -967,7 +970,7 @@ def _validate_run(
     """Recompute immutable run identity before any local or remote stage."""
     selected_protocol = protocol_for_name(configuration.protocol_name)
     if configuration.dataset_sha256 != DATASET_SHA256:
-        raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v7")
+        raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v8")
     if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256:
         raise BenchmarkRunError("run manifest item hash changed")
     if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256:
@@ -977,7 +980,7 @@ def _validate_run(
     if manifest.tier != configuration.tier:
         raise BenchmarkRunError("run manifest tier changed")
     if configuration.dataset_commit != DATASET_COMMIT:
-        raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v7")
+        raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v8")
     if configuration.adapter_version != ADAPTER_VERSION:
         raise BenchmarkRunError("run adapter version differs from current code")
     if _models_hash(values=documents) != configuration.documents_sha256:
@@ -1308,6 +1311,8 @@ def _answer_one(
     tool_latency_ms = 0
     agent_call_count = 0
     reader_attempts = 0
+    first_step_retries = 0
+    invalid_completion_attempts = 0
     prior_calls = sum(record.agent_call_count for record in state.answers.values())
     for _ in range(max_agent_calls_per_question):
         if prior_calls + agent_call_count >= max_agent_calls:
@@ -1351,6 +1356,7 @@ def _answer_one(
                 retrieval_succeeded=bool(trace),
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts + int(bool(trace)),
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms + call_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1372,6 +1378,7 @@ def _answer_one(
                 retrieval_succeeded=bool(trace),
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms + call_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1385,6 +1392,7 @@ def _answer_one(
             if error.usage is not None:
                 usages.append(error.usage)
                 state.evaluator_cost_usd += error.usage.cost_usd
+            invalid_completion_attempts += 1
             if trace:
                 reader_attempts += 1
             if agent_observation is not None:
@@ -1394,13 +1402,14 @@ def _answer_one(
                     outcome="provider_error",
                 )
             can_retry = (
-                bool(trace)
-                and reader_attempts <= answer_reader_retry_budget
+                invalid_completion_attempts <= answer_reader_retry_budget
                 and agent_call_count < max_agent_calls_per_question
                 and prior_calls + agent_call_count < max_agent_calls
                 and state.evaluator_cost_usd < max_evaluator_cost_usd
             )
             if can_retry:
+                if not trace:
+                    first_step_retries += 1
                 continue
             return _failed_answer(
                 question=question,
@@ -1410,6 +1419,7 @@ def _answer_one(
                 retrieval_succeeded=bool(trace),
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1436,6 +1446,7 @@ def _answer_one(
                 retrieval_succeeded=bool(trace),
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts + int(bool(trace)),
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms + call_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1472,6 +1483,7 @@ def _answer_one(
                 reader_attempts=(
                     reader_attempts + int(step.action == "answer" and bool(trace))
                 ),
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1495,12 +1507,13 @@ def _answer_one(
                     retrieval_succeeded=False,
                     agent_call_count=agent_call_count,
                     reader_attempts=reader_attempts,
+                    first_step_retries=first_step_retries,
                     reader_latency_ms=agent_latency_ms,
                     tool_calls=tuple(trace),
                     usages=tuple(usages),
                 )
             answer = step.answer or ""
-            if len(answer.split()) > 6:
+            if len(answer.split()) > 20:
                 if agent_observation is not None:
                     agent_observation.finish(
                         usage=response.usage,
@@ -1510,11 +1523,12 @@ def _answer_one(
                 return _failed_answer(
                     question=question,
                     kind="invalid_response",
-                    message="answer agent exceeded the six-word answer limit",
+                    message="answer agent exceeded the twenty-word answer limit",
                     retrieval_latency_ms=tool_latency_ms,
                     retrieval_succeeded=True,
                     agent_call_count=agent_call_count,
                     reader_attempts=reader_attempts + 1,
+                    first_step_retries=first_step_retries,
                     reader_latency_ms=agent_latency_ms,
                     claims=_claims_from_trace(
                         trace=tuple(trace), doc_sessions=doc_sessions
@@ -1547,6 +1561,7 @@ def _answer_one(
                 reader_called=True,
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts + 1,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 generated_answer=answer,
                 reader_usage=_aggregate_usage(usages=tuple(usages)),
@@ -1560,6 +1575,7 @@ def _answer_one(
                 retrieval_succeeded=bool(trace),
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1576,6 +1592,7 @@ def _answer_one(
                 retrieval_succeeded=True,
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1611,6 +1628,7 @@ def _answer_one(
                 retrieval_succeeded=False,
                 agent_call_count=agent_call_count,
                 reader_attempts=reader_attempts,
+                first_step_retries=first_step_retries,
                 reader_latency_ms=agent_latency_ms,
                 claims=_claims_from_trace(
                     trace=tuple(trace), doc_sessions=doc_sessions
@@ -1639,6 +1657,7 @@ def _answer_one(
         retrieval_succeeded=bool(trace),
         agent_call_count=agent_call_count,
         reader_attempts=reader_attempts,
+        first_step_retries=first_step_retries,
         reader_latency_ms=agent_latency_ms,
         claims=_claims_from_trace(trace=tuple(trace), doc_sessions=doc_sessions),
         tool_calls=tuple(trace),
@@ -1809,6 +1828,7 @@ def _failed_answer(
     retrieval_succeeded: bool,
     agent_call_count: int,
     reader_attempts: int = 0,
+    first_step_retries: int = 0,
     reader_latency_ms: int | None = None,
     claims: tuple[RetrievedClaim, ...] = (),
     tool_calls: tuple[ToolCallRecord, ...] = (),
@@ -1832,6 +1852,7 @@ def _failed_answer(
         reader_called=agent_call_count > 0,
         agent_call_count=agent_call_count,
         reader_attempts=reader_attempts,
+        first_step_retries=first_step_retries,
         reader_latency_ms=reader_latency_ms,
         reader_usage=_aggregate_usage(usages=usages) if usages else None,
         failure=_failure(kind=kind, message=message),