benchmark: RS-LoCoMo-Full-v5-strong — the strong-agent protocol variant

benchmarks/locomo/cli.py

@@ -8,6 +8,8 @@
 from pathlib import Path
 import sys
 
+from benchmarks.locomo.protocol import DEFAULT_PROTOCOL_KEY
+from benchmarks.locomo.protocol import PROTOCOL_REGISTRY
 from benchmarks.locomo.runner import answer_sample
 from benchmarks.locomo.runner import BenchmarkRunError
 from benchmarks.locomo.runner import ingest_sample
@@ -27,7 +29,10 @@ def main(argv: list[str] | None = None) -> int:
     try:
         if args.command == "prepare":
             configuration = prepare_run(
-                dataset_path=args.dataset, tier=args.tier, output=args.output
+                dataset_path=args.dataset,
+                tier=args.tier,
+                output=args.output,
+                protocol=args.protocol,
             )
             print(configuration.model_dump_json())
             return 0
@@ -118,6 +123,9 @@ def _parser() -> argparse.ArgumentParser:
         "--tier", choices=("smoke", "development", "publication"), required=True
     )
     prepare.add_argument("--output", type=Path, required=True)
+    prepare.add_argument(
+        "--protocol", choices=tuple(PROTOCOL_REGISTRY), default=DEFAULT_PROTOCOL_KEY
+    )
 
     ingest = commands.add_parser(
         "ingest", help="upload one sample to a clean isolated deployment"

benchmarks/locomo/model.py

@@ -1,4 +1,4 @@
-"""Typed values for the full-system RS-LoCoMo-Full-v5 protocol."""
+"""Typed values for the full-system RS-LoCoMo-Full-v5 protocols."""
 
 from __future__ import annotations
 
@@ -23,6 +23,10 @@
 Category = Literal[1, 2, 3, 4, 5]
 RetainedCategory = Literal[1, 2, 3, 4]
 Tier = Literal["smoke", "development", "publication"]
+ProtocolKey = Literal["full-v5", "full-v5-strong"]
+ProtocolName = Literal["RS-LoCoMo-Full-v5", "RS-LoCoMo-Full-v5-strong"]
+AnswerAgentModel = Literal["openai/gpt-4o-mini", "openai/gpt-5.6-luna"]
+JudgeModel = Literal["openai/gpt-5.6-luna"]
 FailureKind = Literal[
     "readiness", "tool", "reader", "judge", "accounting", "invalid_response", "missing"
 ]
@@ -108,7 +112,7 @@ class QuestionManifest(FrozenModel):
 class RunConfiguration(FrozenModel):
     """Immutable identity of one prepared benchmark run."""
 
-    protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5"
+    protocol_name: ProtocolName = "RS-LoCoMo-Full-v5"
     adapter_version: NonEmpty
     prepared_at: datetime
     repository_revision: NonEmpty
@@ -124,8 +128,8 @@ class RunConfiguration(FrozenModel):
     max_tool_calls_per_question: Literal[8] = 8
     max_agent_calls_per_question: Literal[9] = 9
     knowledge_mode: Literal["not_composed"] = "not_composed"
-    answer_agent_model: Literal["openai/gpt-4o-mini"] = "openai/gpt-4o-mini"
-    judge_model: Literal["openai/gpt-5.6-luna"] = "openai/gpt-5.6-luna"
+    answer_agent_model: AnswerAgentModel = "openai/gpt-4o-mini"
+    judge_model: JudgeModel = "openai/gpt-5.6-luna"
     answer_agent_temperature: float = Field(default=0.0, ge=0, le=2)
     judge_temperature: float = Field(default=0.0, ge=0, le=2)
     judge_repetitions: Literal[1] = 1
@@ -318,6 +322,8 @@ class RunState(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
+    protocol_name: ProtocolName
+    protocol_fingerprint: NonEmpty
     ingests: dict[str, IngestRecord] = Field(default_factory=dict)
     readiness: dict[str, PipelineReadinessReport] = Field(default_factory=dict)
     answers: dict[str, AnswerRecord] = Field(default_factory=dict)
@@ -350,7 +356,7 @@ class SessionDiagnosticSummary(FrozenModel):
 class RunSummary(FrozenModel):
     """Publication-ready local aggregate with no hidden denominator."""
 
-    protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5"
+    protocol_name: ProtocolName = "RS-LoCoMo-Full-v5"
     protocol_fingerprint: NonEmpty
     tier: Tier
     questions: int = Field(ge=1)

benchmarks/locomo/protocol.py

@@ -7,20 +7,28 @@
 import hashlib
 import json
 import string
+from types import MappingProxyType
 from typing import Final
+from typing import Mapping
 
 from nltk.stem import PorterStemmer
 import regex
 
+from benchmarks.locomo.model import AnswerAgentModel
 from benchmarks.locomo.model import AnswerAgentStep
+from benchmarks.locomo.model import JudgeModel
 from benchmarks.locomo.model import JudgeOutput
 from benchmarks.locomo.model import LoCoMoSample
 from benchmarks.locomo.model import LoCoMoSession
+from benchmarks.locomo.model import ProtocolKey
+from benchmarks.locomo.model import ProtocolName
 from benchmarks.locomo.model import RetainedCategory
 from benchmarks.locomo.model import ToolCallRecord
 from rememberstack.model import ToolDescriptor
 
 PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v5"
+STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v5-strong"
+DEFAULT_PROTOCOL_KEY: Final = "full-v5"
 ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07"
 MAX_TOOL_CALLS: Final = 8
 MAX_AGENT_CALLS: Final = 9
@@ -41,6 +49,7 @@
 )
 EXPECTED_PROJECTION_PLANES: Final = ("P2_graph", "P3_corpusfs")
 ANSWER_AGENT_MODEL: Final = "openai/gpt-4o-mini"
+STRONG_ANSWER_AGENT_MODEL: Final = "openai/gpt-5.6-luna"
 JUDGE_MODEL: Final = "openai/gpt-5.6-luna"
 TEMPERATURE: Final = 0.0
 
@@ -91,6 +100,77 @@
 Gold answer: {gold_answer}
 Generated answer: {generated_answer}"""
 
+
+@dataclass(frozen=True)
+class LoCoMoProtocol:
+    """One fully typed, immutable LoCoMo protocol pin."""
+
+    key: ProtocolKey
+    name: ProtocolName
+    answer_agent_model: AnswerAgentModel
+    judge_model: JudgeModel
+    answer_prompt_template: str
+    judge_prompt_template: str
+    answer_schema: type[AnswerAgentStep]
+    judge_schema: type[JudgeOutput]
+    tool_catalog_sha256: str
+    max_tool_calls_per_question: int
+    max_agent_calls_per_question: int
+    answer_agent_temperature: float
+    judge_temperature: float
+    judge_repetitions: int
+
+
+_FULL_V5 = LoCoMoProtocol(
+    key="full-v5",
+    name=PROTOCOL_NAME,
+    answer_agent_model=ANSWER_AGENT_MODEL,
+    judge_model=JUDGE_MODEL,
+    answer_prompt_template=ANSWER_AGENT_PROMPT_TEMPLATE,
+    judge_prompt_template=JUDGE_PROMPT_TEMPLATE,
+    answer_schema=AnswerAgentStep,
+    judge_schema=JudgeOutput,
+    tool_catalog_sha256=EXPECTED_TOOL_CATALOG_SHA256,
+    max_tool_calls_per_question=MAX_TOOL_CALLS,
+    max_agent_calls_per_question=MAX_AGENT_CALLS,
+    answer_agent_temperature=TEMPERATURE,
+    judge_temperature=TEMPERATURE,
+    judge_repetitions=1,
+)
+_FULL_V5_STRONG = LoCoMoProtocol(
+    key="full-v5-strong",
+    name=STRONG_PROTOCOL_NAME,
+    answer_agent_model=STRONG_ANSWER_AGENT_MODEL,
+    judge_model=JUDGE_MODEL,
+    answer_prompt_template=ANSWER_AGENT_PROMPT_TEMPLATE,
+    judge_prompt_template=JUDGE_PROMPT_TEMPLATE,
+    answer_schema=AnswerAgentStep,
+    judge_schema=JudgeOutput,
+    tool_catalog_sha256=EXPECTED_TOOL_CATALOG_SHA256,
+    max_tool_calls_per_question=MAX_TOOL_CALLS,
+    max_agent_calls_per_question=MAX_AGENT_CALLS,
+    answer_agent_temperature=TEMPERATURE,
+    judge_temperature=TEMPERATURE,
+    judge_repetitions=1,
+)
+
+PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType(
+    {_FULL_V5.key: _FULL_V5, _FULL_V5_STRONG.key: _FULL_V5_STRONG}
+)
+
+
+def protocol_for_key(key: ProtocolKey) -> LoCoMoProtocol:
+    """Resolve the one protocol explicitly selected during preparation."""
+    return PROTOCOL_REGISTRY[key]
+
+
+def protocol_for_name(name: ProtocolName) -> LoCoMoProtocol:
+    """Resolve a persisted protocol name for immutable-pin validation."""
+    return next(
+        protocol for protocol in PROTOCOL_REGISTRY.values() if protocol.name == name
+    )
+
+
 _DIALOG_ID = regex.compile(r"D([0-9]+):[0-9]+")
 _EXACT_DIALOG_ID = regex.compile(r"^D[0-9]+:[0-9]+$")
 _ARTICLES = regex.compile(r"\b(a|an|the|and)\b")