From 9214bba2c0059884056da675f0eb39d386955d08 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Wed, 29 Jul 2026 18:13:30 +0200 Subject: [PATCH] feat: add strong-agent LoCoMo protocol --- benchmarks/locomo/cli.py | 10 +- benchmarks/locomo/model.py | 16 +- benchmarks/locomo/protocol.py | 80 +++++++++ benchmarks/locomo/runner.py | 178 ++++++++++++++----- plan/designs/locomo_benchmark_design.md | 9 + src/tests/benchmarks/test_locomo_protocol.py | 70 +++++++- src/tests/benchmarks/test_locomo_runner.py | 92 ++++++++-- 7 files changed, 391 insertions(+), 64 deletions(-) diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index e5b3c0c9..d446c127 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -8,6 +8,8 @@ from pathlib import Path import sys +from benchmarks.locomo.protocol import DEFAULT_PROTOCOL_KEY +from benchmarks.locomo.protocol import PROTOCOL_REGISTRY from benchmarks.locomo.runner import answer_sample from benchmarks.locomo.runner import BenchmarkRunError from benchmarks.locomo.runner import ingest_sample @@ -27,7 +29,10 @@ def main(argv: list[str] | None = None) -> int: try: if args.command == "prepare": configuration = prepare_run( - dataset_path=args.dataset, tier=args.tier, output=args.output + dataset_path=args.dataset, + tier=args.tier, + output=args.output, + protocol=args.protocol, ) print(configuration.model_dump_json()) return 0 @@ -118,6 +123,9 @@ def _parser() -> argparse.ArgumentParser: "--tier", choices=("smoke", "development", "publication"), required=True ) prepare.add_argument("--output", type=Path, required=True) + prepare.add_argument( + "--protocol", choices=tuple(PROTOCOL_REGISTRY), default=DEFAULT_PROTOCOL_KEY + ) ingest = commands.add_parser( "ingest", help="upload one sample to a clean isolated deployment" diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index f736ca20..25dd0a0a 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v5 protocol.""" +"""Typed values for the full-system RS-LoCoMo-Full-v5 protocols.""" from __future__ import annotations @@ -23,6 +23,10 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] +ProtocolKey = Literal["full-v5", "full-v5-strong"] +ProtocolName = Literal["RS-LoCoMo-Full-v5", "RS-LoCoMo-Full-v5-strong"] +AnswerAgentModel = Literal["openai/gpt-4o-mini", "openai/gpt-5.6-luna"] +JudgeModel = Literal["openai/gpt-5.6-luna"] FailureKind = Literal[ "readiness", "tool", "reader", "judge", "accounting", "invalid_response", "missing" ] @@ -108,7 +112,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v5" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -124,8 +128,8 @@ class RunConfiguration(FrozenModel): max_tool_calls_per_question: Literal[8] = 8 max_agent_calls_per_question: Literal[9] = 9 knowledge_mode: Literal["not_composed"] = "not_composed" - answer_agent_model: Literal["openai/gpt-4o-mini"] = "openai/gpt-4o-mini" - judge_model: Literal["openai/gpt-5.6-luna"] = "openai/gpt-5.6-luna" + answer_agent_model: AnswerAgentModel = "openai/gpt-4o-mini" + judge_model: JudgeModel = "openai/gpt-5.6-luna" answer_agent_temperature: float = Field(default=0.0, ge=0, le=2) judge_temperature: float = Field(default=0.0, ge=0, le=2) judge_repetitions: Literal[1] = 1 @@ -318,6 +322,8 @@ class RunState(BaseModel): model_config = ConfigDict(extra="forbid") + protocol_name: ProtocolName + protocol_fingerprint: NonEmpty ingests: dict[str, IngestRecord] = Field(default_factory=dict) readiness: dict[str, PipelineReadinessReport] = Field(default_factory=dict) answers: dict[str, AnswerRecord] = Field(default_factory=dict) @@ -350,7 +356,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: Literal["RS-LoCoMo-Full-v5"] = "RS-LoCoMo-Full-v5" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v5" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index ea0458fe..03914025 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -7,20 +7,28 @@ import hashlib import json import string +from types import MappingProxyType from typing import Final +from typing import Mapping from nltk.stem import PorterStemmer import regex +from benchmarks.locomo.model import AnswerAgentModel from benchmarks.locomo.model import AnswerAgentStep +from benchmarks.locomo.model import JudgeModel from benchmarks.locomo.model import JudgeOutput from benchmarks.locomo.model import LoCoMoSample from benchmarks.locomo.model import LoCoMoSession +from benchmarks.locomo.model import ProtocolKey +from benchmarks.locomo.model import ProtocolName from benchmarks.locomo.model import RetainedCategory from benchmarks.locomo.model import ToolCallRecord from rememberstack.model import ToolDescriptor PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v5" +STRONG_PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v5-strong" +DEFAULT_PROTOCOL_KEY: Final = "full-v5" ADAPTER_VERSION: Final = "locomo-full-adapter-2026.07" MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 @@ -41,6 +49,7 @@ ) EXPECTED_PROJECTION_PLANES: Final = ("P2_graph", "P3_corpusfs") ANSWER_AGENT_MODEL: Final = "openai/gpt-4o-mini" +STRONG_ANSWER_AGENT_MODEL: Final = "openai/gpt-5.6-luna" JUDGE_MODEL: Final = "openai/gpt-5.6-luna" TEMPERATURE: Final = 0.0 @@ -91,6 +100,77 @@ Gold answer: {gold_answer} Generated answer: {generated_answer}""" + +@dataclass(frozen=True) +class LoCoMoProtocol: + """One fully typed, immutable LoCoMo protocol pin.""" + + key: ProtocolKey + name: ProtocolName + answer_agent_model: AnswerAgentModel + judge_model: JudgeModel + answer_prompt_template: str + judge_prompt_template: str + answer_schema: type[AnswerAgentStep] + judge_schema: type[JudgeOutput] + tool_catalog_sha256: str + max_tool_calls_per_question: int + max_agent_calls_per_question: int + answer_agent_temperature: float + judge_temperature: float + judge_repetitions: int + + +_FULL_V5 = LoCoMoProtocol( + key="full-v5", + name=PROTOCOL_NAME, + answer_agent_model=ANSWER_AGENT_MODEL, + judge_model=JUDGE_MODEL, + answer_prompt_template=ANSWER_AGENT_PROMPT_TEMPLATE, + judge_prompt_template=JUDGE_PROMPT_TEMPLATE, + answer_schema=AnswerAgentStep, + judge_schema=JudgeOutput, + tool_catalog_sha256=EXPECTED_TOOL_CATALOG_SHA256, + max_tool_calls_per_question=MAX_TOOL_CALLS, + max_agent_calls_per_question=MAX_AGENT_CALLS, + answer_agent_temperature=TEMPERATURE, + judge_temperature=TEMPERATURE, + judge_repetitions=1, +) +_FULL_V5_STRONG = LoCoMoProtocol( + key="full-v5-strong", + name=STRONG_PROTOCOL_NAME, + answer_agent_model=STRONG_ANSWER_AGENT_MODEL, + judge_model=JUDGE_MODEL, + answer_prompt_template=ANSWER_AGENT_PROMPT_TEMPLATE, + judge_prompt_template=JUDGE_PROMPT_TEMPLATE, + answer_schema=AnswerAgentStep, + judge_schema=JudgeOutput, + tool_catalog_sha256=EXPECTED_TOOL_CATALOG_SHA256, + max_tool_calls_per_question=MAX_TOOL_CALLS, + max_agent_calls_per_question=MAX_AGENT_CALLS, + answer_agent_temperature=TEMPERATURE, + judge_temperature=TEMPERATURE, + judge_repetitions=1, +) + +PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( + {_FULL_V5.key: _FULL_V5, _FULL_V5_STRONG.key: _FULL_V5_STRONG} +) + + +def protocol_for_key(key: ProtocolKey) -> LoCoMoProtocol: + """Resolve the one protocol explicitly selected during preparation.""" + return PROTOCOL_REGISTRY[key] + + +def protocol_for_name(name: ProtocolName) -> LoCoMoProtocol: + """Resolve a persisted protocol name for immutable-pin validation.""" + return next( + protocol for protocol in PROTOCOL_REGISTRY.values() if protocol.name == name + ) + + _DIALOG_ID = regex.compile(r"D([0-9]+):[0-9]+") _EXACT_DIALOG_ID = regex.compile(r"^D[0-9]+:[0-9]+$") _ARTICLES = regex.compile(r"\b(a|an|the|and)\b") diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index c9c5616a..699010b5 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -31,6 +31,7 @@ from benchmarks.locomo.dataset import load_manifest from benchmarks.locomo.dataset import manifest_bytes_hash from benchmarks.locomo.dataset import validate_manifest +from benchmarks.locomo.model import AnswerAgentModel from benchmarks.locomo.model import AnswerAgentStep from benchmarks.locomo.model import AnswerRecord from benchmarks.locomo.model import BenchmarkFailure @@ -43,6 +44,7 @@ from benchmarks.locomo.model import LoCoMoQuestion from benchmarks.locomo.model import PreflightProbe from benchmarks.locomo.model import PreparedDocument +from benchmarks.locomo.model import ProtocolKey from benchmarks.locomo.model import QuestionManifest from benchmarks.locomo.model import RetainedCategory from benchmarks.locomo.model import RetrievedClaim @@ -53,17 +55,16 @@ from benchmarks.locomo.model import ToolCallRecord from benchmarks.locomo.protocol import ADAPTER_VERSION from benchmarks.locomo.protocol import ANSWER_AGENT_MODEL -from benchmarks.locomo.protocol import ANSWER_AGENT_PROMPT_TEMPLATE +from benchmarks.locomo.protocol import DEFAULT_PROTOCOL_KEY from benchmarks.locomo.protocol import EXPECTED_PIPELINE_STAGES from benchmarks.locomo.protocol import EXPECTED_PROJECTION_PLANES -from benchmarks.locomo.protocol import EXPECTED_TOOL_CATALOG_SHA256 from benchmarks.locomo.protocol import JUDGE_MODEL -from benchmarks.locomo.protocol import JUDGE_PROMPT_TEMPLATE from benchmarks.locomo.protocol import MAX_AGENT_CALLS from benchmarks.locomo.protocol import MAX_TOOL_CALLS from benchmarks.locomo.protocol import official_f1 from benchmarks.locomo.protocol import prompt_sha256 -from benchmarks.locomo.protocol import PROTOCOL_NAME +from benchmarks.locomo.protocol import protocol_for_key +from benchmarks.locomo.protocol import protocol_for_name from benchmarks.locomo.protocol import render_answer_agent_prompt from benchmarks.locomo.protocol import render_judge_prompt from benchmarks.locomo.protocol import render_session @@ -129,8 +130,15 @@ def configured_values(self) -> tuple[str, str, str] | None: return public_key, secret_key, host -def prepare_run(*, dataset_path: Path, tier: str, output: Path) -> RunConfiguration: +def prepare_run( + *, + dataset_path: Path, + tier: str, + output: Path, + protocol: ProtocolKey = DEFAULT_PROTOCOL_KEY, +) -> RunConfiguration: """Validate, fingerprint, and render a local run without remote calls.""" + selected_protocol = protocol_for_key(protocol) dataset = load_dataset(dataset_path) manifest = load_manifest(tier) questions = validate_manifest(dataset=dataset, manifest=manifest) @@ -147,7 +155,7 @@ def prepare_run(*, dataset_path: Path, tier: str, output: Path) -> RunConfigurat ) revision = _repository_revision() base = { - "protocol_name": PROTOCOL_NAME, + "protocol_name": selected_protocol.name, "adapter_version": ADAPTER_VERSION, "repository_revision": revision, "dataset_commit": DATASET_COMMIT, @@ -158,19 +166,25 @@ def prepare_run(*, dataset_path: Path, tier: str, output: Path) -> RunConfigurat "documents_sha256": _models_hash(values=documents), "item_count": len(manifest.item_ids), "sample_ids": sample_ids, - "max_tool_calls_per_question": MAX_TOOL_CALLS, - "max_agent_calls_per_question": MAX_AGENT_CALLS, + "max_tool_calls_per_question": (selected_protocol.max_tool_calls_per_question), + "max_agent_calls_per_question": ( + selected_protocol.max_agent_calls_per_question + ), "knowledge_mode": "not_composed", - "answer_agent_model": ANSWER_AGENT_MODEL, - "judge_model": JUDGE_MODEL, - "answer_agent_temperature": TEMPERATURE, - "judge_temperature": TEMPERATURE, - "judge_repetitions": 1, - "tool_catalog_sha256": EXPECTED_TOOL_CATALOG_SHA256, - "answer_prompt_sha256": prompt_sha256(template=ANSWER_AGENT_PROMPT_TEMPLATE), - "judge_prompt_sha256": prompt_sha256(template=JUDGE_PROMPT_TEMPLATE), - "answer_schema_sha256": schema_sha256(model=AnswerAgentStep), - "judge_schema_sha256": schema_sha256(model=JudgeOutput), + "answer_agent_model": selected_protocol.answer_agent_model, + "judge_model": selected_protocol.judge_model, + "answer_agent_temperature": selected_protocol.answer_agent_temperature, + "judge_temperature": selected_protocol.judge_temperature, + "judge_repetitions": selected_protocol.judge_repetitions, + "tool_catalog_sha256": selected_protocol.tool_catalog_sha256, + "answer_prompt_sha256": prompt_sha256( + template=selected_protocol.answer_prompt_template + ), + "judge_prompt_sha256": prompt_sha256( + template=selected_protocol.judge_prompt_template + ), + "answer_schema_sha256": schema_sha256(model=selected_protocol.answer_schema), + "judge_schema_sha256": schema_sha256(model=selected_protocol.judge_schema), } configuration = RunConfiguration( **base, @@ -181,7 +195,13 @@ def prepare_run(*, dataset_path: Path, tier: str, output: Path) -> RunConfigurat _atomic_model(path=output / _RUN_FILE, value=configuration) _atomic_model(path=output / _MANIFEST_FILE, value=manifest) _atomic_models(path=output / _DOCUMENTS_FILE, values=documents) - _atomic_model(path=output / _STATE_FILE, value=RunState()) + _atomic_model( + path=output / _STATE_FILE, + value=RunState( + protocol_name=configuration.protocol_name, + protocol_fingerprint=configuration.protocol_fingerprint, + ), + ) return configuration @@ -193,7 +213,10 @@ class ProviderPreflightError(BenchmarkRunError): def preflight_provider( - *, provider: ModelProviderPort, embedding_model: str + *, + provider: ModelProviderPort, + embedding_model: str, + answer_agent_model: AnswerAgentModel = ANSWER_AGENT_MODEL, ) -> tuple[str, ...]: """Prove the credential and both model kinds work before spending real time. @@ -209,20 +232,20 @@ def preflight_provider( try: response = provider.generate( request=ModelRequest( - model=ANSWER_AGENT_MODEL, prompt=_PREFLIGHT_PROMPT, temperature=0.0 + model=answer_agent_model, prompt=_PREFLIGHT_PROMPT, temperature=0.0 ), response_type=PreflightProbe, ) except (OpenRouterProviderError, ProviderAccountingError) as error: raise ProviderPreflightError( - f"chat model {ANSWER_AGENT_MODEL} is not usable: {error}" + f"chat model {answer_agent_model} is not usable: {error}" ) from error if not response.output.ok: raise ProviderPreflightError( - f"chat model {ANSWER_AGENT_MODEL} answered the probe with ok=false;" + f"chat model {answer_agent_model} answered the probe with ok=false;" " the provider is reachable but not usable for this run" ) - checks.append(f"chat {ANSWER_AGENT_MODEL}: ok") + checks.append(f"chat {answer_agent_model}: ok") try: provider.embed( @@ -284,7 +307,9 @@ def ingest_sample( " preflight cannot check the model the pipeline will actually use" ) for line in preflight_provider( - provider=provider, embedding_model=embedding_model + provider=provider, + embedding_model=embedding_model, + answer_agent_model=context.configuration.answer_agent_model, ): print(f"preflight: {line}", file=sys.stderr) if max_documents < len(documents): @@ -413,7 +438,9 @@ def answer_sample( if question.item_id not in context.state.answers ) called = sum(record.agent_call_count for record in context.state.answers.values()) - worst_case = called + len(remaining) * MAX_AGENT_CALLS + worst_case = ( + called + len(remaining) * context.configuration.max_agent_calls_per_question + ) if worst_case > max_agent_calls: raise ExecutionGuardError( f"max-agent-calls {max_agent_calls} cannot cover at most" @@ -440,6 +467,16 @@ def answer_sample( state=context.state, max_agent_calls=max_agent_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, + answer_agent_model=context.configuration.answer_agent_model, + answer_agent_temperature=( + context.configuration.answer_agent_temperature + ), + max_tool_calls_per_question=( + context.configuration.max_tool_calls_per_question + ), + max_agent_calls_per_question=( + context.configuration.max_agent_calls_per_question + ), ) else: with tracer.question( @@ -454,6 +491,16 @@ def answer_sample( state=context.state, max_agent_calls=max_agent_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, + answer_agent_model=context.configuration.answer_agent_model, + answer_agent_temperature=( + context.configuration.answer_agent_temperature + ), + max_tool_calls_per_question=( + context.configuration.max_tool_calls_per_question + ), + max_agent_calls_per_question=( + context.configuration.max_agent_calls_per_question + ), question_trace=question_trace, ) if question_trace is not None: @@ -527,6 +574,8 @@ def judge_sample( state=context.state, max_judge_calls=max_judge_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, + judge_model=context.configuration.judge_model, + judge_temperature=context.configuration.judge_temperature, ) else: with tracer.question( @@ -539,6 +588,8 @@ def judge_sample( state=context.state, max_judge_calls=max_judge_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, + judge_model=context.configuration.judge_model, + judge_temperature=context.configuration.judge_temperature, question_trace=question_trace, ) if question_trace is not None: @@ -609,6 +660,7 @@ def summarize_run(*, run_dir: Path) -> RunSummary: diagnostic_complete.append(float(diagnostic.complete)) usages = _all_usages(state=context.state) summary = RunSummary( + protocol_name=context.configuration.protocol_name, protocol_fingerprint=context.configuration.protocol_fingerprint, tier=context.configuration.tier, questions=len(questions), @@ -740,7 +792,12 @@ def _load_run(*, run_dir: Path) -> _RunContext: dataset=dataset, questions=questions, ) - _validate_state(state=state, documents=documents, questions=questions) + _validate_state( + configuration=configuration, + state=state, + documents=documents, + questions=questions, + ) return _RunContext( configuration=configuration, manifest=manifest, @@ -761,6 +818,7 @@ def _validate_run( questions: tuple[LoCoMoQuestion, ...], ) -> None: """Recompute immutable run identity before any local or remote stage.""" + selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v5") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: @@ -775,11 +833,6 @@ def _validate_run( raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v5") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") - if ( - configuration.answer_agent_temperature != TEMPERATURE - or configuration.judge_temperature != TEMPERATURE - ): - raise BenchmarkRunError("run temperature differs from RS-LoCoMo-Full-v5") if _models_hash(values=documents) != configuration.documents_sha256: raise BenchmarkRunError("prepared document manifest changed") if len(questions) != configuration.item_count: @@ -819,15 +872,28 @@ def _validate_run( } if _canonical_hash(base) != configuration.protocol_fingerprint: raise BenchmarkRunError("run protocol fingerprint changed") - current_hashes = { - "tool_catalog_sha256": EXPECTED_TOOL_CATALOG_SHA256, - "answer_prompt_sha256": prompt_sha256(template=ANSWER_AGENT_PROMPT_TEMPLATE), - "judge_prompt_sha256": prompt_sha256(template=JUDGE_PROMPT_TEMPLATE), - "answer_schema_sha256": schema_sha256(model=AnswerAgentStep), - "judge_schema_sha256": schema_sha256(model=JudgeOutput), + current_pin = { + "answer_agent_model": selected_protocol.answer_agent_model, + "judge_model": selected_protocol.judge_model, + "max_tool_calls_per_question": (selected_protocol.max_tool_calls_per_question), + "max_agent_calls_per_question": ( + selected_protocol.max_agent_calls_per_question + ), + "answer_agent_temperature": selected_protocol.answer_agent_temperature, + "judge_temperature": selected_protocol.judge_temperature, + "judge_repetitions": selected_protocol.judge_repetitions, + "tool_catalog_sha256": selected_protocol.tool_catalog_sha256, + "answer_prompt_sha256": prompt_sha256( + template=selected_protocol.answer_prompt_template + ), + "judge_prompt_sha256": prompt_sha256( + template=selected_protocol.judge_prompt_template + ), + "answer_schema_sha256": schema_sha256(model=selected_protocol.answer_schema), + "judge_schema_sha256": schema_sha256(model=selected_protocol.judge_schema), } - for field, actual in current_hashes.items(): - if getattr(configuration, field) != actual: + for field, expected in current_pin.items(): + if getattr(configuration, field) != expected: raise BenchmarkRunError(f"current {field} differs from prepared run") _validate_documents( run_dir=run_dir, @@ -881,11 +947,17 @@ def _validate_documents( def _validate_state( *, + configuration: RunConfiguration, state: RunState, documents: tuple[PreparedDocument, ...], questions: tuple[LoCoMoQuestion, ...], ) -> None: """Reject unknown, mismatched, or unaccounted checkpoint records.""" + if ( + state.protocol_name != configuration.protocol_name + or state.protocol_fingerprint != configuration.protocol_fingerprint + ): + raise BenchmarkRunError("run state protocol pin differs from run configuration") document_map = {document.source_ref: document for document in documents} question_map = {question.item_id: question for question in questions} if not set(state.ingests) <= set(document_map): @@ -1063,6 +1135,10 @@ def _answer_one( state: RunState, max_agent_calls: int, max_evaluator_cost_usd: Decimal, + answer_agent_model: AnswerAgentModel = ANSWER_AGENT_MODEL, + answer_agent_temperature: float = TEMPERATURE, + max_tool_calls_per_question: int = MAX_TOOL_CALLS, + max_agent_calls_per_question: int = MAX_AGENT_CALLS, question_trace: QuestionTrace | None = None, ) -> AnswerRecord: """Let a bounded agent choose ordinary public recipes, then answer.""" @@ -1072,7 +1148,7 @@ def _answer_one( agent_latency_ms = 0 tool_latency_ms = 0 prior_calls = sum(record.agent_call_count for record in state.answers.values()) - for _ in range(MAX_AGENT_CALLS): + for _ in range(max_agent_calls_per_question): if prior_calls + len(usages) >= max_agent_calls: raise ExecutionGuardError( "answer-agent call ceiling reached before next call" @@ -1086,13 +1162,15 @@ def _answer_one( agent_observation = ( None if question_trace is None - else question_trace.start_agent_call(model=ANSWER_AGENT_MODEL) + else question_trace.start_agent_call(model=answer_agent_model) ) started = time.monotonic_ns() try: response = provider.generate( request=ModelRequest( - model=ANSWER_AGENT_MODEL, prompt=prompt, temperature=TEMPERATURE + model=answer_agent_model, + prompt=prompt, + temperature=answer_agent_temperature, ), response_type=AnswerAgentStep, ) @@ -1280,7 +1358,7 @@ def _answer_one( tool_calls=tuple(trace), usages=tuple(usages), ) - if len(trace) >= MAX_TOOL_CALLS: + if len(trace) >= max_tool_calls_per_question: return _failed_answer( question=question, kind="invalid_response", @@ -1364,6 +1442,8 @@ def _judge_answer( state: RunState, max_judge_calls: int, max_evaluator_cost_usd: Decimal, + judge_model: str = JUDGE_MODEL, + judge_temperature: float = TEMPERATURE, question_trace: QuestionTrace | None = None, ) -> JudgeRecord: """Return a local wrong for answer failures or invoke the configured judge.""" @@ -1376,6 +1456,8 @@ def _judge_answer( state=state, max_judge_calls=max_judge_calls, max_evaluator_cost_usd=max_evaluator_cost_usd, + judge_model=judge_model, + judge_temperature=judge_temperature, question_trace=question_trace, ) @@ -1388,6 +1470,8 @@ def _judge_one( state: RunState, max_judge_calls: int, max_evaluator_cost_usd: Decimal, + judge_model: str = JUDGE_MODEL, + judge_temperature: float = TEMPERATURE, question_trace: QuestionTrace | None = None, ) -> JudgeRecord: """Invoke the judge once; every call failure becomes a visible wrong.""" @@ -1400,19 +1484,19 @@ def _judge_one( judge_observation = ( None if question_trace is None - else question_trace.start_judge_call(model=JUDGE_MODEL) + else question_trace.start_judge_call(model=judge_model) ) started = time.monotonic_ns() try: response = provider.generate( request=ModelRequest( - model=JUDGE_MODEL, + model=judge_model, prompt=render_judge_prompt( question=question.question, gold_answer=question.answer or "", generated_answer=answer.generated_answer or "", ), - temperature=TEMPERATURE, + temperature=judge_temperature, ), response_type=JudgeOutput, ) diff --git a/plan/designs/locomo_benchmark_design.md b/plan/designs/locomo_benchmark_design.md index 236860e0..1cb8c410 100644 --- a/plan/designs/locomo_benchmark_design.md +++ b/plan/designs/locomo_benchmark_design.md @@ -61,6 +61,15 @@ The tool-catalog hash changed, so the protocol version bumps — the v4 smoke scores (glm-4.7-flash arm) were taken against the pre-truncation catalog and are not directly comparable. +**v5-strong variant (2026-07-29):** `RS-LoCoMo-Full-v5-strong` changes only +the answer agent to `openai/gpt-5.6-luna`. It exists because three smoke +passes on a healthy store (coarse evidence-session recall 0.5, with the gold +evidence at rank 1) scored only 1–2/8 with `openai/gpt-4o-mini`, which looped +past the tool-call limit or returned invalid responses. Scores from +`RS-LoCoMo-Full-v5` and `RS-LoCoMo-Full-v5-strong` are never comparable. The +weak-agent `RS-LoCoMo-Full-v5` protocol remains the default measurement of +what a harness consumer experiences. + ### 2.1 Why v2+ uses a stronger judge `RS-LoCoMo-Full-v1` used `openai/gpt-4o-mini` for both the answer agent and the judge. The diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 779b5105..7f575aa5 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -4,7 +4,9 @@ from datetime import timezone import hashlib import json +from pathlib import Path +from benchmarks.locomo import cli from benchmarks.locomo.model import AnswerAgentStep from benchmarks.locomo.model import LoCoMoQuestion from benchmarks.locomo.model import LoCoMoSample @@ -12,9 +14,11 @@ from benchmarks.locomo.model import LoCoMoTurn from benchmarks.locomo.model import ToolCallRecord from benchmarks.locomo.protocol import ANSWER_AGENT_PROMPT_TEMPLATE +from benchmarks.locomo.protocol import DEFAULT_PROTOCOL_KEY from benchmarks.locomo.protocol import EXPECTED_TOOL_CATALOG_SHA256 from benchmarks.locomo.protocol import official_f1 from benchmarks.locomo.protocol import PROTOCOL_NAME +from benchmarks.locomo.protocol import PROTOCOL_REGISTRY from benchmarks.locomo.protocol import render_answer_agent_prompt from benchmarks.locomo.protocol import render_judge_prompt from benchmarks.locomo.protocol import render_session @@ -132,8 +136,9 @@ def test_frozen_tool_catalog_hash_matches_stock_full_system_recipes() -> None: def test_protocol_is_v5_and_answer_prompt_has_loop_guards() -> None: - """v4 fingerprint: protocol name plus the answer-loop discipline lines.""" + """The default v5 identity and answer-loop discipline remain unchanged.""" assert PROTOCOL_NAME == "RS-LoCoMo-Full-v5" + assert DEFAULT_PROTOCOL_KEY == "full-v5" prompt = ANSWER_AGENT_PROMPT_TEMPLATE assert "never repeat a tool call with the same tool AND the same" in prompt assert "switch tools rather than retrying" in prompt @@ -142,6 +147,69 @@ def test_protocol_is_v5_and_answer_prompt_has_loop_guards() -> None: assert 'answering "Unknown"' in prompt +def test_typed_protocol_registry_changes_only_answer_agent_identity() -> None: + assert tuple(PROTOCOL_REGISTRY) == ("full-v5", "full-v5-strong") + default = PROTOCOL_REGISTRY["full-v5"] + strong = PROTOCOL_REGISTRY["full-v5-strong"] + + assert default.name == "RS-LoCoMo-Full-v5" + assert default.answer_agent_model == "openai/gpt-4o-mini" + assert strong.name == "RS-LoCoMo-Full-v5-strong" + assert strong.answer_agent_model == "openai/gpt-5.6-luna" + identical_fields = ( + "judge_model", + "answer_prompt_template", + "judge_prompt_template", + "answer_schema", + "judge_schema", + "tool_catalog_sha256", + "max_tool_calls_per_question", + "max_agent_calls_per_question", + "answer_agent_temperature", + "judge_temperature", + "judge_repetitions", + ) + assert all( + getattr(default, field) == getattr(strong, field) for field in identical_fields + ) + + +@pytest.mark.parametrize( + ("extra_args", "expected"), + (((), "full-v5"), (("--protocol", "full-v5-strong"), "full-v5-strong")), +) +def test_prepare_cli_selects_protocol_only_at_prepare( + extra_args: tuple[str, ...], expected: str, monkeypatch: pytest.MonkeyPatch +) -> None: + selected: list[object] = [] + + class _Prepared: + def model_dump_json(self) -> str: + return "{}" + + def fake_prepare_run(**values: object) -> _Prepared: + selected.append(values["protocol"]) + return _Prepared() + + monkeypatch.setattr(cli, "prepare_run", fake_prepare_run) + + exit_code = cli.main( + [ + "prepare", + "--dataset", + str(Path("synthetic.json")), + "--tier", + "smoke", + "--output", + str(Path("run")), + *extra_args, + ] + ) + + assert exit_code == 0 + assert selected == [expected] + + def test_judge_never_receives_tool_trace() -> None: prompt = render_judge_prompt( question="Where?", gold_answer="Prague", generated_answer="Prague" diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 826ab843..6e851a51 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -25,10 +25,13 @@ from benchmarks.locomo.model import LoCoMoSample from benchmarks.locomo.model import LoCoMoSession from benchmarks.locomo.model import LoCoMoTurn +from benchmarks.locomo.model import ProtocolKey from benchmarks.locomo.model import QuestionManifest from benchmarks.locomo.model import RunState from benchmarks.locomo.protocol import ANSWER_AGENT_MODEL from benchmarks.locomo.protocol import EXPECTED_PIPELINE_STAGES +from benchmarks.locomo.protocol import JUDGE_MODEL +from benchmarks.locomo.protocol import PROTOCOL_NAME from benchmarks.locomo.runner import _answer_one from benchmarks.locomo.runner import _judge_one from benchmarks.locomo.runner import answer_sample @@ -72,7 +75,7 @@ def test_agent_calls_public_recipe_then_answers() -> None: provider=provider, tools=(_tool(),), doc_sessions={}, - state=RunState(), + state=_run_state(), max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), ) @@ -103,7 +106,7 @@ def test_answer_without_consulting_memory_is_rejected() -> None: provider=provider, tools=(_tool(),), doc_sessions={}, - state=RunState(), + state=_run_state(), max_agent_calls=9, max_evaluator_cost_usd=Decimal("1"), ) @@ -118,7 +121,7 @@ def test_answer_without_consulting_memory_is_rejected() -> None: def test_agent_and_judge_share_one_run_absolute_cost_threshold() -> None: client, raw_client = _memory_client() provider = _CostProvider(cost=Decimal("0.30")) - state = RunState() + state = _run_state() try: answer = _answer_one( question=_question(), @@ -149,7 +152,7 @@ def test_agent_and_judge_share_one_run_absolute_cost_threshold() -> None: def test_a_call_that_crosses_the_cost_threshold_is_recorded_then_stops() -> None: client, raw_client = _memory_client() provider = _CostProvider(cost=Decimal("0.70")) - state = RunState() + state = _run_state() try: answer = _answer_one( question=_question(), @@ -172,17 +175,30 @@ def test_a_call_that_crosses_the_cost_threshold_is_recorded_then_stops() -> None assert provider.models == [ANSWER_AGENT_MODEL] -def test_staged_mock_run_checks_readiness_and_resumes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("protocol", "answer_agent_model"), + (("full-v5", "openai/gpt-4o-mini"), ("full-v5-strong", "openai/gpt-5.6-luna")), +) +def test_staged_mock_run_uses_prepared_protocol_and_resumes( + protocol: ProtocolKey, + answer_agent_model: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: _patch_prepared_inputs(monkeypatch=monkeypatch) run_dir = tmp_path / "run" - prepare_run(dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir) + prepare_run( + dataset_path=tmp_path / "synthetic.json", + tier="smoke", + output=run_dir, + protocol=protocol, + ) raw_client = httpx.Client( base_url="http://memory.test", transport=httpx.MockTransport(_run_transport) ) client = MemoryClient(client=raw_client) - provider = FakeModelProvider(generate_router=_tool_answer_and_judge) + preflight_provider = _PreflightProvider() + provider = _CostProvider(cost=Decimal(0)) try: ingests = ingest_sample( run_dir=run_dir, @@ -191,7 +207,7 @@ def test_staged_mock_run_checks_readiness_and_resumes( execute=True, isolated_deployment_confirmation="conv-test", client=client, - provider=_PreflightProvider(), + provider=preflight_provider, ) first_answers = answer_sample( run_dir=run_dir, @@ -235,7 +251,8 @@ def test_staged_mock_run_checks_readiness_and_resumes( assert len(ingests) == 1 assert first_answers == second_answers assert first_judges == second_judges - assert len(provider.generated_prompts) == 3 + assert preflight_provider.models == [answer_agent_model] + assert provider.models == [answer_agent_model, answer_agent_model, JUDGE_MODEL] summary = summarize_run(run_dir=run_dir) assert summary.judge_correct == 1 assert summary.official_f1 == 1 @@ -610,6 +627,53 @@ def test_missing_records_remain_in_full_manifest_denominator( assert summary.failures == {"missing_answer": 1, "missing_judge": 1} +def test_prepared_protocol_pins_and_fingerprints_are_distinct( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _patch_prepared_inputs(monkeypatch=monkeypatch) + weak_dir = tmp_path / "weak" + strong_dir = tmp_path / "strong" + + weak = prepare_run( + dataset_path=tmp_path / "synthetic.json", tier="smoke", output=weak_dir + ) + strong = prepare_run( + dataset_path=tmp_path / "synthetic.json", + tier="smoke", + output=strong_dir, + protocol="full-v5-strong", + ) + + assert weak.protocol_name == "RS-LoCoMo-Full-v5" + assert weak.answer_agent_model == "openai/gpt-4o-mini" + assert weak.protocol_fingerprint == ( + "e532af4e7b349df3532388356eabe7acec474f778ba9fdcc01b397c3abe140d6" + ) + assert strong.protocol_name == "RS-LoCoMo-Full-v5-strong" + assert strong.answer_agent_model == "openai/gpt-5.6-luna" + assert strong.protocol_fingerprint == ( + "5014757da1a0e75bf256dc2126a71adfe83747bfff2439519640c3cfc41e9979" + ) + assert strong.protocol_fingerprint != weak.protocol_fingerprint + + weak_state = RunState.model_validate_json( + (weak_dir / "state.json").read_text(encoding="utf-8") + ) + strong_state = RunState.model_validate_json( + (strong_dir / "state.json").read_text(encoding="utf-8") + ) + assert (weak_state.protocol_name, weak_state.protocol_fingerprint) == ( + weak.protocol_name, + weak.protocol_fingerprint, + ) + assert (strong_state.protocol_name, strong_state.protocol_fingerprint) == ( + strong.protocol_name, + strong.protocol_fingerprint, + ) + assert summarize_run(run_dir=weak_dir).protocol_name == weak.protocol_name + assert summarize_run(run_dir=strong_dir).protocol_name == strong.protocol_name + + def test_protocol_mutation_is_rejected( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -739,12 +803,14 @@ def __init__(self, *, fail: bool = False) -> None: self.fail = fail self.embed_calls = 0 self.generate_calls = 0 + self.models: list[str] = [] def generate( self, *, request: ModelRequest, response_type: type[ResponseT] ) -> GeneratedResponse[ResponseT]: """Return the tiny structured probe answer.""" self.generate_calls += 1 + self.models.append(request.model) if self.fail: raise OpenRouterProviderError("OpenRouter /chat/completions returned 401") return GeneratedResponse( @@ -821,6 +887,12 @@ def embed(self, *, request: EmbeddingRequest) -> EmbeddingResponse: raise AssertionError(f"unexpected embed call: {request.model}") +def _run_state() -> RunState: + return RunState( + protocol_name=PROTOCOL_NAME, protocol_fingerprint="synthetic-test-fingerprint" + ) + + def _patch_prepared_inputs(*, monkeypatch: pytest.MonkeyPatch) -> None: dataset = _synthetic_dataset() item_ids = ("conv-test/qa/0000",)