diff --git a/tests/test_twin_model_guard.py b/tests/test_twin_model_guard.py new file mode 100644 index 0000000..5940a3c --- /dev/null +++ b/tests/test_twin_model_guard.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from zwill.edsl_integration import ( + DEFAULT_EXPORT_MODEL, + TWIN_VALIDATION_TARGETS, + superseded_twin_model_labels, + superseded_twin_model_warning, +) + + +def test_superseded_models_are_flagged_and_current_models_are_not() -> None: + specs = [ + ("gpt-4.1", "openai"), + ("gpt-4o", None), + ("claude-3-opus", "anthropic"), + ("gemini-1.5-pro", "google"), + ("gpt-5.5", "openai"), + ("claude-opus-4-8", "anthropic"), + ("gemini-2.5-pro", "google"), + ] + flagged = superseded_twin_model_labels(specs) + assert flagged == ["openai:gpt-4.1", "gpt-4o", "anthropic:claude-3-opus", "google:gemini-1.5-pro"] + + +def test_warning_fires_only_for_superseded_models() -> None: + # The exact model the research-agent run used. + warning = superseded_twin_model_warning([("gpt-4.1", "openai")]) + assert warning is not None + assert warning["code"] == "superseded_twin_model" + assert "openai:gpt-4.1" in warning["message"] + assert DEFAULT_EXPORT_MODEL in warning["message"] # points at the strong default + + # A current frontier model raises nothing. + assert superseded_twin_model_warning([("gpt-5.5", "openai")]) is None + + +def test_default_export_model_is_a_current_model() -> None: + # The omitted-model default must not itself be flagged as superseded. + assert superseded_twin_model_labels([(DEFAULT_EXPORT_MODEL, "openai")]) == [] + + +def test_validation_targets_cover_the_model_bearing_exports() -> None: + assert TWIN_VALIDATION_TARGETS == { + "twin-probability-job", + "rank-utility-twin-job", + "numeric-twin-job", + } diff --git a/zwill/edsl_integration.py b/zwill/edsl_integration.py index 3ada617..6a6bd12 100644 --- a/zwill/edsl_integration.py +++ b/zwill/edsl_integration.py @@ -349,10 +349,13 @@ def option_key(index: int) -> str: return key +DEFAULT_EXPORT_MODEL = "gpt-5.5" + + def parse_model_specs(args: argparse.Namespace) -> list[tuple[str, str | None]]: values = normalize_name_list(args.model) + normalize_name_list(args.models) if not values: - values = ["gpt-5.5"] + values = [DEFAULT_EXPORT_MODEL] specs = [] for value in values: @@ -421,6 +424,50 @@ def model_label(service_name: str | None, model_name: str | None) -> str: return str(model_name or "") +# Model families that are clearly superseded for twin validation as of this +# release. Validating a twin on one understates its capability (a research-agent +# run once validated on gpt-4.1 while frontier was gpt-5.5), so the twin-job +# export warns. Advisory only, never blocking. Keep current as models evolve. +SUPERSEDED_TWIN_MODEL_MARKERS = ( + "gpt-4o", "gpt-4.1", "gpt-4.5", "gpt-4-", "gpt-4", "gpt-3.5", "gpt-35", + "claude-3", "claude-2", "claude-instant", + "gemini-1", "gemini-pro", + "text-davinci", "text-bison", "davinci", "babbage", "ada", +) +RECOMMENDED_TWIN_MODELS = ("openai:gpt-5.5", "anthropic:claude-opus-4-8", "google:gemini-2.5-pro") + +# --target values whose export bakes in the model that will predict held-out +# answers, i.e. where model choice is validation-critical. +TWIN_VALIDATION_TARGETS = frozenset({"twin-probability-job", "rank-utility-twin-job", "numeric-twin-job"}) + + +def superseded_twin_model_labels(specs: list[tuple[str, str | None]]) -> list[str]: + """Labels among ``specs`` that match a superseded model family.""" + flagged: list[str] = [] + for model_name, service_name in specs: + lowered = str(model_name).lower() + if any(marker in lowered for marker in SUPERSEDED_TWIN_MODEL_MARKERS): + flagged.append(model_label(service_name, model_name)) + return flagged + + +def superseded_twin_model_warning(specs: list[tuple[str, str | None]]) -> dict[str, Any] | None: + """A loud warning envelope when a twin will be validated on a weak model.""" + superseded = superseded_twin_model_labels(specs) + if not superseded: + return None + plural = "is a superseded model" if len(superseded) == 1 else "are superseded models" + return { + "code": "superseded_twin_model", + "message": ( + f"This twin validation will run on {', '.join(superseded)}, which {plural} for this release — " + f"results may understate twin capability. Use a current frontier model (e.g. {RECOMMENDED_TWIN_MODELS[0]}) " + f"via --model unless you are deliberately benchmarking an older model. zwill defaults to " + f"{DEFAULT_EXPORT_MODEL} when --model is omitted." + ), + } + + def build_edsl_agent_list_dict(survey_name: str, args: argparse.Namespace) -> dict[str, Any]: from .twin_jobs import uncoded_metadata_keys @@ -1245,15 +1292,29 @@ def cmd_edsl_export(args: argparse.Namespace) -> None: ), } ) + + # For validation-critical exports, name the model the twin will run on and + # warn loudly if it is a superseded model (understating twin capability). + model_labels = None + if args.target in TWIN_VALIDATION_TARGETS: + specs = parse_model_specs(args) + model_labels = [model_label(service_name, model_name) for model_name, service_name in specs] + model_warning = superseded_twin_model_warning(specs) + if model_warning: + warnings.append(model_warning) + + export_data = { + "target": args.target, + "survey": args.survey, + "scenario_count": scenario_count, + "model_count": len(export_dict.get("models", []) or []) or None, + } + if model_labels is not None: + export_data["models"] = model_labels emit_raw_export( "zwill edsl-export", args, output, - { - "target": args.target, - "survey": args.survey, - "scenario_count": scenario_count, - "model_count": len(export_dict.get("models", []) or []) or None, - }, + export_data, warnings=warnings, ) diff --git a/zwill/guides/agent-workflow.md b/zwill/guides/agent-workflow.md index 10682a9..e6b6f83 100644 --- a/zwill/guides/agent-workflow.md +++ b/zwill/guides/agent-workflow.md @@ -36,6 +36,12 @@ noise?" Do not report a positive result from a bare twin run. local embeddings, or accept a weaker readout. - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it and error on every call; EDSL omits it automatically. +- Validate twins on a **current frontier model**. `edsl-export --target + twin-probability-job` defaults to `gpt-5.5` when `--model` is omitted, and names + the chosen model in its output; if you pass a superseded model (e.g. `gpt-4o`, + `gpt-4.1`, `claude-3-*`) it emits a `superseded_twin_model` warning, because a + weak model understates twin capability and makes the whole validation + uninformative. Only use an older model when you are deliberately benchmarking it. ## Where outputs go