Warn when a twin is validated on a superseded model

tests/test_twin_model_guard.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from zwill.edsl_integration import (
+    DEFAULT_EXPORT_MODEL,
+    TWIN_VALIDATION_TARGETS,
+    superseded_twin_model_labels,
+    superseded_twin_model_warning,
+)
+
+
+def test_superseded_models_are_flagged_and_current_models_are_not() -> None:
+    specs = [
+        ("gpt-4.1", "openai"),
+        ("gpt-4o", None),
+        ("claude-3-opus", "anthropic"),
+        ("gemini-1.5-pro", "google"),
+        ("gpt-5.5", "openai"),
+        ("claude-opus-4-8", "anthropic"),
+        ("gemini-2.5-pro", "google"),
+    ]
+    flagged = superseded_twin_model_labels(specs)
+    assert flagged == ["openai:gpt-4.1", "gpt-4o", "anthropic:claude-3-opus", "google:gemini-1.5-pro"]
+
+
+def test_warning_fires_only_for_superseded_models() -> None:
+    # The exact model the research-agent run used.
+    warning = superseded_twin_model_warning([("gpt-4.1", "openai")])
+    assert warning is not None
+    assert warning["code"] == "superseded_twin_model"
+    assert "openai:gpt-4.1" in warning["message"]
+    assert DEFAULT_EXPORT_MODEL in warning["message"]  # points at the strong default
+
+    # A current frontier model raises nothing.
+    assert superseded_twin_model_warning([("gpt-5.5", "openai")]) is None
+
+
+def test_default_export_model_is_a_current_model() -> None:
+    # The omitted-model default must not itself be flagged as superseded.
+    assert superseded_twin_model_labels([(DEFAULT_EXPORT_MODEL, "openai")]) == []
+
+
+def test_validation_targets_cover_the_model_bearing_exports() -> None:
+    assert TWIN_VALIDATION_TARGETS == {
+        "twin-probability-job",
+        "rank-utility-twin-job",
+        "numeric-twin-job",
+    }

zwill/edsl_integration.py

@@ -349,10 +349,13 @@ def option_key(index: int) -> str:
     return key
 
 
+DEFAULT_EXPORT_MODEL = "gpt-5.5"
+
+
 def parse_model_specs(args: argparse.Namespace) -> list[tuple[str, str | None]]:
     values = normalize_name_list(args.model) + normalize_name_list(args.models)
     if not values:
-        values = ["gpt-5.5"]
+        values = [DEFAULT_EXPORT_MODEL]
 
     specs = []
     for value in values:
@@ -421,6 +424,50 @@ def model_label(service_name: str | None, model_name: str | None) -> str:
     return str(model_name or "")
 
 
+# Model families that are clearly superseded for twin validation as of this
+# release. Validating a twin on one understates its capability (a research-agent
+# run once validated on gpt-4.1 while frontier was gpt-5.5), so the twin-job
+# export warns. Advisory only, never blocking. Keep current as models evolve.
+SUPERSEDED_TWIN_MODEL_MARKERS = (
+    "gpt-4o", "gpt-4.1", "gpt-4.5", "gpt-4-", "gpt-4", "gpt-3.5", "gpt-35",
+    "claude-3", "claude-2", "claude-instant",
+    "gemini-1", "gemini-pro",
+    "text-davinci", "text-bison", "davinci", "babbage", "ada",
+)
+RECOMMENDED_TWIN_MODELS = ("openai:gpt-5.5", "anthropic:claude-opus-4-8", "google:gemini-2.5-pro")
+
+# --target values whose export bakes in the model that will predict held-out
+# answers, i.e. where model choice is validation-critical.
+TWIN_VALIDATION_TARGETS = frozenset({"twin-probability-job", "rank-utility-twin-job", "numeric-twin-job"})
+
+
+def superseded_twin_model_labels(specs: list[tuple[str, str | None]]) -> list[str]:
+    """Labels among ``specs`` that match a superseded model family."""
+    flagged: list[str] = []
+    for model_name, service_name in specs:
+        lowered = str(model_name).lower()
+        if any(marker in lowered for marker in SUPERSEDED_TWIN_MODEL_MARKERS):
+            flagged.append(model_label(service_name, model_name))
+    return flagged
+
+
+def superseded_twin_model_warning(specs: list[tuple[str, str | None]]) -> dict[str, Any] | None:
+    """A loud warning envelope when a twin will be validated on a weak model."""
+    superseded = superseded_twin_model_labels(specs)
+    if not superseded:
+        return None
+    plural = "is a superseded model" if len(superseded) == 1 else "are superseded models"
+    return {
+        "code": "superseded_twin_model",
+        "message": (
+            f"This twin validation will run on {', '.join(superseded)}, which {plural} for this release — "
+            f"results may understate twin capability. Use a current frontier model (e.g. {RECOMMENDED_TWIN_MODELS[0]}) "
+            f"via --model unless you are deliberately benchmarking an older model. zwill defaults to "
+            f"{DEFAULT_EXPORT_MODEL} when --model is omitted."
+        ),
+    }
+
+
 
 def build_edsl_agent_list_dict(survey_name: str, args: argparse.Namespace) -> dict[str, Any]:
     from .twin_jobs import uncoded_metadata_keys
@@ -1245,15 +1292,29 @@ def cmd_edsl_export(args: argparse.Namespace) -> None:
                 ),
             }
         )
+
+    # For validation-critical exports, name the model the twin will run on and
+    # warn loudly if it is a superseded model (understating twin capability).
+    model_labels = None
+    if args.target in TWIN_VALIDATION_TARGETS:
+        specs = parse_model_specs(args)
+        model_labels = [model_label(service_name, model_name) for model_name, service_name in specs]
+        model_warning = superseded_twin_model_warning(specs)
+        if model_warning:
+            warnings.append(model_warning)
+
+    export_data = {
+        "target": args.target,
+        "survey": args.survey,
+        "scenario_count": scenario_count,
+        "model_count": len(export_dict.get("models", []) or []) or None,
+    }
+    if model_labels is not None:
+        export_data["models"] = model_labels
     emit_raw_export(
         "zwill edsl-export",
         args,
         output,
-        {
-            "target": args.target,
-            "survey": args.survey,
-            "scenario_count": scenario_count,
-            "model_count": len(export_dict.get("models", []) or []) or None,
-        },
+        export_data,
         warnings=warnings,
     )

zwill/guides/agent-workflow.md

@@ -36,6 +36,12 @@ noise?" Do not report a positive result from a bare twin run.
   local embeddings, or accept a weaker readout.
 - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it
   and error on every call; EDSL omits it automatically.
+- Validate twins on a **current frontier model**. `edsl-export --target
+  twin-probability-job` defaults to `gpt-5.5` when `--model` is omitted, and names
+  the chosen model in its output; if you pass a superseded model (e.g. `gpt-4o`,
+  `gpt-4.1`, `claude-3-*`) it emits a `superseded_twin_model` warning, because a
+  weak model understates twin capability and makes the whole validation
+  uninformative. Only use an older model when you are deliberately benchmarking it.
 
 ## Where outputs go