Embedder auto: try Expected Parrot first, with a bounded failover

tests/test_twin_baseline.py

@@ -252,7 +252,7 @@ def test_twin_baseline_cli_stores_predictions_and_flows_through_report(tmp_path,
     assert report_path.exists()
 
 
-def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, capsys) -> None:
+def test_resolve_baseline_embedder_ep_first_with_failover(monkeypatch, capsys) -> None:
     import zwill.twin_baseline_commands as tbc
     from zwill.twin_baseline import DEFAULT_EMBEDDING_MODEL
 
@@ -263,23 +263,30 @@ def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, ca
     for choice in ("sentence-transformers", "local", "st", "hashing", "lexical"):
         assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL))
 
-    # auto prefers local sentence-transformers when installed...
-    monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True)
-    assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL))
-
-    # ...and even with an Expected Parrot key set, auto must NOT pick the remote
-    # endpoint (which can hang) — it stays on the local model.
+    # auto with an Expected Parrot key + a HEALTHY endpoint -> use the remote first.
     monkeypatch.setenv("EXPECTED_PARROT_API_KEY", "x")
+    healthy = lambda *a, **k: (lambda texts: [[1.0] for _ in texts])  # noqa: E731
+    monkeypatch.setattr(tbc, "edsl_embedder", healthy)
+    embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
+    assert embedder(["x"]) == [[1.0]]  # the remote embedder was chosen
+
+    # auto with an UNAVAILABLE endpoint (probe raises) -> fail over fast to local.
+    def _dead(*a, **k):
+        def _embed(texts):
+            raise RuntimeError("endpoint down")
+        return _embed
+
+    monkeypatch.setattr(tbc, "edsl_embedder", _dead)
+    monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True)
     embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
     assert embedder.__qualname__.startswith("sentence_transformers_embedder")
+    assert "did not respond" in capsys.readouterr().err
 
-    # auto with no keys and no local embeddings -> the built-in lexical embedder
-    # (always runs) plus a warning, never an error / hang.
+    # auto, endpoint down, no keys, no local embeddings -> built-in lexical embedder
+    # (always runs) plus a warning; never an error or hang.
     monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False)
     monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: False)
     embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
-    assert callable(embedder)
-    # It actually embeds (zero-dependency, instant).
     vectors = embedder(["hello world", "hello"])
     assert len(vectors) == 2 and len(vectors[0]) == len(vectors[1]) > 0
     assert "built-in lexical embedder" in capsys.readouterr().err

zwill/cli_parser.py

@@ -378,7 +378,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None:
     p.add_argument("--sample-respondents", type=int, help="Optional random respondent sample size (debugging).")
     p.add_argument("--seed", type=int, help="Seed for --sample-respondents.")
     p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model name.")
-    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.")
+    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).")
     p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the logistic model.")
     p.add_argument("--job-id", help="Override the derived baseline job id.")
     p.add_argument("--replace", action="store_true", help="Overwrite existing predictions for this job id.")
@@ -399,7 +399,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None:
     p.add_argument("--skip-leakage-audit", action="store_true", help="Do not run the context leakage audit.")
     p.add_argument("--skip-bootstrap", action="store_true", help="Do not compute bootstrap confidence intervals.")
     p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model for the baseline.")
-    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.")
+    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).")
     p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the baseline logistic model.")
     p.add_argument("--leakage-threshold", type=float, default=0.7, help="Cramer's V threshold for flagging leakage.")
     p.add_argument("--min-pair-rows", type=int, default=30, help="Minimum co-answered respondents for a leakage pair.")

zwill/guides/agent-workflow.md

@@ -27,14 +27,14 @@ noise?" Do not report a positive result from a bare twin run.
 - Running twins uses **Expected Parrot remote inference**: `EXPECTED_PARROT_API_KEY`
   in a `.env` that `zwill edsl-run` can find (it loads the nearest `.env`).
 - The **conditional baseline** embeds question/option text and is an XGBoost model.
-  `--embedder auto` (default) picks a *reliable, local* backend so the gated
-  `--require-baseline` validation never hangs: a direct `OPENAI_API_KEY`, else a
-  local sentence-transformers model when installed (`pip install
-  'zwill[conditional-baseline]'`), else a built-in lexical embedder that always
-  runs (weaker — it leans on covariates). The remote Expected Parrot embeddings
-  endpoint is **not** in `auto` (it can hang when unavailable); reach it explicitly
-  with `--embedder edsl`. For the strongest baseline, install the extra so `auto`
-  uses the semantic sentence-transformers model.
+  `--embedder auto` (default) tries the **Expected Parrot embeddings endpoint
+  first**, behind a short health probe so an unavailable endpoint **fails over in
+  seconds instead of hanging** the gated `--require-baseline` validation. It then
+  falls back to a direct `OPENAI_API_KEY`, then a local sentence-transformers model
+  when installed (`pip install 'zwill[conditional-baseline]'`), then a built-in
+  lexical embedder that always runs (weaker — it leans on covariates). Force a
+  backend with `--embedder edsl|openai|sentence-transformers|hashing` (`edsl` has
+  no failover). Install the extra so the fallback is the strong semantic model.
 - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it
   and error on every call; EDSL omits it automatically.
 - Validate twins on a **current frontier model**. `edsl-export --target

zwill/twin_baseline_commands.py

@@ -16,18 +16,50 @@
     sentence_transformers_embedder,
 )
 
+# How long to wait for a one-text health probe before deciding the Expected
+# Parrot embeddings endpoint is unavailable and failing over to a local backend.
+EP_PROBE_TIMEOUT_SECONDS = 20.0
+
+
+def _probe_embedder(factory: Any, *, timeout: float) -> Embedder | None:
+    """Return the embedder if a tiny probe embed succeeds within ``timeout``.
+
+    Runs the probe on a daemon thread so a hung/slow endpoint fails over quickly
+    (bounded by ``timeout``) instead of stalling the whole validation, and never
+    blocks process exit. Returns None on error, timeout, or an empty result.
+    """
+    import threading
+
+    try:
+        embedder = factory()
+    except Exception:  # pragma: no cover - construction rarely fails
+        return None
+    result: dict[str, Any] = {}
+
+    def _run() -> None:
+        try:
+            result["vectors"] = embedder(["health check"])
+        except Exception as exc:  # noqa: BLE001
+            result["error"] = exc
+
+    thread = threading.Thread(target=_run, daemon=True)
+    thread.start()
+    thread.join(timeout)
+    if thread.is_alive() or "error" in result or not result.get("vectors"):
+        return None
+    return embedder
+
 
 def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder:
     """Pick the embedding backend for the conditional baseline.
 
-    `--embedder auto` (the default) prefers *reliable, local* backends so the
-    gated `--require-baseline` validation never hangs: a direct OpenAI key, then
-    a local sentence-transformers model, then a zero-dependency built-in lexical
-    embedder that always works (weaker, so it leans on covariates). The remote
-    Expected Parrot embeddings endpoint is intentionally NOT in the auto path --
-    it can hang the validation when unavailable -- but stays reachable via
-    `--embedder edsl`. `openai`, `sentence-transformers`/`local`, and
-    `hashing`/`lexical` force a backend.
+    `--embedder auto` (the default) tries the Expected Parrot embeddings endpoint
+    first (behind a bounded health probe, so an unavailable endpoint fails over in
+    seconds instead of hanging the validation), then a direct OpenAI key, then a
+    local sentence-transformers model, then a zero-dependency built-in lexical
+    embedder that always works (weaker -- it leans on covariates). `edsl`,
+    `openai`, `sentence-transformers`/`local`, and `hashing`/`lexical` force a
+    backend (`edsl` with no failover).
     """
     choice = (getattr(args, "embedder", None) or "auto").lower()
     # A local sentence-transformers model uses its own default, not the OpenAI one.
@@ -40,7 +72,18 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder:
         return openai_embedder(model=embedding_model)
     if choice in {"hashing", "lexical", "builtin", "hash"}:
         return hashing_embedder()
-    # auto -- reliable/local only; never the remote endpoint (opt in with --embedder edsl).
+    # auto -- Expected Parrot endpoint first, but health-probed so it can't hang.
+    if os.environ.get("EXPECTED_PARROT_API_KEY"):
+        remote = _probe_embedder(
+            lambda: edsl_embedder(model=embedding_model), timeout=EP_PROBE_TIMEOUT_SECONDS
+        )
+        if remote is not None:
+            return remote
+        print(
+            "warning: the Expected Parrot embeddings endpoint did not respond within "
+            f"{EP_PROBE_TIMEOUT_SECONDS:.0f}s; falling back to a local embedder for the conditional baseline.",
+            file=sys.stderr,
+        )
     if os.environ.get("OPENAI_API_KEY"):
         return openai_embedder(model=embedding_model)
     if sentence_transformers_available():