Make the baseline embedder auto-path reliable so --require-baseline never hangs

tests/test_twin_baseline.py

@@ -252,29 +252,34 @@ def test_twin_baseline_cli_stores_predictions_and_flows_through_report(tmp_path,
     assert report_path.exists()
 
 
-def test_resolve_baseline_embedder_sentence_transformers_and_auto_fallback(monkeypatch) -> None:
-    import pytest
-
+def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, capsys) -> None:
     import zwill.twin_baseline_commands as tbc
-    from zwill.errors import ZwillError
     from zwill.twin_baseline import DEFAULT_EMBEDDING_MODEL
 
     monkeypatch.delenv("OPENAI_API_KEY", raising=False)
     monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False)
 
-    # forced local backend resolves to a callable embedder without any API key
-    # (the model is loaded lazily inside the embedder, not at resolve time)
-    for choice in ("sentence-transformers", "local", "st"):
-        embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL)
-        assert callable(embedder)
+    # Forced local / hashing backends resolve to a callable without any API key.
+    for choice in ("sentence-transformers", "local", "st", "hashing", "lexical"):
+        assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL))
 
-    # auto with no keys but sentence-transformers installed -> local fallback
+    # auto prefers local sentence-transformers when installed...
     monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True)
     assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL))
 
-    # auto with no keys and no local embeddings -> a helpful error, not a crash
+    # ...and even with an Expected Parrot key set, auto must NOT pick the remote
+    # endpoint (which can hang) — it stays on the local model.
+    monkeypatch.setenv("EXPECTED_PARROT_API_KEY", "x")
+    embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
+    assert embedder.__qualname__.startswith("sentence_transformers_embedder")
+
+    # auto with no keys and no local embeddings -> the built-in lexical embedder
+    # (always runs) plus a warning, never an error / hang.
+    monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False)
     monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: False)
-    with pytest.raises(ZwillError) as exc:
-        tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
-    hint = getattr(exc.value, "hint", "") or ""
-    assert "local-embeddings" in hint or "sentence-transformers" in hint
+    embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)
+    assert callable(embedder)
+    # It actually embeds (zero-dependency, instant).
+    vectors = embedder(["hello world", "hello"])
+    assert len(vectors) == 2 and len(vectors[0]) == len(vectors[1]) > 0
+    assert "built-in lexical embedder" in capsys.readouterr().err

zwill/cli_parser.py

@@ -378,7 +378,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None:
     p.add_argument("--sample-respondents", type=int, help="Optional random respondent sample size (debugging).")
     p.add_argument("--seed", type=int, help="Seed for --sample-respondents.")
     p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model name.")
-    p.add_argument("--embedder", choices=["auto", "openai", "edsl", "sentence-transformers"], default="auto", help="Embedding backend: 'auto' uses OpenAI when OPENAI_API_KEY is set, else Expected Parrot, else local sentence-transformers if installed; 'edsl' forces Expected Parrot; 'sentence-transformers' forces a local model (no API key; needs the [local-embeddings] extra).")
+    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.")
     p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the logistic model.")
     p.add_argument("--job-id", help="Override the derived baseline job id.")
     p.add_argument("--replace", action="store_true", help="Overwrite existing predictions for this job id.")
@@ -399,7 +399,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None:
     p.add_argument("--skip-leakage-audit", action="store_true", help="Do not run the context leakage audit.")
     p.add_argument("--skip-bootstrap", action="store_true", help="Do not compute bootstrap confidence intervals.")
     p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model for the baseline.")
-    p.add_argument("--embedder", choices=["auto", "openai", "edsl", "sentence-transformers"], default="auto", help="Embedding backend for the baseline: 'auto' uses OpenAI when OPENAI_API_KEY is set, else Expected Parrot, else local sentence-transformers if installed; 'edsl' forces Expected Parrot; 'sentence-transformers' forces a local model (no API key; needs the [local-embeddings] extra).")
+    p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.")
     p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the baseline logistic model.")
     p.add_argument("--leakage-threshold", type=float, default=0.7, help="Cramer's V threshold for flagging leakage.")
     p.add_argument("--min-pair-rows", type=int, default=30, help="Minimum co-answered respondents for a leakage pair.")

zwill/guides/agent-workflow.md

@@ -26,14 +26,15 @@ noise?" Do not report a positive result from a bare twin run.
   editable checkout). Running twins uses EDSL.
 - Running twins uses **Expected Parrot remote inference**: `EXPECTED_PARROT_API_KEY`
   in a `.env` that `zwill edsl-run` can find (it loads the nearest `.env`).
-- The **conditional baseline** embeds question/option text. It picks an embedding
-  backend from `--embedder auto` (default): a direct `OPENAI_API_KEY`, then Expected
-  Parrot (`EXPECTED_PARROT_API_KEY`, `--embedder edsl`), then a local
-  sentence-transformers model when installed (`--embedder sentence-transformers`, no
-  API key — `pip install 'zwill[conditional-baseline]'`). The baseline itself is an
-  XGBoost model, so install that extra to run it fully offline. Without any backend
-  the baseline is skipped and the comparison loses its point — set a key or install
-  local embeddings, or accept a weaker readout.
+- The **conditional baseline** embeds question/option text and is an XGBoost model.
+  `--embedder auto` (default) picks a *reliable, local* backend so the gated
+  `--require-baseline` validation never hangs: a direct `OPENAI_API_KEY`, else a
+  local sentence-transformers model when installed (`pip install
+  'zwill[conditional-baseline]'`), else a built-in lexical embedder that always
+  runs (weaker — it leans on covariates). The remote Expected Parrot embeddings
+  endpoint is **not** in `auto` (it can hang when unavailable); reach it explicitly
+  with `--embedder edsl`. For the strongest baseline, install the extra so `auto`
+  uses the semantic sentence-transformers model.
 - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it
   and error on every call; EDSL omits it automatically.
 - Validate twins on a **current frontier model**. `edsl-export --target

zwill/twin_baseline.py

@@ -205,6 +205,28 @@ def embed(texts: list[str]) -> list[list[float]]:
     return embed
 
 
+def hashing_embedder(*, dim: int = 256) -> Embedder:
+    """Zero-dependency lexical embedder (bag-of-words feature hashing).
+
+    No model download, no network, instant — so the gated ``--require-baseline``
+    validation can always run. It captures lexical overlap, not meaning, so the
+    conditional baseline built on it leans mostly on respondent covariates; it is
+    a weaker bar than a semantic embedder but always available.
+    """
+
+    def embed(texts: list[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for text in texts:
+            vector = [0.0] * dim
+            for token in str(text).lower().split():
+                bucket = int(hashlib.sha1(token.encode()).hexdigest(), 16) % dim
+                vector[bucket] += 1.0
+            vectors.append(vector)
+        return vectors
+
+    return embed
+
+
 def _load_xgboost() -> Any:
     """Lazy-import XGBClassifier so importing this module never requires xgboost."""
     try:

zwill/twin_baseline_commands.py

@@ -10,6 +10,7 @@
     baseline_job_id,
     build_conditional_baseline_predictions,
     edsl_embedder,
+    hashing_embedder,
     openai_embedder,
     sentence_transformers_available,
     sentence_transformers_embedder,
@@ -19,12 +20,14 @@
 def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder:
     """Pick the embedding backend for the conditional baseline.
 
-    `--embedder auto` (the default) prefers a direct OpenAI key, then Expected
-    Parrot (remote EDSL, needs only EXPECTED_PARROT_API_KEY), then a local
-    sentence-transformers model when it is installed -- so the gated
-    `--require-baseline` validation can run with compute and no API keys at all.
-    `openai`, `edsl`/`expected-parrot`, and `sentence-transformers`/`local` force
-    a backend.
+    `--embedder auto` (the default) prefers *reliable, local* backends so the
+    gated `--require-baseline` validation never hangs: a direct OpenAI key, then
+    a local sentence-transformers model, then a zero-dependency built-in lexical
+    embedder that always works (weaker, so it leans on covariates). The remote
+    Expected Parrot embeddings endpoint is intentionally NOT in the auto path --
+    it can hang the validation when unavailable -- but stays reachable via
+    `--embedder edsl`. `openai`, `sentence-transformers`/`local`, and
+    `hashing`/`lexical` force a backend.
     """
     choice = (getattr(args, "embedder", None) or "auto").lower()
     # A local sentence-transformers model uses its own default, not the OpenAI one.
@@ -35,23 +38,21 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder:
         return edsl_embedder(model=embedding_model)
     if choice == "openai":
         return openai_embedder(model=embedding_model)
-    # auto
+    if choice in {"hashing", "lexical", "builtin", "hash"}:
+        return hashing_embedder()
+    # auto -- reliable/local only; never the remote endpoint (opt in with --embedder edsl).
     if os.environ.get("OPENAI_API_KEY"):
         return openai_embedder(model=embedding_model)
-    if os.environ.get("EXPECTED_PARROT_API_KEY"):
-        return edsl_embedder(model=embedding_model)
     if sentence_transformers_available():
         return sentence_transformers_embedder(model=local_model)
-    raise ZwillError(
-        "missing_dependency",
-        "No embedding backend available for the conditional baseline.",
-        hint=(
-            "Set OPENAI_API_KEY (direct OpenAI), or EXPECTED_PARROT_API_KEY "
-            "(route through Expected Parrot, --embedder edsl), or install local "
-            "embeddings with `pip install 'zwill[local-embeddings]'` "
-            "(--embedder sentence-transformers). Or pass --skip-baseline."
-        ),
+    print(
+        "warning: no semantic embedding backend available; using the built-in lexical embedder "
+        "for the conditional baseline (weaker — it leans on covariates). Install "
+        "`zwill[local-embeddings]` or set OPENAI_API_KEY for a semantic baseline, or pass "
+        "`--embedder sentence-transformers` once installed.",
+        file=sys.stderr,
     )
+    return hashing_embedder()
 
 
 def selected_baseline_heldout_questions(args: Any, questions: list[dict[str, Any]]) -> list[str]: