From 310a9a5e3f1a246f9ec8c220ca8858017e93596d Mon Sep 17 00:00:00 2001 From: John Horton Date: Fri, 10 Jul 2026 07:22:36 -0400 Subject: [PATCH] Make the baseline embedder auto-path reliable so --require-baseline never hangs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A research-agent run stalled `twin-validate --require-baseline` for 600s and gave up (ran `--skip-baseline`), losing the whole point of the validation. Root cause: `--embedder auto` preferred the REMOTE Expected Parrot embeddings endpoint whenever EXPECTED_PARROT_API_KEY was set — which it always is for twin runs — and that endpoint is unavailable/404ing, so it hung. The XGBoost compute itself is fine (~22s at full Pew scale: 69 questions, 200 respondents, 8 held-out, 544k training rows). Fixes: - auto now prefers reliable, local backends and NEVER the remote endpoint: direct OPENAI_API_KEY, else local sentence-transformers if installed, else a new zero-dependency built-in lexical (hashing) embedder that always runs. The remote Expected Parrot embeddings stay reachable via explicit `--embedder edsl`. - add hashing_embedder: bag-of-words feature hashing, no model download, no network, instant. Weaker than a semantic embedder (leans on covariates) but the baseline always runs — so `--require-baseline` can't hang or be skipped. Emits a stderr warning when auto falls back to it. - `--embedder` gains a `hashing` choice; help + guide updated to describe the new auto order. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_twin_baseline.py | 35 ++++++++++++++++++------------- zwill/cli_parser.py | 4 ++-- zwill/guides/agent-workflow.md | 17 ++++++++------- zwill/twin_baseline.py | 22 ++++++++++++++++++++ zwill/twin_baseline_commands.py | 37 +++++++++++++++++---------------- 5 files changed, 72 insertions(+), 43 deletions(-) diff --git a/tests/test_twin_baseline.py b/tests/test_twin_baseline.py index 3ddff5b..7fac6cd 100644 --- a/tests/test_twin_baseline.py +++ b/tests/test_twin_baseline.py @@ -252,29 +252,34 @@ def test_twin_baseline_cli_stores_predictions_and_flows_through_report(tmp_path, assert report_path.exists() -def test_resolve_baseline_embedder_sentence_transformers_and_auto_fallback(monkeypatch) -> None: - import pytest - +def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, capsys) -> None: import zwill.twin_baseline_commands as tbc - from zwill.errors import ZwillError from zwill.twin_baseline import DEFAULT_EMBEDDING_MODEL monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False) - # forced local backend resolves to a callable embedder without any API key - # (the model is loaded lazily inside the embedder, not at resolve time) - for choice in ("sentence-transformers", "local", "st"): - embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL) - assert callable(embedder) + # Forced local / hashing backends resolve to a callable without any API key. + for choice in ("sentence-transformers", "local", "st", "hashing", "lexical"): + assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL)) - # auto with no keys but sentence-transformers installed -> local fallback + # auto prefers local sentence-transformers when installed... monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True) assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)) - # auto with no keys and no local embeddings -> a helpful error, not a crash + # ...and even with an Expected Parrot key set, auto must NOT pick the remote + # endpoint (which can hang) — it stays on the local model. + monkeypatch.setenv("EXPECTED_PARROT_API_KEY", "x") + embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) + assert embedder.__qualname__.startswith("sentence_transformers_embedder") + + # auto with no keys and no local embeddings -> the built-in lexical embedder + # (always runs) plus a warning, never an error / hang. + monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False) monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: False) - with pytest.raises(ZwillError) as exc: - tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) - hint = getattr(exc.value, "hint", "") or "" - assert "local-embeddings" in hint or "sentence-transformers" in hint + embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) + assert callable(embedder) + # It actually embeds (zero-dependency, instant). + vectors = embedder(["hello world", "hello"]) + assert len(vectors) == 2 and len(vectors[0]) == len(vectors[1]) > 0 + assert "built-in lexical embedder" in capsys.readouterr().err diff --git a/zwill/cli_parser.py b/zwill/cli_parser.py index 86da61f..a9c8d77 100644 --- a/zwill/cli_parser.py +++ b/zwill/cli_parser.py @@ -378,7 +378,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--sample-respondents", type=int, help="Optional random respondent sample size (debugging).") p.add_argument("--seed", type=int, help="Seed for --sample-respondents.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model name.") - p.add_argument("--embedder", choices=["auto", "openai", "edsl", "sentence-transformers"], default="auto", help="Embedding backend: 'auto' uses OpenAI when OPENAI_API_KEY is set, else Expected Parrot, else local sentence-transformers if installed; 'edsl' forces Expected Parrot; 'sentence-transformers' forces a local model (no API key; needs the [local-embeddings] extra).") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the logistic model.") p.add_argument("--job-id", help="Override the derived baseline job id.") p.add_argument("--replace", action="store_true", help="Overwrite existing predictions for this job id.") @@ -399,7 +399,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--skip-leakage-audit", action="store_true", help="Do not run the context leakage audit.") p.add_argument("--skip-bootstrap", action="store_true", help="Do not compute bootstrap confidence intervals.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model for the baseline.") - p.add_argument("--embedder", choices=["auto", "openai", "edsl", "sentence-transformers"], default="auto", help="Embedding backend for the baseline: 'auto' uses OpenAI when OPENAI_API_KEY is set, else Expected Parrot, else local sentence-transformers if installed; 'edsl' forces Expected Parrot; 'sentence-transformers' forces a local model (no API key; needs the [local-embeddings] extra).") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the baseline logistic model.") p.add_argument("--leakage-threshold", type=float, default=0.7, help="Cramer's V threshold for flagging leakage.") p.add_argument("--min-pair-rows", type=int, default=30, help="Minimum co-answered respondents for a leakage pair.") diff --git a/zwill/guides/agent-workflow.md b/zwill/guides/agent-workflow.md index e6b6f83..47923b3 100644 --- a/zwill/guides/agent-workflow.md +++ b/zwill/guides/agent-workflow.md @@ -26,14 +26,15 @@ noise?" Do not report a positive result from a bare twin run. editable checkout). Running twins uses EDSL. - Running twins uses **Expected Parrot remote inference**: `EXPECTED_PARROT_API_KEY` in a `.env` that `zwill edsl-run` can find (it loads the nearest `.env`). -- The **conditional baseline** embeds question/option text. It picks an embedding - backend from `--embedder auto` (default): a direct `OPENAI_API_KEY`, then Expected - Parrot (`EXPECTED_PARROT_API_KEY`, `--embedder edsl`), then a local - sentence-transformers model when installed (`--embedder sentence-transformers`, no - API key — `pip install 'zwill[conditional-baseline]'`). The baseline itself is an - XGBoost model, so install that extra to run it fully offline. Without any backend - the baseline is skipped and the comparison loses its point — set a key or install - local embeddings, or accept a weaker readout. +- The **conditional baseline** embeds question/option text and is an XGBoost model. + `--embedder auto` (default) picks a *reliable, local* backend so the gated + `--require-baseline` validation never hangs: a direct `OPENAI_API_KEY`, else a + local sentence-transformers model when installed (`pip install + 'zwill[conditional-baseline]'`), else a built-in lexical embedder that always + runs (weaker — it leans on covariates). The remote Expected Parrot embeddings + endpoint is **not** in `auto` (it can hang when unavailable); reach it explicitly + with `--embedder edsl`. For the strongest baseline, install the extra so `auto` + uses the semantic sentence-transformers model. - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it and error on every call; EDSL omits it automatically. - Validate twins on a **current frontier model**. `edsl-export --target diff --git a/zwill/twin_baseline.py b/zwill/twin_baseline.py index d832c5f..ada6a2f 100644 --- a/zwill/twin_baseline.py +++ b/zwill/twin_baseline.py @@ -205,6 +205,28 @@ def embed(texts: list[str]) -> list[list[float]]: return embed +def hashing_embedder(*, dim: int = 256) -> Embedder: + """Zero-dependency lexical embedder (bag-of-words feature hashing). + + No model download, no network, instant — so the gated ``--require-baseline`` + validation can always run. It captures lexical overlap, not meaning, so the + conditional baseline built on it leans mostly on respondent covariates; it is + a weaker bar than a semantic embedder but always available. + """ + + def embed(texts: list[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + vector = [0.0] * dim + for token in str(text).lower().split(): + bucket = int(hashlib.sha1(token.encode()).hexdigest(), 16) % dim + vector[bucket] += 1.0 + vectors.append(vector) + return vectors + + return embed + + def _load_xgboost() -> Any: """Lazy-import XGBClassifier so importing this module never requires xgboost.""" try: diff --git a/zwill/twin_baseline_commands.py b/zwill/twin_baseline_commands.py index bf7eda1..243e174 100644 --- a/zwill/twin_baseline_commands.py +++ b/zwill/twin_baseline_commands.py @@ -10,6 +10,7 @@ baseline_job_id, build_conditional_baseline_predictions, edsl_embedder, + hashing_embedder, openai_embedder, sentence_transformers_available, sentence_transformers_embedder, @@ -19,12 +20,14 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: """Pick the embedding backend for the conditional baseline. - `--embedder auto` (the default) prefers a direct OpenAI key, then Expected - Parrot (remote EDSL, needs only EXPECTED_PARROT_API_KEY), then a local - sentence-transformers model when it is installed -- so the gated - `--require-baseline` validation can run with compute and no API keys at all. - `openai`, `edsl`/`expected-parrot`, and `sentence-transformers`/`local` force - a backend. + `--embedder auto` (the default) prefers *reliable, local* backends so the + gated `--require-baseline` validation never hangs: a direct OpenAI key, then + a local sentence-transformers model, then a zero-dependency built-in lexical + embedder that always works (weaker, so it leans on covariates). The remote + Expected Parrot embeddings endpoint is intentionally NOT in the auto path -- + it can hang the validation when unavailable -- but stays reachable via + `--embedder edsl`. `openai`, `sentence-transformers`/`local`, and + `hashing`/`lexical` force a backend. """ choice = (getattr(args, "embedder", None) or "auto").lower() # A local sentence-transformers model uses its own default, not the OpenAI one. @@ -35,23 +38,21 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: return edsl_embedder(model=embedding_model) if choice == "openai": return openai_embedder(model=embedding_model) - # auto + if choice in {"hashing", "lexical", "builtin", "hash"}: + return hashing_embedder() + # auto -- reliable/local only; never the remote endpoint (opt in with --embedder edsl). if os.environ.get("OPENAI_API_KEY"): return openai_embedder(model=embedding_model) - if os.environ.get("EXPECTED_PARROT_API_KEY"): - return edsl_embedder(model=embedding_model) if sentence_transformers_available(): return sentence_transformers_embedder(model=local_model) - raise ZwillError( - "missing_dependency", - "No embedding backend available for the conditional baseline.", - hint=( - "Set OPENAI_API_KEY (direct OpenAI), or EXPECTED_PARROT_API_KEY " - "(route through Expected Parrot, --embedder edsl), or install local " - "embeddings with `pip install 'zwill[local-embeddings]'` " - "(--embedder sentence-transformers). Or pass --skip-baseline." - ), + print( + "warning: no semantic embedding backend available; using the built-in lexical embedder " + "for the conditional baseline (weaker — it leans on covariates). Install " + "`zwill[local-embeddings]` or set OPENAI_API_KEY for a semantic baseline, or pass " + "`--embedder sentence-transformers` once installed.", + file=sys.stderr, ) + return hashing_embedder() def selected_baseline_heldout_questions(args: Any, questions: list[dict[str, Any]]) -> list[str]: