From f518c898d857166d9032dce2cdc16b5630daa191 Mon Sep 17 00:00:00 2001 From: John Horton Date: Fri, 10 Jul 2026 07:28:45 -0400 Subject: [PATCH] Embedder auto: try Expected Parrot first, but with a bounded failover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correction to the prior fix, which dropped the Expected Parrot endpoint from the auto path entirely. The intent is to use EP first (it's the platform), then fall back to local — the real problem was only that an unavailable endpoint HUNG the gated validation for 600s. auto now: Expected Parrot endpoint first, behind a bounded one-text health probe on a daemon thread; if it doesn't respond within EP_PROBE_TIMEOUT_SECONDS (20s) or errors, fail over in seconds to a direct OpenAI key, then local sentence-transformers, then the built-in lexical embedder that always runs. So EP is preferred but can never stall the run. `--embedder edsl` still forces EP with no failover. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_twin_baseline.py | 29 ++++++++++------ zwill/cli_parser.py | 4 +-- zwill/guides/agent-workflow.md | 16 ++++----- zwill/twin_baseline_commands.py | 61 ++++++++++++++++++++++++++++----- 4 files changed, 80 insertions(+), 30 deletions(-) diff --git a/tests/test_twin_baseline.py b/tests/test_twin_baseline.py index 7fac6cd..1b99edb 100644 --- a/tests/test_twin_baseline.py +++ b/tests/test_twin_baseline.py @@ -252,7 +252,7 @@ def test_twin_baseline_cli_stores_predictions_and_flows_through_report(tmp_path, assert report_path.exists() -def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, capsys) -> None: +def test_resolve_baseline_embedder_ep_first_with_failover(monkeypatch, capsys) -> None: import zwill.twin_baseline_commands as tbc from zwill.twin_baseline import DEFAULT_EMBEDDING_MODEL @@ -263,23 +263,30 @@ def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, ca for choice in ("sentence-transformers", "local", "st", "hashing", "lexical"): assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL)) - # auto prefers local sentence-transformers when installed... - monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True) - assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)) - - # ...and even with an Expected Parrot key set, auto must NOT pick the remote - # endpoint (which can hang) — it stays on the local model. + # auto with an Expected Parrot key + a HEALTHY endpoint -> use the remote first. monkeypatch.setenv("EXPECTED_PARROT_API_KEY", "x") + healthy = lambda *a, **k: (lambda texts: [[1.0] for _ in texts]) # noqa: E731 + monkeypatch.setattr(tbc, "edsl_embedder", healthy) + embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) + assert embedder(["x"]) == [[1.0]] # the remote embedder was chosen + + # auto with an UNAVAILABLE endpoint (probe raises) -> fail over fast to local. + def _dead(*a, **k): + def _embed(texts): + raise RuntimeError("endpoint down") + return _embed + + monkeypatch.setattr(tbc, "edsl_embedder", _dead) + monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True) embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) assert embedder.__qualname__.startswith("sentence_transformers_embedder") + assert "did not respond" in capsys.readouterr().err - # auto with no keys and no local embeddings -> the built-in lexical embedder - # (always runs) plus a warning, never an error / hang. + # auto, endpoint down, no keys, no local embeddings -> built-in lexical embedder + # (always runs) plus a warning; never an error or hang. monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False) monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: False) embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) - assert callable(embedder) - # It actually embeds (zero-dependency, instant). vectors = embedder(["hello world", "hello"]) assert len(vectors) == 2 and len(vectors[0]) == len(vectors[1]) > 0 assert "built-in lexical embedder" in capsys.readouterr().err diff --git a/zwill/cli_parser.py b/zwill/cli_parser.py index a9c8d77..c39d920 100644 --- a/zwill/cli_parser.py +++ b/zwill/cli_parser.py @@ -378,7 +378,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--sample-respondents", type=int, help="Optional random respondent sample size (debugging).") p.add_argument("--seed", type=int, help="Seed for --sample-respondents.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model name.") - p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the logistic model.") p.add_argument("--job-id", help="Override the derived baseline job id.") p.add_argument("--replace", action="store_true", help="Overwrite existing predictions for this job id.") @@ -399,7 +399,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--skip-leakage-audit", action="store_true", help="Do not run the context leakage audit.") p.add_argument("--skip-bootstrap", action="store_true", help="Do not compute bootstrap confidence intervals.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model for the baseline.") - p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the baseline logistic model.") p.add_argument("--leakage-threshold", type=float, default=0.7, help="Cramer's V threshold for flagging leakage.") p.add_argument("--min-pair-rows", type=int, default=30, help="Minimum co-answered respondents for a leakage pair.") diff --git a/zwill/guides/agent-workflow.md b/zwill/guides/agent-workflow.md index 47923b3..9db6e46 100644 --- a/zwill/guides/agent-workflow.md +++ b/zwill/guides/agent-workflow.md @@ -27,14 +27,14 @@ noise?" Do not report a positive result from a bare twin run. - Running twins uses **Expected Parrot remote inference**: `EXPECTED_PARROT_API_KEY` in a `.env` that `zwill edsl-run` can find (it loads the nearest `.env`). - The **conditional baseline** embeds question/option text and is an XGBoost model. - `--embedder auto` (default) picks a *reliable, local* backend so the gated - `--require-baseline` validation never hangs: a direct `OPENAI_API_KEY`, else a - local sentence-transformers model when installed (`pip install - 'zwill[conditional-baseline]'`), else a built-in lexical embedder that always - runs (weaker — it leans on covariates). The remote Expected Parrot embeddings - endpoint is **not** in `auto` (it can hang when unavailable); reach it explicitly - with `--embedder edsl`. For the strongest baseline, install the extra so `auto` - uses the semantic sentence-transformers model. + `--embedder auto` (default) tries the **Expected Parrot embeddings endpoint + first**, behind a short health probe so an unavailable endpoint **fails over in + seconds instead of hanging** the gated `--require-baseline` validation. It then + falls back to a direct `OPENAI_API_KEY`, then a local sentence-transformers model + when installed (`pip install 'zwill[conditional-baseline]'`), then a built-in + lexical embedder that always runs (weaker — it leans on covariates). Force a + backend with `--embedder edsl|openai|sentence-transformers|hashing` (`edsl` has + no failover). Install the extra so the fallback is the strong semantic model. - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it and error on every call; EDSL omits it automatically. - Validate twins on a **current frontier model**. `edsl-export --target diff --git a/zwill/twin_baseline_commands.py b/zwill/twin_baseline_commands.py index 243e174..d305cfa 100644 --- a/zwill/twin_baseline_commands.py +++ b/zwill/twin_baseline_commands.py @@ -16,18 +16,50 @@ sentence_transformers_embedder, ) +# How long to wait for a one-text health probe before deciding the Expected +# Parrot embeddings endpoint is unavailable and failing over to a local backend. +EP_PROBE_TIMEOUT_SECONDS = 20.0 + + +def _probe_embedder(factory: Any, *, timeout: float) -> Embedder | None: + """Return the embedder if a tiny probe embed succeeds within ``timeout``. + + Runs the probe on a daemon thread so a hung/slow endpoint fails over quickly + (bounded by ``timeout``) instead of stalling the whole validation, and never + blocks process exit. Returns None on error, timeout, or an empty result. + """ + import threading + + try: + embedder = factory() + except Exception: # pragma: no cover - construction rarely fails + return None + result: dict[str, Any] = {} + + def _run() -> None: + try: + result["vectors"] = embedder(["health check"]) + except Exception as exc: # noqa: BLE001 + result["error"] = exc + + thread = threading.Thread(target=_run, daemon=True) + thread.start() + thread.join(timeout) + if thread.is_alive() or "error" in result or not result.get("vectors"): + return None + return embedder + def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: """Pick the embedding backend for the conditional baseline. - `--embedder auto` (the default) prefers *reliable, local* backends so the - gated `--require-baseline` validation never hangs: a direct OpenAI key, then - a local sentence-transformers model, then a zero-dependency built-in lexical - embedder that always works (weaker, so it leans on covariates). The remote - Expected Parrot embeddings endpoint is intentionally NOT in the auto path -- - it can hang the validation when unavailable -- but stays reachable via - `--embedder edsl`. `openai`, `sentence-transformers`/`local`, and - `hashing`/`lexical` force a backend. + `--embedder auto` (the default) tries the Expected Parrot embeddings endpoint + first (behind a bounded health probe, so an unavailable endpoint fails over in + seconds instead of hanging the validation), then a direct OpenAI key, then a + local sentence-transformers model, then a zero-dependency built-in lexical + embedder that always works (weaker -- it leans on covariates). `edsl`, + `openai`, `sentence-transformers`/`local`, and `hashing`/`lexical` force a + backend (`edsl` with no failover). """ choice = (getattr(args, "embedder", None) or "auto").lower() # A local sentence-transformers model uses its own default, not the OpenAI one. @@ -40,7 +72,18 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: return openai_embedder(model=embedding_model) if choice in {"hashing", "lexical", "builtin", "hash"}: return hashing_embedder() - # auto -- reliable/local only; never the remote endpoint (opt in with --embedder edsl). + # auto -- Expected Parrot endpoint first, but health-probed so it can't hang. + if os.environ.get("EXPECTED_PARROT_API_KEY"): + remote = _probe_embedder( + lambda: edsl_embedder(model=embedding_model), timeout=EP_PROBE_TIMEOUT_SECONDS + ) + if remote is not None: + return remote + print( + "warning: the Expected Parrot embeddings endpoint did not respond within " + f"{EP_PROBE_TIMEOUT_SECONDS:.0f}s; falling back to a local embedder for the conditional baseline.", + file=sys.stderr, + ) if os.environ.get("OPENAI_API_KEY"): return openai_embedder(model=embedding_model) if sentence_transformers_available():