diff --git a/tests/test_twin_baseline.py b/tests/test_twin_baseline.py index 7fac6cd..1b99edb 100644 --- a/tests/test_twin_baseline.py +++ b/tests/test_twin_baseline.py @@ -252,7 +252,7 @@ def test_twin_baseline_cli_stores_predictions_and_flows_through_report(tmp_path, assert report_path.exists() -def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, capsys) -> None: +def test_resolve_baseline_embedder_ep_first_with_failover(monkeypatch, capsys) -> None: import zwill.twin_baseline_commands as tbc from zwill.twin_baseline import DEFAULT_EMBEDDING_MODEL @@ -263,23 +263,30 @@ def test_resolve_baseline_embedder_prefers_local_and_never_hangs(monkeypatch, ca for choice in ("sentence-transformers", "local", "st", "hashing", "lexical"): assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder=choice), DEFAULT_EMBEDDING_MODEL)) - # auto prefers local sentence-transformers when installed... - monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True) - assert callable(tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL)) - - # ...and even with an Expected Parrot key set, auto must NOT pick the remote - # endpoint (which can hang) — it stays on the local model. + # auto with an Expected Parrot key + a HEALTHY endpoint -> use the remote first. monkeypatch.setenv("EXPECTED_PARROT_API_KEY", "x") + healthy = lambda *a, **k: (lambda texts: [[1.0] for _ in texts]) # noqa: E731 + monkeypatch.setattr(tbc, "edsl_embedder", healthy) + embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) + assert embedder(["x"]) == [[1.0]] # the remote embedder was chosen + + # auto with an UNAVAILABLE endpoint (probe raises) -> fail over fast to local. + def _dead(*a, **k): + def _embed(texts): + raise RuntimeError("endpoint down") + return _embed + + monkeypatch.setattr(tbc, "edsl_embedder", _dead) + monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: True) embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) assert embedder.__qualname__.startswith("sentence_transformers_embedder") + assert "did not respond" in capsys.readouterr().err - # auto with no keys and no local embeddings -> the built-in lexical embedder - # (always runs) plus a warning, never an error / hang. + # auto, endpoint down, no keys, no local embeddings -> built-in lexical embedder + # (always runs) plus a warning; never an error or hang. monkeypatch.delenv("EXPECTED_PARROT_API_KEY", raising=False) monkeypatch.setattr(tbc, "sentence_transformers_available", lambda: False) embedder = tbc.resolve_baseline_embedder(argparse.Namespace(embedder="auto"), DEFAULT_EMBEDDING_MODEL) - assert callable(embedder) - # It actually embeds (zero-dependency, instant). vectors = embedder(["hello world", "hello"]) assert len(vectors) == 2 and len(vectors[0]) == len(vectors[1]) > 0 assert "built-in lexical embedder" in capsys.readouterr().err diff --git a/zwill/cli_parser.py b/zwill/cli_parser.py index a9c8d77..c39d920 100644 --- a/zwill/cli_parser.py +++ b/zwill/cli_parser.py @@ -378,7 +378,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--sample-respondents", type=int, help="Optional random respondent sample size (debugging).") p.add_argument("--seed", type=int, help="Seed for --sample-respondents.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model name.") - p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the conditional baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the logistic model.") p.add_argument("--job-id", help="Override the derived baseline job id.") p.add_argument("--replace", action="store_true", help="Overwrite existing predictions for this job id.") @@ -399,7 +399,7 @@ def add_report_build_args(parser: argparse.ArgumentParser) -> None: p.add_argument("--skip-leakage-audit", action="store_true", help="Do not run the context leakage audit.") p.add_argument("--skip-bootstrap", action="store_true", help="Do not compute bootstrap confidence intervals.") p.add_argument("--embedding-model", default="text-embedding-3-small", help="Embedding model for the baseline.") - p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) uses a direct OpenAI key, else a local sentence-transformers model if installed, else a built-in lexical embedder that always runs — it never uses the remote endpoint, so it can't hang. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' opts into the remote Expected Parrot embeddings.") + p.add_argument("--embedder", choices=["auto", "openai", "sentence-transformers", "hashing", "edsl"], default="auto", help="Embedding backend for the baseline. 'auto' (default) tries the Expected Parrot endpoint first behind a short health probe (so an unavailable endpoint fails over in seconds instead of hanging), then a direct OpenAI key, then a local sentence-transformers model, then a built-in lexical embedder that always runs. 'sentence-transformers' forces the local model ([local-embeddings] extra); 'hashing' forces the zero-dependency lexical embedder; 'edsl' forces the remote Expected Parrot embeddings (no failover).") p.add_argument("--l2", type=float, default=1.0, help="L2 regularization strength for the baseline logistic model.") p.add_argument("--leakage-threshold", type=float, default=0.7, help="Cramer's V threshold for flagging leakage.") p.add_argument("--min-pair-rows", type=int, default=30, help="Minimum co-answered respondents for a leakage pair.") diff --git a/zwill/guides/agent-workflow.md b/zwill/guides/agent-workflow.md index 47923b3..9db6e46 100644 --- a/zwill/guides/agent-workflow.md +++ b/zwill/guides/agent-workflow.md @@ -27,14 +27,14 @@ noise?" Do not report a positive result from a bare twin run. - Running twins uses **Expected Parrot remote inference**: `EXPECTED_PARROT_API_KEY` in a `.env` that `zwill edsl-run` can find (it loads the nearest `.env`). - The **conditional baseline** embeds question/option text and is an XGBoost model. - `--embedder auto` (default) picks a *reliable, local* backend so the gated - `--require-baseline` validation never hangs: a direct `OPENAI_API_KEY`, else a - local sentence-transformers model when installed (`pip install - 'zwill[conditional-baseline]'`), else a built-in lexical embedder that always - runs (weaker — it leans on covariates). The remote Expected Parrot embeddings - endpoint is **not** in `auto` (it can hang when unavailable); reach it explicitly - with `--embedder edsl`. For the strongest baseline, install the extra so `auto` - uses the semantic sentence-transformers model. + `--embedder auto` (default) tries the **Expected Parrot embeddings endpoint + first**, behind a short health probe so an unavailable endpoint **fails over in + seconds instead of hanging** the gated `--require-baseline` validation. It then + falls back to a direct `OPENAI_API_KEY`, then a local sentence-transformers model + when installed (`pip install 'zwill[conditional-baseline]'`), then a built-in + lexical embedder that always runs (weaker — it leans on covariates). Force a + backend with `--embedder edsl|openai|sentence-transformers|hashing` (`edsl` has + no failover). Install the extra so the fallback is the strong semantic model. - Do **not** pass `temperature` to models. Newer Anthropic/OpenAI models reject it and error on every call; EDSL omits it automatically. - Validate twins on a **current frontier model**. `edsl-export --target diff --git a/zwill/twin_baseline_commands.py b/zwill/twin_baseline_commands.py index 243e174..d305cfa 100644 --- a/zwill/twin_baseline_commands.py +++ b/zwill/twin_baseline_commands.py @@ -16,18 +16,50 @@ sentence_transformers_embedder, ) +# How long to wait for a one-text health probe before deciding the Expected +# Parrot embeddings endpoint is unavailable and failing over to a local backend. +EP_PROBE_TIMEOUT_SECONDS = 20.0 + + +def _probe_embedder(factory: Any, *, timeout: float) -> Embedder | None: + """Return the embedder if a tiny probe embed succeeds within ``timeout``. + + Runs the probe on a daemon thread so a hung/slow endpoint fails over quickly + (bounded by ``timeout``) instead of stalling the whole validation, and never + blocks process exit. Returns None on error, timeout, or an empty result. + """ + import threading + + try: + embedder = factory() + except Exception: # pragma: no cover - construction rarely fails + return None + result: dict[str, Any] = {} + + def _run() -> None: + try: + result["vectors"] = embedder(["health check"]) + except Exception as exc: # noqa: BLE001 + result["error"] = exc + + thread = threading.Thread(target=_run, daemon=True) + thread.start() + thread.join(timeout) + if thread.is_alive() or "error" in result or not result.get("vectors"): + return None + return embedder + def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: """Pick the embedding backend for the conditional baseline. - `--embedder auto` (the default) prefers *reliable, local* backends so the - gated `--require-baseline` validation never hangs: a direct OpenAI key, then - a local sentence-transformers model, then a zero-dependency built-in lexical - embedder that always works (weaker, so it leans on covariates). The remote - Expected Parrot embeddings endpoint is intentionally NOT in the auto path -- - it can hang the validation when unavailable -- but stays reachable via - `--embedder edsl`. `openai`, `sentence-transformers`/`local`, and - `hashing`/`lexical` force a backend. + `--embedder auto` (the default) tries the Expected Parrot embeddings endpoint + first (behind a bounded health probe, so an unavailable endpoint fails over in + seconds instead of hanging the validation), then a direct OpenAI key, then a + local sentence-transformers model, then a zero-dependency built-in lexical + embedder that always works (weaker -- it leans on covariates). `edsl`, + `openai`, `sentence-transformers`/`local`, and `hashing`/`lexical` force a + backend (`edsl` with no failover). """ choice = (getattr(args, "embedder", None) or "auto").lower() # A local sentence-transformers model uses its own default, not the OpenAI one. @@ -40,7 +72,18 @@ def resolve_baseline_embedder(args: Any, embedding_model: str) -> Embedder: return openai_embedder(model=embedding_model) if choice in {"hashing", "lexical", "builtin", "hash"}: return hashing_embedder() - # auto -- reliable/local only; never the remote endpoint (opt in with --embedder edsl). + # auto -- Expected Parrot endpoint first, but health-probed so it can't hang. + if os.environ.get("EXPECTED_PARROT_API_KEY"): + remote = _probe_embedder( + lambda: edsl_embedder(model=embedding_model), timeout=EP_PROBE_TIMEOUT_SECONDS + ) + if remote is not None: + return remote + print( + "warning: the Expected Parrot embeddings endpoint did not respond within " + f"{EP_PROBE_TIMEOUT_SECONDS:.0f}s; falling back to a local embedder for the conditional baseline.", + file=sys.stderr, + ) if os.environ.get("OPENAI_API_KEY"): return openai_embedder(model=embedding_model) if sentence_transformers_available():