diff --git a/.env.example b/.env.example
index dc8e9c7..36d4b31 100644
--- a/.env.example
+++ b/.env.example
@@ -7,18 +7,27 @@
GEMINI_API_KEY=your-gemini-api-key
TAVILY_API_KEY=tvly-your-tavily-api-key
-# --- Optional (only needed when LLM_PROVIDER=groq) ---
+# --- Optional (only needed when the matching LLM_PROVIDER is selected) ---
GROQ_API_KEY=your-groq-api-key
+# Cerebras: ~1M tokens/day free tier (~10x Groq) — used to run the full eval in one
+# session. Get a key at https://cloud.cerebras.ai/
+CEREBRAS_API_KEY=your-cerebras-api-key
-# --- Provider selection: "gemini" (default) or "groq" ---
+# --- Provider selection: "gemini" (default), "groq", or "cerebras" ---
LLM_PROVIDER=gemini
# --- Model ids (current free-tier defaults shown) ---
GEMINI_MODEL=gemini-2.5-flash-lite
GROQ_MODEL=llama-3.1-8b-instant
+# Cerebras model availability varies by account/tier — check models.list().
+CEREBRAS_MODEL=gpt-oss-120b
# --- Research / qualification / grounding tunables ---
-RESEARCH_MAX_QUERIES=4
+# Research depth is leaned out by default (cheaper/faster; fits free-tier token
+# caps). RESEARCH_MAX_PAGE_CHARS is the biggest token lever — see ADR-0012.
+RESEARCH_MAX_QUERIES=3
+RESEARCH_MAX_PAGE_CHARS=3500
+RESEARCH_MAX_FACTS_PER_SOURCE=5
QUALIFY_THRESHOLD=0.5
GROUNDEDNESS_THRESHOLD=0.9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9592ed0..7c0e152 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,145 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
_Nothing yet — P5 (storage and review app: production `Store` backends and a
human-review UI over the queue) is next._
+## [0.9.0] — 2026-06-14
+
+**Qualifier fix + held-out validation.** The temperature-0 re-baseline exposed six
+qualification false positives (F1 0.769); a diagnostic traced them to an under-firing
+negative-signal veto and `industry=unknown` being dropped from the score (see
+ADR-0015).
+
+### Changed
+
+- **`nodes/qualify.py` — reliable negative-signal veto.** The assessor prompt now
+ judges NEGATIVE signals from the company's described business (so a negation like
+ "not fintech" fires when the facts show design tools / dev infra / ML hosting,
+ instead of defaulting to `unknown`). Structural attributes and positive signals
+ still follow "don't guess → unknown".
+- **`nodes/qualify.py` — industry is a required component.** When the ICP names target
+ industries, an `unknown` or `no_match` industry scores 0.0 and stays in the
+ denominator (instead of being dropped), so a fintech ICP can't qualify a company it
+ can't place in-industry. Size/region unknowns are still dropped.
+
+### Added
+
+- **`examples/eval_companies_holdout.json`** — an 8-company held-out validation set
+ (clear good-fit, clear bad-fit incl. an incumbent bank, two borderline) to test
+ whether qualifier changes generalize vs. overfit.
+
+### Eval (cerebras/gpt-oss-120b, temperature 0, 2026-06-14)
+
+- **Held-out (n=8, headline): F1 1.0** (TP=4, FP=0, TN=4, FN=0) — every unseen company
+ correct, including an incumbent bank and two borderlines.
+- **Original 17 (development): F1 0.769 → 0.947** — all six false positives eliminated
+ (precision 0.625 → 1.0). Both sets improved, so the fix generalizes.
+- **Trade-off, flagged:** one new false negative on the dev set (`ramp.com`, flaky
+ `industry=unknown` on a real fintech; recall 1.0 → 0.9). Draft-gate overreach
+ (drafter over-claiming, caught by the gate) noted as a separate finding. See
+ `docs/evals.md`.
+
+## [0.8.0] — 2026-06-14
+
+**Draft grounding decoupled from phrasing.** The first full Cerebras eval revealed
+that validating draft hooks as **verbatim source substrings** was brittle: faithful
+paraphrases were discarded, so most qualified companies scored
+`groundedness=0.0` with empty verdicts — an artifact of the matching layer, not of
+grounding. Source-text grounding belongs at extraction; the draft should ground by
+*selecting facts* and the verify gate should judge the *body*'s faithfulness (see
+ADR-0014).
+
+### Changed
+
+- **Draft node** (`nodes/draft.py`) — grounds by **fact-selection**: the model is
+ shown the claimable (first-party) facts as a numbered list and returns the **ids**
+ it grounded the email in. `hooks_used` is the canonical claim text of those facts,
+ grounded by construction. No more verbatim-hook substring matching or fuzzy
+ matching of paraphrased text back to facts.
+- **Verify node** (`nodes/verify.py`) — now a **structural** check (every hook
+ re-resolves to a first-party fact; else `structural`) plus **one** body-faithfulness
+ judge (`judge_body`) over the draft **body** against the selected facts, rating each
+ body claim `faithful` / `overreach` / `unsupported`. Passes iff grounded, body
+ non-empty, judge ran, and no `unsupported` (and no `overreach` under
+ `FAITHFULNESS_STRICT`). `groundedness_score` redefined as
+ `faithful_body_claims / total_body_claims`. `judge_faithfulness` is replaced by
+ `judge_body`.
+- **`VerificationResult` / `ClaimVerdict`** — `claim_verdicts` now describe **body**
+ claims (with the supporting fact cited by id); `tier_breakdown` counts the grounding
+ hooks by tier; failure reasons shrink to `unsupported` / `overreach` / `structural`
+ / `judge-error`.
+- **No fact-id leakage in drafts** — the draft prompt forbids fact ids in the
+ subject/body, with a regex backstop (`_strip_fact_ids`) that removes any leaked
+ `(Fact N)` token (the ids are for the `facts_used` field only).
+- **Deterministic gate-critical calls** — `LLMClient.complete` / `complete_json` take
+ an optional `temperature` (sent only when set; provider default otherwise), and the
+ **qualify, draft, and verify-judge** calls pass `temperature=0` so the headline
+ numbers are a reproducible run, not a lucky sample.
+
+### Added
+
+- **`redraft` eval command** (`evals/run_eval.py`) — re-runs only draft + verify for
+ already-qualified companies, reusing cached research **and** each record's frozen
+ qualification verdict (the qualification matrix is never recomputed).
+
+### Eval — canonical temperature-0 re-baseline (cerebras/gpt-oss-120b, 2026-06-14, n=17)
+
+One clean full run with qualify/draft/verify at temperature 0 (research reused from
+cache). Replaces the earlier mixed table; this run is canonical.
+
+- Qualification: TP=10, FP=6, TN=1, FN=0 → **F1 0.769** (precision 0.625, recall 1.0,
+ accuracy 0.647).
+- Draft-gate pass-rate **9/16 = 0.5625**, mean groundedness/faithfulness **0.8615**,
+ own_site live re-verifiability **0.9032 (56/62)**.
+- Reproducibility honesty: an earlier *non-deterministic* run scored F1 0.870; at
+ temperature 0 the negative-signal veto under-fires consistently (figma/huggingface/
+ jpmorganchase become false positives), so the reproducible number is lower. See
+ `docs/evals.md` for the full Known Limitations (six FPs; veto-hardening /
+ industry-gating fix deferred to a held-out set; residual temp-0 variance).
+
+## [0.7.0] — 2026-06-13
+
+**Cerebras provider.** Adds a third LLM provider so the eval can run end-to-end in
+one session on Cerebras's ~1M tokens/day free tier (~10x Groq) (see ADR-0013).
+
+### Added
+
+- `CerebrasClient` (`clients/llm.py`) — OpenAI-compatible via `cerebras-cloud-sdk`;
+ mirrors `GroqClient` (JSON mode + lenient parsing + error normalization to
+ `LLMError`). Default model `gpt-oss-120b` (available models vary by Cerebras
+ account/tier — check `models.list()`).
+- `CEREBRAS_API_KEY` (optional) and `CEREBRAS_MODEL` settings; `get_llm_client()`
+ now selects `"cerebras"`. `Settings.active_model` resolves the model id for the
+ active provider.
+- `CONTEXT_TOKEN_CAP` (8192) and `trim_to_token_budget()` — bound the qualify/draft
+ facts payloads so no single prompt exceeds Cerebras's free-tier context window.
+- `cerebras-cloud-sdk` runtime dependency (lazily imported — only needed when the
+ provider is selected).
+
+### Changed
+
+- `LLM_PROVIDER` now accepts `"cerebras"` in addition to `"gemini"` / `"groq"`; the
+ smoke check and eval report use `Settings.active_model` so the right model is named.
+
+## [0.6.0] — 2026-06-13
+
+**Lean research depth (default).** Research is leaner by default — cheaper, faster,
+and within free-tier token caps — at the same depth the eval and production both
+run, so there is no eval-vs-prod mismatch (see ADR-0012).
+
+### Added
+
+- `RESEARCH_MAX_PAGE_CHARS` (default `3500`) — truncates each source's text fed to
+ the extractor; the biggest token lever (was ~12,000). `extract_facts` truncates
+ once and runs the evidence-substring check against that same truncated text, so
+ groundedness is preserved.
+- `RESEARCH_MAX_FACTS_PER_SOURCE` (default `5`) setting (was a fixed `8`).
+
+### Changed
+
+- `RESEARCH_MAX_QUERIES` default 4 → **3**.
+- Research depth is now configured via `Settings` (the three knobs above) and
+ passed through `run_research` to the extractor; the old module constants survive
+ only as lean fallback defaults. Per-company research token use roughly halves.
+
## [0.5.0] — 2026-06-13
**P4 — Eval harness.** The offline evaluation that produces the headline numbers,
diff --git a/README.md b/README.md
index 664a9b1..8f72d48 100644
--- a/README.md
+++ b/README.md
@@ -1,105 +1,250 @@
# pitch-pilot
-> An autonomous **SDR** agent: give it a company **domain**, and it researches the
-> company, qualifies it against your Ideal Customer Profile, drafts grounded
-> outreach, and verifies every claim against a real source — then logs it for a
-> human to review. **Nothing is auto-sent.**
+**An autonomous SDR agent that turns a company domain into source-cited outreach —
+every claim is first-party-sourced and faithfulness-judged before a human sees it,
+so there's no hallucinated personalization.**
-## Why it's different — groundedness
+Give it a domain and pitch-pilot researches the company, qualifies it against your
+Ideal Customer Profile, drafts outreach grounded only in cited facts, verifies every
+claim against its source, and queues the result for human approval. **It never
+auto-sends.**
-The hero feature is **groundedness**: no fact exists without a `source_url`. The
-core [`Fact`](src/pitch_pilot/models/fact.py) type refuses to be constructed
-without an `http(s)` source, so every claim the agent makes is traceable to a page
-*by design* — not by a hopeful post-hoc check. Outreach is drafted only from
-grounded facts, every claim is re-verified against its source, and only drafts
-above a groundedness threshold reach the human-review queue.
+_Last updated: 2026-06-14 · Full documentation: [`docs/`](docs/index.md)_
-- ✅ Every fact carries a `source_url` (enforced at construction)
-- ✅ Nothing auto-sends — qualified leads land in a review queue
-- 🚫 No LinkedIn scraping (out of scope by design)
+## Results
-## Architecture (P0)
+> `cerebras/gpt-oss-120b`, gate-critical calls at **temperature 0**, 2026-06-14. The
+> qualifier fix ([ADR-0015](docs/decisions.md)) was developed on the original 17 and
+> validated on a **held-out** set it never saw. Every number is from
+> [`docs/evals.md`](docs/evals.md), the source of truth.
-Hybrid: a **deterministic outer graph** (`research → qualify → draft → verify →
-log`) wrapping an **agentic research sub-loop**. See
-[`docs/architecture.md`](docs/architecture.md) and
-[`docs/decisions.md`](docs/decisions.md).
+**Held-out validation — the headline (n=8 unseen companies):**
-This phase (P0) ships the foundation — typed data contracts, swappable provider
-clients, typed config, a smoke test, and unit tests. The LangGraph pipeline and
-node logic land in P1.
+| Stage | Metric | Result |
+| --- | --- | --- |
+| **Qualification** (n=8) | Accuracy / Precision / Recall / **F1** | 1.0 / 1.0 / 1.0 / **1.0** |
+| | Confusion — TP / FP / TN / FN | 4 / 0 / 4 / 0 |
+| **Drafting** | Draft-gate pass-rate | 0.75 (3 of 4) |
+| **Grounding** | Mean groundedness ¹ | 0.9583 |
-## Setup (Windows / PowerShell)
+All eight unseen companies landed correctly — three good-fit fintechs qualified, two
+non-fintech tools and an incumbent bank disqualified, and both borderlines (coinbase,
+shopify) went the labeled way.
-```powershell
-# 1. Create and activate a virtual environment (Python 3.11+)
-py -3.11 -m venv .venv
-.\.venv\Scripts\Activate.ps1
+**The fix's evidence — original 17, before → after** (same set, same temp-0 config,
+only the qualifier changed):
-# 2. Install the package (editable) + dev tools
-python -m pip install --upgrade pip
-pip install -e ".[dev]"
+| Metric (n=17) | Before | After |
+| --- | --- | --- |
+| **Qualification F1** | 0.769 | **0.947** |
+| Precision / Recall | 0.625 / 1.0 | 1.0 / 0.9 |
+| Confusion TP/FP/TN/FN | 10/6/1/0 | 9/0/7/1 |
-# 3. Configure your keys
-Copy-Item .env.example .env
-# then edit .env and fill in at least GEMINI_API_KEY and TAVILY_API_KEY
-```
+The fix eliminated **all six** false positives (precision 0.625 → 1.0) via a
+reliably-firing negative-signal veto plus a required-industry penalty. **Both sets
+improved, so it generalizes rather than overfitting.** The cost, stated honestly: one
+new false negative on the dev set — `ramp.com`, when the assessor flakily returned
+`industry=unknown` for a real fintech (recall 1.0 → 0.9). The held-out set did not
+show this. Full provenance, the live-re-verifiability caveat, and the draft-gate
+overreach finding are in [Evaluation](docs/evals.md).
-> **Windows `.env` gotcha:** don't create `.env` with `Set-Content -Encoding utf8`
-> — it writes a UTF-8 BOM that corrupts the *first* key so it silently fails to
-> load. `Copy-Item .env.example .env` is safe (or use `-Encoding utf8NoBOM` /
-> a normal editor).
+¹ Groundedness = faithful body-claims ÷ total body-claims; under the strict gate it
+equals the faithfulness score (same numerator) — one signal, not two.
-
-macOS / Linux
+## Demo
-```bash
-python3.11 -m venv .venv && source .venv/bin/activate
-python -m pip install --upgrade pip
-pip install -e ".[dev]"
-cp .env.example .env
-```
-
+Verbatim CLI output (trimmed only where marked `[...]`), same config as the tables
+above: `cerebras/gpt-oss-120b`, eval ICP (`examples/eval_icp.json`), gate-critical
+calls at temperature 0. Research is reused from cache; qualify/draft/verify run live.
+Cerebras is not bit-deterministic even at temperature 0, so each demo is one fresh
+sample.
-## Configuration
+_[Live demo: coming after deploy]_
-| Variable | Required | Default | Description |
-| --- | --- | --- | --- |
-| `GEMINI_API_KEY` | ✅ | — | Google Gen AI (Gemini) API key |
-| `TAVILY_API_KEY` | ✅ | — | Tavily search API key |
-| `GROQ_API_KEY` | optional | — | Groq API key (only if `LLM_PROVIDER=groq`) |
-| `LLM_PROVIDER` | optional | `gemini` | Active LLM provider: `gemini` or `groq` |
-| `GEMINI_MODEL` | optional | `gemini-2.5-flash-lite` | Gemini model id |
-| `GROQ_MODEL` | optional | `llama-3.1-8b-instant` | Groq model id |
-| `RESEARCH_MAX_QUERIES` | optional | `4` | Max search queries per research run |
-| `GROUNDEDNESS_THRESHOLD` | optional | `0.9` | Min groundedness score for a draft to pass |
+**1. A qualified lead that clears the gate — `mercury.com`:**
-Required keys are validated at startup: if one is missing, pitch-pilot fails
-immediately with a `ConfigError` that names it.
+```text
+PS> python -m pitch_pilot.cli run mercury.com --icp examples/eval_icp.json
-## Smoke test (P0 acceptance gate)
+Running pipeline for mercury.com (provider = cerebras, icp = examples/eval_icp.json) ...
-Proves all three external dependencies work with your keys:
+Research: 64 grounded facts from 17 sources (4 queries).
-```powershell
-python -m pitch_pilot.cli smoke
+== Qualification ==
+ QUALIFIED — fit score 0.83
+ Fit score 0.83 >= threshold 0.50; industry=match, size=unknown, region=match; matched 2/4 positive signal(s).
+ matched: processes online payments or transactions at scale, recently raised growth funding
+
+== Draft ==
+ Subject: Helping Mercury streamline global payments and cash management
+
+ I noticed Mercury’s fintech platform offers free checking and savings accounts with zero minimums and up to 3.60% yield, which is a great foundation for growing businesses. Your payments product lets users send money worldwide with no fees on USD payments, and the instant issuance of cards gives teams immediate control over spend. Since Mercury helps manage business finances at every stage of growth, I think our integrated finance‑automation solution could further reduce manual effort and enhance your existing tools.
+
+ Grounded hooks: Mercury offers free checking and savings accounts with zero minimums and up to 3.60% yield through Treasury by Mercury Advisory. | Mercury payments allow sending money worldwide with no fees on USD payments. | [...]
+
+== Verification ==
+ groundedness 1.00 (4/4 verified) · faithfulness 1.00 — PASS
+ claims by source tier: own_site=5
+ - tier=own_site substring_ok=yes faithfulness=faithful
+ claim: Mercury’s fintech platform offers free checking and savings accounts with zero minimums and up to 3.60% yield
+ source: https://mercury.com
+ - tier=own_site substring_ok=yes faithfulness=faithful
+ claim: Mercury’s payments product lets users send money worldwide with no fees on USD payments
+ source: https://mercury.com
+ [... 2 more faithful own_site claims ...]
+
+== Logged ==
+ outcome: ready
+ written to: pitch_pilot_store.jsonl
+ (pitch-pilot never auto-sends — a human approves before anything goes out.)
```
-It runs one Tavily search, one LLM completion, and one page fetch, printing a
-clear ✅ / ❌ for each. (Also available as the `pitch-pilot smoke` console script.)
+**2. The groundedness gate rejecting a bad claim — `wise.com`:**
+
+Wise qualifies cleanly, but its draft turned the fact "In Q4, 65% of transfers **were**
+instant" into "65% of transfers **now** complete in under 20 seconds" — dropping the
+"Q4" qualifier and present-tensing a past-quarter stat. The judge rated that claim
+`overreach`, the draft **failed**, and the lead was routed to human review instead of
+`ready` — the gate catching exactly the kind of subtle over-claim it is built for.
+
+```text
+PS> python -m pitch_pilot.cli run wise.com --icp examples/eval_icp.json
-## Tests
+Running pipeline for wise.com (provider = cerebras, icp = examples/eval_icp.json) ...
-Unit tests are fully mocked — **no network and no API keys required**:
+Research: 65 grounded facts from 17 sources (4 queries).
+
+== Qualification ==
+ QUALIFIED — fit score 0.83
+ Fit score 0.83 >= threshold 0.50; industry=match, size=unknown, region=match; matched 2/4 positive signal(s).
+ matched: processes online payments or transactions at scale, regulatory or compliance exposure
+
+== Draft ==
+ Subject: Supporting Wise’s fast, low‑cost international payments
+
+ I’ve been impressed by Wise’s personal international account that lets users send, spend and convert money as if they were local. The claim that customers can save up to 45% on global transfers really stands out. I also saw that 65% of transfers now complete in under 20 seconds, showing how fast your service is. I’d love to explore how we could support that momentum.
+
+ Grounded hooks: Wise offers a personal international account for sending, spending and converting money like a local. | Wise claims users can save up to 45% when sending money globally. | [...]
+
+== Verification ==
+ groundedness 0.67 (2/3 verified) · faithfulness 0.67 — FAIL
+ claims by source tier: own_site=3
+ - tier=own_site substring_ok=yes faithfulness=faithful
+ claim: Wise has a personal international account that lets users send, spend and convert money as if they were local.
+ source: https://wise.com
+ - tier=own_site substring_ok=yes faithfulness=faithful
+ claim: Customers can save up to 45% on global transfers.
+ source: https://wise.com
+ - tier=own_site substring_ok=yes faithfulness=overreach
+ claim: 65% of transfers now complete in under 20 seconds.
+ source: https://wise.com/gb/blog/q4-2024-mission-update-speed
+ failures:
+ ❌ overreach: 65% of transfers now complete in under 20 seconds.
+
+== Logged ==
+ outcome: review
+ written to: pitch_pilot_store.review.jsonl
+ (pitch-pilot never auto-sends — a human approves before anything goes out.)
+```
+
+## What it does
+
+A deterministic five-step loop over a single domain:
+
+`research → qualify → draft → verify → log`
+
+**Research** runs an agentic, RAG-style retrieval sub-loop (the LLM plans queries →
+search → fetch → extract cited facts). **Qualify** scores the company against a
+declarative ICP. **Draft** writes outreach grounded only in first-party facts.
+**Verify** audits that draft against its sources. **Log** files the lead for a human
+as `ready`, `review` (needs edits), or `disqualified` — never sending anything.
+
+## The differentiator — groundedness
+
+Most "AI SDR" tools generate fluent outreach that is confidently wrong: invented
+funding rounds, misattributed quotes, hallucinated headcounts. pitch-pilot makes
+that structurally hard, in four layers:
+
+1. **Extraction-time grounding.** The atomic unit of research is a typed `Fact` that
+ *cannot be constructed without an `http(s)` source URL*, and the extractor keeps
+ only claims whose verbatim evidence is a literal substring of the fetched page.
+ An ungrounded fact is unrepresentable — not caught after the fact, but impossible.
+2. **Source tiering.** Every fact is tagged `own_site` / `authoritative` /
+ `third_party_snippet` by how durable and trustworthy its source is.
+3. **First-party-only drafting.** Outreach may be grounded *only* in `own_site` /
+ `authoritative` facts. The model selects which facts to stand on **by id**, so the
+ hooks are grounded by construction — it can paraphrase freely, but it cannot
+ fabricate.
+4. **LLM faithfulness judge.** A judge reads the drafted body against the selected
+ facts and rates every claim `faithful` / `overreach` / `unsupported`. A draft
+ passes only if nothing is unsupported (and nothing overreaches, under strict mode).
+
+The payoff is outreach you can audit sentence by sentence. Deep dive:
+[Groundedness methodology](docs/groundedness.md).
+
+## Architecture
+
+```mermaid
+flowchart LR
+ domain(["domain"]) --> research["research
(agentic sub-loop)"]
+ research --> qualify{"qualify"}
+ qualify -- "disqualified" --> log["log"]
+ qualify -- "qualified" --> draft["draft"]
+ draft --> verify{"verify
(groundedness gate)"}
+ verify -- "pass / fail" --> log
+ log --> out(["outcome: ready / review / disqualified"])
+```
+
+**Hybrid by design:** a *deterministic outer graph* runs the fixed business steps in
+a known, auditable order, while an *agentic sub-loop* runs inside the research step —
+where open-ended exploration actually helps. (See [ADR-0003](docs/decisions.md).)
+
+**Stack:** Python 3.11+ · **LangGraph** (outer graph) · **pydantic v2** (typed
+contracts) · pluggable LLMs — **Cerebras / Groq / Gemini** (swappable behind one
+interface) · Tavily search · httpx + selectolax fetch. Runs entirely on free tiers
+(**$0**). More in [Architecture](docs/architecture.md) and [Pipeline](docs/pipeline.md).
+
+## Limitations
+
+Deliberate scope, stated plainly:
+
+- **Small samples.** Held-out n=8, development n=17, human-proposed labels, a single
+ run each. F1 1.0 on eight companies is encouraging, not conclusive — a larger
+ held-out set is the next step before the headline is bankable.
+- **One residual qualification miss + run-to-run variance.** Making `industry=unknown`
+ count against a company (the fix) costs recall when the assessor *flakily* fails to
+ confirm a real fintech's industry — it cost one false negative (`ramp.com`) on the
+ dev set. More broadly, Cerebras is not bit-deterministic even at temperature 0, so a
+ company's draft/verdict can vary between runs; these are single runs, not averages.
+- **Draft-gate overreach (noted, not fixed this pass).** The gate rejects a real share
+ of *qualified* drafts (≈44% pre-fix, ≈22–25% after) — the **drafter** over-claims
+ beyond the cited facts and the gate correctly catches it. Tightening the draft prompt
+ is the next step ([details](docs/evals.md)).
+- **Human-in-the-loop.** It never auto-sends; every lead lands in a review queue.
+- **No LinkedIn scraping** — out of scope by design.
+- **Lead discovery is future work** — today you supply the domain.
+
+## Quickstart (Windows / PowerShell)
```powershell
-pytest
+py -3.11 -m venv .venv
+.\.venv\Scripts\Activate.ps1
+pip install -e ".[dev]"
+Copy-Item .env.example .env # then add GEMINI_API_KEY + TAVILY_API_KEY (other keys optional)
+python -m pitch_pilot.cli smoke # verifies search + LLM + fetch with your keys
+python -m pitch_pilot.cli run ramp.com --icp examples/eval_icp.json
```
-## Project layout
+Unit tests are fully mocked — **no keys, no network**: `pytest`. Full setup and the
+Windows `.env` gotcha: [Getting Started](docs/getting-started.md).
+
+## Documentation
-See [`docs/architecture.md`](docs/architecture.md) for the annotated directory
-tree and the end-to-end data flow.
+- **[Full docs site](docs/index.md)** — narrative guides + an API reference
+ auto-generated from docstrings (`mkdocs serve`).
+- **[Groundedness methodology](docs/groundedness.md)** — the hero guarantee in depth.
+- **[Evaluation](docs/evals.md)** — dataset, labeling rubric, metrics, and the numbers above.
+- **[Design decisions (ADRs)](docs/decisions.md)** — why it is built this way.
## License
diff --git a/docs/components/clients.md b/docs/components/clients.md
index 6237e08..a8b2748 100644
--- a/docs/components/clients.md
+++ b/docs/components/clients.md
@@ -1,6 +1,8 @@
# Clients
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/clients/`
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/clients/`
+>
+> P4-era: a Cerebras provider was added so the eval can run on its ~1M tokens/day free tier (see ADR-0013).
The `clients` package is pitch-pilot's swappable external-service layer. Every call that leaves the process — an LLM completion, a web search, an HTTP page fetch — goes through a small interface defined here. The rest of the pipeline depends on those interfaces, never on a vendor SDK, so providers can be swapped by configuration and the network can be mocked at a single seam in tests.
@@ -12,7 +14,7 @@ There are three concerns, each with its own interface:
| Web search | `SearchClient` (Protocol) | `TavilyClient` | `get_search_client()` |
| Page fetch | `fetch_page()` (plain function) | httpx + selectolax | — |
-All three are re-exported from the package root (`pitch_pilot.clients`), alongside `GroqClient`, `LLMError`, and `LLMJSONError`. See the API Reference (in the nav) for full signatures.
+All three are re-exported from the package root (`pitch_pilot.clients`), alongside `GroqClient`, `CerebrasClient`, `LLMError`, and `LLMJSONError`. See the API Reference (in the nav) for full signatures.
## Lazy SDK imports
@@ -20,6 +22,7 @@ Every concrete client imports its vendor SDK **lazily**, inside the method that
- `GeminiClient` imports `from google import genai` (and `google.genai.types`) only when it builds its client or makes a call.
- `GroqClient` imports `from groq import Groq` only on first use.
+- `CerebrasClient` imports `from cerebras.cloud.sdk import Cerebras` only on first use.
- `TavilyClient` imports `from tavily import TavilyClient` only on first use.
Two consequences follow, and they are the reason for the pattern:
@@ -35,8 +38,12 @@ Each client also caches its underlying SDK client after first construction (an `
| Method | Returns | Notes |
| --- | --- | --- |
-| `complete(system, user)` | `str` | Free-text completion, stripped of surrounding whitespace. |
-| `complete_json(system, user)` | `dict` | A parsed JSON **object**. Raises `LLMJSONError` on bad JSON. |
+| `complete(system, user, temperature=None)` | `str` | Free-text completion, stripped of surrounding whitespace. |
+| `complete_json(system, user, temperature=None)` | `dict` | A parsed JSON **object**. Raises `LLMJSONError` on bad JSON. |
+
+`temperature` is optional (the provider default when `None`); it is sent only when
+set, so it stays backward-compatible. Gate-critical calls (draft, verify judge) pass
+`0.0` for reproducible output.
Both take a `system` prompt (role/behavior) and a `user` prompt (the request). The error hierarchy is `LLMError(RuntimeError)` with `LLMJSONError(LLMError)` for JSON-parse failures specifically.
@@ -58,14 +65,19 @@ It raises `LLMJSONError` when the text is empty, is not valid JSON, or parses to
**`GroqClient`** — backed by the official Groq SDK, which is OpenAI-compatible. Both methods call `client.chat.completions.create(...)` with `system` and `user` messages. `complete_json` sets `response_format={"type": "json_object"}` and, because Groq's JSON mode *requires* an explicit instruction, injects a "respond with a single valid JSON object" line into the system message (the shared `_json_system` helper). Default model: `llama-3.1-8b-instant`.
+**`CerebrasClient`** — backed by the `cerebras-cloud-sdk` (OpenAI-compatible, base URL `https://api.cerebras.ai/v1`). It mirrors `GroqClient` exactly — same `chat.completions.create(...)` call, same `response_format` + `_json_system` for `complete_json`, same lenient parsing. **Why it exists:** budget. The full eval needs many tokens, and Cerebras's free tier allows ~1M tokens/day (~10x Groq's ~100k), enough to run the whole eval set in one session (see [Evaluation](../evals.md) and [ADR-0013](../decisions.md)). Default model: `gpt-oss-120b` — **available models vary by account/tier** (check the SDK's `models.list()`); Llama-3.3-70B is not on every free-tier account. **Context cap:** its free tier limits a single request to **8,192 tokens**; prompt builders bound their variable-length payloads via `trim_to_token_budget` / `CONTEXT_TOKEN_CAP` so no request exceeds it.
+
### Factory: `get_llm_client()`
`get_llm_client(settings=None)` selects the provider from `Settings.llm_provider` (defaulting to the cached process settings when none is passed):
- `gemini` → `GeminiClient(api_key=gemini_api_key, model=gemini_model)`.
- `groq` → `GroqClient(api_key=groq_api_key, model=groq_model)`, but raises `ValueError` if `GROQ_API_KEY` is not set.
+- `cerebras` → `CerebrasClient(api_key=cerebras_api_key, model=cerebras_model)`, but raises `ValueError` if `CEREBRAS_API_KEY` is not set.
- Anything else → `ValueError` naming the unknown provider.
+`Settings.active_model` resolves the model id for the current provider (used by the smoke check and the eval report so the right model is named).
+
Provider name and model defaults are validated in [`configuration.md`](../configuration.md); `llm_provider` is normalized to lowercase and restricted to `gemini` or `groq` at config-load time.
## SearchClient
diff --git a/docs/components/evals.md b/docs/components/evals.md
index 0b4d18c..b520032 100644
--- a/docs/components/evals.md
+++ b/docs/components/evals.md
@@ -1,6 +1,6 @@
# Eval Harness
-> **Last updated:** 2026-06-13 · **Source files:** `evals/run_eval.py`, `evals/metrics.py`
+> **Last updated:** 2026-06-14 · **Source files:** `evals/run_eval.py`, `evals/metrics.py`
The eval harness produces the headline numbers in [Evaluation](../evals.md). It
lives outside the package (`evals/`, not `src/`) because it is tooling, not library
@@ -9,11 +9,12 @@ It is designed around free-tier rate limits: **cache, checkpoint, back off, resu
## `run_eval.py` — the runner
-Three commands via `python -m evals.run_eval `:
+Four commands via `python -m evals.run_eval `:
| Command | What it does | Network |
| --- | --- | --- |
| `run` (default) | Evaluate each company: research (cached) → qualify → draft → verify; checkpoint each result; write a report; print aggregates. | LLM + search |
+| `redraft` | Re-run **only** draft + verify for already-qualified companies, reusing cached research **and** each record's frozen qualification verdict; rewrite the results file in place. Lets the draft/verify logic change without re-researching or perturbing the qualification matrix. | LLM only |
| `recheck` | Re-fetch each used claim's source and confirm the evidence still appears → live-verifiability by tier. | HTTP fetch only |
| `report` | Recompute metrics from existing results (+ recheck cache) and rewrite the report. | none |
diff --git a/docs/components/nodes.md b/docs/components/nodes.md
index 7793069..7516f04 100644
--- a/docs/components/nodes.md
+++ b/docs/components/nodes.md
@@ -1,8 +1,8 @@
# Nodes
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/nodes/`, `src/pitch_pilot/graph/state.py`
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/nodes/`, `src/pitch_pilot/graph/state.py`
>
-> P3 hardened the verify node (first-party tier + substring + LLM faithfulness judge) and made Policy B claim-gating the rule in the draft node.
+> P3 hardened the verify node and made Policy B claim-gating the rule in the draft node. **0.8.0** decoupled grounding from phrasing: the draft grounds by *selecting facts* (no verbatim-hook substring check), and verify judges the **body**'s faithfulness to the selected facts (see [ADR-0014](../decisions.md)).
**Status: all five nodes implemented (research in P1; qualify, draft, verify, log in P2).** Each node lives in its own module under `src/pitch_pilot/nodes/`, and `build_pipeline()` wires them into the outer graph (see [`graph.md`](graph.md) and [`../pipeline.md`](../pipeline.md)). Every node follows the same shape: a pure `run_*(...)` function that takes its dependencies as arguments (trivially testable offline) plus a thin `*_node(state)` graph adapter that can be handed injected clients.
@@ -30,8 +30,8 @@ The `PipelineState` contract itself — including the `status` and `errors` book
| --- | --- | --- | --- |
| `research_node` | `company` | `research` (`ResearchResult`) | **Implemented (P1).** Runs the agentic research loop (seed → plan → search → extract Facts), bounded by `RESEARCH_MAX_QUERIES`. Tags each fact with a `source_tier`. |
| `qualify_node` | `research`, `icp` | `qualification` (`QualificationResult`) | **Implemented (P2).** Hybrid: LLM assesses signals vs facts; deterministic code scores + vetoes. Conditional edge stops disqualified leads (→ log), sends qualified ones to draft. |
-| `draft_node` | `research`, `qualification` | `draft` (`Draft`) | **Implemented (P2).** Writes outreach only from grounded facts; tier-gates numerics; validates hooks back to facts. |
-| `verify_node` | `draft`, `research` | `verification` (`VerificationResult`) | **Hardened (P3).** A claim verifies only if first-party-sourced + substring-anchored + judged `faithful` by the LLM. Draft passes only if every claim verifies. Network-free except the judge call. |
+| `draft_node` | `research`, `qualification` | `draft` (`Draft`) | **Implemented (P2; 0.8.0 selection).** Writes free-prose outreach grounded only in first-party facts the model **selects by id**; `hooks_used` are those facts (grounded by construction). |
+| `verify_node` | `draft`, `research` | `verification` (`VerificationResult`) | **Hardened (P3; 0.8.0 body judge).** Re-resolves hooks to first-party facts (structural), then an LLM judge rates the **body**'s claims `faithful`/`overreach`/`unsupported` against the selected facts. Network-free except the judge call. |
| `log_node` | whole `PipelineState` | persisted `Lead` | **Implemented (P2).** Saves `ready` / enqueues `review` / saves `disqualified` via the Store. Never auto-sends. |
| `discover_node` | — (seed source) | candidate domains | Future seam (P6) that sources new candidate domains to seed runs. |
@@ -67,23 +67,30 @@ the control flow is not a fixed query list. One run proceeds as:
result, run the extractor on its content (the result URL is the source).
5. **Reflect.** De-duplicate by claim, accumulate facts, record the query, loop.
-The dimensions and caps are module constants in `nodes/research.py`:
-`RESEARCH_DIMENSIONS`, `MAX_FACTS_PER_SOURCE`, `SEARCH_RESULTS_PER_QUERY`, and
-`MAX_TEXT_CHARS`.
+**Research depth is tunable and leaned out by default** (cheaper, faster, and
+fits free-tier token caps — see [ADR-0012](../decisions.md)):
+`RESEARCH_MAX_QUERIES` (3), `RESEARCH_MAX_PAGE_CHARS` (3500 — the biggest token
+lever), and `RESEARCH_MAX_FACTS_PER_SOURCE` (5) are [`Settings`](../configuration.md)
+that `run_research` passes through to the extractor. `RESEARCH_DIMENSIONS` and
+`SEARCH_RESULTS_PER_QUERY` remain module constants; `MAX_TEXT_CHARS` /
+`MAX_FACTS_PER_SOURCE` survive only as lean fallback defaults for direct
+`extract_facts` calls.
### The extractor — the groundedness guard
-`extract_facts(text, source_url, source_title, llm, company_domain=None)` is where
-groundedness is enforced at research time. It prompts the model to return **only**
-claims the provided text explicitly supports, each with a verbatim `evidence`
-snippet, a `category`, and a `confidence`. The system prompt forbids using outside
-knowledge. Then, for every candidate, the extractor checks that the `evidence`
-actually appears in the source text (a whitespace- and case-insensitive substring
-match) and **drops** — and logs — any candidate that fails. Surviving facts are
-tagged with a `source_tier` via `classify_source_tier(url, company_domain)`. This
-is a cheap but effective anti-hallucination layer; see
-[groundedness.md](../groundedness.md). Extraction is capped at
-`MAX_FACTS_PER_SOURCE` per source so one page can't dominate.
+`extract_facts(text, source_url, source_title, llm, company_domain=None, *,
+max_page_chars=…, max_facts_per_source=…)` is where groundedness is enforced at
+research time. It first **truncates the source text to `max_page_chars`**, then
+prompts the model to return **only** claims that truncated text explicitly
+supports, each with a verbatim `evidence` snippet, a `category`, and a
+`confidence`. The system prompt forbids using outside knowledge. For every
+candidate, the extractor checks that the `evidence` actually appears in **that same
+truncated text** (a whitespace- and case-insensitive substring match) and
+**drops** — and logs — any candidate that fails: the model can only ground claims
+in the text we verify against, so truncation never weakens the guarantee. Surviving
+facts are tagged with a `source_tier` via `classify_source_tier(url,
+company_domain)`. Extraction stops after `max_facts_per_source` facts so one page
+can't dominate. See [groundedness.md](../groundedness.md).
### Robustness
@@ -110,52 +117,68 @@ and a summary line (facts, distinct sources, and the LLM-chosen queries that ran
- The **LLM assesses**, for each ICP attribute (industry, region, employee count)
and each positive/negative signal, whether the facts support it —
`match` / `no_match` / `unknown`, citing the supporting fact. It does *not* decide
- qualification.
+ qualification. Negative signals are judged from the company's described business
+ (so a negation like "not fintech" fires when the facts show design tools / dev infra
+ / ML hosting, rather than defaulting to `unknown` — [ADR-0015](../decisions.md)).
- **Deterministic code scores.** A weighted blend of industry / size / region /
- positive-signals yields a fit score in `[0, 1]`; *unknown* structural components
- are dropped and the weights renormalized, so a research gap never penalizes.
- Positive signals score as `matched / total`.
+ positive-signals yields a fit score in `[0, 1]`. *Size/region* unknowns are dropped
+ and the weights renormalized (a research gap there shouldn't penalize), but
+ **industry is required when the ICP names target industries**: an `unknown` or
+ `no_match` industry scores 0.0 and stays in the denominator, so a fintech ICP can't
+ qualify a company it can't place in-industry. Positive signals score as
+ `matched / total`.
- **A matched negative signal is a hard veto** — it forces `qualified = False`
regardless of score.
- The company qualifies iff `score >= QUALIFY_THRESHOLD` and nothing vetoed.
Unknowns are never guessed: an unknown signal appears in neither `matched_signals`
-nor `missed_signals`.
+nor `missed_signals`. The assessment call runs at `temperature=0` so the verdict (and
+the headline F1) is reproducible.
-## The draft node (Policy B since P3)
+## The draft node (Policy B since P3; selection since 0.8.0)
`run_draft(research, qualification, llm, settings) -> Draft` writes the outreach
-email, with two groundedness enforcement layers (see
-[groundedness.md](../groundedness.md)):
+email, grounding it by **fact-selection** (see [groundedness.md](../groundedness.md)
+and [ADR-0014](../decisions.md)):
- **First-party claim pool.** Only `own_site` / `authoritative` facts are offered
- to the model as *claimable*. `third_party_snippet` facts are passed in a separate
- *context-only* section — usable for tone or framing, never as a stated claim or a
- hook. (P2 gated only hard numerics; P3's Policy B extends this to **all** claims.)
-- **Validated outputs.** Each hook the model returns is matched back to a claimable
- fact; anything that doesn't map to one is discarded, so `hooks_used` is always a
- subset of the first-party research facts.
-
-## The verify node (P3 — the real gate)
-
-`run_verification(draft, research, llm, settings) -> VerificationResult` audits
-every claim (the draft's `hooks_used`) through four checks (see the
-[Groundedness methodology](../groundedness.md)). A claim is **verified** iff it is:
-
-1. **backed** by a `Fact`;
-2. **first-party** — tier `own_site` or `authoritative` (Policy B; a
- `third_party_snippet` backing is a hard `volatile-source` failure);
-3. **substring-anchored** — the backing fact carries verbatim `evidence`
- (`substring_ok`); and
-4. **faithful** — `judge_faithfulness(claim, evidence, llm)` returns `faithful`
- (or `overreach` when `FAITHFULNESS_STRICT` is off). The judge fails closed.
-
-The draft **passes only if every claim verifies.** Each failure is recorded in
-`flagged_claims` with its reason (`unbacked` / `volatile-source` / `not-substring`
-/ `overreach` / `unsupported`), and the per-claim audit trail is returned in
-`VerificationResult.claim_verdicts`, alongside `groundedness_score`,
-`faithfulness_score`, and `tier_breakdown`. The node is **network-free except for
-the judge call** — independent live re-verification of sources is an eval-time
+ to the model as *claimable*, on **numbered** lines. `third_party_snippet` facts are
+ passed in a separate *context-only* section — usable for tone or framing, never as
+ a stated claim. (P2 gated only hard numerics; P3's Policy B extends this to **all**
+ claims.)
+- **Selection by id, not by copy.** The model writes the body as free prose (it may
+ paraphrase) and returns the **ids** of the claimable facts it grounded the email
+ in. `hooks_used` is the canonical claim text of those facts — a subset of the
+ first-party research facts, grounded **by construction**. The draft layer does
+ **not** substring- or fuzzy-match hook text against the source.
+- **No id leakage in the prose.** The fact ids are a selection mechanism for
+ `facts_used` only; the prompt forbids them in the subject/body, and a regex
+ backstop (`_strip_fact_ids`) removes any `(Fact N)`-style token that slips through.
+- **Deterministic drafting.** The draft call runs at `temperature=0` for reproducible
+ output.
+
+## The verify node (P3 hardening; 0.8.0 body-faithfulness)
+
+`run_verification(draft, research, llm, settings) -> VerificationResult` audits the
+draft in two parts (see the [Groundedness methodology](../groundedness.md)):
+
+1. **Structural — the grounding facts.** Each `hook_used` is re-resolved to a
+ first-party `Fact`; any that fails to resolve is a `structural` failure (an
+ invariant violation — hooks are first-party by construction).
+2. **Faithfulness — the body.** One LLM judge (`judge_body`, run at `temperature=0`)
+ reads the draft **body** and the selected facts, extracts every factual claim the
+ body makes about the company, and rates each `faithful` / `overreach` /
+ `unsupported`. The judge fails closed (any error or malformed response fails the
+ draft).
+
+The draft **passes** iff it has a grounded hook, a non-empty body, the judge ran,
+and no body claim is `unsupported` (and none is `overreach` under
+`FAITHFULNESS_STRICT`). Each failure is recorded in `flagged_claims` with its reason
+(`structural` / `overreach` / `unsupported` / `judge-error`); the per-body-claim
+audit trail is returned in `VerificationResult.claim_verdicts`, alongside
+`groundedness_score` (faithful body claims / total), `faithfulness_score`, and
+`tier_breakdown` (the grounding hooks by tier). The node is **network-free except
+for the judge call** — independent live re-verification of sources is an eval-time
metric (P4), not part of the per-run path.
## The log node (P2)
diff --git a/docs/configuration.md b/docs/configuration.md
index 4477737..fdd356f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1,6 +1,6 @@
# Configuration
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/config.py`, `.env.example`
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/config.py`, `.env.example`
All runtime configuration lives in a single, validated `Settings` object built on Pydantic Settings. pitch-pilot reads its configuration once at startup and fails loudly if anything required is missing or invalid — it never starts a run half-configured. For a step-by-step setup walkthrough, see [Getting Started](getting-started.md).
@@ -45,15 +45,19 @@ One row per field defined in `config.py`:
| `GEMINI_API_KEY` | Yes | — | Google Gen AI (Gemini) API key. |
| `TAVILY_API_KEY` | Yes | — | Tavily search API key (used by the research sub-loop). |
| `GROQ_API_KEY` | Only when `LLM_PROVIDER=groq` | `None` | Groq API key. |
-| `LLM_PROVIDER` | No | `gemini` | Active LLM provider — `"gemini"` or `"groq"`. |
+| `CEREBRAS_API_KEY` | Only when `LLM_PROVIDER=cerebras` | `None` | Cerebras API key. Its ~1M tokens/day free tier (~10x Groq) is used to run the full eval in one session (ADR-0013). |
+| `LLM_PROVIDER` | No | `gemini` | Active LLM provider — `"gemini"`, `"groq"`, or `"cerebras"`. |
| `GEMINI_MODEL` | No | `gemini-2.5-flash-lite` | Gemini model id. |
| `GROQ_MODEL` | No | `llama-3.1-8b-instant` | Groq model id. |
-| `RESEARCH_MAX_QUERIES` | No | `4` | Max search queries per research run. |
+| `CEREBRAS_MODEL` | No | `gpt-oss-120b` | Cerebras model id. Available models vary by account/tier — check the provider's `models.list()`. |
+| `RESEARCH_MAX_QUERIES` | No | `3` | Max search queries per research run. |
+| `RESEARCH_MAX_PAGE_CHARS` | No | `3500` | Max characters of each source's text fed to the extractor. The biggest token lever; truncation preserves the evidence-substring check (ADR-0012). |
+| `RESEARCH_MAX_FACTS_PER_SOURCE` | No | `5` | Max facts extracted from a single source, so one page can't dominate. |
| `QUALIFY_THRESHOLD` | No | `0.5` | Minimum fit score for a company to qualify against the ICP. A matched negative signal vetoes qualification regardless. |
-| `GROUNDEDNESS_THRESHOLD` | No | `0.9` | Minimum groundedness score for a draft to pass verification. With first-party-only enforcement a passing draft scores 1.0, so this is effectively a floor; kept for transparency. |
+| `GROUNDEDNESS_THRESHOLD` | No | `0.9` | Floor kept for transparency/future tuning. The verify gate is **all-or-nothing on faithfulness** (pass iff no body claim is `unsupported`, and none `overreach` under strict), not a score cutoff; under strict mode a passing draft scores 1.0. |
| `FAITHFULNESS_STRICT` | No | `true` | When `true`, an `overreach` faithfulness verdict fails the verify gate; when `false`, only `unsupported` fails. `unsupported` always fails. |
-The two LLM providers and their keys/models are consumed when building model clients — see [components/clients.md](components/clients.md).
+The LLM providers and their keys/models are consumed when building model clients — see [components/clients.md](components/clients.md).
## Validation rules
@@ -61,14 +65,17 @@ These constraints are enforced when `Settings` loads; violating any of them prod
| Rule | Field |
| --- | --- |
-| Must be `"gemini"` or `"groq"` (trimmed and lower-cased before checking). | `LLM_PROVIDER` |
+| Must be `"gemini"`, `"groq"`, or `"cerebras"` (trimmed and lower-cased before checking). | `LLM_PROVIDER` |
| Must be an integer `>= 1`. | `RESEARCH_MAX_QUERIES` |
+| Must be an integer `>= 500`. | `RESEARCH_MAX_PAGE_CHARS` |
+| Must be an integer `>= 1`. | `RESEARCH_MAX_FACTS_PER_SOURCE` |
| Must be a float in the closed range `[0, 1]`. | `QUALIFY_THRESHOLD` |
| Must be a float in the closed range `[0, 1]`. | `GROUNDEDNESS_THRESHOLD` |
| Boolean (`true`/`false`, case-insensitive; also `1`/`0`). | `FAITHFULNESS_STRICT` |
| Required only when `LLM_PROVIDER=groq`; otherwise optional and defaults to `None`. | `GROQ_API_KEY` |
+| Required only when `LLM_PROVIDER=cerebras`; otherwise optional and defaults to `None`. | `CEREBRAS_API_KEY` |
-The `LLM_PROVIDER` validator normalizes its input, so `GEMINI`, ` gemini `, and `Gemini` all resolve to the canonical `"gemini"`.
+The `LLM_PROVIDER` validator normalizes its input, so `GEMINI`, ` gemini `, and `Gemini` all resolve to the canonical `"gemini"` (likewise `groq` / `cerebras`).
## Sample `.env`
@@ -79,18 +86,22 @@ Copy `.env.example` to `.env` and fill in your keys. The defaults below match th
GEMINI_API_KEY=your-gemini-api-key
TAVILY_API_KEY=tvly-your-tavily-api-key
-# --- Optional (only needed when LLM_PROVIDER=groq) ---
+# --- Optional (only needed when the matching LLM_PROVIDER is selected) ---
GROQ_API_KEY=your-groq-api-key
+CEREBRAS_API_KEY=your-cerebras-api-key
-# --- Provider selection: "gemini" (default) or "groq" ---
+# --- Provider selection: "gemini" (default), "groq", or "cerebras" ---
LLM_PROVIDER=gemini
# --- Model ids (current free-tier defaults shown) ---
GEMINI_MODEL=gemini-2.5-flash-lite
GROQ_MODEL=llama-3.1-8b-instant
+CEREBRAS_MODEL=gpt-oss-120b
# --- Research / qualification / grounding tunables ---
-RESEARCH_MAX_QUERIES=4
+RESEARCH_MAX_QUERIES=3
+RESEARCH_MAX_PAGE_CHARS=3500
+RESEARCH_MAX_FACTS_PER_SOURCE=5
QUALIFY_THRESHOLD=0.5
GROUNDEDNESS_THRESHOLD=0.9
FAITHFULNESS_STRICT=true
diff --git a/docs/data-models.md b/docs/data-models.md
index 9250b20..3ad3892 100644
--- a/docs/data-models.md
+++ b/docs/data-models.md
@@ -1,6 +1,6 @@
# Data Models
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/models/`
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/models/`
pitch-pilot's data contracts are a set of [pydantic](https://docs.pydantic.dev/) models — one model per artifact that flows through the pipeline. They are the typed boundary between nodes: each node reads some models off the shared state and writes others back. This page documents every model and field exactly as defined in `src/pitch_pilot/models/`.
@@ -108,38 +108,38 @@ Each fact is also tagged with a **`source_tier`** that records how trustworthy i
## Draft
-`Draft` is the outreach email produced for a lead. The [`draft_node`](pipeline.md) writes it only from grounded `Fact` objects, and every hook it claims to use is validated back against the facts — so `hooks_used` is always a subset of the real research facts. Hard-numeric claims (funding, headcount, …) from `third_party_snippet` facts are withheld from drafting entirely. It is then checked by the verification step before a human reviews it. pitch-pilot never sends a `Draft` automatically.
+`Draft` is the outreach email produced for a lead. The [`draft_node`](pipeline.md) grounds it by **selecting facts**: the model is shown the first-party (`own_site`/`authoritative`) facts and returns the ids of the ones it grounded the email in, so `hooks_used` is always a subset of the real research facts (grounded by construction — see [ADR-0014](decisions.md)). The body is free prose the model may paraphrase. It is then checked by the verification step before a human reviews it. pitch-pilot never sends a `Draft` automatically.
| Field | Type | Default | Purpose |
| --- | --- | --- | --- |
| `subject` | `str` | required | The email subject line. |
-| `body` | `str` | required | The email body. |
-| `hooks_used` | `list[str]` | `[]` (empty list) | The exact `Fact` claims the draft referenced — each traces back to a grounded source. Validated against the research facts, so a hook the model invents is discarded. |
+| `body` | `str` | required | The email body (free prose; the verify node judges its faithfulness). |
+| `hooks_used` | `list[str]` | `[]` (empty list) | The canonical claim text of the first-party `Fact` objects the model selected to ground the email — each traces back to a grounded source. Grounded by construction (selection is by fact id), so an invented selection resolves to nothing and is dropped. |
## VerificationResult
-`VerificationResult` is the groundedness audit of a `Draft` and the enforcement point for the hero guarantee. The [`verify_node`](pipeline.md) checks each claim (the draft's `hooks_used`) and **passes the draft only if every claim is *verified*** — backed by a first-party (`own_site`/`authoritative`) `Fact`, substring-anchored, and judged `faithful` by the LLM (or `overreach` when `FAITHFULNESS_STRICT` is off). The full methodology, and how each score is defined, lives in [groundedness.md](groundedness.md).
+`VerificationResult` is the groundedness audit of a `Draft` and the enforcement point for the hero guarantee. The [`verify_node`](pipeline.md) re-resolves the draft's hooks to first-party facts (structural), then runs an LLM judge over the **body**'s claims against those facts. It **passes the draft only if** it is grounded, the body is non-empty, the judge ran, and no body claim is `unsupported` (and none is `overreach` when `FAITHFULNESS_STRICT`). The full methodology, and how each score is defined, lives in [groundedness.md](groundedness.md).
| Field | Type | Default | Purpose |
| --- | --- | --- | --- |
-| `groundedness_score` | `float` | required | Fraction of claims fully verified (`grounded_claims / total_claims`), in `[0.0, 1.0]`. Reported even when the draft passes. |
-| `faithfulness_score` | `float` | `0.0` | Fraction of claims the judge rated `faithful` (`faithful_claims / total_claims`), in `[0.0, 1.0]`. |
-| `total_claims` | `int` | required | Total number of claims checked (the draft's hooks). |
-| `grounded_claims` | `int` | required | Number of claims that are fully verified (the numerator of `groundedness_score`). |
-| `tier_breakdown` | `dict[str, int]` | `{}` | Count of claims per backing source tier, e.g. `{"own_site": 2, "unbacked": 1}`. |
-| `claim_verdicts` | `list[ClaimVerdict]` | `[]` (empty list) | The per-claim audit trail (see `ClaimVerdict` below). |
-| `flagged_claims` | `list[str]` | `[]` (empty list) | Failure lines for claims that did not verify, each prefixed with the reason: `unbacked:` / `volatile-source:` / `not-substring:` / `overreach:` / `unsupported:`. |
-| `passed` | `bool` | required | True only if there is at least one claim and **every** claim is verified. |
+| `groundedness_score` | `float` | required | Fraction of body claims verified (`grounded_claims / total_claims`), in `[0.0, 1.0]`; a claim is verified when judged `faithful` (or `overreach` when not strict). `1.0` for a grounded body with no checkable claim; `0.0` for an ungrounded/empty-body draft. |
+| `faithfulness_score` | `float` | `0.0` | Fraction of body claims the judge rated `faithful` (`faithful_claims / total_claims`), in `[0.0, 1.0]`. |
+| `total_claims` | `int` | required | Number of body claims the judge extracted and checked. |
+| `grounded_claims` | `int` | required | Number of body claims that count as verified (the numerator of `groundedness_score`). |
+| `tier_breakdown` | `dict[str, int]` | `{}` | Count of the draft's grounding facts (hooks) per source tier, e.g. `{"own_site": 2}`. |
+| `claim_verdicts` | `list[ClaimVerdict]` | `[]` (empty list) | The per-body-claim audit trail (see `ClaimVerdict` below). |
+| `flagged_claims` | `list[str]` | `[]` (empty list) | Failure lines, each prefixed with the reason: `structural:` / `overreach:` / `unsupported:` / `judge-error:`. |
+| `passed` | `bool` | required | True only if the draft is grounded, the body is non-empty, the judge ran, and no body claim failed. |
## ClaimVerdict
-`ClaimVerdict` is the per-claim audit trail produced by the verify node — one per claim the draft stands on, pass or fail — so a reviewer can see exactly why each claim was accepted or rejected.
+`ClaimVerdict` is the per-body-claim audit trail produced by the verify node — one per factual claim the body makes about the company, pass or fail — so a reviewer can see which fact (if any) backs each claim and how the judge rated it.
| Field | Type | Default | Purpose |
| --- | --- | --- | --- |
-| `claim` | `str` | required | The draft claim under audit (a `Draft` hook). |
-| `fact_used` | `str \| None` | `None` | The claim text of the backing `Fact` chosen to support it; `None` if unbacked. |
-| `source_url` | `str \| None` | `None` | The backing fact's source URL; `None` if unbacked. |
-| `tier` | `str \| None` | `None` | The backing fact's source tier; `None` if unbacked. |
-| `substring_ok` | `bool` | `False` | Whether the backing fact carries a verbatim `evidence` snippet (the extraction-time substring guard held). |
-| `faithfulness` | `Literal \| None` | `None` | The judge verdict (`faithful`/`overreach`/`unsupported`), or `None` when the claim failed an earlier check and was not judged. |
+| `claim` | `str` | required | The body claim under audit (extracted by the faithfulness judge). |
+| `fact_used` | `str \| None` | `None` | The claim text of the supporting `Fact` the judge cited; `None` when `unsupported`. |
+| `source_url` | `str \| None` | `None` | The supporting fact's source URL; `None` when unsupported. |
+| `tier` | `str \| None` | `None` | The supporting fact's source tier; `None` when unsupported. |
+| `substring_ok` | `bool` | `False` | Whether the supporting fact carries a verbatim `evidence` snippet (the extraction-time substring guard held). |
+| `faithfulness` | `Literal \| None` | `None` | The judge verdict for this body claim (`faithful`/`overreach`/`unsupported`). |
diff --git a/docs/decisions.md b/docs/decisions.md
index 412e7d7..ad9d68a 100644
--- a/docs/decisions.md
+++ b/docs/decisions.md
@@ -266,3 +266,168 @@ appended at the bottom. Format: **Date · Status · Context · Decision · Conse
verification. The 8B model malforms the structured JSON under load, so headline
numbers are produced on a capable model (Groq 70B or Gemini), recorded alongside
the numbers.
+
+## ADR-0012 — Research depth is leaned out by default (quota + cost/latency), and the eval runs at shipping depth
+
+- **Date:** 2026-06-13
+- **Status:** Accepted
+- **Context:** The first full-eval attempt showed one company's research consuming
+ ~40k tokens — feeding the extractor up to ~12,000 characters per source across
+ ~13 extraction calls. That blew Groq's free-tier **100k tokens/day** cap after
+ ~2 companies, stalling the eval ([ADR-0011](#adr-0011-the-eval-set-includes-negatives--sparse-the-harness-caches-checkpoints-and-backs-off)).
+ But the deeper point is that this cost is real in production too: most of those
+ characters are boilerplate/nav/footer text that adds little grounded signal. A
+ leaner research pass is both the quota fix **and** a genuine cost/latency
+ improvement — and it must be the *default*, so the eval measures the same depth we
+ actually ship (no eval-vs-prod mismatch).
+- **Decision:** Make research depth tunable via `Settings` and adopt **leaner
+ defaults** as the shipping configuration:
+ - `RESEARCH_MAX_QUERIES` 4 → **3**;
+ - `RESEARCH_MAX_PAGE_CHARS` (new) → **3500** — truncate each source's text fed
+ to the extractor (was ~12,000). This is the dominant token lever;
+ - `RESEARCH_MAX_FACTS_PER_SOURCE` 8 → **5**.
+ Crucially, truncation happens **once**, before extraction, and the
+ evidence-substring grounding check runs against that *same* truncated text — so
+ the model can only ground claims in what we verify, and groundedness is fully
+ preserved.
+- **Consequences:** Per-company research token use drops by roughly half (extraction
+ input falls from ~3,000 to ~900 tokens/call), so more companies fit in the daily
+ cap and every run is cheaper and faster — at full shipping fidelity. The trade-off
+ is slightly thinner research (fewer queries, fewer facts/source, less page text),
+ which can lower recall on facts buried deep in a long page; we accept that for the
+ cost/latency/quota win and because the highest-value facts are near the top of a
+ company's own pages. The caps are env-tunable for runs that want more depth.
+
+## ADR-0013 — Add Cerebras as a provider so the eval runs in one session on its ~1M tokens/day free tier
+
+- **Date:** 2026-06-13
+- **Status:** Accepted
+- **Context:** Even after leaning out research depth ([ADR-0012](#adr-0012-research-depth-is-leaned-out-by-default-quota--costlatency-and-the-eval-runs-at-shipping-depth)),
+ the full eval (~17 companies) needs more tokens than Groq's free **100k/day** cap
+ allows in a single day, so a complete, reproducible run kept spilling across days
+ (and stalling on per-minute limits). Gemini's free tier is even tighter (20
+ requests/day). We wanted to finish the eval in one session without paying.
+- **Decision:** Add **Cerebras** as a third LLM provider. Its free tier allows
+ **~1M tokens/day (~10x Groq)** — enough to run the whole eval set in one session.
+ `CerebrasClient` uses the OpenAI-compatible `cerebras-cloud-sdk` and mirrors
+ `GroqClient` (same JSON-mode + lenient parsing, same error normalization to
+ `LLMError`). Its free tier caps a single request at **8,192 tokens**, so prompt
+ builders bound their variable-length facts payloads (`trim_to_token_budget` /
+ `CONTEXT_TOKEN_CAP`) to stay under it; the lean page cap already keeps extraction
+ prompts small.
+ - **Model note:** available models vary by Cerebras account/tier. Our free-tier
+ account does **not** offer Llama-3.3-70B (only `gpt-oss-120b` and `zai-glm-4.7`),
+ so the default is **`gpt-oss-120b`** (a capable model that handles the structured
+ JSON well). Comparability is preserved a different way: the full eval is re-run
+ **fresh and internally consistent on this one model**, and the earlier
+ full-depth Groq partial is **discarded** (its cache cleared) rather than mixed —
+ so the headline numbers come from a single model + depth, recorded with the run.
+- **Consequences:** The eval can run end-to-end in one session on a capable 70B
+ model at no cost, and the provider seam ([ADR-0002](#adr-0002-external-providers-sit-behind-thin-client-interfaces))
+ made this a pure addition — one new client + one factory branch, no pipeline
+ changes. The trade-offs: a new optional dependency (`cerebras-cloud-sdk`, lazily
+ imported so it is only needed when the provider is selected), and a tighter 8,192-
+ token context window that the prompt-budget guard must respect (a useful
+ discipline regardless of provider).
+
+## ADR-0014 — Draft grounding is decoupled: source-substring lives at extraction; draft grounding = fact-selection + a body-faithfulness judge
+
+- **Date:** 2026-06-14
+- **Status:** Accepted
+- **Context:** The P3 draft/verify design asked the model to copy a fact's text
+ **verbatim** into `hooks`, then validated each hook as a substring of the source
+ and ran a faithfulness judge per claim↔evidence pair. This was brittle in exactly
+ the wrong direction: a *faithful paraphrase* — the normal, desirable way to write
+ outreach — was discarded because it was not a verbatim copy, while the check added
+ no real safety (the fact it pointed at had already passed the extraction-time
+ substring guard). The first full Cerebras run showed the damage: most qualified
+ companies produced `draft_passed=false`, `groundedness=0.0` with **empty**
+ `claim_verdicts` — the hooks simply failed to match, so there was nothing left to
+ judge. The numbers were an artifact of the matching layer, not of grounding.
+- **Decision:** Move the source-text grounding check to where it belongs —
+ **extraction**, where a `Fact`'s `evidence` is already substring-verified against
+ its source ([ADR-0007](#adr-0007-facts-carry-verbatim-evidence-verified-by-a-substring-check))
+ — and split the draft-time guarantee into two parts:
+ 1. **Draft grounding = fact-selection.** The model is shown the claimable
+ (first-party, Policy B) facts as a **numbered** list and returns the **ids** of
+ the facts it grounded the email in. `hooks_used` is the canonical claim text of
+ those facts, so each hook is a real first-party fact **by construction**. The
+ body is free prose the model may paraphrase. The draft layer does **not**
+ re-substring-check or fuzzy-match hook text against the source.
+ 2. **Verify = structural assert + body-faithfulness judge.** The verify node
+ re-resolves each hook to a first-party fact (a `structural` failure if any does
+ not — an invariant violation), then runs **one** LLM judge over the draft
+ **body** against the selected facts, extracting every factual claim the body
+ makes about the company and rating it `faithful` / `overreach` / `unsupported`.
+ The draft passes iff it is grounded, the body is non-empty, the judge ran, and
+ no body claim is `unsupported` (and none is `overreach` under
+ `FAITHFULNESS_STRICT`). `groundedness_score` is redefined as
+ `faithful_body_claims / total_body_claims` (verified / total under strict mode).
+ - **Why this is stronger than verbatim-hook validation.** It checks the thing that
+ actually matters — *does the prose the human will send overstate the sources?* —
+ instead of penalizing paraphrase. Source-text grounding is not weakened: it
+ still happens, once, at extraction, against the exact text we verify. And the
+ judge now audits the **body** a reviewer reads, not a verbatim hook list the
+ reader never sees.
+- **Re-running honestly.** Only **draft + verify** were re-run (new `redraft`
+ command), reusing cached research **and** the frozen qualification verdicts, so the
+ qualification matrix is untouched (TP=10, FP=3, TN=4, FN=0; F1 0.870) and the new
+ draft/verify numbers are real and comparable. On `cerebras/gpt-oss-120b`
+ (2026-06-14): draft-gate pass-rate **11/13 = 0.846**, mean groundedness **0.936**,
+ own_site live re-verifiability **0.90 (45/50)**. The three qualification false
+ positives and the *industry-gating* idea that would address two of them are
+ recorded as **future work** in [Evaluation](evals.md) — deliberately **not**
+ applied now, because tuning the scorer on the same 17-company set and reporting the
+ improved number on that set would not be defensible.
+- **Consequences:** Grounding is now robust to paraphrase, the per-run cost drops
+ (one body-judge call instead of one per hook), and the reasons the gate can emit
+ shrink to `unsupported` / `overreach` / `structural` / `judge-error`. The
+ `ClaimVerdict` audit trail now describes **body** claims (with the supporting fact
+ cited by id), and `VerificationResult.tier_breakdown` counts the grounding hooks
+ by tier. The substring-anchoring guarantee is unchanged — it simply lives in one
+ place now.
+
+## ADR-0015 — Qualifier: reliable negative-signal veto + industry as a required component
+
+- **Date:** 2026-06-14
+- **Status:** Accepted
+- **Context:** The temperature-0 re-baseline ([Evaluation](evals.md)) exposed six
+ qualification false positives (F1 0.769, precision 0.625): non-fintech companies
+ (`figma`, `huggingface`, `linear`, `vercel`, `nilenso`) and an incumbent bank
+ (`jpmorganchase`) all qualified. A diagnostic on the raw assessor output found two
+ root causes:
+ 1. **The negative-signal veto under-fired.** Negative signals are phrased as
+ negations ("not fintech or no money movement"), and the assessor — told "if the
+ facts don't establish it, mark it `unknown`; don't guess" — defaulted negations
+ to `unknown` because no fact *literally states* the negation. So the hard veto
+ rarely triggered, and it was flaky run-to-run.
+ 2. **`industry=unknown` was dropped from the score.** Unknown structural components
+ were renormalized out, so a company whose industry could not be confirmed was not
+ penalized; one matched positive signal then cleared the 0.5 threshold (0.53).
+- **Decision:** Two targeted changes in `nodes/qualify.py`:
+ 1. **Judge negative signals from the described business, not literal negation.** The
+ assessor prompt now instructs: for a NEGATIVE signal, decide whether the
+ disqualifying condition is *true of the company given what the facts reveal about
+ its core business* (e.g. "not fintech" matches when the business is design tools
+ / dev infra / ML hosting; "large incumbent bank" matches a major established
+ bank) — use `unknown` only when the facts genuinely don't reveal the domain.
+ "Don't guess" still governs structural attributes and positive signals.
+ 2. **Industry is a required component for an industry-specified ICP.** When
+ `icp.industries` is non-empty, an `unknown` *or* `no_match` industry scores 0.0
+ and stays in the denominator (instead of being dropped), so a fintech ICP does
+ not qualify a company it cannot place in-industry. Size/region unknowns are still
+ dropped — research gaps there are noise, not disqualifying.
+- **Validation (the part that makes this defensible).** The fix was developed against
+ the original 17 and then validated on a **held-out** set of 8 unseen companies
+ (`examples/eval_companies_holdout.json`). Both improved: original 17 F1 **0.769 →
+ 0.947** (all six FPs eliminated, precision → 1.0); held-out F1 **1.0** (4/4 fit,
+ 4/4 not — including an incumbent bank and two borderlines). Improving on a set it
+ was never tuned on is the evidence it generalizes rather than overfits.
+- **Consequences / trade-off:** Making `industry=unknown` count against a company
+ introduced **one false negative** on the dev set — `ramp.com`, when the assessor
+ flakily returned `industry=unknown` for a genuine fintech, fell below threshold
+ (recall 1.0 → 0.9). This is the deliberate precision/recall trade (we accept a rare
+ flaky-industry miss to stop the systematic non-fintech FPs); the held-out set did
+ not exhibit it. A confidence/retry on the industry assessment is a candidate
+ follow-up. The draft-gate overreach rate (drafter over-claiming, caught correctly by
+ the gate) is noted in [Evaluation](evals.md) as a separate, not-yet-fixed finding.
diff --git a/docs/evals.md b/docs/evals.md
index 1094680..abf9aff 100644
--- a/docs/evals.md
+++ b/docs/evals.md
@@ -1,6 +1,6 @@
# Evaluation
-> **Last updated:** 2026-06-13 · **Source files:** `evals/`, `examples/eval_icp.json`, `examples/eval_companies.json`
+> **Last updated:** 2026-06-14 · **Source files:** `evals/`, `examples/eval_icp.json`, `examples/eval_companies.json`
An agent's quality should be *measured, not vibed*. This page defines the eval
dataset, the **labeling rubric** that makes the ground truth defensible, the
@@ -30,6 +30,11 @@ Two files under `examples/`:
jpmorganchase (incumbent-bank negative signal).
- **sparse:** nilenso → not_qualified; fampay, jupiter.money → qualified
(sparse good-fit edge cases) `[VERIFY thin]`.
+- **`eval_companies_holdout.json`** — a **held-out** validation set of 8 companies
+ **not** in the file above (stripe, gocardless, chime, canva, datadoghq, wellsfargo,
+ coinbase, shopify): clear good-fit, clear bad-fit (non-fintech + an incumbent bank),
+ and two borderline. It is the clean test of whether a qualifier change generalizes
+ rather than overfitting the development 17.
## Labeling rubric
@@ -68,8 +73,18 @@ The harness (`evals/run_eval.py`) is built for free-tier reality:
(honoring a provider-supplied retry-after); a company that stays rate-limited is
recorded as an error and retried next run, never aborting the whole run.
-Qualify → draft → verify are cheap and re-run each eval. See
-[ADR-0011](decisions.md) and the [eval harness component](components/evals.md).
+Qualify → draft → verify are cheap and re-run each eval. The **`redraft`** command
+re-runs *only* draft + verify for the already-qualified companies, reusing cached
+research **and** each record's frozen qualification verdict — so the draft/verify
+logic can change without paying for research or perturbing the qualification matrix.
+See [ADR-0011](decisions.md), [ADR-0014](decisions.md), and the
+[eval harness component](components/evals.md).
+
+**Reproducibility.** The three gate-critical LLM calls — qualify, draft, and the
+verify faithfulness judge — run at **temperature 0** so the headline numbers are a
+reproducible run, not a lucky sample. (Cerebras is not bit-deterministic even at
+temperature 0, so a small residual run-to-run variance remains; the table below is
+one canonical run, reported as such.)
### Metrics
@@ -77,9 +92,9 @@ Qualify → draft → verify are cheap and re-run each eval. See
| --- | --- |
| **Qualification accuracy / precision / recall / F1** | Predicted `qualified` vs the verified `label` (positive class = `qualified`). |
| **Draft gate pass-rate** | Of leads the pipeline qualified, the share producing a draft that passes the verify gate. |
-| **Mean groundedness** | Mean `groundedness_score` over drafted companies (verified claims / total). |
-| **Mean faithfulness** | Mean `faithfulness_score` (claims judged `faithful` / total). |
-| **Failure modes** | Counts of verify failures by reason (`unbacked` / `volatile-source` / `not-substring` / `overreach` / `unsupported`) — proves the gate *catches*, not just passes. |
+| **Mean groundedness** | Mean `groundedness_score` over drafted companies. Under the 0.8.0 faithfulness-based definition (see [Groundedness](groundedness.md)) this is *verified body claims / total body claims* — `faithful` body claims (plus `overreach` when not strict) over every factual claim the body makes about the company. |
+| **Mean faithfulness** | Mean `faithfulness_score` — body claims judged `faithful` / total body claims. |
+| **Failure modes** | Counts of verify failures by reason (`unsupported` / `overreach` / `structural` / `judge-error`) — proves the gate *catches*, not just passes. |
| **Live re-verifiability by tier** | From the separate `recheck` command: re-fetch each used claim's source and confirm the evidence still appears, reported **by source tier**. The honest, network-bounded metric ([ADR-0010](decisions.md)). |
| **Degradation** | Mean facts/company for `sparse` vs `good_fit`. |
@@ -87,38 +102,118 @@ Qualify → draft → verify are cheap and re-run each eval. See
```powershell
python -m evals.run_eval --limit 3 # evaluate (cached, checkpointed, resumable)
+python -m evals.run_eval redraft # re-run draft+verify only (qualification frozen)
python -m evals.run_eval recheck # live re-verifiability by tier (network-bounded)
python -m evals.run_eval report # recompute metrics + rewrite the report
+
+# Held-out validation (live research for the 8 unseen companies):
+python -m evals.run_eval run --companies examples/eval_companies_holdout.json --run-id holdout
+python -m evals.run_eval recheck --run-id holdout
+python -m evals.run_eval report --run-id holdout
```
-Use a capable model — the 8B model malforms the structured JSON. Set
-`LLM_PROVIDER=groq` with `GROQ_MODEL=llama-3.3-70b-versatile`, or use Gemini. Each
-run writes a dated report to `evals/reports/eval-.md`.
+Use a capable model — the 8B model malforms the structured JSON. The headline run
+uses `LLM_PROVIDER=cerebras` with `CEREBRAS_MODEL=gpt-oss-120b` (its ~1M tokens/day
+free tier runs the whole set in one session); `GROQ_MODEL=llama-3.3-70b-versatile`
+or Gemini also work. Each run writes a dated report to `evals/reports/eval-.md`.
## Current numbers
-!!! warning "Partial, positives-only baseline — not the headline"
- The numbers below are from a **partial smoke run of 4 `good_fit` companies**
- (ramp, brex, mercury, checkout) on `groq/llama-3.3-70b-versatile`, with
- **human-unverified labels**. Because the slice contains only positives,
- qualification precision/recall/F1 are trivially 1.0 and are **not** a meaningful
- headline yet — a real baseline needs the negatives and sparse companies (and
- verified labels). The draft-gate, groundedness, faithfulness, failure-mode, and
- live-re-verifiability figures are already informative.
-
-| Metric | Target | Latest (partial) | Model | As of |
-| --- | --- | --- | --- | --- |
-| Qualification F1 | ≥ 0.80 | 1.0 _(positives-only; not meaningful yet)_ | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Draft gate pass-rate | report-only | 0.75 (3/4) | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Mean groundedness | ≥ 0.90 | 0.94 | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Mean faithfulness | ≥ 0.90 | 0.94 | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Failure modes (gate caught) | report-only | `overreach: 1` | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Live re-verifiability (own_site) | report-only | 1.0 (15/15) | groq/llama-3.3-70b-versatile | 2026-06-13 |
-| Facts/company (good_fit) | report-only | 62.25 | groq/llama-3.3-70b-versatile | 2026-06-13 |
-
-The full dated report is at `evals/reports/eval-2026-06-13.md`. Notably the gate
-**caught an `overreach` claim** in one draft (checkout) and routed it to review —
-evidence that verification catches, not just passes.
-
-> When a full, human-verified run completes, replace this table, update its **As
-> of** date and the **Model** column (documentation protocol, rule 7).
+All runs `cerebras/gpt-oss-120b`, gate-critical calls (qualify, draft, verify judge)
+at **temperature 0**, **2026-06-14**. The qualifier fix (reliable negative-signal
+veto + industry as a required component — [ADR-0015](decisions.md)) was developed
+against the original 17 and then validated on a **held-out** set it never saw.
+
+### Headline — held-out validation (n=8, clean test of generalization)
+
+Eight **new** companies (not in the original 17): clear good-fit fintechs, clear
+bad-fit (non-fintech + an incumbent bank), and two borderline. See
+`examples/eval_companies_holdout.json`.
+
+| Metric | Result | Model | As of |
+| --- | --- | --- | --- |
+| **Qualification F1** | **1.0** (precision 1.0, recall 1.0, accuracy 1.0; TP=4, FP=0, TN=4, FN=0) | cerebras/gpt-oss-120b | 2026-06-14 |
+| Draft gate pass-rate | 0.75 (3/4) | cerebras/gpt-oss-120b | 2026-06-14 |
+| Mean groundedness / faithfulness | 0.9583 (over 4 drafts) | cerebras/gpt-oss-120b | 2026-06-14 |
+| Live re-verifiability (own_site) | 0.40 (8/20) — see caveat | cerebras/gpt-oss-120b | 2026-06-14 |
+
+Every held-out company landed correctly: `stripe` / `gocardless` / `chime` qualified;
+`canva` / `datadoghq` (non-fintech) and `wellsfargo` (incumbent bank) disqualified;
+and both borderlines went the labeled way — `coinbase` (crypto exchange, in-ICP)
+qualified at 0.69, `shopify` (commerce, not fintech) disqualified at 0.37. The veto
+and industry rule generalize to companies the fix was never tuned on.
+
+> **Held-out re-verifiability caveat.** The 0.40 (8/20) looks alarming but is a
+> **fetch-access** artifact, not fabrication: 10 of the 12 misses are `dead` — the
+> source returned HTTP 403 (bot-blocking) on re-fetch (chime, canva, coinbase block
+> automated fetchers), so the evidence couldn't be re-checked at all. Only 2 were
+> genuinely absent. The original-17 sources (well-known, fetchable) give the cleaner
+> durability signal of **0.9231 (36/39)** below.
+
+### Fix evidence — original 17, before vs. after (development set)
+
+Same set, same temperature-0 config; only the qualifier changed.
+
+| Metric (n=17) | Before fix | After fix |
+| --- | --- | --- |
+| **Qualification F1** | **0.769** | **0.9474** |
+| precision / recall / accuracy | 0.625 / 1.0 / 0.647 | 1.0 / 0.9 / 0.9412 |
+| Confusion TP / FP / TN / FN | 10 / 6 / 1 / 0 | 9 / 0 / 7 / 1 |
+| Draft gate pass-rate | 0.5625 (9/16) | 0.7778 (7/9) |
+| Mean groundedness / faithfulness | 0.8615 | 0.95 |
+| Live re-verifiability (own_site) | 0.9032 (56/62) | 0.9231 (36/39) |
+
+The fix eliminated **all six** false positives (precision 0.625 → 1.0): the
+non-fintech companies (`figma`, `huggingface`, `linear`, `vercel`, `nilenso`) and the
+incumbent bank (`jpmorganchase`) now disqualify via a reliably-firing "not fintech" /
+"incumbent bank" veto plus the industry penalty.
+
+**Both sets improved (original 0.769 → 0.947, held-out 1.0), so the fix generalizes
+rather than overfitting the 17.**
+
+**The recall cost, stated honestly.** The after-fix original-17 has **one false
+negative — `ramp.com`** (recall 1.0 → 0.9). On that run the assessor flakily returned
+`industry=unknown` for Ramp (a clear fintech), and because industry is now a required
+component, the unknown scored 0.0 and dropped Ramp below threshold (0.28). This is the
+deliberate trade-off: making `industry=unknown` count against a company fixes the
+non-fintech false positives but, when the assessor *flakily* fails to confirm a real
+fintech's industry, it can wrongly reject it. The held-out set did not exhibit this
+(all four good-fits, including borderline `coinbase`, qualified), but it is a real
+residual risk of the assessor's run-to-run variance.
+
+**Draft-gate overreach — a noted finding (not fixed this pass).** Across runs the gate
+rejects a meaningful share of *qualified* drafts (≈44% on the pre-fix 16, ≈22–25%
+after). Spot-checking the flagged claims (e.g. "single-point solution for compliance",
+"strong foundation for accelerating product delivery", AI-capability embellishments),
+these are genuine over-claims the **drafter** adds beyond the cited facts — the gate is
+correctly catching drafter overreach, not being pedantic. Tightening the draft prompt
+to claim less is the natural next step.
+
+**On the two grounding scores.** Under the default strict gate, `groundedness_score`
+and `faithfulness_score` share the same numerator (faithful body claims), so they are
+**numerically identical** and reported as **one** signal — not two independent pieces
+of evidence. They diverge only with `FAITHFULNESS_STRICT=false`.
+
+Reports: held-out at `evals/reports/eval-2026-06-14-holdout.md`; original-17 after-fix
+at `evals/reports/eval-2026-06-14-original17-after.md`.
+
+### Known limitations
+
+These are honest gaps, documented rather than tuned away:
+
+- **Small samples.** Held-out n=8, development n=17, human-proposed labels, a single
+ run each. F1 1.0 on eight companies is encouraging, not conclusive — a larger
+ held-out set is needed before the headline is bankable.
+- **Residual run-to-run variance.** Cerebras is not bit-deterministic even at
+ temperature 0, so a company can vary between runs — including the `ramp.com` false
+ negative above (flaky `industry=unknown` on a real fintech) and draft gate outcomes.
+- **One residual qualification miss type.** The industry rule depends on the assessor
+ confirming industry; when it flakily returns `unknown` for a genuine fintech, the
+ company is wrongly rejected. A confidence/retry on the industry assessment, or a
+ softer penalty, is a candidate follow-up — to be validated on a held-out set.
+- **Labels are human-proposed.** The ground-truth labels (both sets) still need a
+ human pass against the rubric before the numbers are fully trustworthy.
+
+> When the qualification logic changes or a new model is used, re-run on **both** the
+> development and held-out sets, then replace these tables and bump the **As of** date
+> and **Model** (documentation protocol, rule 7).
diff --git a/docs/groundedness.md b/docs/groundedness.md
index 7fbe91f..f92b335 100644
--- a/docs/groundedness.md
+++ b/docs/groundedness.md
@@ -1,12 +1,21 @@
# Groundedness — methodology
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/models/fact.py`, `src/pitch_pilot/nodes/research.py`, `src/pitch_pilot/nodes/draft.py`, `src/pitch_pilot/nodes/verify.py`, `src/pitch_pilot/models/verification.py`
-
-Groundedness is the hero feature. **No fact exists without a `source_url`, and no
-claim reaches a human without being first-party-sourced, substring-anchored, and
-judged to be faithfully supported by its evidence.** This page is the methodology:
-exactly what is checked, where, and what each reported number means — so the
-headline groundedness claim is defensible rather than a vibe.
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/models/fact.py`, `src/pitch_pilot/nodes/research.py`, `src/pitch_pilot/nodes/draft.py`, `src/pitch_pilot/nodes/verify.py`, `src/pitch_pilot/models/verification.py`
+
+Groundedness is the hero feature. **No fact exists without a `source_url`; every
+fact is substring-anchored to verbatim source text at the moment it is born; an
+outreach draft may be grounded only in first-party facts; and the draft's body is
+independently judged faithful to those facts before it reaches a human.** This page
+is the methodology: exactly what is checked, where, and what each reported number
+means — so the headline groundedness claim is defensible rather than a vibe.
+
+!!! note "0.8.0 — grounding decoupled from phrasing"
+ Source-text grounding (the verbatim-substring check) lives at **extraction**,
+ where a fact is matched against its source. The **draft** layer grounds outreach
+ by *selecting which facts* to stand on (by id), and the **verify** layer judges
+ the body's *faithfulness* to those facts. The draft no longer re-substring-checks
+ (or fuzzy-matches) paraphrased hook text against the source — a brittle check
+ that discarded faithful paraphrases. See [ADR-0014](decisions.md).
## Why it matters
@@ -19,8 +28,9 @@ should trace to a real page a reviewer can click through and confirm.
## The four checks
-Groundedness is enforced in four layers, at two different times — three at
-research/draft time, one at verification:
+Groundedness is enforced in four layers, at two different times — three by
+construction (extraction, tiering, draft fact-selection), one by judgement at
+verification:
### Layer 1 — Extraction-time substring guard
@@ -32,7 +42,11 @@ unrepresentable (see [ADR-0001](decisions.md)). Beyond that, when the research
evidence is not literally present in the source** (a whitespace- and
case-insensitive substring match). This filters claims the model tried to smuggle
in from prior knowledge, at the moment facts are born (see
-[ADR-0007](decisions.md)).
+[ADR-0007](decisions.md)). The source text is truncated to
+`RESEARCH_MAX_PAGE_CHARS` **before** extraction, and the substring check runs
+against that *same* truncated text — so leaning out research depth ([ADR-0012](decisions.md))
+shrinks tokens without weakening the guarantee: the model can only ground claims in
+the text we verify against.
### Layer 2 — Source tiering
@@ -56,61 +70,66 @@ enforces this on the way in — only first-party facts are offered as *claimable
hook — and the verify node enforces it again on the way out (see
[ADR-0010](decisions.md)).
-### Layer 3 — Substring re-check at verify
+### Layer 3 — Draft fact-selection (grounding by construction)
-At verification each draft claim (a `Draft` hook) is resolved back to its backing
-`Fact`, and `substring_ok` records whether that fact carries the verbatim evidence
-snippet that Layer 1 proved was present in the live source. A claim whose backing
-fact has no such evidence fails as `not-substring`. This re-affirms — at gate time,
-without re-fetching — that the claim is anchored to verbatim source text, not just
-loosely associated with a URL.
+The [draft node](components/nodes.md) offers the model only the *claimable*
+(first-party) facts, **numbered**, and the model returns the **ids** of the facts it
+grounded the email in. `Draft.hooks_used` is the canonical claim text of those
+selected facts. Because each id resolves to a real first-party `Fact` that already
+passed Layer 1, every hook is grounded **by construction** — no substring or fuzzy
+re-check of hook text is performed (that check belongs at extraction, not here). The
+verify node re-resolves the hooks and treats any that fail to map to a first-party
+fact as a `structural` failure (an invariant violation that should never occur).
-### Layer 4 — LLM faithfulness judge
+### Layer 4 — LLM body-faithfulness judge
-Presence is not support: evidence can be a genuine verbatim snippet and still not
-back the claim, or back only a weaker version of it. So the verify node runs an
-**LLM judge** on every claim↔evidence pair (`judge_faithfulness`), which returns:
+A grounded set of facts is necessary but not sufficient: the free-prose **body**
+could still overstate or invent. So the verify node runs a single **LLM judge**
+(`judge_body`) over the draft body against the selected facts. It extracts every
+factual claim the body makes *about the company* and rates each:
-- **faithful** — the evidence directly supports the claim as stated;
-- **overreach** — the evidence partially supports it but the claim
+- **faithful** — a selected fact directly supports the claim as stated;
+- **overreach** — a fact partially supports it but the claim
generalizes/exaggerates beyond it;
-- **unsupported** — the evidence does not support the claim.
+- **unsupported** — no selected fact supports the claim.
-The judge **fails closed**: any judge error is treated as `unsupported`. This is
-the one network call the verify node makes; it does **not** re-fetch sources.
+The judge **fails closed**: any judge error (or a malformed response) is treated as
+a failure and the draft does not pass. This is the one network call the verify node
+makes; it does **not** re-fetch sources.
## The gate
-A claim is **verified** iff *all* of the following hold:
+The draft **passes** iff *all* of the following hold:
```text
-backed by a Fact AND tier in {own_site, authoritative} AND substring_ok
- AND faithfulness == "faithful"
- (or "overreach" when FAITHFULNESS_STRICT is false)
+at least one grounded hook (first-party fact) AND non-empty body
+ AND the faithfulness judge ran successfully
+ AND no body claim is "unsupported"
+ (and none is "overreach" when FAITHFULNESS_STRICT)
```
-The draft **passes only if every claim is verified.** Each failing claim is
-recorded in `flagged_claims` with its specific reason — `unbacked`,
-`volatile-source`, `not-substring`, `overreach`, or `unsupported` — and the full
-per-claim audit trail is returned in
-[`VerificationResult.claim_verdicts`](data-models.md). An empty draft (no claims)
-does not pass.
+Each failing claim is recorded in `flagged_claims` with its specific reason —
+`structural`, `overreach`, `unsupported`, or `judge-error` — and the full
+per-body-claim audit trail is returned in
+[`VerificationResult.claim_verdicts`](data-models.md). An ungrounded or empty-body
+draft does not pass.
## The metrics
| Metric | Definition | Where computed |
| --- | --- | --- |
-| `groundedness_score` | `verified_claims / total_claims` — the fraction of claims that cleared **all four** checks. Reported even when the draft passes. | Per run, in the verify node |
-| `faithfulness_score` | `faithful_claims / total_claims` — the fraction the judge rated `faithful` (an `overreach` claim that passes under lenient mode is *not* counted here). | Per run, in the verify node |
-| `tier_breakdown` | Count of claims by backing source tier (e.g. `{own_site: 2}`), `unbacked` for claims with no fact. | Per run, in the verify node |
-| **independent live re-verification** | For a sample of facts, re-fetch the `source_url` and re-confirm the evidence is still present on the live page — reported **by tier**. This is an **eval-time** metric (P4), measured offline against a labeled set; it is deliberately **not** in the per-run hot path (see [ADR-0010](decisions.md)). | Offline, [Evaluation](evals.md) (P4) |
+| `groundedness_score` | `verified_claims / total_body_claims` — the fraction of body claims that count as verified, where verified means the judge rated the claim `faithful` (or `overreach` when `FAITHFULNESS_STRICT` is off). Under the default strict mode this equals `faithful_claims / total_body_claims`. `1.0` for a grounded body with no checkable company claim; `0.0` for an ungrounded or empty-body draft. | Per run, in the verify node |
+| `faithfulness_score` | `faithful_claims / total_body_claims` — the fraction of body claims the judge rated `faithful` (an `overreach` claim that passes under lenient mode is *not* counted here). | Per run, in the verify node |
+| `tier_breakdown` | Count of the draft's **grounding facts** (hooks) by source tier, e.g. `{own_site: 2}`. | Per run, in the verify node |
+| **independent live re-verification** | For each used grounding fact, re-fetch the `source_url` and re-confirm the evidence is still present on the live page — reported **by tier**. This is an **eval-time** metric (P4), measured offline against a labeled set; it is deliberately **not** in the per-run hot path (see [ADR-0010](decisions.md)). | Offline, [Evaluation](evals.md) (P4) |
## The honest headline
-What pitch-pilot guarantees per run is precise and bounded: **every claim in a
-passing draft is first-party-sourced, substring-anchored to verbatim evidence, and
-judged faithful to that evidence.** What it does *not* claim per run is that every
-cited page is still live and unchanged — *live re-verifiability* is reported
+What pitch-pilot guarantees per run is precise and bounded: **a passing draft is
+grounded only in first-party facts that are each substring-anchored to verbatim
+source text (at extraction), and every factual claim its body makes about the
+company is judged faithful to those facts.** What it does *not* claim per run is that
+every cited page is still live and unchanged — *live re-verifiability* is reported
**separately, by tier, at eval time**, because the honest finding from P1 is that
first-party sources re-verify far more reliably than third-party ones. Stating the
two separately is the point: we do not launder a faithfulness number into a
@@ -118,9 +137,12 @@ durability promise.
!!! note "Status"
Layers 1 (P1) and 2 (P2) and the draft-side of Policy B (P2) shipped earlier.
- Layer 3, Layer 4, the verify-side Policy B enforcement, and the enriched
- metrics are **P3**. The eval-time independent live re-verification is **P4**
- (see [Evaluation](evals.md)).
+ The original Layer 4 faithfulness judge, verify-side Policy B enforcement, and
+ enriched metrics were **P3**. The eval-time independent live re-verification is
+ **P4**. **0.8.0** decoupled grounding from phrasing: draft grounding became
+ fact-selection (Layer 3), the faithfulness judge moved to the draft **body**
+ against the selected facts (Layer 4), and the brittle draft-time substring
+ re-check was removed (see [ADR-0014](decisions.md)).
## What the gate does — and doesn't — do
diff --git a/docs/index.md b/docs/index.md
index bc0500f..c68886a 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,6 +1,6 @@
# pitch-pilot
-> **Last updated:** 2026-06-05 · **Source files:** `README.md`, `src/pitch_pilot/`
+> **Last updated:** 2026-06-14 · **Source files:** `README.md`, `src/pitch_pilot/`
**pitch-pilot** is an autonomous [SDR](glossary.md) (Sales Development Rep) agent.
Give it a company **domain** and it researches the company, qualifies it against
@@ -26,6 +26,24 @@ groundedness score clears a configurable threshold.
The result is outreach you can trust and audit: every sentence traces back to a
page. See [Groundedness](groundedness.md) for the deep dive.
+## Results at a glance
+
+`cerebras/gpt-oss-120b`, gate-critical calls at temperature 0, 2026-06-14. The
+qualifier fix ([ADR-0015](decisions.md)) was developed on the original 17 and
+validated on a **held-out** set it never saw:
+
+- **Held-out (n=8, headline): Qualification F1 1.0** (precision 1.0, recall 1.0;
+ TP/FP/TN/FN = 4/0/4/0) — every unseen company landed correctly, including an
+ incumbent bank and two borderlines.
+- **Original 17 (development): F1 0.769 → 0.947** after the fix (all six false
+ positives eliminated; one new false negative from a flaky industry assessment).
+- **Mean groundedness 0.95–0.96**, equal to the faithfulness score under the strict
+ gate (one signal, not two).
+
+Both sets improved, so the fix generalizes rather than overfitting. Full provenance,
+before/after tables, the live-re-verifiability caveat, and the honest limitations:
+[Evaluation](evals.md).
+
## What it does — and refuses to do
| Does | Refuses |
@@ -47,7 +65,7 @@ runs inside the research step where open-ended exploration actually helps.
┌──────────┐ ┌──────────┐ ┌────────┐ ┌────────┐ ┌────────┐
│ RESEARCH │──▶│ QUALIFY │──▶│ DRAFT │──▶│ VERIFY │──▶│ LOG │
└──────────┘ └────┬─────┘ └────────┘ └───┬────┘ └────────┘
- agentic loop │ gate: disqualified → stop │ gate: score ≥ threshold
+ agentic loop │ gate: disqualified → stop │ gate: body claims faithful?
```
Read more in [Architecture](architecture.md) and [Pipeline](pipeline.md).
@@ -76,7 +94,8 @@ Full instructions: [Getting Started](getting-started.md).
## Project status
-P0 (the foundation) is complete: typed contracts, swappable clients, configuration,
-a working smoke test, tests, and this documentation site. The live pipeline and
-node logic land in [P1](roadmap.md). This project is built in public as a
+The end-to-end pipeline is shipping (P0–P4): the agentic research loop, ICP
+qualification, grounded drafting, the groundedness verification gate, and the
+offline [evaluation harness](evals.md) with the numbers above. Storage backends and
+a human-review app are next ([P5](roadmap.md)). This project is built in public as a
portfolio-grade reference for a grounded, agentic outbound system.
diff --git a/docs/pipeline.md b/docs/pipeline.md
index 9944655..83b392d 100644
--- a/docs/pipeline.md
+++ b/docs/pipeline.md
@@ -1,6 +1,6 @@
# Pipeline
-> **Last updated:** 2026-06-13 · **Source files:** `src/pitch_pilot/graph/`, `src/pitch_pilot/nodes/`
+> **Last updated:** 2026-06-14 · **Source files:** `src/pitch_pilot/graph/`, `src/pitch_pilot/nodes/`
The pipeline is the deterministic outer graph that orchestrates a run. It is
assembled with [LangGraph](glossary.md) on top of the typed
@@ -13,8 +13,9 @@ assembled with [LangGraph](glossary.md) on top of the typed
`python -m pitch_pilot.cli run ` command runs the whole thing. The
`research_node` and its [agentic sub-loop](#the-agentic-research-sub-loop) were
delivered in [P1](roadmap.md), and [P3](roadmap.md) hardened the `verify` node
- into the real groundedness gate (first-party tier + substring + LLM faithfulness
- judge — see [Groundedness](groundedness.md)). The graph shape is stable.
+ into the real groundedness gate (later refined in 0.8.0 to a structural hook
+ check + an LLM body-faithfulness judge — see [Groundedness](groundedness.md)).
+ The graph shape is stable.
## The graph
@@ -80,25 +81,28 @@ state with its slice filled in. State accumulates; nothing is discarded.
- **Conditional edge:** if `qualified` is `False` → **`log_node`** (stop early);
if `True` → **`draft_node`**.
-### `draft_node` — write grounded outreach ✅ implemented (P2)
+### `draft_node` — write grounded outreach ✅ implemented (P2; selection in 0.8.0)
- **Reads:** `research`, `qualification`
- **Writes:** `draft: Draft`
-- **What it does:** Composes a subject + body **only from grounded facts**.
- Hard-numeric claims from `third_party_snippet` facts are withheld, and every hook
- the model returns is validated back against the facts — so `hooks_used` is always
- a subset of the research facts. See [Groundedness → Layer 3](groundedness.md).
+- **What it does:** Writes a subject + free-prose body and grounds it by
+ **selecting facts**: only first-party facts are offered (numbered), and the model
+ returns the **ids** it grounded the email in, so `hooks_used` is always a subset of
+ the research facts (grounded by construction). See
+ [Groundedness → Layer 3](groundedness.md).
-### `verify_node` — the groundedness gate ✅ hardened (P3)
+### `verify_node` — the groundedness gate ✅ hardened (P3; body judge in 0.8.0)
- **Reads:** `draft`, `research`
- **Writes:** `verification: VerificationResult`
-- **What it does:** Audits each claim (the draft's `hooks_used`) through four
- checks — backed, first-party tier (Policy B), substring-anchored, and an **LLM
- faithfulness judge**. The draft **passes only if every claim verifies.** Failures
- are recorded by reason (`unbacked` / `volatile-source` / `not-substring` /
- `overreach` / `unsupported`) with a per-claim audit trail. Network-free except
- the judge call. See the [Groundedness methodology](groundedness.md).
+- **What it does:** Re-resolves the draft's hooks to first-party facts
+ (structural), then runs **one LLM judge** over the draft **body** against those
+ facts, rating each body claim `faithful` / `overreach` / `unsupported`. The draft
+ **passes** iff grounded, body non-empty, judge ran, and no body claim is
+ `unsupported` (and none `overreach` under `FAITHFULNESS_STRICT`). Failures are
+ recorded by reason (`structural` / `overreach` / `unsupported` / `judge-error`)
+ with a per-body-claim audit trail. Network-free except the judge call. See the
+ [Groundedness methodology](groundedness.md).
- **Edge:** always proceeds to `log_node`, which decides the outcome from the
verification verdict.
diff --git a/docs/roadmap.md b/docs/roadmap.md
index 2ac54d2..e1376eb 100644
--- a/docs/roadmap.md
+++ b/docs/roadmap.md
@@ -1,6 +1,6 @@
# Roadmap
-> **Last updated:** 2026-06-13 · **Source files:** project-wide
+> **Last updated:** 2026-06-14 · **Source files:** project-wide
pitch-pilot is built in phases. Each phase is shippable and leaves the project in
a runnable state. Legend: ✅ done · 🟡 in progress · ⬜ planned.
@@ -10,8 +10,8 @@ a runnable state. Legend: ✅ done · 🟡 in progress · ⬜ planned.
| **P0 — Foundation** | ✅ done | Installable scaffold: typed contracts, swappable clients, fail-loud config, smoke test, unit tests, and this docs site. |
| **P1 — Agentic research** | ✅ done | The bounded, LLM-driven research loop (seed → plan → search → extract grounded `Fact`s, capped by `RESEARCH_MAX_QUERIES`), the `Fact.evidence` substring grounding check, a `research_node` graph adapter, and the `research` CLI command. |
| **P2 — Pipeline** | ✅ done | The deterministic LangGraph outer graph (`build_pipeline()`) and the node functions (`qualify → draft → verify → log`) wired on top of `PipelineState`, plus `Fact.source_tier`, the `pitch-pilot run` CLI, and a basic verification gate. |
-| **P3 — Verification & scoring** | ✅ done | The real groundedness gate: Policy B (first-party-only claims), an LLM faithfulness judge, the `groundedness_score` / `faithfulness_score` / `tier_breakdown` metrics and per-claim audit trail, and the documented [methodology](groundedness.md). |
-| **P4 — Evaluation** | ✅ done | The labeled eval set (positives + negatives + sparse, with a defensible rubric), the rate-limit-resilient harness (`evals/run_eval.py` — cache, checkpoint, backoff, resume), the metrics module, the independent live re-check by tier, and the dated report. Numbers table fills once labels are human-verified. |
+| **P3 — Verification & scoring** | ✅ done | The real groundedness gate: Policy B (first-party-only claims), an LLM faithfulness judge, the `groundedness_score` / `faithfulness_score` / `tier_breakdown` metrics and per-claim audit trail, and the documented [methodology](groundedness.md). _Refined in 0.8.0: draft grounding decoupled from phrasing — fact-selection + a body-faithfulness judge ([ADR-0014](decisions.md))._ |
+| **P4 — Evaluation** | ✅ done | The labeled eval set (positives + negatives + sparse, with a defensible rubric), the rate-limit-resilient harness (`evals/run_eval.py` — cache, checkpoint, backoff, resume, `redraft`), the metrics module, the independent live re-check by tier, and the dated report. Headline numbers recorded in [Evaluation](evals.md) (cerebras/gpt-oss-120b, 2026-06-14); labels remain human-proposed. |
| **P5 — Storage & review app** | ⬜ planned (next) | Production `Store` backends (HubSpot, Google Sheets) and a human-review UI over the queue (`app/`). |
| **P6 — Discovery** | ⬜ planned | The `discover_node` seam that sources candidate domains (inbound lists, look-alikes, market maps) under $0 constraints. |
| **P7 — Hardening & deploy** | ⬜ planned | Observability, rate limiting, packaging, the live docs site, and CI/CD. |
diff --git a/evals/metrics.py b/evals/metrics.py
index 2bafb30..01b49dd 100644
--- a/evals/metrics.py
+++ b/evals/metrics.py
@@ -28,8 +28,11 @@
from collections import Counter
# The verify gate's failure reasons, in reporting order. Counting these proves the
-# gate *catches* bad claims, not just that it passes good ones.
-FAILURE_REASONS = ("unbacked", "volatile-source", "not-substring", "overreach", "unsupported")
+# gate *catches* bad claims, not just that it passes good ones. Under the P5 gate
+# hooks are grounded by construction, so the gate's failures are body-faithfulness
+# verdicts (``unsupported`` / ``overreach``), a never-should-happen ``structural``
+# violation, and a fail-closed ``judge-error``.
+FAILURE_REASONS = ("unsupported", "overreach", "structural", "judge-error")
def ok_results(results: list[dict]) -> list[dict]:
diff --git a/evals/reports/eval-2026-06-13.md b/evals/reports/eval-2026-06-13.md
index eeaa389..a55ac75 100644
--- a/evals/reports/eval-2026-06-13.md
+++ b/evals/reports/eval-2026-06-13.md
@@ -1,4 +1,4 @@
-> **Run date:** 2026-06-13 · **Model:** `groq/llama-3.3-70b-versatile` · **Source:** `evals/run_eval.py`
+> **Run date:** 2026-06-13 · **Model:** `cerebras/gpt-oss-120b` · **Source:** `evals/run_eval.py`
# pitch-pilot eval — 2026-06-13
@@ -6,19 +6,31 @@
## Aggregates
-- **Companies:** 4 evaluated, 0 error(s) of 4
-- **Qualification:** accuracy 1.0, precision 1.0, recall 1.0, F1 1.0 (tp=4, fp=0, tn=0, fn=0)
-- **Draft gate pass-rate:** 3/4 = 0.75
-- **Mean groundedness:** 0.9375 · **mean faithfulness:** 0.9375 (over 4 drafts)
-- **Failure modes:** {'unbacked': 0, 'volatile-source': 0, 'not-substring': 0, 'overreach': 1, 'unsupported': 0}
-- **Facts/company by category (degradation):** {'good_fit': 62.25}
-- **Live re-verifiability by tier:** own_site 1.0 (15/15)
+- **Companies:** 17 evaluated, 0 error(s) of 17
+- **Qualification:** accuracy 0.8235, precision 0.7692, recall 1.0, F1 0.8696 (tp=10, fp=3, tn=4, fn=0)
+- **Draft gate pass-rate:** 1/13 = 0.0769
+- **Mean groundedness:** 0.2615 · **mean faithfulness:** 0.2615 (over 13 drafts)
+- **Failure modes:** {'unbacked': 0, 'volatile-source': 0, 'not-substring': 0, 'overreach': 4, 'unsupported': 9}
+- **Facts/company by category (degradation):** {'bad_fit': 54.5, 'good_fit': 61.75, 'sparse': 50.33}
## Per-company
| domain | category | label (truth) | predicted | score | draft | grounded | faithful | facts |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| `ramp.com` | good_fit | qualified | qualified | 1.0 | pass | 1.0 | 1.0 | 64 |
-| `brex.com` | good_fit | qualified | qualified | 0.8958 | pass | 1.0 | 1.0 | 65 |
-| `mercury.com` | good_fit | qualified | qualified | 1.0 | pass | 1.0 | 1.0 | 69 |
-| `checkout.com` | good_fit | qualified | qualified | 0.625 | fail | 0.75 | 0.75 | 51 |
+| `ramp.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.0 | 0.0 | 68 |
+| `brex.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.0 | 0.0 | 60 |
+| `mercury.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.0 | 0.0 | 64 |
+| `checkout.com` | good_fit | qualified | qualified | 0.625 | fail | 0.0 | 0.0 | 56 |
+| `plaid.com` | good_fit | qualified | qualified | 1.0 | fail | 0.6667 | 0.6667 | 65 |
+| `razorpay.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.0 | 0.0 | 59 |
+| `monzo.com` | good_fit | qualified | qualified | 0.6667 | fail | 0.3333 | 0.3333 | 57 |
+| `wise.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.5 | 0.5 | 65 |
+| `notion.so` | bad_fit | not_qualified | not_qualified | 0.25 | — | None | None | 56 |
+| `figma.com` | bad_fit | not_qualified | not_qualified | 0.2833 | — | None | None | 60 |
+| `linear.app` | bad_fit | not_qualified | qualified | 0.5312 | fail | 0.4 | 0.4 | 55 |
+| `vercel.com` | bad_fit | not_qualified | qualified | 0.5312 | fail | 0.0 | 0.0 | 56 |
+| `huggingface.co` | bad_fit | not_qualified | not_qualified | 0.5312 | — | None | None | 44 |
+| `jpmorganchase.com` | bad_fit | not_qualified | not_qualified | 0.75 | — | None | None | 56 |
+| `nilenso.com` | sparse | not_qualified | qualified | 0.6667 | pass | 1.0 | 1.0 | 47 |
+| `fampay.in` | sparse | qualified | qualified | 0.8333 | fail | 0.0 | 0.0 | 51 |
+| `jupiter.money` | sparse | qualified | qualified | 0.9167 | fail | 0.5 | 0.5 | 53 |
diff --git a/evals/reports/eval-2026-06-14-holdout.md b/evals/reports/eval-2026-06-14-holdout.md
new file mode 100644
index 0000000..a2e6b4e
--- /dev/null
+++ b/evals/reports/eval-2026-06-14-holdout.md
@@ -0,0 +1,28 @@
+> **Run date:** 2026-06-14 · **Model:** `cerebras/gpt-oss-120b` · **Source:** `evals/run_eval.py`
+
+# pitch-pilot eval — 2026-06-14
+
+> **Labels are human-proposed** (see `examples/eval_companies.json` and the rubric in `docs/evals.md`). Verify before trusting these numbers.
+
+## Aggregates
+
+- **Companies:** 8 evaluated, 0 error(s) of 8
+- **Qualification:** accuracy 1.0, precision 1.0, recall 1.0, F1 1.0 (tp=4, fp=0, tn=4, fn=0)
+- **Draft gate pass-rate:** 3/4 = 0.75
+- **Mean groundedness:** 0.9583 · **mean faithfulness:** 0.9583 (over 4 drafts)
+- **Failure modes:** {'unsupported': 0, 'overreach': 1, 'structural': 0, 'judge-error': 0}
+- **Facts/company by category (degradation):** {'bad_fit': 56.67, 'borderline': 56.5, 'good_fit': 55.67}
+- **Live re-verifiability by tier:** own_site 0.4 (8/20)
+
+## Per-company
+
+| domain | category | label (truth) | predicted | score | draft | grounded | faithful | facts |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| `stripe.com` | good_fit | qualified | qualified | 0.8333 | pass | 1.0 | 1.0 | 57 |
+| `gocardless.com` | good_fit | qualified | qualified | 0.8333 | pass | 1.0 | 1.0 | 61 |
+| `chime.com` | good_fit | qualified | qualified | 0.75 | fail | 0.8333 | 0.8333 | 49 |
+| `canva.com` | bad_fit | not_qualified | not_qualified | 0.1042 | — | None | None | 54 |
+| `datadoghq.com` | bad_fit | not_qualified | not_qualified | 0.2 | — | None | None | 57 |
+| `wellsfargo.com` | bad_fit | not_qualified | not_qualified | 0.3667 | — | None | None | 59 |
+| `coinbase.com` | borderline | qualified | qualified | 0.6875 | pass | 1.0 | 1.0 | 64 |
+| `shopify.com` | borderline | not_qualified | not_qualified | 0.3667 | — | None | None | 49 |
diff --git a/evals/reports/eval-2026-06-14-original17-after.md b/evals/reports/eval-2026-06-14-original17-after.md
new file mode 100644
index 0000000..82eff81
--- /dev/null
+++ b/evals/reports/eval-2026-06-14-original17-after.md
@@ -0,0 +1,37 @@
+> **Run date:** 2026-06-14 · **Model:** `cerebras/gpt-oss-120b` · **Source:** `evals/run_eval.py`
+
+# pitch-pilot eval — 2026-06-14
+
+> **Labels are human-proposed** (see `examples/eval_companies.json` and the rubric in `docs/evals.md`). Verify before trusting these numbers.
+
+## Aggregates
+
+- **Companies:** 17 evaluated, 0 error(s) of 17
+- **Qualification:** accuracy 0.9412, precision 1.0, recall 0.9, F1 0.9474 (tp=9, fp=0, tn=7, fn=1)
+- **Draft gate pass-rate:** 7/9 = 0.7778
+- **Mean groundedness:** 0.95 · **mean faithfulness:** 0.95 (over 9 drafts)
+- **Failure modes:** {'unsupported': 1, 'overreach': 1, 'structural': 0, 'judge-error': 0}
+- **Facts/company by category (degradation):** {'bad_fit': 54.5, 'good_fit': 61.75, 'sparse': 50.33}
+- **Live re-verifiability by tier:** own_site 0.9231 (36/39)
+
+## Per-company
+
+| domain | category | label (truth) | predicted | score | draft | grounded | faithful | facts |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| `ramp.com` | good_fit | qualified | not_qualified | 0.2833 | — | None | None | 68 |
+| `brex.com` | good_fit | qualified | qualified | 0.75 | pass | 1.0 | 1.0 | 60 |
+| `mercury.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.8 | 0.8 | 64 |
+| `checkout.com` | good_fit | qualified | qualified | 0.625 | pass | 1.0 | 1.0 | 56 |
+| `plaid.com` | good_fit | qualified | qualified | 1.0 | pass | 1.0 | 1.0 | 65 |
+| `razorpay.com` | good_fit | qualified | qualified | 0.8333 | fail | 0.75 | 0.75 | 59 |
+| `monzo.com` | good_fit | qualified | qualified | 0.6667 | pass | 1.0 | 1.0 | 57 |
+| `wise.com` | good_fit | qualified | qualified | 0.8333 | pass | 1.0 | 1.0 | 65 |
+| `notion.so` | bad_fit | not_qualified | not_qualified | 0.1042 | — | None | None | 56 |
+| `figma.com` | bad_fit | not_qualified | not_qualified | 0.2833 | — | None | None | 60 |
+| `linear.app` | bad_fit | not_qualified | not_qualified | 0.2833 | — | None | None | 55 |
+| `vercel.com` | bad_fit | not_qualified | not_qualified | 0.2 | — | None | None | 56 |
+| `huggingface.co` | bad_fit | not_qualified | not_qualified | 0.2833 | — | None | None | 44 |
+| `jpmorganchase.com` | bad_fit | not_qualified | not_qualified | 0.75 | — | None | None | 56 |
+| `nilenso.com` | sparse | not_qualified | not_qualified | 0.6667 | — | None | None | 47 |
+| `fampay.in` | sparse | qualified | qualified | 0.9167 | pass | 1.0 | 1.0 | 51 |
+| `jupiter.money` | sparse | qualified | qualified | 0.9167 | pass | 1.0 | 1.0 | 53 |
diff --git a/evals/reports/eval-2026-06-14.md b/evals/reports/eval-2026-06-14.md
new file mode 100644
index 0000000..a2e6b4e
--- /dev/null
+++ b/evals/reports/eval-2026-06-14.md
@@ -0,0 +1,28 @@
+> **Run date:** 2026-06-14 · **Model:** `cerebras/gpt-oss-120b` · **Source:** `evals/run_eval.py`
+
+# pitch-pilot eval — 2026-06-14
+
+> **Labels are human-proposed** (see `examples/eval_companies.json` and the rubric in `docs/evals.md`). Verify before trusting these numbers.
+
+## Aggregates
+
+- **Companies:** 8 evaluated, 0 error(s) of 8
+- **Qualification:** accuracy 1.0, precision 1.0, recall 1.0, F1 1.0 (tp=4, fp=0, tn=4, fn=0)
+- **Draft gate pass-rate:** 3/4 = 0.75
+- **Mean groundedness:** 0.9583 · **mean faithfulness:** 0.9583 (over 4 drafts)
+- **Failure modes:** {'unsupported': 0, 'overreach': 1, 'structural': 0, 'judge-error': 0}
+- **Facts/company by category (degradation):** {'bad_fit': 56.67, 'borderline': 56.5, 'good_fit': 55.67}
+- **Live re-verifiability by tier:** own_site 0.4 (8/20)
+
+## Per-company
+
+| domain | category | label (truth) | predicted | score | draft | grounded | faithful | facts |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| `stripe.com` | good_fit | qualified | qualified | 0.8333 | pass | 1.0 | 1.0 | 57 |
+| `gocardless.com` | good_fit | qualified | qualified | 0.8333 | pass | 1.0 | 1.0 | 61 |
+| `chime.com` | good_fit | qualified | qualified | 0.75 | fail | 0.8333 | 0.8333 | 49 |
+| `canva.com` | bad_fit | not_qualified | not_qualified | 0.1042 | — | None | None | 54 |
+| `datadoghq.com` | bad_fit | not_qualified | not_qualified | 0.2 | — | None | None | 57 |
+| `wellsfargo.com` | bad_fit | not_qualified | not_qualified | 0.3667 | — | None | None | 59 |
+| `coinbase.com` | borderline | qualified | qualified | 0.6875 | pass | 1.0 | 1.0 | 64 |
+| `shopify.com` | borderline | not_qualified | not_qualified | 0.3667 | — | None | None | 49 |
diff --git a/evals/run_eval.py b/evals/run_eval.py
index 8146753..92fe97f 100644
--- a/evals/run_eval.py
+++ b/evals/run_eval.py
@@ -10,6 +10,11 @@
* ``run`` (default) — evaluate each company (research[cached] → qualify → draft →
verify), checkpoint results, write a report, print aggregates.
+* ``redraft`` — re-run **only** draft + verify for already-qualified companies,
+ reusing cached research **and** each record's frozen qualification verdict (the
+ qualification matrix is never recomputed). Rewrites the results file in place.
+ Use this to refresh draft/verify numbers after changing the draft or verify logic
+ without paying for research or perturbing qualification.
* ``recheck`` — the honest, network-bounded metric: re-fetch each used claim's
source and confirm the evidence still appears, reporting live-verifiability
**by tier**. Run separately from ``run``.
@@ -26,6 +31,7 @@
import datetime
import json
import logging
+import os
import re
import time
from pathlib import Path
@@ -36,6 +42,7 @@
from pitch_pilot.config import ConfigError, Settings, get_settings
from pitch_pilot.models.icp import ICP, load_icp
from pitch_pilot.models.lead import Company
+from pitch_pilot.models.qualification import QualificationResult
from pitch_pilot.models.research import ResearchResult
from pitch_pilot.nodes.draft import run_draft
from pitch_pilot.nodes.qualify import run_qualification
@@ -129,11 +136,11 @@ def _attempt(self, fn):
self.retries += 1
attempt += 1
- def complete(self, system: str, user: str) -> str:
- return self._attempt(lambda: self.inner.complete(system, user))
+ def complete(self, system: str, user: str, **kwargs) -> str:
+ return self._attempt(lambda: self.inner.complete(system, user, **kwargs))
- def complete_json(self, system: str, user: str) -> dict:
- return self._attempt(lambda: self.inner.complete_json(system, user))
+ def complete_json(self, system: str, user: str, **kwargs) -> dict:
+ return self._attempt(lambda: self.inner.complete_json(system, user, **kwargs))
# ---------------------------------------------------------------------------
@@ -311,6 +318,69 @@ def run_eval(companies: list[dict], icp: ICP, *, llm, search, settings: Settings
return read_jsonl(results_file)
+# ---------------------------------------------------------------------------
+# Re-run draft + verify only (qualification frozen).
+# ---------------------------------------------------------------------------
+def _qual_from_record(record: dict) -> QualificationResult:
+ """Reconstruct the frozen `QualificationResult` from an existing result record.
+
+ Only ``reason`` actually feeds drafting; the other fields are carried so the
+ object faithfully mirrors the recorded verdict. The verdict is never recomputed.
+ """
+ return QualificationResult(
+ qualified=bool(record.get("predicted_qualified")),
+ score=float(record.get("score") or 0.0),
+ reason=str(record.get("qual_reason") or ""),
+ matched_signals=list(record.get("matched_signals") or []),
+ missed_signals=list(record.get("missed_signals") or []),
+ )
+
+
+def redraft(records: list[dict], *, llm, settings: Settings) -> list[dict]:
+ """Re-run only draft + verify for already-qualified companies; return new records.
+
+ Research is loaded from cache and the qualification verdict is copied verbatim
+ from the existing record (never recomputed), so the qualification matrix is
+ untouched. Records that were disqualified, errored, or lack cached research are
+ passed through unchanged. Each redrafted record's draft/verify fields are replaced
+ with the freshly computed values.
+ """
+ out: list[dict] = []
+ for record in records:
+ if record.get("status") != "ok" or not record.get("predicted_qualified"):
+ out.append(record)
+ continue
+ domain = record.get("domain")
+ research = load_cached_research(domain)
+ if research is None:
+ logger.warning("redraft: no cached research for %s; leaving record unchanged", domain)
+ out.append(record)
+ continue
+ if hasattr(llm, "reset"):
+ llm.reset()
+ qual = _qual_from_record(record)
+ draft = run_draft(research, qual, llm, settings)
+ ver = run_verification(draft, research, llm, settings)
+ if getattr(llm, "gave_up", False):
+ logger.warning("redraft: rate-limited on %s; leaving record unchanged", domain)
+ out.append(record)
+ continue
+ updated = {
+ **record,
+ "from_cache": True,
+ "hooks_used": list(draft.hooks_used),
+ "draft_passed": ver.passed,
+ "groundedness_score": ver.groundedness_score,
+ "faithfulness_score": ver.faithfulness_score,
+ "tier_breakdown": ver.tier_breakdown,
+ "claim_verdicts": [cv.model_dump() for cv in ver.claim_verdicts],
+ "flagged_claims": ver.flagged_claims,
+ }
+ out.append(updated)
+ _print_company_line(updated)
+ return out
+
+
# ---------------------------------------------------------------------------
# Independent live re-check (the honest, network-bounded metric).
# ---------------------------------------------------------------------------
@@ -483,14 +553,16 @@ def build_report(results: list[dict], agg: dict, model_label: str, run_date: str
# CLI.
# ---------------------------------------------------------------------------
def _model_label(settings: Settings) -> str:
- model = settings.gemini_model if settings.llm_provider == "gemini" else settings.groq_model
- return f"{settings.llm_provider}/{model}"
+ return f"{settings.llm_provider}/{settings.active_model}"
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="pitch-pilot eval harness.")
- parser.add_argument("command", nargs="?", default="run", choices=["run", "recheck", "report"],
- help="run (default), recheck (live re-verify), or report (recompute).")
+ parser.add_argument("command", nargs="?", default="run",
+ choices=["run", "redraft", "recheck", "report"],
+ help="run (default), redraft (re-run draft+verify only, "
+ "qualification frozen), recheck (live re-verify), or "
+ "report (recompute).")
parser.add_argument("--limit", type=int, default=None, help="Max NEW companies to process this run.")
parser.add_argument("--resume", action="store_true", default=True,
help="Skip companies already evaluated (default).")
@@ -515,6 +587,26 @@ def main(argv: list[str] | None = None) -> int:
model_label = _model_label(settings)
run_date = datetime.date.today().isoformat()
+ if args.command == "redraft":
+ records = dedupe_results(read_jsonl(results_file))
+ if not records:
+ print(f"No results at {results_file}. Run the eval first.")
+ return 1
+ max_retries = int(os.environ.get("EVAL_MAX_RETRIES", "8"))
+ base_delay = float(os.environ.get("EVAL_BASE_DELAY", "10"))
+ llm = RetryingLLM(get_llm_client(settings), max_retries=max_retries, base_delay=base_delay)
+ print(f"Redraft run '{run_id}' on {model_label} — re-running draft+verify for "
+ f"qualified companies (qualification frozen).\n")
+ updated = redraft(records, llm=llm, settings=settings)
+ results_file.parent.mkdir(parents=True, exist_ok=True)
+ with results_file.open("w", encoding="utf-8") as fh:
+ for record in updated:
+ fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+ n_qual = sum(1 for r in updated if r.get("status") == "ok" and r.get("predicted_qualified"))
+ print(f"\nRewrote {results_file} ({n_qual} qualified companies redrafted).")
+ print("Run `python -m evals.run_eval recheck` then `report` for the new numbers.")
+ return 0
+
if args.command == "recheck":
results = read_jsonl(results_file)
if not results:
@@ -550,7 +642,11 @@ def main(argv: list[str] | None = None) -> int:
# --- run ---
icp = load_icp(args.icp)
companies = load_companies(args.companies)
- llm = RetryingLLM(get_llm_client(settings))
+ # Patient retry budget: a single call grinds through per-minute TPM limits
+ # (which reset every ~60s) instead of giving up. Tunable via env for slower tiers.
+ max_retries = int(os.environ.get("EVAL_MAX_RETRIES", "8"))
+ base_delay = float(os.environ.get("EVAL_BASE_DELAY", "10"))
+ llm = RetryingLLM(get_llm_client(settings), max_retries=max_retries, base_delay=base_delay)
search = get_search_client(settings)
print(f"Eval run '{run_id}' on {model_label} — {len(companies)} companies "
f"(limit={args.limit}, resume={args.resume})\n")
diff --git a/examples/eval_companies_holdout.json b/examples/eval_companies_holdout.json
new file mode 100644
index 0000000..1625853
--- /dev/null
+++ b/examples/eval_companies_holdout.json
@@ -0,0 +1,17 @@
+{
+ "_WARNING": "HELD-OUT validation set — companies NOT in examples/eval_companies.json. Labels are proposed against the rubric in docs/evals.md and must be human-verified before trusting any metric. This set is the clean test of whether the qualifier fix (ADR-0015) generalizes vs. overfits the original 17.",
+ "labeling_rubric": "docs/evals.md",
+ "icp": "examples/eval_icp.json",
+ "companies": [
+ {"domain": "stripe.com", "category": "good_fit", "label": "qualified", "rationale": "Payments infrastructure; canonical fintech with money movement at scale."},
+ {"domain": "gocardless.com", "category": "good_fit", "label": "qualified", "rationale": "Bank-debit / recurring-payments fintech (UK), in the 50-1000 band."},
+ {"domain": "chime.com", "category": "good_fit", "label": "qualified", "rationale": "US neobank — consumer banking and payments."},
+
+ {"domain": "canva.com", "category": "bad_fit", "label": "not_qualified", "rationale": "Design platform, not fintech / no money movement (parallels figma)."},
+ {"domain": "datadoghq.com", "category": "bad_fit", "label": "not_qualified", "rationale": "Observability / developer infrastructure, not fintech."},
+ {"domain": "wellsfargo.com", "category": "bad_fit", "label": "not_qualified", "rationale": "Large incumbent bank that builds fraud in-house — matches a negative signal."},
+
+ {"domain": "coinbase.com", "category": "borderline", "label": "qualified", "rationale": "Crypto exchange — explicitly in the ICP industries, money movement + heavy regulatory exposure. Borderline only on size (headcount over the 1000 band); should still qualify on industry + signals."},
+ {"domain": "shopify.com", "category": "borderline", "label": "not_qualified", "rationale": "Primarily an e-commerce SaaS platform (has Shopify Payments, so it does move money) but not a fintech company; well outside the size band. Borderline call — leaning not_qualified as a commerce, not fintech, business."}
+ ]
+}
diff --git a/pyproject.toml b/pyproject.toml
index 41f64dc..df4ae51 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "pitch-pilot"
-version = "0.5.0"
+version = "0.9.0"
description = "An autonomous SDR agent that researches a company, qualifies it against an ICP, drafts grounded outreach, and verifies every claim against a source."
readme = "README.md"
requires-python = ">=3.11"
@@ -26,6 +26,7 @@ dependencies = [
"langgraph>=1.0",
"google-genai>=1.0",
"groq>=1.0",
+ "cerebras-cloud-sdk>=1.0",
"tavily-python>=0.5",
"httpx>=0.27",
"selectolax>=0.3",
diff --git a/src/pitch_pilot/cli.py b/src/pitch_pilot/cli.py
index 652897a..3d34257 100644
--- a/src/pitch_pilot/cli.py
+++ b/src/pitch_pilot/cli.py
@@ -97,7 +97,7 @@ def check_llm() -> str:
from pitch_pilot.clients.llm import get_llm_client
client = get_llm_client(settings)
- model = settings.gemini_model if settings.llm_provider == "gemini" else settings.groq_model
+ model = settings.active_model
reply = client.complete(
system="You are a terse assistant. Reply with exactly what is asked and nothing else.",
user="Reply with the single word: OK",
diff --git a/src/pitch_pilot/clients/__init__.py b/src/pitch_pilot/clients/__init__.py
index 4dd8466..b0b5366 100644
--- a/src/pitch_pilot/clients/__init__.py
+++ b/src/pitch_pilot/clients/__init__.py
@@ -8,6 +8,7 @@
from pitch_pilot.clients.fetch import fetch_page
from pitch_pilot.clients.llm import (
+ CerebrasClient,
GeminiClient,
GroqClient,
LLMClient,
@@ -21,6 +22,7 @@
"LLMClient",
"GeminiClient",
"GroqClient",
+ "CerebrasClient",
"get_llm_client",
"LLMError",
"LLMJSONError",
diff --git a/src/pitch_pilot/clients/llm.py b/src/pitch_pilot/clients/llm.py
index a2edddd..8663ba1 100644
--- a/src/pitch_pilot/clients/llm.py
+++ b/src/pitch_pilot/clients/llm.py
@@ -26,6 +26,36 @@
"markdown code fences."
)
+#: Smallest free-tier context window we target — Cerebras's free tier caps a single
+#: request at 8,192 tokens (input + output). Prompt builders keep well under this by
+#: budgeting their variable-length payloads (see `trim_to_token_budget` and ADR-0013).
+CONTEXT_TOKEN_CAP = 8192
+
+
+def trim_to_token_budget(lines: list[str], max_tokens: int) -> list[str]:
+ """Keep leading ``lines`` whose cumulative size stays within ``max_tokens``.
+
+ Uses a cheap ``~4 chars/token`` estimate. This bounds variable-length prompt
+ payloads (e.g. a facts list) so no single request exceeds a provider's context
+ cap, regardless of how many — or how long — the items are.
+
+ Args:
+ lines: Candidate prompt lines, in priority order.
+ max_tokens: Token budget for the combined lines.
+
+ Returns:
+ The longest leading prefix of ``lines`` that fits the budget.
+ """
+ budget_chars = max_tokens * 4
+ out: list[str] = []
+ total = 0
+ for line in lines:
+ total += len(line) + 1 # +1 for the joining newline
+ if total > budget_chars:
+ break
+ out.append(line)
+ return out
+
class LLMError(RuntimeError):
"""Base error for LLM client failures."""
@@ -39,12 +69,19 @@ class LLMJSONError(LLMError):
class LLMClient(Protocol):
"""The provider-neutral LLM interface the pipeline depends on."""
- def complete(self, system: str, user: str) -> str:
- """Return a free-text completion for the given system + user prompts."""
+ def complete(self, system: str, user: str, temperature: float | None = None) -> str:
+ """Return a free-text completion for the given system + user prompts.
+
+ ``temperature`` overrides the provider default when set; gate-critical calls
+ (draft, verify judge) pass ``0.0`` for reproducibility.
+ """
...
- def complete_json(self, system: str, user: str) -> dict:
- """Return a parsed JSON object. Raises `LLMJSONError` on bad JSON."""
+ def complete_json(self, system: str, user: str, temperature: float | None = None) -> dict:
+ """Return a parsed JSON object. Raises `LLMJSONError` on bad JSON.
+
+ ``temperature`` overrides the provider default when set (see `complete`).
+ """
...
@@ -150,12 +187,13 @@ def _ensure_client(self):
self._client = genai.Client(api_key=self._api_key)
return self._client
- def complete(self, system: str, user: str) -> str:
+ def complete(self, system: str, user: str, temperature: float | None = None) -> str:
"""Return a free-text completion.
Args:
system: System instruction (role/behavior).
user: User prompt (the request).
+ temperature: Sampling temperature; the provider default when ``None``.
Returns:
The model's text response, stripped of surrounding whitespace.
@@ -166,17 +204,20 @@ def complete(self, system: str, user: str) -> str:
from google.genai import types
client = self._ensure_client()
+ config_kwargs: dict = {"system_instruction": system}
+ if temperature is not None:
+ config_kwargs["temperature"] = temperature
try:
response = client.models.generate_content(
model=self._model,
contents=user,
- config=types.GenerateContentConfig(system_instruction=system),
+ config=types.GenerateContentConfig(**config_kwargs),
)
except Exception as exc: # normalize vendor SDK errors to LLMError
raise LLMError(f"Gemini request failed: {exc}") from exc
return (response.text or "").strip()
- def complete_json(self, system: str, user: str) -> dict:
+ def complete_json(self, system: str, user: str, temperature: float | None = None) -> dict:
"""Return a parsed JSON object using Gemini's JSON output mode.
Sets ``response_mime_type="application/json"`` and parses the result
@@ -185,6 +226,7 @@ def complete_json(self, system: str, user: str) -> dict:
Args:
system: System instruction.
user: User prompt.
+ temperature: Sampling temperature; the provider default when ``None``.
Returns:
The parsed JSON object.
@@ -196,14 +238,17 @@ def complete_json(self, system: str, user: str) -> dict:
from google.genai import types
client = self._ensure_client()
+ config_kwargs: dict = {
+ "system_instruction": _json_system(system),
+ "response_mime_type": "application/json",
+ }
+ if temperature is not None:
+ config_kwargs["temperature"] = temperature
try:
response = client.models.generate_content(
model=self._model,
contents=user,
- config=types.GenerateContentConfig(
- system_instruction=_json_system(system),
- response_mime_type="application/json",
- ),
+ config=types.GenerateContentConfig(**config_kwargs),
)
except Exception as exc: # normalize vendor SDK errors to LLMError
raise LLMError(f"Gemini request failed: {exc}") from exc
@@ -232,12 +277,13 @@ def _ensure_client(self):
self._client = Groq(api_key=self._api_key)
return self._client
- def complete(self, system: str, user: str) -> str:
+ def complete(self, system: str, user: str, temperature: float | None = None) -> str:
"""Return a free-text completion.
Args:
system: System message (role/behavior).
user: User message (the request).
+ temperature: Sampling temperature; the provider default when ``None``.
Returns:
The model's text response, stripped of surrounding whitespace.
@@ -246,19 +292,22 @@ def complete(self, system: str, user: str) -> str:
LLMError: if the provider request fails.
"""
client = self._ensure_client()
+ kwargs: dict = {
+ "model": self._model,
+ "messages": [
+ {"role": "system", "content": system},
+ {"role": "user", "content": user},
+ ],
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
try:
- response = client.chat.completions.create(
- model=self._model,
- messages=[
- {"role": "system", "content": system},
- {"role": "user", "content": user},
- ],
- )
+ response = client.chat.completions.create(**kwargs)
except Exception as exc: # normalize vendor SDK errors to LLMError
raise LLMError(f"Groq request failed: {exc}") from exc
return (response.choices[0].message.content or "").strip()
- def complete_json(self, system: str, user: str) -> dict:
+ def complete_json(self, system: str, user: str, temperature: float | None = None) -> dict:
"""Return a parsed JSON object using Groq's JSON mode.
Uses ``response_format={"type": "json_object"}`` and injects an explicit
@@ -267,6 +316,7 @@ def complete_json(self, system: str, user: str) -> dict:
Args:
system: System message.
user: User message.
+ temperature: Sampling temperature; the provider default when ``None``.
Returns:
The parsed JSON object.
@@ -276,20 +326,119 @@ def complete_json(self, system: str, user: str) -> dict:
LLMJSONError: if the response is not a valid JSON object.
"""
client = self._ensure_client()
+ kwargs: dict = {
+ "model": self._model,
+ "messages": [
+ {"role": "system", "content": _json_system(system)},
+ {"role": "user", "content": user},
+ ],
+ "response_format": {"type": "json_object"},
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
try:
- response = client.chat.completions.create(
- model=self._model,
- messages=[
- {"role": "system", "content": _json_system(system)},
- {"role": "user", "content": user},
- ],
- response_format={"type": "json_object"},
- )
+ response = client.chat.completions.create(**kwargs)
except Exception as exc: # normalize vendor SDK errors (e.g. json_validate_failed) to LLMError
raise LLMError(f"Groq request failed: {exc}") from exc
return _loads_json_lenient(response.choices[0].message.content or "")
+class CerebrasClient:
+ """`LLMClient` backed by the Cerebras Cloud SDK (OpenAI-compatible API).
+
+ Cerebras exposes an OpenAI-compatible chat-completions API; the official
+ ``cerebras-cloud-sdk`` defaults to ``https://api.cerebras.ai/v1``. We use it the
+ same way as `GroqClient`, reusing the shared lenient JSON parsing. Its appeal for
+ the eval is budget: the free tier allows ~1M tokens/day (≈10x Groq's), enough to
+ run the whole eval set in one session (see ADR-0013). Available models vary by
+ account — the default is ``gpt-oss-120b``; check the provider's ``models.list()``.
+ """
+
+ def __init__(self, api_key: str, model: str) -> None:
+ """Store credentials and model id; the SDK client is built on first use.
+
+ Args:
+ api_key: Cerebras API key.
+ model: Cerebras model id to call (e.g. ``gpt-oss-120b``).
+ """
+ self._api_key = api_key
+ self._model = model
+ self._client = None # built lazily on first use
+
+ def _ensure_client(self):
+ """Lazily construct and cache the underlying ``cerebras.cloud.sdk.Cerebras`` client."""
+ if self._client is None:
+ from cerebras.cloud.sdk import Cerebras
+
+ self._client = Cerebras(api_key=self._api_key)
+ return self._client
+
+ def complete(self, system: str, user: str, temperature: float | None = None) -> str:
+ """Return a free-text completion.
+
+ Args:
+ system: System message (role/behavior).
+ user: User message (the request).
+ temperature: Sampling temperature; the provider default when ``None``.
+
+ Returns:
+ The model's text response, stripped of surrounding whitespace.
+
+ Raises:
+ LLMError: if the provider request fails.
+ """
+ client = self._ensure_client()
+ kwargs: dict = {
+ "model": self._model,
+ "messages": [
+ {"role": "system", "content": system},
+ {"role": "user", "content": user},
+ ],
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
+ try:
+ response = client.chat.completions.create(**kwargs)
+ except Exception as exc: # normalize vendor SDK errors to LLMError
+ raise LLMError(f"Cerebras request failed: {exc}") from exc
+ return (response.choices[0].message.content or "").strip()
+
+ def complete_json(self, system: str, user: str, temperature: float | None = None) -> dict:
+ """Return a parsed JSON object using Cerebras's JSON mode.
+
+ Uses ``response_format={"type": "json_object"}`` and injects an explicit
+ "respond with JSON" instruction, then parses leniently.
+
+ Args:
+ system: System message.
+ user: User message.
+ temperature: Sampling temperature; the provider default when ``None``.
+
+ Returns:
+ The parsed JSON object.
+
+ Raises:
+ LLMError: if the provider request fails.
+ LLMJSONError: if the response is not a valid JSON object.
+ """
+ client = self._ensure_client()
+ kwargs: dict = {
+ "model": self._model,
+ "messages": [
+ {"role": "system", "content": _json_system(system)},
+ {"role": "user", "content": user},
+ ],
+ "response_format": {"type": "json_object"},
+ }
+ if temperature is not None:
+ kwargs["temperature"] = temperature
+ try:
+ response = client.chat.completions.create(**kwargs)
+ except Exception as exc: # normalize vendor SDK errors to LLMError
+ raise LLMError(f"Cerebras request failed: {exc}") from exc
+ return _loads_json_lenient(response.choices[0].message.content or "")
+
+
def get_llm_client(settings: Settings | None = None) -> LLMClient:
"""Return the configured `LLMClient` based on ``LLM_PROVIDER``.
@@ -297,8 +446,8 @@ def get_llm_client(settings: Settings | None = None) -> LLMClient:
settings: Settings to use; defaults to `get_settings`.
Raises:
- ValueError: if the provider is ``groq`` but no ``GROQ_API_KEY`` is set,
- or if the provider name is unrecognized.
+ ValueError: if the selected provider's API key is missing, or if the
+ provider name is unrecognized.
"""
settings = settings or get_settings()
provider = settings.llm_provider
@@ -313,4 +462,11 @@ def get_llm_client(settings: Settings | None = None) -> LLMClient:
)
return GroqClient(api_key=settings.groq_api_key, model=settings.groq_model)
- raise ValueError(f"Unknown LLM_PROVIDER {provider!r}; expected 'gemini' or 'groq'.")
+ if provider == "cerebras":
+ if not settings.cerebras_api_key:
+ raise ValueError(
+ "LLM_PROVIDER=cerebras but CEREBRAS_API_KEY is not set. Add CEREBRAS_API_KEY to your .env."
+ )
+ return CerebrasClient(api_key=settings.cerebras_api_key, model=settings.cerebras_model)
+
+ raise ValueError(f"Unknown LLM_PROVIDER {provider!r}; expected 'gemini', 'groq', or 'cerebras'.")
diff --git a/src/pitch_pilot/config.py b/src/pitch_pilot/config.py
index 9b5527e..d7de744 100644
--- a/src/pitch_pilot/config.py
+++ b/src/pitch_pilot/config.py
@@ -16,7 +16,7 @@
from pydantic import Field, ValidationError, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
-_VALID_PROVIDERS = {"gemini", "groq"}
+_VALID_PROVIDERS = {"gemini", "groq", "cerebras"}
class ConfigError(RuntimeError):
@@ -34,17 +34,31 @@ class Settings(BaseSettings):
gemini_api_key: Google Gen AI (Gemini) API key. **Required.**
tavily_api_key: Tavily search API key. **Required.**
groq_api_key: Groq API key. Required only when ``llm_provider`` is ``"groq"``.
- llm_provider: Active LLM provider — ``"gemini"`` (default) or ``"groq"``.
+ cerebras_api_key: Cerebras API key. Required only when ``llm_provider`` is
+ ``"cerebras"`` (its ~1M tokens/day free tier is used for the eval).
+ llm_provider: Active LLM provider — ``"gemini"`` (default), ``"groq"``, or
+ ``"cerebras"``.
gemini_model: Gemini model id (default ``gemini-2.5-flash-lite``).
groq_model: Groq model id (default ``llama-3.1-8b-instant``).
- research_max_queries: Max search queries per research run (``>= 1``, default 4).
+ cerebras_model: Cerebras model id (default ``gpt-oss-120b``; available models
+ vary by account/tier — check the provider's ``models.list()``).
+ research_max_queries: Max search queries per research run (``>= 1``, default 3).
+ research_max_page_chars: Max characters of each fetched page / search-result
+ content fed to the extractor (``>= 500``, default 3500). The biggest
+ token lever — truncating the source text keeps cost and latency (and
+ free-tier token usage) down. The evidence-substring check runs against
+ this truncated text, so groundedness is preserved.
+ research_max_facts_per_source: Max facts extracted from a single source
+ (``>= 1``, default 5), so one page can't dominate a run.
qualify_threshold: Minimum fit score, in ``[0, 1]``, for a company to
qualify against the ICP (default 0.5). A matched negative signal vetoes
qualification regardless of this score.
- groundedness_threshold: Minimum groundedness score, in ``[0, 1]``, for a
- draft to pass verification (default 0.9). With first-party-only
- enforcement (P3) a passing draft scores 1.0, so this is effectively a
- floor; it is kept for transparency and future tuning.
+ groundedness_threshold: A floor in ``[0, 1]`` kept for transparency and
+ future tuning (default 0.9). The verify gate is **all-or-nothing on
+ faithfulness** — a draft passes iff no body claim is ``unsupported`` (and
+ none is ``overreach`` under ``faithfulness_strict``), not by comparing
+ ``groundedness_score`` to this threshold; under strict mode a passing
+ draft scores 1.0.
faithfulness_strict: When ``True`` (default), the verification gate treats
an ``"overreach"`` faithfulness verdict as a failure; when ``False``,
only ``"unsupported"`` fails. ``"unsupported"`` always fails.
@@ -65,15 +79,26 @@ class Settings(BaseSettings):
groq_api_key: str | None = Field(
default=None, description="Groq API key. Required only when LLM_PROVIDER=groq."
)
+ cerebras_api_key: str | None = Field(
+ default=None, description="Cerebras API key. Required only when LLM_PROVIDER=cerebras."
+ )
# --- Provider + model selection ---
- llm_provider: str = Field(default="gemini", description='Active LLM provider: "gemini" or "groq".')
+ llm_provider: str = Field(default="gemini", description='Active LLM provider: "gemini", "groq", or "cerebras".')
gemini_model: str = Field(default="gemini-2.5-flash-lite", description="Gemini model id.")
groq_model: str = Field(default="llama-3.1-8b-instant", description="Groq model id.")
+ cerebras_model: str = Field(default="gpt-oss-120b", description="Cerebras model id (availability varies by account; check models.list()).")
# --- Tunables ---
research_max_queries: int = Field(
- default=4, ge=1, description="Max number of search queries per research run."
+ default=3, ge=1, description="Max number of search queries per research run."
+ )
+ research_max_page_chars: int = Field(
+ default=3500, ge=500,
+ description="Max characters of each source's text fed to the extractor (token lever).",
+ )
+ research_max_facts_per_source: int = Field(
+ default=5, ge=1, description="Max facts extracted from a single source."
)
qualify_threshold: float = Field(
default=0.5, ge=0.0, le=1.0, description="Minimum fit score for a company to qualify against the ICP."
@@ -96,6 +121,15 @@ def _validate_provider(cls, value: str) -> str:
)
return normalized
+ @property
+ def active_model(self) -> str:
+ """The model id for the currently selected ``llm_provider``."""
+ return {
+ "gemini": self.gemini_model,
+ "groq": self.groq_model,
+ "cerebras": self.cerebras_model,
+ }[self.llm_provider]
+
@lru_cache(maxsize=1)
def get_settings() -> Settings:
diff --git a/src/pitch_pilot/models/draft.py b/src/pitch_pilot/models/draft.py
index 313e8cd..36b4552 100644
--- a/src/pitch_pilot/models/draft.py
+++ b/src/pitch_pilot/models/draft.py
@@ -15,9 +15,11 @@ class Draft(BaseModel):
Attributes:
subject: The email subject line.
body: The email body.
- hooks_used: The angles/hooks the draft leaned on, e.g.
- ``["recent funding", "open SDR roles"]`` — useful for review and
- for tracing each hook back to a source.
+ hooks_used: The canonical claim text of the first-party `Fact` objects the
+ draft was grounded in — the facts the model selected by id, e.g.
+ ``["Acme raised a $20M Series B", "Acme is hiring SDRs"]``. Grounded by
+ construction (each is a real own_site/authoritative fact), so each hook
+ traces straight back to a source.
"""
subject: str
diff --git a/src/pitch_pilot/models/verification.py b/src/pitch_pilot/models/verification.py
index a7bfe15..a46c2a6 100644
--- a/src/pitch_pilot/models/verification.py
+++ b/src/pitch_pilot/models/verification.py
@@ -16,22 +16,25 @@
class ClaimVerdict(BaseModel):
- """The full audit trail for one claim in a draft.
+ """The full audit trail for one claim the draft body makes about the company.
- One of these is produced per claim the draft stands on, whether it passed or
- failed, so a reviewer can see exactly why each claim was accepted or rejected.
+ Under the 0.8.0 gate the faithfulness judge extracts each factual claim the draft
+ **body** makes about the company and rates it against the facts the draft is
+ grounded in. One ``ClaimVerdict`` is produced per such body claim — whether it was
+ judged faithful or not — so a reviewer can see exactly which fact (if any) backs
+ each claim and how the judge rated it.
Attributes:
- claim: The draft claim under audit (a `Draft` hook).
- fact_used: The claim text of the backing `Fact` chosen to support it, or
- ``None`` if no fact backs the claim.
- source_url: The backing fact's source URL, or ``None`` if unbacked.
- tier: The backing fact's `SourceTier`, or ``None`` if unbacked.
- substring_ok: Whether the backing fact carries a verbatim ``evidence``
- snippet (the extraction-time substring guard held). ``False`` when the
- backing fact has no evidence.
- faithfulness: The LLM judge's verdict for this claim↔evidence pair, or
- ``None`` when the claim failed an earlier check and was not judged.
+ claim: The body claim under audit (extracted by the faithfulness judge).
+ fact_used: The claim text of the supporting `Fact` the judge cited, or
+ ``None`` when the claim is ``unsupported``.
+ source_url: The supporting fact's source URL, or ``None`` when unsupported.
+ tier: The supporting fact's `SourceTier`, or ``None`` when unsupported.
+ substring_ok: Whether the supporting fact carries a verbatim ``evidence``
+ snippet (the extraction-time substring guard held). ``False`` when there
+ is no supporting fact.
+ faithfulness: The LLM judge's verdict for this body claim
+ (``faithful`` / ``overreach`` / ``unsupported``).
"""
claim: str
@@ -43,32 +46,37 @@ class ClaimVerdict(BaseModel):
class VerificationResult(BaseModel):
- """The result of auditing every claim in a `Draft` against its sources.
+ """The result of auditing a `Draft`: structural grounding plus body faithfulness.
- This is the enforcement point for the hero guarantee. Under the P3 gate a draft
- passes only if **every** claim is *verified*: backed by a first-party
- (``own_site`` / ``authoritative``) `Fact` with a verbatim evidence snippet that
- an LLM judge rates as faithfully supporting the claim (see the groundedness
- methodology docs for the full picture).
+ This is the enforcement point for the hero guarantee. Under the 0.8.0 gate a draft
+ passes only if it is grounded in at least one first-party (``own_site`` /
+ ``authoritative``) `Fact`, has a non-empty body, the faithfulness judge ran, and
+ **no body claim is ``unsupported``** (and none is ``overreach`` when
+ ``FAITHFULNESS_STRICT``). Source-text substring grounding lives at extraction, so
+ the hooks are grounded by construction; this result audits the body's *support*
+ (see the groundedness methodology docs for the full picture).
Attributes:
- groundedness_score: Fraction of claims that are fully verified, in
- ``[0, 1]`` (``verified_claims / total_claims``). Reported even when the
- draft passes.
- faithfulness_score: Fraction of claims the judge rated ``"faithful"``, in
- ``[0, 1]`` (``faithful_claims / total_claims``).
- total_claims: Total number of claims checked (the draft's hooks).
- grounded_claims: Number of claims that are fully verified (the numerator of
- ``groundedness_score``).
- tier_breakdown: Count of claims per backing source tier, e.g.
- ``{"own_site": 2, "unbacked": 1}``.
- claim_verdicts: The per-claim audit trail (see `ClaimVerdict`).
- flagged_claims: Human-readable failure lines for the claims that did **not**
- verify, each prefixed with the specific reason (``unbacked:`` /
- ``volatile-source:`` / ``not-substring:`` / ``overreach:`` /
- ``unsupported:``).
- passed: ``True`` only if there is at least one claim and every claim is
- verified.
+ groundedness_score: Fraction of body claims that count as verified, in
+ ``[0, 1]`` (``verified_claims / total_body_claims``), where a claim is
+ verified when the judge rates it ``faithful`` (or ``overreach`` when
+ ``FAITHFULNESS_STRICT`` is off). Under the default strict mode this equals
+ ``faithful_claims / total_body_claims``. ``0.0`` when the draft is
+ ungrounded or its body is empty; ``1.0`` when a grounded body makes no
+ checkable company claim.
+ faithfulness_score: Fraction of body claims the judge rated ``"faithful"``, in
+ ``[0, 1]`` (``faithful_claims / total_body_claims``).
+ total_claims: Number of body claims the judge extracted and checked.
+ grounded_claims: Number of body claims that count as verified (the numerator
+ of ``groundedness_score``).
+ tier_breakdown: Count of the draft's grounding facts (hooks) per source tier,
+ e.g. ``{"own_site": 2, "authoritative": 1}``.
+ claim_verdicts: The per-body-claim audit trail (see `ClaimVerdict`).
+ flagged_claims: Human-readable failure lines, each prefixed with the specific
+ reason (``structural:`` / ``overreach:`` / ``unsupported:`` /
+ ``judge-error:``).
+ passed: ``True`` only if the draft is grounded, the body is non-empty, the
+ judge ran, and no body claim failed.
"""
groundedness_score: float = Field(ge=0.0, le=1.0)
diff --git a/src/pitch_pilot/nodes/__init__.py b/src/pitch_pilot/nodes/__init__.py
index 046933a..fc5020c 100644
--- a/src/pitch_pilot/nodes/__init__.py
+++ b/src/pitch_pilot/nodes/__init__.py
@@ -11,9 +11,9 @@
``QualificationResult`` (P2).
* ``draft_node`` / `run_draft` — write grounded outreach from facts →
``Draft`` (P2).
- * ``verify_node`` / `run_verification` / `judge_faithfulness` — the groundedness
- gate over a draft (first-party tier + substring + LLM faithfulness judge) →
- ``VerificationResult`` (P2 basic gate, hardened in P3).
+ * ``verify_node`` / `run_verification` / `judge_body` — the groundedness gate
+ over a draft (structural first-party hook check + LLM body-faithfulness judge)
+ → ``VerificationResult`` (P2 basic gate, hardened in P3, decoupled in P5).
* ``log_node`` / `log_lead` — persist the lead + enqueue it for human review
(P2).
@@ -30,7 +30,7 @@
research_node,
run_research,
)
-from pitch_pilot.nodes.verify import judge_faithfulness, run_verification, verify_node
+from pitch_pilot.nodes.verify import judge_body, run_verification, verify_node
__all__ = [
"run_research",
@@ -43,7 +43,7 @@
"draft_node",
"run_verification",
"verify_node",
- "judge_faithfulness",
+ "judge_body",
"log_lead",
"log_node",
]
diff --git a/src/pitch_pilot/nodes/draft.py b/src/pitch_pilot/nodes/draft.py
index defa019..ea0c16f 100644
--- a/src/pitch_pilot/nodes/draft.py
+++ b/src/pitch_pilot/nodes/draft.py
@@ -1,35 +1,41 @@
-"""The draft node — write grounded outreach from the research facts.
+"""The draft node — write grounded outreach by SELECTING which facts to stand on.
-This node writes the outreach email. The hard rule, straight from the project's
-hero guarantee, is that **every hook the draft leans on must be a real `Fact`** —
-the model may phrase things, but it may not introduce a claim that the research
-did not establish. P3 sharpens that rule with **Policy B (first-party-only for
-claims)**: a stated draft claim may rest only on an ``own_site`` or
-``authoritative`` fact.
+This node writes the outreach email. The hero guarantee is that **every fact the
+draft is grounded in is a real, source-backed `Fact`** — and, under **Policy B**,
+only a first-party (``own_site`` / ``authoritative``) fact.
-Two enforcement layers make that true regardless of what the LLM returns:
+Release 0.8.0 decouples *grounding* from *phrasing* (see ADR-0014). The earlier design asked
+the model to copy fact text verbatim into ``hooks`` and then validated each hook as
+a substring of the source. That made grounding brittle: a perfectly faithful
+paraphrase was thrown away because it was not a verbatim copy. The substring check
+belongs at **extraction** (where the research node already verifies each fact's
+``evidence`` is a literal substring of its source), not at the draft layer.
+
+So the draft layer now works like this:
1. **Tier-gating the claim pool.** Only ``own_site`` / ``authoritative`` facts are
- offered to the model as *claimable*. ``third_party_snippet`` facts are passed in
- a separate **context-only** section — the model may use them for tone or framing,
- but they can never become a hook. This is the direct lesson of the P1
- validation: search-snippet sources were the ones that failed live
- re-verification, so they must never carry a stated claim.
-2. **Validating the outputs.** Whatever ``hooks`` the model claims it used are
- matched back against the *claimable* (first-party) facts; anything that does not
- map to one is discarded. So ``Draft.hooks_used`` is always a subset of the
- first-party research facts, by construction.
-
-The body's prose is *not* trusted blindly — the verify node independently audits
-the draft's claims against sources. This node's job is to produce a grounded
-draft and an honest list of the facts it stood on.
+ offered to the model as *claimable*, each on a **numbered** line.
+ ``third_party_snippet`` facts are passed in a separate **context-only** section —
+ usable for tone or framing, never as a stated claim.
+2. **Selection by reference, not by copy.** The model writes the email as free prose
+ (it may paraphrase the facts naturally) and returns the **ids** of the claimable
+ facts it grounded the email in. ``Draft.hooks_used`` is then those selected facts'
+ canonical claim text. Because the ids resolve to real first-party facts, every
+ hook is grounded **by construction** — each already passed the extraction-time
+ evidence-substring check. No hook text is re-substring-checked, and a paraphrased
+ body is never fuzzy-matched back to a fact.
+
+The body's prose is *not* trusted blindly — the verify node independently judges the
+body's claims for faithfulness against the selected facts. This node's job is to
+produce a grounded draft and an honest list of the facts it stood on.
"""
from __future__ import annotations
import logging
+import re
-from pitch_pilot.clients.llm import LLMClient, LLMError, get_llm_client
+from pitch_pilot.clients.llm import LLMClient, LLMError, get_llm_client, trim_to_token_budget
from pitch_pilot.config import Settings, get_settings
from pitch_pilot.graph.state import PipelineState
from pitch_pilot.models.draft import Draft
@@ -42,25 +48,50 @@
_MAX_FACTS_IN_PROMPT = 30
_MAX_FACT_CHARS = 240
+# Per-block token budget. The draft prompt renders two blocks (claimable + context),
+# so each is bounded so their sum stays well under a small (8192-token) free-tier
+# context window even with many/long facts (see ADR-0013).
+_FACTS_TOKEN_BUDGET = 1500
# Policy B: only these tiers may carry a stated draft claim.
_CLAIMABLE_TIERS = ("own_site", "authoritative")
+# Gate-critical call: draft deterministically for reproducible output (see verify).
+_DRAFT_TEMPERATURE = 0.0
+
+# Backstop for fact-id leakage. The numbered ids are a selection mechanism for the
+# ``facts_used`` field only; the model is told never to put them in the prose, and
+# this strips any that slip through (e.g. ``(Fact 15)`` / ``[fact 2]``).
+_FACT_ID_RE = re.compile(r"\s*[\(\[]\s*facts?\s*#?\s*\d+\s*[\)\]]", re.IGNORECASE)
+
+
+def _strip_fact_ids(text: str) -> str:
+ """Remove any leaked fact-id citations like ``(Fact 15)`` from draft prose."""
+ cleaned = _FACT_ID_RE.sub("", text)
+ cleaned = re.sub(r"[ \t]{2,}", " ", cleaned) # collapse doubled spaces
+ cleaned = re.sub(r"\s+([.,;:!?])", r"\1", cleaned) # tidy space-before-punctuation
+ return cleaned.strip()
+
_DRAFT_SYSTEM = (
- "You are an SDR writing a short, specific cold outreach email. You are given "
- "CLAIMABLE FACTS about a company and (sometimes) CONTEXT-ONLY facts, plus a note "
- "on why the company qualified. Write a concise, genuine email (subject + body, "
- "3-5 sentences).\n\n"
- "Strict rules:\n"
- "- Every concrete claim about the company MUST come from the CLAIMABLE FACTS. "
- "Never invent or add information from outside knowledge.\n"
+ "You are an SDR writing a short, specific cold outreach email. You are given a "
+ "NUMBERED list of CLAIMABLE FACTS about a company and (sometimes) CONTEXT-ONLY "
+ "facts, plus a note on why the company qualified. Write a concise, genuine email "
+ "(subject + body, 3-5 sentences).\n\n"
+ "Rules:\n"
+ "- Ground every concrete claim about the company in the CLAIMABLE FACTS. You may "
+ "paraphrase them naturally — you need NOT copy them verbatim — but never state "
+ "anything about the company the CLAIMABLE FACTS do not support, and never use "
+ "outside knowledge.\n"
"- CONTEXT-ONLY facts may inform tone or framing but must NEVER be stated as a "
- "claim and must NEVER appear in 'hooks'.\n"
- "- In 'hooks', list the exact claim text of every CLAIMABLE FACT you referenced, "
- "copied verbatim.\n"
+ "claim and must NEVER appear in 'facts_used'.\n"
+ "- In 'facts_used', return the integer ids (from the numbered CLAIMABLE FACTS) of "
+ "every fact your email is grounded in.\n"
+ "- The fact id numbers are ONLY for the 'facts_used' field. NEVER write an id in "
+ "the subject or body — no '(Fact 3)', 'Fact 3', '[2]', etc. The email must read "
+ "as natural prose a prospect would see.\n"
"- Do not be pushy; no fake urgency; no placeholders like [Name].\n\n"
- 'Respond with a JSON object: {"subject": "...", "body": "...", "hooks": '
- '["", ...]}'
+ 'Respond with a JSON object: {"subject": "...", "body": "...", "facts_used": '
+ "[, ...]}"
)
@@ -79,12 +110,28 @@ def _context_facts(facts: list[Fact]) -> list[Fact]:
return [f for f in facts if f.source_tier == "third_party_snippet"]
-def _facts_block(facts: list[Fact]) -> str:
- """Render facts as compact, tier-tagged lines for the draft prompt."""
- return "\n".join(
- f"- [{fact.source_tier}] {fact.claim[:_MAX_FACT_CHARS]}"
- for fact in facts[:_MAX_FACTS_IN_PROMPT]
- )
+def _shown_claimable(claimable: list[Fact]) -> list[Fact]:
+ """The claimable facts actually shown to the model, in id order (1-based position).
+
+ Capped to ``_MAX_FACTS_IN_PROMPT`` and then to the per-block token budget. The id
+ a fact is given in the prompt is its 1-based index in this list, so the same list
+ is used both to render the prompt and to resolve the model's selected ids.
+ """
+ capped = claimable[:_MAX_FACTS_IN_PROMPT]
+ lines = [f"[{i}] {fact.claim[:_MAX_FACT_CHARS]}" for i, fact in enumerate(capped, start=1)]
+ kept = trim_to_token_budget(lines, _FACTS_TOKEN_BUDGET)
+ return capped[: len(kept)]
+
+
+def _numbered_block(facts: list[Fact]) -> str:
+ """Render facts as numbered, id-tagged lines (the ids the model selects by)."""
+ return "\n".join(f"[{i}] {fact.claim[:_MAX_FACT_CHARS]}" for i, fact in enumerate(facts, start=1))
+
+
+def _context_block(facts: list[Fact]) -> str:
+ """Render context-only facts as compact tier-tagged lines, bounded to a budget."""
+ lines = [f"- [{fact.source_tier}] {fact.claim[:_MAX_FACT_CHARS]}" for fact in facts[:_MAX_FACTS_IN_PROMPT]]
+ return "\n".join(trim_to_token_budget(lines, _FACTS_TOKEN_BUDGET))
def _draft_user_prompt(
@@ -93,24 +140,72 @@ def _draft_user_prompt(
context: list[Fact],
qualification: QualificationResult | None,
) -> str:
- """Build the draft prompt: who the company is, why it qualified, and which facts it may use."""
+ """Build the draft prompt: who the company is, why it qualified, and the numbered
+ claimable facts it may ground the email in (plus context-only background)."""
why = qualification.reason if qualification else "(not provided)"
+ shown = _shown_claimable(claimable)
context_block = (
- "\n\nCONTEXT-ONLY facts (background; NEVER state as a claim or use as a hook):\n"
- + _facts_block(context)
+ "\n\nCONTEXT-ONLY facts (background only; NEVER state as a claim or select):\n"
+ + _context_block(context)
if context
else ""
)
return (
f"COMPANY: {company_name}\n"
f"WHY IT QUALIFIED: {why}\n\n"
- "CLAIMABLE FACTS (the only facts you may state and put in hooks):\n"
- f"{_facts_block(claimable)}"
+ "CLAIMABLE FACTS (numbered; you may ground the email ONLY in these):\n"
+ f"{_numbered_block(shown)}"
f"{context_block}\n\n"
- "Write the outreach email now."
+ "Write the outreach email now, then list in 'facts_used' the ids of the "
+ "CLAIMABLE FACTS you grounded it in."
)
+def _as_id(item: object) -> int | None:
+ """Coerce a model-returned selection into a 1-based fact id, or ``None``.
+
+ Accepts an int (``3``) or a digit string, optionally bracketed (``"3"``,
+ ``"[3]"``). Booleans and anything else return ``None``.
+ """
+ if isinstance(item, bool):
+ return None
+ if isinstance(item, int):
+ return item
+ if isinstance(item, str):
+ match = re.fullmatch(r"\s*\[?(\d+)\]?\s*", item)
+ if match:
+ return int(match.group(1))
+ return None
+
+
+def _selected_hooks(raw: object, shown: list[Fact], claimable: list[Fact]) -> list[str]:
+ """Resolve the model's selected fact references to canonical claim text.
+
+ Selection is **by id** (1-based position in the shown claimable list), so each
+ hook is a real first-party fact by construction — no substring or fuzzy match.
+ As a defensive fallback for models that echo a fact's claim verbatim instead of
+ its id, an *exact* (normalized) claim match is also accepted; a paraphrase is
+ never matched. The result is de-duplicated, preserving the model's order.
+ """
+ if not isinstance(raw, list):
+ return []
+ by_norm_claim = {_normalize(fact.claim): fact for fact in claimable}
+ hooks: list[str] = []
+ seen: set[str] = set()
+ for item in raw:
+ fact: Fact | None = None
+ idx = _as_id(item)
+ if idx is not None and 1 <= idx <= len(shown):
+ fact = shown[idx - 1]
+ elif isinstance(item, str):
+ fact = by_norm_claim.get(_normalize(item)) # verbatim echo, not paraphrase
+ if fact is None or fact.claim in seen:
+ continue
+ seen.add(fact.claim)
+ hooks.append(fact.claim)
+ return hooks
+
+
def run_draft(
research: ResearchResult | None,
qualification: QualificationResult | None,
@@ -119,11 +214,13 @@ def run_draft(
) -> Draft:
"""Write a grounded outreach `Draft` from the research facts (Policy B).
- The LLM drafts the email; this function constrains and audits it: only
- first-party (``own_site`` / ``authoritative``) facts are offered as claimable,
- third-party-snippet facts are passed as context only, and every hook the model
- returns is matched back against the claimable facts so ``hooks_used`` is always
- a subset of the first-party research facts (see the module docstring).
+ The LLM drafts the email as free prose and selects, *by id*, which claimable
+ facts it grounded the email in. Only first-party (``own_site`` /
+ ``authoritative``) facts are offered as claimable; third-party-snippet facts are
+ passed as context only. ``hooks_used`` is the canonical claim text of the
+ selected facts, so it is always a subset of the first-party research facts and is
+ grounded by construction — the draft layer does **not** substring- or
+ fuzzy-match hook text against the source (see the module docstring and ADR-0014).
Args:
research: The research whose ``facts`` ground the draft.
@@ -147,25 +244,18 @@ def run_draft(
company_name = research.company.name or research.company.domain if research else "the company"
try:
payload = llm.complete_json(
- _DRAFT_SYSTEM, _draft_user_prompt(company_name, claimable, context, qualification)
+ _DRAFT_SYSTEM,
+ _draft_user_prompt(company_name, claimable, context, qualification),
+ temperature=_DRAFT_TEMPERATURE,
)
except LLMError as exc:
logger.warning("draft LLM call failed: %s", exc)
return Draft(subject="", body="", hooks_used=[])
- subject = str(payload.get("subject", "")).strip()
- body = str(payload.get("body", "")).strip()
-
- # Validate hooks: keep only those that map to a real CLAIMABLE fact (verbatim,
- # whitespace/case-insensitive), de-duplicated, using the canonical fact text.
- claimable_by_norm = {_normalize(fact.claim): fact.claim for fact in claimable}
- hooks_used: list[str] = []
- seen: set[str] = set()
- for hook in payload.get("hooks", []) if isinstance(payload.get("hooks"), list) else []:
- key = _normalize(str(hook))
- if key in claimable_by_norm and key not in seen:
- seen.add(key)
- hooks_used.append(claimable_by_norm[key])
+ # Strip any leaked fact-id tokens the prompt told the model to keep out (backstop).
+ subject = _strip_fact_ids(str(payload.get("subject", "")).strip())
+ body = _strip_fact_ids(str(payload.get("body", "")).strip())
+ hooks_used = _selected_hooks(payload.get("facts_used"), _shown_claimable(claimable), claimable)
return Draft(subject=subject, body=body, hooks_used=hooks_used)
diff --git a/src/pitch_pilot/nodes/qualify.py b/src/pitch_pilot/nodes/qualify.py
index 0d6b04a..fa4227b 100644
--- a/src/pitch_pilot/nodes/qualify.py
+++ b/src/pitch_pilot/nodes/qualify.py
@@ -9,14 +9,19 @@
Two principles shape it:
-* **Unknowns are never guessed.** When the facts do not establish a signal, the
- assessment marks it ``"unknown"`` rather than assuming it is present or absent.
- Unknown *structural* attributes (industry, size, region) are dropped from the
- score and the remaining weights are renormalized, so a company is not punished
- for a research gap.
+* **Unknowns are never guessed.** When the facts do not establish a *positive*
+ signal or a *size/region* attribute, the assessment marks it ``"unknown"`` and
+ that component is dropped from the score (weights renormalized), so a research
+ gap there does not punish the company.
+* **Industry is required when the ICP names target industries.** Then an
+ ``unknown`` or ``no_match`` industry scores 0.0 and stays in the denominator —
+ a fintech-specific ICP must not qualify a company it cannot place in-industry
+ (see ADR-0015).
* **A matched negative signal is a hard veto.** No fit score, however high, can
qualify a company that matches an ICP negative signal (a competitor, a
- non-profit, …).
+ non-profit, …). Negative signals are judged from the company's described
+ business, so negations like "not fintech" fire reliably rather than defaulting
+ to ``unknown``.
The scoring weights and the veto live here in code (not in the prompt), so the
verdict is reproducible and explainable from the same assessment every time.
@@ -26,7 +31,7 @@
import logging
-from pitch_pilot.clients.llm import LLMClient, LLMError, get_llm_client
+from pitch_pilot.clients.llm import LLMClient, LLMError, get_llm_client, trim_to_token_budget
from pitch_pilot.config import Settings, get_settings
from pitch_pilot.graph.state import PipelineState
from pitch_pilot.models.fact import Fact
@@ -41,9 +46,16 @@
# the facts leave unknown are dropped and the remaining weights renormalized.
_WEIGHTS = {"industry": 0.35, "size": 0.25, "region": 0.15, "positives": 0.25}
-# How many facts to show the assessor, and how long each line may be.
+# Gate-critical call: assess deterministically so the qualification verdict (and the
+# headline F1) is reproducible, not a nondeterministic snapshot.
+_QUALIFY_TEMPERATURE = 0.0
+
+# How many facts to show the assessor, how long each line may be, and a token
+# budget for the whole facts block so the prompt fits a small (8192-token)
+# free-tier context window even with many/long facts (see ADR-0013).
_MAX_FACTS_IN_PROMPT = 40
_MAX_FACT_CHARS = 240
+_FACTS_TOKEN_BUDGET = 3000
_QUALIFY_SYSTEM = (
"You are the qualification assessor for an SDR agent. You are given an Ideal "
@@ -53,8 +65,19 @@
"qualifies — downstream code does that.\n\n"
"Strict rules:\n"
"- Judge ONLY from the provided FACTS. Never use outside knowledge.\n"
- "- If the FACTS do not establish something, its status is 'unknown'. Do not "
+ "- For structural attributes (industry, region, employee_count) and POSITIVE "
+ "signals: if the FACTS do not establish it, the status is 'unknown'. Do not "
"guess 'match' or 'no_match'.\n"
+ "- NEGATIVE signals describe a disqualifying condition. Judge whether that "
+ "condition is TRUE of this company from what the FACTS reveal about its core "
+ "business — reason from the business the facts describe; do NOT default to "
+ "'unknown' just because no fact literally states the negation. For example "
+ "'not fintech or no money movement' is 'match' when the facts show the business "
+ "is outside payments/banking/lending (e.g. design tools, developer "
+ "infrastructure, analytics, ML model hosting); 'large incumbent bank that builds "
+ "fraud in-house' is 'match' when the facts describe a major established bank. Use "
+ "'unknown' for a negative only when the facts genuinely don't reveal what the "
+ "company does.\n"
"- For every match, cite the supporting fact text in 'evidence_fact' (copied "
"from the FACTS); otherwise use an empty string.\n"
"- For employee_count, return an integer 'value' only if a fact states it, "
@@ -71,12 +94,12 @@
def _facts_block(facts: list[Fact]) -> str:
- """Render facts as compact, source-tagged lines for the assessor prompt."""
- lines = []
- for fact in facts[:_MAX_FACTS_IN_PROMPT]:
- claim = fact.claim[:_MAX_FACT_CHARS]
- lines.append(f"- [{fact.source_tier}] {claim} (source: {fact.source_url})")
- return "\n".join(lines)
+ """Render facts as compact, source-tagged lines, bounded to a token budget."""
+ lines = [
+ f"- [{fact.source_tier}] {fact.claim[:_MAX_FACT_CHARS]} (source: {fact.source_url})"
+ for fact in facts[:_MAX_FACTS_IN_PROMPT]
+ ]
+ return "\n".join(trim_to_token_budget(lines, _FACTS_TOKEN_BUDGET))
def _qualify_user_prompt(icp: ICP, facts: list[Fact]) -> str:
@@ -97,7 +120,9 @@ def _qualify_user_prompt(icp: ICP, facts: list[Fact]) -> str:
def _assess(icp: ICP, facts: list[Fact], llm: LLMClient) -> dict:
"""Ask the LLM to assess the ICP against the facts; return ``{}`` on failure."""
try:
- return llm.complete_json(_QUALIFY_SYSTEM, _qualify_user_prompt(icp, facts))
+ return llm.complete_json(
+ _QUALIFY_SYSTEM, _qualify_user_prompt(icp, facts), temperature=_QUALIFY_TEMPERATURE
+ )
except LLMError as exc:
logger.warning("qualification assessor LLM call failed: %s", exc)
return {}
@@ -202,8 +227,19 @@ def run_qualification(
)
# --- Weighted, renormalized fit score over the components we actually know ---
+ # Industry is a REQUIRED attribute when the ICP names target industries: an
+ # unconfirmed (``unknown``) or mismatched industry scores 0.0 and stays in the
+ # denominator, so a research gap on industry counts against the company instead
+ # of being dropped. A fintech-specific ICP must not qualify a company it cannot
+ # place in-industry (the lesson behind the figma/huggingface false positives —
+ # see ADR-0015). Size/region unknowns are still dropped: gaps there are research
+ # noise, not disqualifying.
+ if icp.industries:
+ industry_component: float | None = 1.0 if industry == "match" else 0.0
+ else:
+ industry_component = _component_score(industry)
components: dict[str, float | None] = {
- "industry": _component_score(industry),
+ "industry": industry_component,
"size": _component_score(size),
"region": _component_score(region),
"positives": positives_component,
diff --git a/src/pitch_pilot/nodes/research.py b/src/pitch_pilot/nodes/research.py
index d2a11e9..251f7cd 100644
--- a/src/pitch_pilot/nodes/research.py
+++ b/src/pitch_pilot/nodes/research.py
@@ -56,11 +56,15 @@
RESEARCH_DIMENSIONS: tuple[str, ...] = ("overview", "news", "hiring", "tech")
_VALID_CATEGORIES = set(RESEARCH_DIMENSIONS)
-# Hard caps that bound a single research run regardless of the planner's choices.
-MAX_FACTS_PER_SOURCE = 8 # cap extraction per page so one source can't dominate
+# Hard caps that bound a single research run. The page-char and per-source caps are
+# the lean *fallback* defaults used when `extract_facts` is called without explicit
+# limits; the shipping defaults live in `Settings` (`research_max_page_chars`,
+# `research_max_facts_per_source`) and `run_research` passes those through. See
+# ADR-0012 for why research depth is leaned out by default.
+MAX_FACTS_PER_SOURCE = 5 # fallback per-source fact cap (Settings overrides)
MAX_EVIDENCE_CHARS = 200 # evidence snippet length cap (mirrors Fact.evidence)
SEARCH_RESULTS_PER_QUERY = 4 # how many hits to extract from per search query
-MAX_TEXT_CHARS = 12_000 # cap the text sent to the extractor (cost / latency)
+MAX_TEXT_CHARS = 3_500 # fallback page-text cap fed to the extractor (Settings overrides)
_WHITESPACE = re.compile(r"\s+")
@@ -199,17 +203,24 @@ def extract_facts(
source_title: str | None,
llm: LLMClient,
company_domain: str | None = None,
+ *,
+ max_page_chars: int = MAX_TEXT_CHARS,
+ max_facts_per_source: int = MAX_FACTS_PER_SOURCE,
) -> list[Fact]:
"""Extract grounded `Fact`s from a single source's text.
This is the groundedness guard. It asks the LLM (via
`LLMClient.complete_json`) to return only claims that the supplied ``text``
explicitly supports, each with a verbatim ``evidence`` snippet. Every
- candidate is then checked: its evidence must actually appear in ``text`` (a
- whitespace- and case-insensitive substring match). Candidates that fail the
+ candidate is then checked: its evidence must actually appear in the source text
+ (a whitespace- and case-insensitive substring match). Candidates that fail the
check — typically claims the model pulled from its own prior knowledge — are
- dropped and logged. Extraction is capped at `MAX_FACTS_PER_SOURCE` facts per
- source.
+ dropped and logged. Extraction is capped at ``max_facts_per_source`` per source.
+
+ The source text is truncated to ``max_page_chars`` **before** extraction, and
+ the substring check runs against that *same* truncated text — so the model only
+ ever sees, and can only ground claims in, the text we actually verify against.
+ This truncation is the dominant token/cost lever (see ADR-0012).
Each surviving fact is tagged with a `SourceTier` via `classify_source_tier`
(``company_domain`` decides what counts as the company's own site).
@@ -221,20 +232,25 @@ def extract_facts(
llm: The LLM client used to perform extraction.
company_domain: The company's own domain, used to tier the source. When
omitted, no source can be recognized as ``"own_site"``.
+ max_page_chars: Truncate the source text to this many characters before
+ extraction (and grounding). Defaults to `MAX_TEXT_CHARS`.
+ max_facts_per_source: Stop after this many grounded facts from this source.
+ Defaults to `MAX_FACTS_PER_SOURCE`.
Returns:
A list of grounded `Fact`s (possibly empty). Never raises for a bad page
or a bad LLM response — those cases yield an empty list.
"""
- text = (text or "").strip()
- if not text:
+ # Truncate ONCE: the model sees exactly the text we will ground against.
+ source_text = (text or "").strip()[:max_page_chars]
+ if not source_text:
return []
user_prompt = (
f"SOURCE URL: {source_url}\n"
f"SOURCE TITLE: {source_title or '(unknown)'}\n\n"
"TEXT:\n"
- f"{text[:MAX_TEXT_CHARS]}"
+ f"{source_text}"
)
try:
payload = llm.complete_json(_EXTRACTOR_SYSTEM, user_prompt)
@@ -242,7 +258,7 @@ def extract_facts(
logger.warning("extractor LLM call failed for %s: %s", source_url, exc)
return []
- normalized_source = _normalize(text)
+ normalized_source = _normalize(source_text)
source_tier = classify_source_tier(source_url, company_domain)
facts: list[Fact] = []
for item in _facts_payload(payload):
@@ -279,8 +295,8 @@ def extract_facts(
logger.info("skipped invalid fact from %s: %s", source_url, exc)
continue
facts.append(fact)
- if len(facts) >= MAX_FACTS_PER_SOURCE:
- logger.info("hit per-source fact cap (%d) for %s", MAX_FACTS_PER_SOURCE, source_url)
+ if len(facts) >= max_facts_per_source:
+ logger.info("hit per-source fact cap (%d) for %s", max_facts_per_source, source_url)
break
return facts
@@ -407,7 +423,11 @@ def accumulate(new_facts: list[Fact]) -> int:
seed_text = ""
result.errors.append(f"seed fetch raised for {seed_url}: {exc}")
if seed_text:
- accumulate(extract_facts(seed_text, seed_url, company.name, llm, company.domain))
+ accumulate(extract_facts(
+ seed_text, seed_url, company.name, llm, company.domain,
+ max_page_chars=settings.research_max_page_chars,
+ max_facts_per_source=settings.research_max_facts_per_source,
+ ))
else:
result.errors.append(f"seed page returned no usable text: {seed_url}")
@@ -442,7 +462,11 @@ def accumulate(new_facts: list[Fact]) -> int:
if not hit.content or not hit.url:
continue
try:
- hit_facts = extract_facts(hit.content, hit.url, hit.title, llm, company.domain)
+ hit_facts = extract_facts(
+ hit.content, hit.url, hit.title, llm, company.domain,
+ max_page_chars=settings.research_max_page_chars,
+ max_facts_per_source=settings.research_max_facts_per_source,
+ )
except Exception as exc: # noqa: BLE001 — extraction must not crash the run
result.errors.append(f"extraction failed for {hit.url}: {exc}")
continue
diff --git a/src/pitch_pilot/nodes/verify.py b/src/pitch_pilot/nodes/verify.py
index c58f086..4970fc3 100644
--- a/src/pitch_pilot/nodes/verify.py
+++ b/src/pitch_pilot/nodes/verify.py
@@ -1,34 +1,38 @@
-"""The verify node — the real groundedness gate over a draft (P3).
+"""The verify node — the real groundedness gate over a draft (0.8.0).
This is the enforcement point for the hero guarantee, applied to the finished
-draft: **every claim the draft stands on must be first-party-sourced,
-substring-anchored, and judged to be faithfully supported by its evidence.** A
-claim is *verified* only when all four hold:
-
-1. **Backed** — the claim maps to a real `Fact` in the research.
-2. **First-party tier** — that fact is ``own_site`` or ``authoritative`` (Policy B:
- a stated claim may never rest on a ``third_party_snippet`` fact).
-3. **Substring-anchored** — the fact carries a verbatim ``evidence`` snippet (the
- extraction-time substring guard held; recorded on the fact).
-4. **Faithful** — an LLM judge rates the claim↔evidence pair ``"faithful"`` (or
- ``"overreach"`` when ``FAITHFULNESS_STRICT`` is off). This is *distinct* from the
- substring check: substring proves the evidence is present, the judge decides
- whether it actually *supports* the claim as stated.
-
-The draft **passes only if every claim is verified.** Each claim that fails is
-recorded with its specific reason (``unbacked`` / ``volatile-source`` /
-``not-substring`` / ``overreach`` / ``unsupported``), and a full per-claim audit
-trail is returned in `VerificationResult.claim_verdicts`.
-
-The node is **network-free except for the LLM judge call** — it does not re-fetch
-sources. Independent live re-verification of each source is an *eval-time* metric
-(P4), reported separately by tier, not part of the per-run hot path (see ADR-0010
-in the decisions log).
+draft. Release 0.8.0 splits the check into a **structural** part (cheap, by construction) and
+a **faithfulness** part (an LLM judge over the body), reflecting that source-text
+grounding now lives at *extraction*, not here (see ADR-0014):
+
+1. **Structural — grounding facts.** Every ``hook_used`` must resolve to a
+ first-party (``own_site`` / ``authoritative``) `Fact` in the research. This is
+ true *by construction* (the draft node only selects such facts by id); the gate
+ re-resolves and asserts it, recording any hook that fails to resolve as a
+ ``structural`` failure (it must never happen).
+2. **Faithfulness — the body.** A single LLM judge reads the draft **body** and the
+ set of selected facts, extracts every factual claim the body makes *about the
+ company*, and rates each ``faithful`` / ``overreach`` / ``unsupported`` against
+ those facts. This judges *support*, which a substring check cannot: a fact can be
+ genuinely sourced yet not back a particular paraphrase in the body.
+
+The draft **passes** only if: there is at least one grounded hook, the body is
+non-empty, there are no structural failures, the judge call succeeded, and **no body
+claim is ``unsupported``** (and none is ``overreach`` when ``FAITHFULNESS_STRICT``).
+Each failing claim is recorded with its reason (``structural`` / ``overreach`` /
+``unsupported`` / ``judge-error``), and a full per-claim audit trail is returned in
+`VerificationResult.claim_verdicts`.
+
+The node is **network-free except for the single LLM judge call** — it does not
+re-fetch sources. Independent live re-verification of each source is an *eval-time*
+metric (P4), reported separately by tier (see ADR-0010 in the decisions log).
"""
from __future__ import annotations
import logging
+import re
+from collections import Counter
from pitch_pilot.clients.llm import LLMClient, LLMError, get_llm_client
from pitch_pilot.config import Settings, get_settings
@@ -41,102 +45,140 @@
logger = logging.getLogger(__name__)
-# Tiers a stated claim may rest on (Policy B). A claim backed only by a
-# third_party_snippet fact is a policy violation, not merely a soft flag.
+# Tiers a stated claim may rest on (Policy B). A hook that resolves to anything else
+# is a structural violation, not merely a soft flag.
_FIRST_PARTY_TIERS = {"own_site", "authoritative"}
-_TIER_RANK = {"own_site": 0, "authoritative": 1, "third_party_snippet": 2}
+_TIER_RANK = {"own_site": 0, "authoritative": 1}
+
+# Gate-critical call: judge deterministically so a draft's verdict is reproducible.
+_JUDGE_TEMPERATURE = 0.0
_FAITHFULNESS_SYSTEM = (
- "You are a strict groundedness judge for an SDR agent. You are given a CLAIM "
- "and an EVIDENCE snippet copied verbatim from a source. Decide ONLY whether the "
- "EVIDENCE supports the CLAIM — never use outside knowledge.\n\n"
+ "You are a strict groundedness judge for an SDR agent. You are given the BODY of "
+ "a cold outreach email and a NUMBERED list of FACTS (each copied from a source, "
+ "with its verbatim evidence). Identify every factual assertion the BODY makes "
+ "ABOUT THE COMPANY (the recipient). Ignore greetings, questions, the sender's own "
+ "product or pitch, and generic pleasantries. For each company claim decide ONLY "
+ "from the FACTS whether it is supported — never use outside knowledge.\n\n"
"Verdicts:\n"
- "- 'faithful': the EVIDENCE directly supports the CLAIM as stated.\n"
- "- 'overreach': the EVIDENCE partially supports it, but the CLAIM generalizes, "
- "exaggerates, or adds beyond what the EVIDENCE actually says.\n"
- "- 'unsupported': the EVIDENCE does not support the CLAIM.\n\n"
- 'Respond with a JSON object: {"verdict": "faithful"|"overreach"|"unsupported", '
- '"reason": ""}'
+ "- 'faithful': a FACT directly supports the claim as stated.\n"
+ "- 'overreach': a FACT partially supports it, but the claim generalizes, "
+ "exaggerates, or adds beyond what the FACT actually says.\n"
+ "- 'unsupported': no FACT supports the claim.\n\n"
+ "For each claim also return 'fact_id': the id of the supporting FACT (or null when "
+ "unsupported).\n\n"
+ 'Respond with a JSON object: {"claims": [{"claim": "...", "verdict": '
+ '"faithful"|"overreach"|"unsupported", "fact_id": , "reason": '
+ '""}]}'
)
-def judge_faithfulness(claim: str, evidence: str, llm: LLMClient) -> dict:
- """Ask the LLM whether ``evidence`` actually supports ``claim``.
+def _as_id(item: object) -> int | None:
+ """Coerce a model-returned fact reference into a 1-based id, or ``None``."""
+ if isinstance(item, bool):
+ return None
+ if isinstance(item, int):
+ return item
+ if isinstance(item, str):
+ match = re.fullmatch(r"\s*\[?(\d+)\]?\s*", item)
+ if match:
+ return int(match.group(1))
+ return None
+
+
+def judge_body(body: str, selected: list[Fact], llm: LLMClient) -> tuple[bool, list[dict]]:
+ """Judge the draft ``body``'s claims for faithfulness against the selected facts.
- This judges *support*, which the substring check cannot: the evidence can be a
- genuine verbatim snippet yet still not back the claim (or back a weaker version
- of it). Fails **closed** — any judge error returns ``"unsupported"`` so a flaky
- model never lets an unverified claim through.
+ One LLM call: the judge extracts each factual claim the body makes about the
+ company and rates it against the selected facts, naming the supporting fact by id.
+ Fails **closed** — any judge error or malformed response returns ``ok=False`` so a
+ flaky model never lets an unjudged body pass the gate.
Args:
- claim: The draft claim under audit.
- evidence: The verbatim evidence snippet from the backing fact.
+ body: The draft body to audit.
+ selected: The first-party facts the draft is grounded in (ids are 1-based).
llm: LLM client used for the judgement.
Returns:
- A dict ``{"verdict": "faithful"|"overreach"|"unsupported", "reason": str}``.
+ ``(ok, claims)`` where ``ok`` is whether the judge call succeeded and
+ ``claims`` is a list of ``{"claim", "verdict", "fact", "reason"}`` dicts
+ (``fact`` is the resolved `Fact` or ``None``).
"""
- user = f"CLAIM: {claim}\n\nEVIDENCE: {evidence}"
+ facts_block = "\n".join(
+ f"[{i}] {fact.claim}" + (f" — evidence: {fact.evidence}" if fact.evidence else "")
+ for i, fact in enumerate(selected, start=1)
+ )
+ user = f"BODY:\n{body}\n\nFACTS:\n{facts_block}"
try:
- payload = llm.complete_json(_FAITHFULNESS_SYSTEM, user)
+ payload = llm.complete_json(_FAITHFULNESS_SYSTEM, user, temperature=_JUDGE_TEMPERATURE)
except LLMError as exc:
- logger.warning("faithfulness judge failed for claim %r: %s", claim, exc)
- return {"verdict": "unsupported", "reason": f"judge call failed: {exc}"}
- verdict = str(payload.get("verdict", "")).strip().lower()
- if verdict not in {"faithful", "overreach", "unsupported"}:
- verdict = "unsupported"
- return {"verdict": verdict, "reason": str(payload.get("reason", "")).strip()}
-
-
-def _candidate_facts(claim: str, facts: list[Fact]) -> list[Fact]:
- """Facts whose claim matches ``claim`` (normalized), highest-tier first.
-
- Unlike a pure backing check this ignores evidence, so a claim backed by a fact
- with *no* evidence still resolves to a candidate — and then fails the substring
- check rather than being reported as unbacked. Within a tier, facts that carry
- evidence sort first.
+ logger.warning("body faithfulness judge failed: %s", exc)
+ return False, []
+
+ raw = payload.get("claims")
+ if not isinstance(raw, list):
+ logger.warning("body faithfulness judge returned no 'claims' list: %r", payload)
+ return False, []
+
+ claims: list[dict] = []
+ for item in raw:
+ if not isinstance(item, dict):
+ continue
+ claim = str(item.get("claim", "")).strip()
+ if not claim:
+ continue
+ verdict = str(item.get("verdict", "")).strip().lower()
+ if verdict not in {"faithful", "overreach", "unsupported"}:
+ verdict = "unsupported"
+ fact: Fact | None = None
+ fid = _as_id(item.get("fact_id"))
+ if verdict != "unsupported" and fid is not None and 1 <= fid <= len(selected):
+ fact = selected[fid - 1]
+ claims.append({"claim": claim, "verdict": verdict, "fact": fact,
+ "reason": str(item.get("reason", "")).strip()})
+ return True, claims
+
+
+def _first_party_by_claim(facts: list[Fact]) -> dict[str, Fact]:
+ """Map each first-party fact's normalized claim to its best-tier fact.
+
+ Within a claim, the highest tier wins (own_site over authoritative) and, at the
+ same tier, a fact carrying evidence is preferred.
"""
- key = _normalize(claim)
- matches = [f for f in facts if _normalize(f.claim) == key]
- return sorted(
- matches,
- key=lambda f: (_TIER_RANK.get(f.source_tier, 3), 0 if f.evidence else 1),
- )
-
-
-def _verify_claim(claim: str, facts: list[Fact], llm: LLMClient, strict: bool) -> tuple[ClaimVerdict, str]:
- """Audit one claim; return its `ClaimVerdict` and a status string.
-
- Status is ``"verified"`` or one of the failure reasons ``unbacked`` /
- ``volatile-source`` / ``not-substring`` / ``overreach`` / ``unsupported``. The
- faithfulness judge is only called once a claim is backed by a first-party fact
- with evidence — there is nothing to judge otherwise.
+ best: dict[str, Fact] = {}
+ for fact in facts:
+ if fact.source_tier not in _FIRST_PARTY_TIERS:
+ continue
+ key = _normalize(fact.claim)
+ current = best.get(key)
+ rank = (_TIER_RANK[fact.source_tier], 0 if fact.evidence else 1)
+ if current is None or rank < (_TIER_RANK[current.source_tier], 0 if current.evidence else 1):
+ best[key] = fact
+ return best
+
+
+def _resolve_hooks(hooks: list[str], facts: list[Fact]) -> tuple[list[Fact], list[str]]:
+ """Resolve each hook to a first-party `Fact`; report any that fail to resolve.
+
+ Hooks are first-party grounded facts by construction (the draft node only selects
+ such facts). This re-resolves them and records any hook that does not map to a
+ first-party fact as a ``structural`` failure — the invariant should hold, so a
+ failure here means an upstream bug, not a model mistake.
"""
- candidates = _candidate_facts(claim, facts)
- if not candidates:
- return ClaimVerdict(claim=claim, substring_ok=False), "unbacked"
-
- chosen = candidates[0]
- verdict = ClaimVerdict(
- claim=claim,
- fact_used=chosen.claim,
- source_url=chosen.source_url,
- tier=chosen.source_tier,
- substring_ok=bool(chosen.evidence),
- )
-
- if chosen.source_tier not in _FIRST_PARTY_TIERS:
- return verdict, "volatile-source"
- if not verdict.substring_ok:
- return verdict, "not-substring"
-
- faithfulness = judge_faithfulness(claim, chosen.evidence, llm)["verdict"]
- verdict.faithfulness = faithfulness # type: ignore[assignment]
- if faithfulness == "faithful":
- return verdict, "verified"
- if faithfulness == "overreach":
- return verdict, ("overreach" if strict else "verified")
- return verdict, "unsupported"
+ by_claim = _first_party_by_claim(facts)
+ selected: list[Fact] = []
+ failures: list[str] = []
+ seen: set[str] = set()
+ for hook in hooks:
+ fact = by_claim.get(_normalize(hook))
+ if fact is None:
+ failures.append(hook)
+ continue
+ if fact.claim in seen:
+ continue
+ seen.add(fact.claim)
+ selected.append(fact)
+ return selected, failures
def run_verification(
@@ -145,47 +187,90 @@ def run_verification(
llm: LLMClient,
settings: Settings,
) -> VerificationResult:
- """Audit a draft's claims and decide whether it passes the groundedness gate.
+ """Audit a draft and decide whether it passes the groundedness gate.
- Each claim in ``draft.hooks_used`` is audited by `_verify_claim`. The draft
- passes only if there is at least one claim and **every** claim is verified.
- Scores are reported even when the draft passes (see the groundedness
- methodology docs for definitions).
+ Resolves the draft's hooks to first-party facts (structural), then runs a single
+ faithfulness judge over the body against those facts. The draft passes only if it
+ has a grounded hook, a non-empty body, no structural failure, a successful judge
+ call, and no ``unsupported`` body claim (and no ``overreach`` when
+ ``faithfulness_strict``). See the groundedness methodology docs for the metric
+ definitions.
Args:
- draft: The `Draft` to audit (its ``hooks_used`` are the claims checked).
+ draft: The `Draft` to audit (``hooks_used`` are the grounding facts; ``body``
+ is what the judge evaluates).
research: The research providing the grounding facts.
llm: LLM client used for the faithfulness judge.
settings: Settings supplying ``faithfulness_strict``.
Returns:
- A `VerificationResult` with the per-claim verdicts, the groundedness and
- faithfulness scores, the tier breakdown, the flagged failures, and the
- pass/fail decision.
+ A `VerificationResult` with the per-body-claim verdicts, the groundedness and
+ faithfulness scores, the tier breakdown of the grounding facts, the flagged
+ failures, and the pass/fail decision.
"""
facts = list(research.facts) if research else []
- claims = list(draft.hooks_used) if draft else []
- total = len(claims)
+ hooks = list(draft.hooks_used) if draft else []
+ body = (draft.body if draft else "") or ""
+ strict = settings.faithfulness_strict
+
+ selected, structural_failures = _resolve_hooks(hooks, facts)
+ tier_breakdown = dict(Counter(fact.source_tier for fact in selected))
+
+ ok, judged = (True, [])
+ if selected and body.strip():
+ ok, judged = judge_body(body, selected, llm)
verdicts: list[ClaimVerdict] = []
- flagged: list[str] = []
- tier_breakdown: dict[str, int] = {}
- verified = 0
+ flagged: list[str] = [f"structural: {hook}" for hook in structural_failures]
faithful = 0
- for claim in claims:
- verdict, status = _verify_claim(claim, facts, llm, settings.faithfulness_strict)
- verdicts.append(verdict)
- tier_breakdown[verdict.tier or "unbacked"] = tier_breakdown.get(verdict.tier or "unbacked", 0) + 1
- if verdict.faithfulness == "faithful":
+ verified = 0
+ for jc in judged:
+ verdict = jc["verdict"]
+ fact: Fact | None = jc["fact"]
+ verdicts.append(ClaimVerdict(
+ claim=jc["claim"],
+ fact_used=fact.claim if fact else None,
+ source_url=fact.source_url if fact else None,
+ tier=fact.source_tier if fact else None,
+ substring_ok=bool(fact and fact.evidence),
+ faithfulness=verdict, # type: ignore[arg-type]
+ ))
+ if verdict == "faithful":
faithful += 1
- if status == "verified":
verified += 1
+ elif verdict == "overreach":
+ if strict:
+ flagged.append(f"overreach: {jc['claim']}")
+ else:
+ verified += 1
else:
- flagged.append(f"{status}: {claim}")
-
- groundedness_score = round(verified / total, 4) if total else 0.0
- faithfulness_score = round(faithful / total, 4) if total else 0.0
- passed = total > 0 and verified == total
+ flagged.append(f"unsupported: {jc['claim']}")
+
+ total = len(judged)
+ judge_failed = bool(selected) and bool(body.strip()) and not ok
+ if judge_failed:
+ flagged.append("judge-error: faithfulness judge failed")
+
+ if not selected or not body.strip() or judge_failed:
+ groundedness_score = 0.0
+ faithfulness_score = 0.0
+ elif total == 0:
+ # A grounded draft whose body makes no checkable company claim: nothing is
+ # unfaithful, so it is vacuously grounded.
+ groundedness_score = 1.0
+ faithfulness_score = 1.0
+ else:
+ groundedness_score = round(verified / total, 4)
+ faithfulness_score = round(faithful / total, 4)
+
+ passed = (
+ bool(selected)
+ and bool(body.strip())
+ and not structural_failures
+ and not judge_failed
+ and all(v.faithfulness != "unsupported" for v in verdicts)
+ and (not strict or all(v.faithfulness != "overreach" for v in verdicts))
+ )
return VerificationResult(
groundedness_score=groundedness_score,
diff --git a/tests/test_clients.py b/tests/test_clients.py
index 2094b1e..8cae80e 100644
--- a/tests/test_clients.py
+++ b/tests/test_clients.py
@@ -12,12 +12,15 @@
from pitch_pilot.clients.fetch import _html_to_text, fetch_page
from pitch_pilot.clients.llm import (
+ CONTEXT_TOKEN_CAP,
+ CerebrasClient,
GeminiClient,
GroqClient,
LLMError,
LLMJSONError,
_loads_json_lenient,
get_llm_client,
+ trim_to_token_budget,
)
from pitch_pilot.config import Settings
@@ -84,6 +87,26 @@ def test_groq_without_key_raises(self):
with pytest.raises(ValueError):
get_llm_client(_settings(llm_provider="groq", groq_api_key=None))
+ def test_selects_cerebras_when_configured(self):
+ client = get_llm_client(_settings(llm_provider="cerebras", cerebras_api_key="k"))
+ assert isinstance(client, CerebrasClient)
+
+ def test_cerebras_without_key_raises(self):
+ with pytest.raises(ValueError):
+ get_llm_client(_settings(llm_provider="cerebras", cerebras_api_key=None))
+
+
+class TestTokenBudget:
+ def test_trim_keeps_prefix_within_budget(self):
+ lines = ["x" * 40 for _ in range(100)] # 40 chars each ~ 10 tokens
+ kept = trim_to_token_budget(lines, max_tokens=50) # ~200 chars budget
+ assert 0 < len(kept) < 100
+ assert (sum(len(s) + 1 for s in kept)) <= 50 * 4
+
+ def test_trim_keeps_all_when_under_budget(self):
+ lines = ["short", "lines"]
+ assert trim_to_token_budget(lines, max_tokens=CONTEXT_TOKEN_CAP) == lines
+
class TestProviderErrorsNormalizeToLLMError:
"""A vendor SDK error (e.g. Groq's json_validate_failed) must surface as LLMError
@@ -118,6 +141,22 @@ class _SDK:
with pytest.raises(LLMError):
client.complete("system", "user")
+ def test_cerebras_complete_json_wraps_sdk_error(self):
+ class _Completions:
+ def create(self, **kwargs):
+ raise RuntimeError("429 context length exceeded")
+
+ class _Chat:
+ completions = _Completions()
+
+ class _SDK:
+ chat = _Chat()
+
+ client = CerebrasClient(api_key="k", model="gpt-oss-120b")
+ client._client = _SDK() # bypass lazy SDK construction
+ with pytest.raises(LLMError):
+ client.complete_json("system", "user")
+
class TestHtmlToText:
def test_strips_scripts_and_styles(self):
diff --git a/tests/test_config.py b/tests/test_config.py
index e23cec3..6287e1f 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -78,7 +78,9 @@ def test_valid_settings_load_with_defaults(monkeypatch, tmp_path):
assert settings.tavily_api_key == "t-key"
assert settings.groq_api_key is None
assert settings.llm_provider == "gemini" # default
- assert settings.research_max_queries == 4 # default
+ assert settings.research_max_queries == 3 # default (leaned out — ADR-0012)
+ assert settings.research_max_page_chars == 3500 # default (token lever)
+ assert settings.research_max_facts_per_source == 5 # default
assert settings.groundedness_threshold == 0.9 # default
assert settings.gemini_model # has a sensible default
assert settings.groq_model
diff --git a/tests/test_draft.py b/tests/test_draft.py
index d142f08..e668f75 100644
--- a/tests/test_draft.py
+++ b/tests/test_draft.py
@@ -1,13 +1,15 @@
"""Unit tests for the draft node. No network access.
The LLM is a fake that returns a fixed draft payload. The behaviors under test
-are the groundedness guarantees the node enforces on the model's output:
-
-* ``hooks_used`` is always a subset of the real research facts — a hook the model
- invents is discarded;
-* under Policy B, NO ``third_party_snippet`` fact can become a hook — only
- ``own_site`` / ``authoritative`` facts are claimable (third-party facts may be
- passed as context but never as a stated claim);
+are the groundedness guarantees the node enforces on the model's output (P5,
+selection-by-id — see ADR-0014):
+
+* ``hooks_used`` resolves the model's selected fact **ids** to real research facts —
+ an out-of-range or invented id is dropped;
+* under Policy B, NO ``third_party_snippet`` fact is even claimable, so it can never
+ become a hook (third-party facts may be passed as context but never selected),
+ even if the model echoes the claim text verbatim;
+* a verbatim echo of a *claimable* fact's claim resolves as a defensive fallback;
* no claimable facts (or an LLM failure) yields an empty draft, not a crash.
"""
@@ -52,60 +54,61 @@ def __init__(self, payload=None, *, raises=False):
self.payload = payload or {}
self.raises = raises
- def complete(self, system, user): # pragma: no cover - unused
+ def complete(self, system, user, temperature=None): # pragma: no cover - unused
return "OK"
- def complete_json(self, system, user):
+ def complete_json(self, system, user, temperature=None):
+ self.temperature = temperature
if self.raises:
raise LLMError("draft model down")
return self.payload
class TestRunDraft:
- def test_hooks_are_a_subset_of_facts(self):
- llm = FakeLLM({
- "subject": "Loved your dev tools",
- "body": "Hi Acme, ...",
- "hooks": [
- "Acme builds developer tools", # real fact
- "Acme is the #1 fintech in the world", # fabricated
- ],
- })
+ def test_selected_ids_resolve_to_facts(self):
+ # claimable = [_OWN_SOFT] -> id 1; id 5 is out of range and is dropped.
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [1, 5]})
draft = run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
assert draft.hooks_used == ["Acme builds developer tools"]
- fact_claims = {"Acme builds developer tools"}
- assert all(h in fact_claims for h in draft.hooks_used)
- def test_refuses_third_party_numeric_but_allows_own_site_numeric(self):
- llm = FakeLLM({
- "subject": "s",
- "body": "b",
- "hooks": ["Acme raised $50M Series B", "Acme has 200 employees"],
- })
+ def test_invalid_selection_yields_no_hooks(self):
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [99]})
+ draft = run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
+ assert draft.hooks_used == []
+
+ def test_only_own_site_numeric_is_claimable_not_third_party(self):
+ # The third-party numeric is not claimable, so id 1 is the own_site fact.
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [1]})
draft = run_draft(_research([_OWN_NUMERIC, _TP_NUMERIC]), _qual(), llm, _settings())
assert "Acme has 200 employees" in draft.hooks_used # own_site numeric: ok
- assert "Acme raised $50M Series B" not in draft.hooks_used # third-party: refused
+ assert "Acme raised $50M Series B" not in draft.hooks_used # third-party: not claimable
- def test_third_party_fact_cannot_be_a_hook_even_when_soft(self):
- # Policy B: a third_party_snippet fact may inform context but never a claim.
- llm = FakeLLM({"subject": "s", "body": "b", "hooks": ["Acme is popular with developers"]})
+ def test_third_party_claim_echo_is_never_a_hook(self):
+ # Policy B: even if the model echoes a third_party_snippet fact's claim text,
+ # it is not claimable and can never become a hook.
+ llm = FakeLLM({"subject": "s", "body": "b",
+ "facts_used": ["Acme is popular with developers"]})
draft = run_draft(_research([_OWN_SOFT, _TP_SOFT]), _qual(), llm, _settings())
- assert "Acme is popular with developers" not in draft.hooks_used
+ assert draft.hooks_used == []
- def test_claim_pool_excludes_all_third_party_facts(self):
- # Only first-party facts are claimable; the model offered both, hooks both.
- llm = FakeLLM({
- "subject": "s", "body": "b",
- "hooks": ["Acme builds developer tools", "Acme is popular with developers"],
- })
+ def test_verbatim_claim_echo_resolves_as_fallback(self):
+ # A model that echoes a claimable fact's exact claim (not its id) still resolves.
+ llm = FakeLLM({"subject": "s", "body": "b",
+ "facts_used": ["Acme builds developer tools"]})
draft = run_draft(_research([_OWN_SOFT, _TP_SOFT]), _qual(), llm, _settings())
assert draft.hooks_used == ["Acme builds developer tools"]
+ def test_hooks_are_deduped_preserving_order(self):
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [2, 1, 1, 2]})
+ draft = run_draft(_research([_OWN_SOFT, _OWN_NUMERIC]), _qual(), llm, _settings())
+ assert draft.hooks_used == ["Acme has 200 employees", "Acme builds developer tools"]
+
def test_subject_and_body_passed_through(self):
- llm = FakeLLM({"subject": " Hello Acme ", "body": " Body here ", "hooks": []})
+ llm = FakeLLM({"subject": " Hello Acme ", "body": " Body here ", "facts_used": []})
draft = run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
assert draft.subject == "Hello Acme"
assert draft.body == "Body here"
+ assert draft.hooks_used == []
def test_no_claimable_facts_yields_empty_draft(self):
# Only third-party facts -> nothing is claimable under Policy B.
@@ -117,6 +120,52 @@ def test_llm_failure_yields_empty_draft(self):
draft = run_draft(_research([_OWN_SOFT]), _qual(), FakeLLM(raises=True), _settings())
assert draft.hooks_used == []
+ def test_body_strips_leaked_fact_id_tokens(self):
+ # The model is told not to, but if it leaks "(Fact N)" into the prose the
+ # node strips it as a backstop, leaving clean, natural text.
+ import re
+ llm = FakeLLM({
+ "subject": "Quick note on your platform",
+ "body": "I saw your unified API (Fact 1) and your hosted page (Fact 2) — impressive.",
+ "facts_used": [1],
+ })
+ draft = run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
+ assert draft.body == "I saw your unified API and your hosted page — impressive."
+ assert not re.search(r"[Ff]act\s*#?\s*\d", draft.body)
+ assert " " not in draft.body # no doubled spaces left behind
+
+ def test_subject_strips_leaked_fact_id_tokens(self):
+ llm = FakeLLM({"subject": "Loved your API [fact 2]", "body": "b", "facts_used": [1]})
+ draft = run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
+ assert "fact" not in draft.subject.lower()
+
+ def test_draft_uses_zero_temperature(self):
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [1]})
+ run_draft(_research([_OWN_SOFT]), _qual(), llm, _settings())
+ assert llm.temperature == 0.0
+
+
+class TestContextCapGuard:
+ def test_large_facts_list_stays_under_context_cap(self):
+ from pitch_pilot.clients.llm import CONTEXT_TOKEN_CAP
+ from pitch_pilot.nodes.draft import (
+ _DRAFT_SYSTEM,
+ _claimable_facts,
+ _context_facts,
+ _draft_user_prompt,
+ )
+
+ own = [Fact(claim=f"Acme ships API product feature {i} at scale",
+ source_url=f"https://acme.com/long/path/page-{i}", evidence="e",
+ source_tier="own_site") for i in range(300)]
+ tp = [Fact(claim=f"Third party says thing {i} about Acme",
+ source_url=f"https://blog.example.com/post-{i}", evidence="e",
+ source_tier="third_party_snippet") for i in range(300)]
+ facts = own + tp
+ prompt = _draft_user_prompt("Acme", _claimable_facts(facts), _context_facts(facts), _qual())
+ est_tokens = (len(_DRAFT_SYSTEM) + len(prompt)) / 4
+ assert est_tokens < CONTEXT_TOKEN_CAP
+
class TestDraftNodeAdapter:
def test_returns_draft_and_status(self):
@@ -129,7 +178,7 @@ def test_returns_draft_and_status(self):
company=Company(domain="acme.com", name="Acme"), icp=icp,
research=_research([_OWN_SOFT]), qualification=_qual(),
)
- llm = FakeLLM({"subject": "s", "body": "b", "hooks": ["Acme builds developer tools"]})
+ llm = FakeLLM({"subject": "s", "body": "b", "facts_used": [1]})
update = draft_node(state, llm=llm, settings=_settings())
assert set(update) == {"draft", "status"}
assert update["status"] == "drafted"
diff --git a/tests/test_evals.py b/tests/test_evals.py
index 78b2a46..4a2c18d 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -43,7 +43,7 @@ def _icp() -> ICP:
"flagged_claims": [], "category": "good_fit", "fact_count": 18},
{"status": "ok", "domain": "c", "label": "not_qualified", "predicted_qualified": True,
"draft_passed": False, "groundedness_score": 0.5, "faithfulness_score": 0.5,
- "flagged_claims": ["unsupported: x", "volatile-source: y"], "category": "bad_fit", "fact_count": 10},
+ "flagged_claims": ["unsupported: x", "overreach: y"], "category": "bad_fit", "fact_count": 10},
{"status": "ok", "domain": "d", "label": "not_qualified", "predicted_qualified": False,
"draft_passed": None, "groundedness_score": None, "faithfulness_score": None,
"flagged_claims": [], "category": "bad_fit", "fact_count": 4},
@@ -73,8 +73,8 @@ def test_mean_scores(self):
def test_failure_modes_counts_by_reason(self):
fm = metrics.failure_modes(_CANNED)
assert fm["unsupported"] == 1
- assert fm["volatile-source"] == 1
- assert fm["unbacked"] == 0
+ assert fm["overreach"] == 1
+ assert fm["structural"] == 0
def test_facts_by_category_degradation(self):
fbc = metrics.facts_by_category(_CANNED)
@@ -220,6 +220,48 @@ def complete_json(self, s, u):
assert all("rate-limited" in r["error"] for r in results)
+class TestRedraft:
+ def test_qualification_is_frozen_and_draft_verify_recomputed(self, monkeypatch, tmp_path):
+ monkeypatch.setattr(run_eval, "CACHE_DIR", tmp_path / "cache")
+ save_cached_research(ResearchResult(company=Company(domain="q.com"), facts=[]))
+ monkeypatch.setattr(run_eval, "run_draft",
+ lambda *a, **k: Draft(subject="s", body="b", hooks_used=["h"]))
+ monkeypatch.setattr(run_eval, "run_verification", lambda *a, **k: _fake_ver())
+
+ records = [
+ {"status": "ok", "domain": "q.com", "label": "qualified", "predicted_qualified": True,
+ "score": 0.83, "qual_reason": "fit", "matched_signals": ["m"], "missed_signals": [],
+ "draft_passed": False, "groundedness_score": 0.0, "fact_count": 10},
+ {"status": "ok", "domain": "d.com", "label": "not_qualified", "predicted_qualified": False,
+ "draft_passed": None, "groundedness_score": None},
+ {"status": "error", "domain": "e.com", "error": "x"},
+ ]
+ out = run_eval.redraft(records, llm=object(), settings=_settings())
+
+ q = next(r for r in out if r["domain"] == "q.com")
+ # qualification fields are preserved verbatim
+ assert (q["predicted_qualified"], q["score"], q["qual_reason"]) == (True, 0.83, "fit")
+ assert q["matched_signals"] == ["m"]
+ # draft + verify are recomputed
+ assert q["draft_passed"] is True and q["groundedness_score"] == 1.0
+ assert q["hooks_used"] == ["h"] and q["from_cache"] is True
+ # disqualified and error records pass through untouched
+ assert next(r for r in out if r["domain"] == "d.com")["draft_passed"] is None
+ assert next(r for r in out if r["domain"] == "e.com")["status"] == "error"
+
+ def test_missing_cache_leaves_record_unchanged(self, monkeypatch, tmp_path):
+ monkeypatch.setattr(run_eval, "CACHE_DIR", tmp_path / "cache")
+
+ def boom(*a, **k):
+ raise AssertionError("must not draft without cached research")
+ monkeypatch.setattr(run_eval, "run_draft", boom)
+
+ records = [{"status": "ok", "domain": "nocache.com", "predicted_qualified": True,
+ "draft_passed": False, "score": 0.7, "qual_reason": "fit"}]
+ out = run_eval.redraft(records, llm=object(), settings=_settings())
+ assert out[0]["draft_passed"] is False # unchanged (no cached research)
+
+
class TestRecheck:
def _setup(self):
research = ResearchResult(company=Company(domain="acme.com"), facts=[
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 5c46af6..19ff795 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -71,17 +71,18 @@ def __init__(self, *, assessment, draft_payload):
self.draft_payload = draft_payload
self.draft_called = False
- def complete(self, system, user): # pragma: no cover - unused
+ def complete(self, system, user, temperature=None): # pragma: no cover - unused
return "OK"
- def complete_json(self, system, user):
+ def complete_json(self, system, user, temperature=None):
if user.startswith("SOURCE URL:"):
url = user.split("SOURCE URL:", 1)[1].splitlines()[0].strip()
return {"facts": SEED_FACTS if url == "https://acme.com" else []}
if user.startswith("ICP:"):
return self.assessment
- if user.startswith("CLAIM:"):
- return {"verdict": "faithful", "reason": "evidence supports the claim"}
+ if user.startswith("BODY:"):
+ return {"claims": [{"claim": "Acme builds developer tools",
+ "verdict": "faithful", "fact_id": 1}]}
if "Write the outreach email" in user:
self.draft_called = True
return self.draft_payload
@@ -129,7 +130,7 @@ def test_disqualified_lead_skips_draft_and_is_saved(self, monkeypatch):
def test_qualified_passing_draft_is_logged_ready(self, monkeypatch):
draft_payload = {"subject": "Loved your tools", "body": "Hi Acme...",
- "hooks": ["Acme builds developer tools"]}
+ "facts_used": [1]}
llm = PipelineLLM(assessment=QUALIFIED_ASSESSMENT, draft_payload=draft_payload)
store = FakeStore()
final = _run(llm, store, monkeypatch)
@@ -143,9 +144,9 @@ def test_qualified_passing_draft_is_logged_ready(self, monkeypatch):
assert store.review == []
def test_qualified_failing_verify_routes_to_review(self, monkeypatch):
- # The draft node drops the fabricated hook, leaving nothing grounded to
- # stand on; verify then fails (no claims) and the lead routes to review.
- draft_payload = {"subject": "s", "body": "b", "hooks": ["Acme is a unicorn"]}
+ # The draft node drops the out-of-range fact id, leaving nothing grounded to
+ # stand on; verify then fails (no hooks) and the lead routes to review.
+ draft_payload = {"subject": "s", "body": "b", "facts_used": [99]}
llm = PipelineLLM(assessment=QUALIFIED_ASSESSMENT, draft_payload=draft_payload)
store = FakeStore()
final = _run(llm, store, monkeypatch)
@@ -162,7 +163,7 @@ class TestBuildPipelineCompiles:
def test_compiles_and_runs_end_to_end(self, monkeypatch):
llm = PipelineLLM(
assessment=QUALIFIED_ASSESSMENT,
- draft_payload={"subject": "s", "body": "b", "hooks": ["Acme builds developer tools"]},
+ draft_payload={"subject": "s", "body": "b", "facts_used": [1]},
)
final = _run(llm, FakeStore(), monkeypatch)
# Every stage produced its artifact.
diff --git a/tests/test_qualify.py b/tests/test_qualify.py
index 39f31eb..12fbf09 100644
--- a/tests/test_qualify.py
+++ b/tests/test_qualify.py
@@ -6,8 +6,9 @@
* a matched negative signal is a hard veto, regardless of fit score;
* the fit score is computed deterministically from the assessment;
-* unknowns are never guessed — neither matched nor missed, and unknown structural
- attributes are dropped from the score, not penalized;
+* unknowns are never guessed — neither matched nor missed; unknown *size/region*
+ are dropped from the score, but an unknown/`no_match` *industry* is penalized
+ (0.0, kept in the denominator) when the ICP names target industries;
* with no facts, the company is not qualified and the LLM is never called.
"""
@@ -57,17 +58,23 @@ def __init__(self, assessment=None, *, raises=False):
self.raises = raises
self.calls = 0
- def complete(self, system, user): # pragma: no cover - unused
+ def complete(self, system, user, temperature=None): # pragma: no cover - unused
return "OK"
- def complete_json(self, system, user):
+ def complete_json(self, system, user, temperature=None):
self.calls += 1
+ self.temperature = temperature
if self.raises:
raise LLMError("assessor down")
return self.assessment
class TestRunQualification:
+ def test_assessor_uses_zero_temperature(self):
+ llm = FakeLLM({"industry": {"status": "unknown"}})
+ run_qualification(_research(), _icp(), llm, _settings())
+ assert llm.temperature == 0.0
+
def test_deterministic_score(self):
assessment = {
"industry": {"status": "match"},
@@ -105,7 +112,7 @@ def test_matched_negative_signal_is_a_hard_veto(self):
def test_unknowns_are_never_guessed(self):
assessment = {
- "industry": {"status": "unknown"},
+ "industry": {"status": "match"},
"region": {"status": "unknown"},
"employee_count": {"value": None},
"positive_signals": [
@@ -118,8 +125,9 @@ def test_unknowns_are_never_guessed(self):
],
}
result = run_qualification(_research(), _icp(), FakeLLM(assessment), _settings())
- # Only the positives component is known: 1 of 2 matched -> 0.5
- assert result.score == 0.5
+ # industry match(.35) + positives 1/2(.25*0.5); region & size unknown -> dropped.
+ # = (0.35 + 0.125) / (0.35 + 0.25) = 0.7917
+ assert result.score == 0.7917
assert result.matched_signals == ["recent funding"]
# The unknown positive is neither matched nor missed (not guessed).
assert "hiring engineers" not in result.matched_signals
@@ -127,6 +135,44 @@ def test_unknowns_are_never_guessed(self):
# Unknown negatives never veto.
assert result.qualified is True
+ def test_industry_unknown_is_penalized_not_dropped(self):
+ # The ICP names target industries, so an unconfirmed industry counts against
+ # the company (0.0, kept in the denominator) — the fix behind the
+ # figma/huggingface false positives (ADR-0015).
+ assessment = {
+ "industry": {"status": "unknown"},
+ "region": {"status": "match"},
+ "employee_count": {"value": None}, # size unknown -> dropped
+ "positive_signals": [
+ {"signal": "recent funding", "status": "match"},
+ {"signal": "hiring engineers", "status": "unknown"},
+ ],
+ "negative_signals": [],
+ }
+ result = run_qualification(_research(), _icp(), FakeLLM(assessment), _settings())
+ # industry 0(.35) + region 1(.15) + positives 1/2(.25*0.5); size dropped.
+ # = (0.15 + 0.125) / (0.35 + 0.15 + 0.25) = 0.3667
+ assert result.score == 0.3667
+ assert result.qualified is False
+
+ def test_industry_dropped_when_icp_names_no_industries(self):
+ # With no target industries, industry is irrelevant and an unknown is dropped.
+ icp = _icp(industries=[])
+ assessment = {
+ "industry": {"status": "unknown"},
+ "region": {"status": "match"},
+ "employee_count": {"value": 100},
+ "positive_signals": [
+ {"signal": "recent funding", "status": "match"},
+ {"signal": "hiring engineers", "status": "match"},
+ ],
+ "negative_signals": [],
+ }
+ result = run_qualification(_research(), icp, FakeLLM(assessment), _settings())
+ # industry dropped; size 1(.25) + region 1(.15) + positives 1(.25) all match -> 1.0
+ assert result.score == 1.0
+ assert result.qualified is True
+
def test_below_threshold_is_not_qualified(self):
assessment = {
"industry": {"status": "no_match"},
@@ -157,6 +203,22 @@ def test_llm_failure_degrades_to_all_unknown(self):
assert result.qualified is False
+class TestContextCapGuard:
+ def test_large_facts_list_stays_under_context_cap(self):
+ from pitch_pilot.clients.llm import CONTEXT_TOKEN_CAP
+ from pitch_pilot.nodes.qualify import _QUALIFY_SYSTEM, _qualify_user_prompt
+
+ facts = [
+ Fact(claim=f"Acme fact number {i} about payments, fraud and growth at scale",
+ source_url=f"https://acme.com/some/fairly/long/path/page-{i}",
+ evidence="e", source_tier="own_site")
+ for i in range(500)
+ ]
+ prompt = _qualify_user_prompt(_icp(), facts)
+ est_tokens = (len(_QUALIFY_SYSTEM) + len(prompt)) / 4 # ~4 chars/token
+ assert est_tokens < CONTEXT_TOKEN_CAP
+
+
class TestQualifyNodeAdapter:
def test_returns_qualification_and_status(self):
assessment = {
diff --git a/tests/test_research.py b/tests/test_research.py
index bddd5e3..4d9b0ce 100644
--- a/tests/test_research.py
+++ b/tests/test_research.py
@@ -174,6 +174,56 @@ def test_per_source_cap_is_enforced(self):
facts = extract_facts(text, "https://acme.com", None, llm)
assert len(facts) == research_module.MAX_FACTS_PER_SOURCE
+ def test_page_text_truncated_to_cap_before_extraction(self):
+ # The model must only see (and be able to ground in) the truncated text.
+ long_text = "ALPHA z " * 100 + " OMEGA"
+ captured: dict = {}
+
+ class CapturingLLM:
+ def complete(self, system, user): # pragma: no cover - unused
+ return ""
+
+ def complete_json(self, system, user):
+ captured["text"] = user.split("TEXT:\n", 1)[1]
+ return {"facts": [{"claim": "a", "evidence": "ALPHA", "category": "overview", "confidence": 0.9}]}
+
+ facts = extract_facts(long_text, "https://acme.com", None, CapturingLLM(), "acme.com",
+ max_page_chars=12)
+ assert len(captured["text"]) <= 12 # truncated before extraction
+ assert "OMEGA" not in captured["text"] # tail beyond the cap is gone
+ assert len(facts) == 1 # evidence within the cap survives
+
+ def test_evidence_beyond_cap_is_dropped(self):
+ # Grounding runs against the truncated text, so a claim whose evidence lives
+ # past the cap is dropped — truncation does not weaken the guarantee.
+ text = "ALPHA marker " + "x" * 200 + " SECRET"
+
+ class LLM:
+ def complete(self, system, user): # pragma: no cover - unused
+ return ""
+
+ def complete_json(self, system, user):
+ return {"facts": [{"claim": "secret", "evidence": "SECRET", "category": "overview", "confidence": 0.9}]}
+
+ facts = extract_facts(text, "https://acme.com", None, LLM(), "acme.com", max_page_chars=20)
+ assert facts == []
+
+ def test_max_facts_per_source_param_overrides_default(self):
+ text = " ".join(f"S{i} is stated." for i in range(10))
+
+ class LLM:
+ def complete(self, system, user): # pragma: no cover - unused
+ return ""
+
+ def complete_json(self, system, user):
+ return {"facts": [
+ {"claim": f"c{i}", "evidence": f"S{i} is stated.", "category": "overview", "confidence": 0.5}
+ for i in range(10)
+ ]}
+
+ facts = extract_facts(text, "https://acme.com", None, LLM(), "acme.com", max_facts_per_source=3)
+ assert len(facts) == 3
+
class TestRunResearch:
def test_every_fact_has_http_source_and_evidence(self, monkeypatch):
diff --git a/tests/test_verify.py b/tests/test_verify.py
index 3222154..e489085 100644
--- a/tests/test_verify.py
+++ b/tests/test_verify.py
@@ -1,28 +1,29 @@
-"""Unit tests for the P3 verify node (the real groundedness gate). No network.
-
-The LLM faithfulness judge is replaced with a fake that returns a scripted verdict
-per claim. The behaviors under test:
-
-* a faithful, first-party, substring-anchored claim verifies and the draft passes;
-* an ``unsupported`` verdict fails the claim and the draft;
-* an ``overreach`` verdict fails under ``FAITHFULNESS_STRICT=True`` and passes when
- it is off;
-* a claim backed only by a ``third_party_snippet`` fact is a hard policy failure
- (and the judge is never even called);
-* a claim whose backing fact has no evidence fails the substring check;
-* an unbacked claim fails;
-* ``groundedness_score``, ``faithfulness_score``, and ``tier_breakdown`` are
- computed correctly across a mix.
+"""Unit tests for the P5 verify node (the real groundedness gate). No network.
+
+The LLM body-faithfulness judge is replaced with a fake that returns scripted
+per-claim verdicts for the draft body. The behaviors under test:
+
+* a faithful body claim grounded in a first-party hook verifies and the draft passes;
+* an ``unsupported`` body claim fails the claim and the draft;
+* an ``overreach`` claim fails under ``FAITHFULNESS_STRICT=True`` and passes when off;
+* a hook that does not resolve to a first-party fact is a ``structural`` failure
+ (and the judge is never called);
+* an empty body, or no grounded hooks, never passes;
+* a judge error / malformed response fails closed;
+* ``groundedness_score``, ``faithfulness_score``, and ``tier_breakdown`` are computed
+ per the P5 definitions (groundedness = faithful body claims / total body claims;
+ tier_breakdown counts the grounding hooks).
"""
from __future__ import annotations
+from pitch_pilot.clients.llm import LLMError
from pitch_pilot.config import Settings
from pitch_pilot.models.draft import Draft
from pitch_pilot.models.fact import Fact
from pitch_pilot.models.lead import Company
from pitch_pilot.models.research import ResearchResult
-from pitch_pilot.nodes.verify import judge_faithfulness, run_verification, verify_node
+from pitch_pilot.nodes.verify import judge_body, run_verification, verify_node
def _settings(**overrides) -> Settings:
@@ -46,34 +47,39 @@ def _research(facts) -> ResearchResult:
evidence="S-1", source_tier="authoritative")
_TP = Fact(claim="Acme is popular with developers", source_url="https://blog.example.com/acme",
evidence="popular with developers", source_tier="third_party_snippet")
-_OWN_NO_EVIDENCE = Fact(claim="Acme is the best", source_url="https://acme.com/x",
- evidence="", source_tier="own_site")
class FakeJudge:
- """Scriptable faithfulness judge: maps claim text -> verdict (default faithful)."""
-
- def __init__(self, verdicts=None, default="faithful"):
- self.verdicts = verdicts or {}
- self.default = default
+ """Scriptable body-faithfulness judge: returns a fixed list of claim verdicts."""
+
+ def __init__(self, claims=None, *, raises=False, malformed=False):
+ self.claims = claims if claims is not None else [
+ {"claim": "body claim", "verdict": "faithful", "fact_id": 1}
+ ]
+ self.raises = raises
+ self.malformed = malformed
self.calls: list[str] = []
- def complete(self, system, user): # pragma: no cover - unused
+ def complete(self, system, user, temperature=None): # pragma: no cover - unused
return "OK"
- def complete_json(self, system, user):
- claim = user.split("CLAIM:", 1)[1].split("\n", 1)[0].strip()
- self.calls.append(claim)
- return {"verdict": self.verdicts.get(claim, self.default), "reason": "test"}
+ def complete_json(self, system, user, temperature=None):
+ self.calls.append(user)
+ self.temperature = temperature
+ if self.raises:
+ raise LLMError("judge down")
+ if self.malformed:
+ return {"not_claims": []}
+ return {"claims": self.claims}
-def _draft(*hooks) -> Draft:
- return Draft(subject="s", body="b", hooks_used=list(hooks))
+def _draft(*hooks, body="Hi there, here is the body.") -> Draft:
+ return Draft(subject="s", body=body, hooks_used=list(hooks))
class TestGate:
- def test_faithful_first_party_claim_verifies_and_passes(self):
- judge = FakeJudge(default="faithful")
+ def test_faithful_claim_verifies_and_passes(self):
+ judge = FakeJudge([{"claim": "Acme builds dev tools", "verdict": "faithful", "fact_id": 1}])
result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]), judge, _settings())
assert result.passed is True
assert result.groundedness_score == 1.0
@@ -82,110 +88,139 @@ def test_faithful_first_party_claim_verifies_and_passes(self):
assert result.flagged_claims == []
assert result.tier_breakdown == {"own_site": 1}
cv = result.claim_verdicts[0]
- assert cv.tier == "own_site" and cv.substring_ok is True and cv.faithfulness == "faithful"
+ assert cv.tier == "own_site" and cv.fact_used == "Acme builds developer tools"
+ assert cv.faithfulness == "faithful" and cv.source_url == "https://acme.com"
def test_authoritative_tier_is_first_party(self):
- result = run_verification(_draft("Acme filed an S-1"), _research([_AUTH]), FakeJudge(), _settings())
+ judge = FakeJudge([{"claim": "They filed an S-1", "verdict": "faithful", "fact_id": 1}])
+ result = run_verification(_draft("Acme filed an S-1"), _research([_AUTH]), judge, _settings())
assert result.passed is True
assert result.tier_breakdown == {"authoritative": 1}
- def test_unsupported_verdict_fails_claim_and_draft(self):
- judge = FakeJudge({"Acme builds developer tools": "unsupported"})
+ def test_unsupported_claim_fails_draft(self):
+ judge = FakeJudge([{"claim": "Acme cures cancer", "verdict": "unsupported", "fact_id": None}])
result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]), judge, _settings())
assert result.passed is False
assert result.groundedness_score == 0.0
assert result.faithfulness_score == 0.0
assert any(f.startswith("unsupported:") for f in result.flagged_claims)
+ cv = result.claim_verdicts[0]
+ assert cv.fact_used is None and cv.tier is None # unsupported -> no backing fact
def test_overreach_fails_when_strict(self):
- judge = FakeJudge({"Acme builds developer tools": "overreach"})
+ judge = FakeJudge([{"claim": "Acme is the #1 dev tool", "verdict": "overreach", "fact_id": 1}])
result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]),
judge, _settings(faithfulness_strict=True))
assert result.passed is False
assert any(f.startswith("overreach:") for f in result.flagged_claims)
assert result.faithfulness_score == 0.0 # overreach is not "faithful"
+ assert result.groundedness_score == 0.0 # ...and not verified under strict
def test_overreach_passes_when_lenient(self):
- judge = FakeJudge({"Acme builds developer tools": "overreach"})
+ judge = FakeJudge([{"claim": "Acme is the #1 dev tool", "verdict": "overreach", "fact_id": 1}])
result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]),
judge, _settings(faithfulness_strict=False))
assert result.passed is True
- assert result.groundedness_score == 1.0 # overreach counts as verified when lenient
- assert result.faithfulness_score == 0.0 # ...but still not "faithful"
+ assert result.groundedness_score == 1.0 # overreach counts as verified when lenient
+ assert result.faithfulness_score == 0.0 # ...but still not "faithful"
assert result.claim_verdicts[0].faithfulness == "overreach"
- def test_third_party_only_claim_is_hard_policy_failure(self):
+ def test_unresolvable_hook_is_structural_failure(self):
+ # The only fact is third-party, so the hook cannot resolve to a first-party
+ # fact -> structural failure, and the body judge is never called.
judge = FakeJudge()
result = run_verification(_draft("Acme is popular with developers"), _research([_TP]), judge, _settings())
assert result.passed is False
- assert any(f.startswith("volatile-source:") for f in result.flagged_claims)
- assert result.tier_breakdown == {"third_party_snippet": 1}
- assert judge.calls == [] # policy fails before we bother judging faithfulness
+ assert any(f.startswith("structural:") for f in result.flagged_claims)
+ assert result.tier_breakdown == {}
+ assert judge.calls == []
def test_own_site_chosen_over_third_party_for_same_claim(self):
- # Same claim from both tiers -> the own_site fact is chosen, so it verifies.
tp_same = Fact(claim="Acme builds developer tools", source_url="https://x.com/acme",
evidence="builds developer tools", source_tier="third_party_snippet")
+ judge = FakeJudge([{"claim": "Acme builds dev tools", "verdict": "faithful", "fact_id": 1}])
result = run_verification(_draft("Acme builds developer tools"),
- _research([tp_same, _OWN]), FakeJudge(), _settings())
+ _research([tp_same, _OWN]), judge, _settings())
assert result.passed is True
- assert result.claim_verdicts[0].tier == "own_site"
+ assert result.tier_breakdown == {"own_site": 1}
+ assert result.claim_verdicts[0].source_url == "https://acme.com"
- def test_substring_mismatch_fails(self):
+ def test_empty_body_does_not_pass(self):
judge = FakeJudge()
- result = run_verification(_draft("Acme is the best"), _research([_OWN_NO_EVIDENCE]), judge, _settings())
+ result = run_verification(_draft("Acme builds developer tools", body=" "),
+ _research([_OWN]), judge, _settings())
assert result.passed is False
- assert any(f.startswith("not-substring:") for f in result.flagged_claims)
- assert result.claim_verdicts[0].substring_ok is False
- assert judge.calls == [] # no evidence -> nothing to judge
+ assert result.groundedness_score == 0.0
+ assert judge.calls == [] # nothing to judge
- def test_unbacked_claim_fails(self):
+ def test_no_hooks_does_not_pass(self):
judge = FakeJudge()
- result = run_verification(_draft("Acme invented time travel"), _research([_OWN]), judge, _settings())
+ result = run_verification(_draft(), _research([_OWN]), judge, _settings())
assert result.passed is False
- assert any(f.startswith("unbacked:") for f in result.flagged_claims)
- assert result.tier_breakdown == {"unbacked": 1}
+ assert result.total_claims == 0
+ assert result.groundedness_score == 0.0
assert judge.calls == []
- def test_empty_draft_does_not_pass(self):
- result = run_verification(_draft(), _research([_OWN]), FakeJudge(), _settings())
+ def test_judge_failure_fails_closed(self):
+ judge = FakeJudge(raises=True)
+ result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]), judge, _settings())
assert result.passed is False
- assert result.total_claims == 0
+ assert any(f.startswith("judge-error:") for f in result.flagged_claims)
assert result.groundedness_score == 0.0
- assert result.faithfulness_score == 0.0
+
+ def test_malformed_judge_fails_closed(self):
+ judge = FakeJudge(malformed=True)
+ result = run_verification(_draft("Acme builds developer tools"), _research([_OWN]), judge, _settings())
+ assert result.passed is False
+ assert any(f.startswith("judge-error:") for f in result.flagged_claims)
def test_scores_and_tier_breakdown_across_a_mix(self):
- # 3 claims: own_site faithful (verified), own_site overreach (fail, strict),
- # third_party (volatile fail). verified=1/3, faithful=1/3.
- judge = FakeJudge({
- "Acme builds developer tools": "faithful",
- "Acme is hiring engineers": "overreach",
- })
- draft = _draft("Acme builds developer tools", "Acme is hiring engineers",
- "Acme is popular with developers")
- result = run_verification(draft, _research([_OWN, _OWN2, _TP]), judge, _settings())
+ # 2 grounding hooks (own_site x2). 3 body claims: faithful, overreach, unsupported.
+ # verified=1/3, faithful=1/3; tier_breakdown counts the hooks, not the claims.
+ judge = FakeJudge([
+ {"claim": "Acme builds dev tools", "verdict": "faithful", "fact_id": 1},
+ {"claim": "Acme is the best employer", "verdict": "overreach", "fact_id": 2},
+ {"claim": "Acme is profitable", "verdict": "unsupported", "fact_id": None},
+ ])
+ draft = _draft("Acme builds developer tools", "Acme is hiring engineers")
+ result = run_verification(draft, _research([_OWN, _OWN2]), judge, _settings())
assert result.total_claims == 3
assert result.grounded_claims == 1
assert result.groundedness_score == 0.3333
assert result.faithfulness_score == 0.3333
- assert result.tier_breakdown == {"own_site": 2, "third_party_snippet": 1}
+ assert result.tier_breakdown == {"own_site": 2}
assert result.passed is False
+ def test_judge_uses_zero_temperature(self):
+ judge = FakeJudge([{"claim": "c", "verdict": "faithful", "fact_id": 1}])
+ run_verification(_draft("Acme builds developer tools"), _research([_OWN]), judge, _settings())
+ assert judge.temperature == 0.0
-class TestJudgeFaithfulness:
+
+class TestJudgeBody:
def test_unknown_verdict_defaults_to_unsupported(self):
class Weird:
- def complete_json(self, s, u):
- return {"verdict": "banana", "reason": "?"}
- assert judge_faithfulness("c", "e", Weird())["verdict"] == "unsupported"
-
- def test_judge_failure_fails_closed(self):
- from pitch_pilot.clients.llm import LLMError
-
+ def complete_json(self, s, u, temperature=None):
+ return {"claims": [{"claim": "c", "verdict": "banana", "fact_id": 1}]}
+ ok, claims = judge_body("body", [_OWN], Weird())
+ assert ok is True
+ assert claims[0]["verdict"] == "unsupported"
+ assert claims[0]["fact"] is None # unsupported -> no backing fact resolved
+
+ def test_judge_failure_returns_not_ok(self):
class Boom:
- def complete_json(self, s, u):
+ def complete_json(self, s, u, temperature=None):
raise LLMError("down")
- assert judge_faithfulness("c", "e", Boom())["verdict"] == "unsupported"
+ ok, claims = judge_body("body", [_OWN], Boom())
+ assert ok is False and claims == []
+
+ def test_resolves_fact_id_to_fact(self):
+ class Good:
+ def complete_json(self, s, u, temperature=None):
+ return {"claims": [{"claim": "c", "verdict": "faithful", "fact_id": 2}]}
+ ok, claims = judge_body("body", [_OWN, _OWN2], Good())
+ assert ok is True
+ assert claims[0]["fact"] is _OWN2
class TestVerifyNodeAdapter: