From 9f73795645153bf06eaa356897ccf6e14577d4ec Mon Sep 17 00:00:00 2001 From: milstan Date: Tue, 28 Jul 2026 12:49:40 +0200 Subject: [PATCH 01/46] =?UTF-8?q?feat(mcp):=20expose=20backend=20MCP-first?= =?UTF-8?q?=20lead=20delivery=20=E2=80=94=20find=5Fnew=5Fleads,=20qualify?= =?UTF-8?q?=5Fleads,=20lead=5Fjob=5Fstatus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three composites over the new job endpoints (POST /1.6/mcp/search, POST /1.6/mcp/qualify, GET /1.6/mcp/jobs/{id}): one ask -> net-new qualified contactable leads; batch qualify + right-contact on known refs; shared cumulative job poll with cursor paging. Submit+poll built into the tools (wait_seconds window, still_running handoff), request_id retry contract, dry_run cost gate, no-silent-spend consent copy. The leadbay_new_leads prompt + heuristics/example-lead-craft snippet carry the fake-leads seed discipline (registry-style buyer description, no invented brand name, no event language, filters for hard bounds) — validated live on staging: same ICP delivered 0/5 junk from a naive query vs 5/5 on-profile from a crafted description-only seed. Cross-routing anti-triggers on pull_leads / extend_lens / bulk_qualify_leads; WORKFLOWS.md rows 46-48 + contracts; persona and naming/deprecation doc in docs/mcp-first-lead-delivery.md. Co-Authored-By: Claude Fable 5 --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 80 +++ WORKFLOWS.md | 60 ++ docs/mcp-first-lead-delivery.md | 206 ++++++ .../src/composite/_composite-file-names.ts | 3 + .../core/src/composite/_mcp-job-helpers.ts | 221 +++++++ packages/core/src/composite/find-new-leads.ts | 259 ++++++++ .../core/src/composite/lead-job-status.ts | 101 +++ packages/core/src/composite/qualify-leads.ts | 227 +++++++ packages/core/src/index.ts | 16 + .../core/src/tool-descriptions.generated.ts | 604 +++++++++++++++++- .../unit/composite/find-new-leads.test.ts | 230 +++++++ .../unit/composite/lead-job-status.test.ts | 124 ++++ .../test/unit/composite/qualify-leads.test.ts | 191 ++++++ packages/mcp/src/prompts.generated.ts | 82 +++ packages/mcp/src/prompts.ts | 20 + packages/mcp/test/audit/routing-block.test.ts | 3 + packages/mcp/test/prompts.test.ts | 1 + .../prompts/leadbay_new_leads.md.tmpl | 97 +++ .../snippets/heuristics/example-lead-craft.md | 48 ++ .../snippets/next-steps/find-new-leads.md | 14 + .../snippets/next-steps/lead-job-status.md | 10 + .../snippets/next-steps/qualify-leads.md | 14 + .../snippets/rendering/lead-delivery-table.md | 73 +++ .../composite/bulk-qualify-leads.md.tmpl | 2 +- .../composite/extend-lens.md.tmpl | 2 + .../composite/find-new-leads.md.tmpl | 111 ++++ .../composite/lead-job-status.md.tmpl | 64 ++ .../composite/pull-leads.md.tmpl | 5 + .../composite/qualify-leads.md.tmpl | 96 +++ 29 files changed, 2960 insertions(+), 4 deletions(-) create mode 100644 .claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md create mode 100644 docs/mcp-first-lead-delivery.md create mode 100644 packages/core/src/composite/_mcp-job-helpers.ts create mode 100644 packages/core/src/composite/find-new-leads.ts create mode 100644 packages/core/src/composite/lead-job-status.ts create mode 100644 packages/core/src/composite/qualify-leads.ts create mode 100644 packages/core/test/unit/composite/find-new-leads.test.ts create mode 100644 packages/core/test/unit/composite/lead-job-status.test.ts create mode 100644 packages/core/test/unit/composite/qualify-leads.test.ts create mode 100644 packages/promptforge/prompts/leadbay_new_leads.md.tmpl create mode 100644 packages/promptforge/snippets/heuristics/example-lead-craft.md create mode 100644 packages/promptforge/snippets/next-steps/find-new-leads.md create mode 100644 packages/promptforge/snippets/next-steps/lead-job-status.md create mode 100644 packages/promptforge/snippets/next-steps/qualify-leads.md create mode 100644 packages/promptforge/snippets/rendering/lead-delivery-table.md create mode 100644 packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl create mode 100644 packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl create mode 100644 packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md new file mode 100644 index 00000000..efc84be1 --- /dev/null +++ b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md @@ -0,0 +1,80 @@ +--- +name: leadbay_new_leads +description: "Guided net-new lead delivery — turn a natural-language need (\"gyms around Dallas that would buy our flooring\") into ICP-perfect NEW companies with qualification evidence and the right contact, via leadbay_find_new_leads. Trigger on \"find me new leads\", \"get me N companies that \", \"we're entering \". Do NOT trigger on \"today's leads\" (leadbay_daily_check_in) or \"qualify these companies I have\" (leadbay_qualify_leads)." +--- + + +## MEMORY + +Before responding, glance at any `_meta.agent_memory.summary` returned by tool calls earlier in this session and reflect its top signals in your reasoning ("Filtering by your stated preference for healthcare"). After any material new signal from the user this conversation (sector, region, deal size, communication style, qualification rule, explicit retraction, or recurrence / scheduling preference such as "I do this every day" or "remind me every morning"), call `leadbay_agent_memory_capture` to persist it: `source:"user_stated"` if literal, `source:"inferred"` with confidence <=6 if inferred. + + +IRON LAW — NO FABRICATION. Every lead id, contact email, custom field id, mapping decision, and tool argument must trace to a value you read from the file the user attached or to an output from a leadbay_* tool call in this session. Do not invent values. Do not "fill in" a missing leadId with a name match. Do not synthesize a CRM id from a guess. If a value is missing, leave the field blank and say so. + + +GATE — DEFER TO TOOL RENDERING. When you call a Leadbay composite that ships its own RENDERING block (every composite in 0.9.0+ does), render the response using that block's recipe verbatim — score bars, glyph palette, column order, hide-list, link priorities, all of it. Do NOT substitute prose, a numbered list, or a different column structure even when an orchestrating prompt's body suggests alternate framing. Prompt-specific commentary (motivational nudges, summaries, next-action recommendations) belongs ABOVE or BELOW the canonical table, never in place of it. + +If the prompt's body and the tool's RENDERING appear to conflict, the tool's RENDERING wins for the structural layout; the prompt's voice wins for the commentary that surrounds it. + + +Find net-new leads for me. My need, in my words: + +> + +If the need above is empty or too vague to name (a) who I sell to and (b) +roughly how many leads I want, ask me ONCE — one short question — then +proceed. Default count when unstated: 10. + +# PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) + +From my words, work out: +- What I SELL and therefore WHO WRITES ME CHECKS — the buyer category, never + the buyer's customers, never my competitors. If my product helps companies + of type X serve audience Y, my buyer is X. +- Hard constraints: geography, size band, sector, exclusions ("no + franchises", "pas de grands groupes" — negatives BIND). +- Contact needs: do I want a person? Which titles? Email, phone, both? +- Buyer archetypes: if my need genuinely spans two different kinds of buyer, + plan one search per archetype — never one blended seed. + +# PHASE 2 — CRAFT THE SEED + +Compose the `example_lead` for each archetype following the craft rules in +the leadbay_find_new_leads description (registry-style description of a +FICTIONAL typical buyer; no invented brand name; no event language; hard +constraints go in `filters`, not prose). Show me the seed description(s) in +one line each — I should recognize my ideal customer in them. + +# PHASE 3 — FREE PREVIEW + +Call `leadbay_find_new_leads` with the seed, `filters`, `count`, +`qualify: false`, no channels — this is FREE — and a `request_id` derived +from the ask + today's date. Render the delivery table and judge fit +honestly: are these the kind of companies I asked for? + +- **On-profile** → offer Phase 4. +- **Off-profile or empty** → read `funnel` + `explain.scope_notes`, tell me + what went wrong in one line (wrong archetype? too narrow a filter? thin + universe?), reshape the seed or filters, and retry under a NEW request_id. + Reshaping is free; do not pay to explore a bad seed. + +# PHASE 4 — PAID DEPTH (only with my explicit go-ahead) + +When I want qualification evidence and/or reachable contacts: +1. Quote first: `dry_run: true` with the exact flags I asked for + (`qualify: true`, `min_ai_score`, `contact_titles`, `title_gate`, + `channels`, `max_cost`) and tell me the worst-case cost in plain money. +2. On my go-ahead, prefer feeding the free preview's deliveries to + `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on + companies already known to match. Run a fresh `qualify: true` search + instead when I asked for more than the preview delivered. +3. While the job runs, poll with `leadbay_lead_job_status` + (`wait_seconds: 60`); report progress, not silence. + +# PHASE 5 — DELIVER + +Render per the lead-delivery table, then ALWAYS the funnel line: matched / +examined / qualified / disqualified / delivered / stop reason / spend. Zero +delivered gets a diagnosis and a concrete next move, never a shrug. Close +with NEXT STEPS from the tool description — and STOP; take no further action +without my say-so. diff --git a/WORKFLOWS.md b/WORKFLOWS.md index 31944c37..61ee75f0 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -57,6 +57,9 @@ The table is the human-readable index. The `yaml expected` + `yaml scenario` blo | 43 | **Enrichment stays active until done (no reprompt)** — the core of product#3866: after the user authorizes a paid enrichment, the agent launches via `leadbay_enrich_titles` (which returns `mode:"launched"` immediately — the job runs async), then STAYS ACTIVE in the same turn: it polls `leadbay_bulk_enrich_status` in a loop until done (`all_done`, or the resolvable set plateaus), and reports the completed enrichment (which contacts got emails/phones, counts, refreshed quota via `leadbay_account_status`) on its own — WITHOUT the user having to ask "is it done yet?". Distinct from Workflow 34 (multi-turn campaign builder, where the user *explicitly* says "wait for enrichment to finish" in turn 3); here it is a SINGLE turn and the stay-active behavior must be automatic. | `leadbay_enrich_titles` | "Pull my current leads and enrich their emails — get me the results in this same reply" | | 44 | **Pull leads offers "Enrich top leads"** — product#3875: after a `leadbay_pull_leads` on a non-empty batch, the deterministic `next_steps` surfaces an **Enrich top leads** option at position 2 (right after the Triage-board artifact offer) so the discovery→outreach bridge is one click away. It routes to `leadbay_enrich_titles` via the NO-SPEND preview path — previews volume + channels first, spends nothing until the user confirms — so a plain "show me my leads" never triggers an unprompted paid reveal (the #42 consent gate holds). | `leadbay_pull_leads`, `leadbay_enrich_titles` | "Show me my top leads for today" | | 45 | **Telemetry enable/disable/status** — product#3879: an in-product control to opt out of / into product-usage telemetry, or check the current setting. `leadbay_set_telemetry` (its `action` argument is `enable`, `disable`, or `status`; default `status`) reads/writes a per-user preference stored on the Leadbay account (`GET /users/me` → `telemetry_enabled`; `POST /users/telemetry`). Telemetry stays ON by default (opt-out). The hosted/web connector honors the flag per-request (a disabled user's events are suppressed). A local/stdio install decides telemetry at startup from `LEADBAY_TELEMETRY_ENABLED` and does not read the account flag, so local opt-out also needs that env var — the tool's copy says so rather than promising local opt-out. | `leadbay_set_telemetry` | "Turn off telemetry — I don't want my usage tracked" | +| 46 | **Net-new lead delivery (one ask → qualified, contactable leads)** — "find me 10 gyms around Dallas that would buy our flooring, with someone I can call". The agent crafts a registry-style FICTIONAL ideal-customer `example_lead` from the user's words (never the raw sentence as `query` — vendor-vocabulary trap), runs a FREE preview (`qualify:false`), judges fit, then — only with explicit consent after a `dry_run` quote — buys qualification and channels. Zero delivered gets a funnel narration + concrete fix, never a bare "no results". Backend: `POST /1.6/mcp/search` job. | `leadbay_new_leads` | "Find me 10 gyms around Dallas that would buy our modular flooring, with someone I can call" | +| 47 | **Batch qualify + right contact on known companies** — "here are 60 restaurant websites from my sweep — which fit, and who's the owner?". `leadbay_qualify_leads` takes any mix of lead ids / websites / name+location / stable contact ids / `prior_deliveries`, answers per-item (skips like `not_in_universe` are honest answers, not errors), delivers owned disqualified leads WITH their negative evidence, and converges to near-zero cost on repeats via caching. Backend: `POST /1.6/mcp/qualify` job. | `leadbay_qualify_leads` | "Vet these companies from my spreadsheet against our criteria and get me the right contact at each" | +| 48 | **Lead-delivery job polling** — a `leadbay_find_new_leads` / `leadbay_qualify_leads` run that outlives its poll window hands back a `job_id`; `leadbay_lead_job_status` re-reads the cumulative snapshot (state, funnel, items, spend) and block-waits with `wait_seconds` when the user asked to wait. | `leadbay_lead_job_status` | "Any results yet from that lead search?" | --- @@ -1020,6 +1023,63 @@ success_criteria: prompt: "Turn off telemetry — I don't want my usage tracked" ``` +```yaml expected +workflow_name: Net-new lead delivery (one ask → qualified, contactable leads) +prompt_name: leadbay_new_leads +required_calls: + - leadbay_find_new_leads +forbidden_calls: + - leadbay_pull_leads + - leadbay_extend_lens +success_criteria: + - "crafted a registry-style example_lead description of the BUYER (a fictional typical gym operator), not the seller's product, and did NOT pass the user's raw sentence as query" + - "left example_lead.name unset (no invented brand name)" + - "first call was FREE (qualify:false, no channels) with a request_id derived from the ask" + - "did NOT launch qualify:true or channels without a dry_run quote and explicit user consent" + - "rendered the delivery table and closed with the honest funnel line (matched/examined/delivered/stop reason/spend)" +``` + +```yaml scenario +prompt: "Find me 10 gyms around Dallas that would buy our modular flooring, with someone I can call" +``` + +```yaml expected +workflow_name: Batch qualify + right contact on known companies +prompt_name: ~ +required_calls: + - leadbay_qualify_leads +forbidden_calls: + - leadbay_find_new_leads + - leadbay_bulk_qualify_leads +success_criteria: + - "passed the user's companies as lead_refs (websites/names), not as a search" + - "requested the Owner/General Manager titles via contact_titles" + - "rendered per-item outcomes including skips (not_in_universe etc.) in plain words — a skip is an answer, not an error" + - "did NOT purchase channels without explicit consent" +``` + +```yaml scenario +prompt: "Here are 3 restaurant websites from my Austin sweep: franklinbbq.com, uchiaustin.com, terry-blacks-bbq.com — which fit our merchant profile, and who's the owner at each?" +``` + +```yaml expected +workflow_name: Lead-delivery job polling +prompt_name: ~ +required_calls: + - leadbay_lead_job_status +forbidden_calls: + - leadbay_bulk_enrich_status + - leadbay_import_status +success_criteria: + - "polled leadbay_lead_job_status with the job_id from the prior delivery" + - "did NOT misroute to the enrichment or import status tools" + - "on a terminal state, rendered the full delivery per the lead-delivery table; on running, reported progress and offered to check again" +``` + +```yaml scenario +prompt: "Any results yet from that lead search you started earlier? Job id is 281d8b55-b357-43ed-aca9-63e50bce84a6" +``` + ## How this stays normative `packages/mcp/test/audit/workflows.test.ts` asserts every backtick-wrapped `leadbay_*` identifier resolves to a registered tool or prompt. Proposed names for not-yet-shipped tools go in italics, not backticks. diff --git a/docs/mcp-first-lead-delivery.md b/docs/mcp-first-lead-delivery.md new file mode 100644 index 00000000..e7f695d4 --- /dev/null +++ b/docs/mcp-first-lead-delivery.md @@ -0,0 +1,206 @@ +# MCP-first lead delivery — personas, use cases, naming + +> Product design for exposing the backend MCP-first endpoints +> (`POST /1.6/mcp/search`, `POST /1.6/mcp/qualify`, `GET /1.6/mcp/jobs/{id}`) +> as first-class MCP tools. Grounded in live staging probes against the five +> test accounts (2026-07-28, `.context/probe/` in the working branch). + +## The two capabilities, in user vocabulary + +| Capability | User sentence | Backend | +|---|---|---| +| **Find new leads** | "Get me 10 *new* companies that look like my ideal customer — qualified, with the right contact and their email." | `POST /mcp/search` → job | +| **Qualify known leads** | "Here are companies I already have — tell me which fit, why, and who to talk to." | `POST /mcp/qualify` → job | + +Both answer in one ask what previously took a chain (pull → select → web-fetch → +poll → enrich → poll), and both are **jobs**: submit returns in <1s, results +stream per-item and are collected by polling. + +Free tier: `qualify: false, channels: []` costs **0** and still returns company ++ fit score + cached research + contact identity. Paid capabilities (fresh AI +qualification ~94c/candidate examined, email 25c / phone 250c on success) are +opt-in flags with a `dry_run` forecast and a `max_cost` cap. + +## Personas + +### P1 — Territory rep, physical product (SnapLock: modular floor tiles → gyms/warehouses, US) +Field seller with a geographic patch; prospecting happens between site visits, +often from the phone. Thinks in places and building types, not in "ICPs". + +- **Asks**: "Find me 10 gyms around Dallas that would buy our flooring, with + someone I can call." · "Any new warehouses opening in my patch?" · "I'm in + Houston Thursday — who's worth a cold visit?" (routes to followups/tour for + known leads, *search* for net-new). +- **Scheduled**: Monday 7am — "5 fresh gym/warehouse leads in my territory with + phone numbers" → call sheet in chat. +- **Artifacts**: printable call sheet; map of new prospects + follow-ups. +- **Params profile**: `example_lead` (a fictional typical gym), `filters.locations`, + `contact_titles: [Owner, Facilities Manager, General Manager]`, + `channels: [phone]`, small `count` (5-10). +- **Happy**: every delivered lead is *actually a gym/warehouse* (not a flooring + vendor), has a name + phone, and the one-line "why it fits". +- **Unhappy**: paying for exploration that delivers 0 (probe: naive query + "gyms in Texas that need durable flooring" spent 165c, delivered 0 — the + pre-screen rejected 30+ vendor-lookalikes). **The seed discipline exists to + prevent exactly this.** + +### P2 — SDR / outbound at a SaaS scale-up (Rippling: HR/payroll → US SaaS 50-2000 emp) +Quota-carrying, volume-oriented, lives in sequences. Needs precise slices and +verified emails; tolerates cost, not junk. + +- **Asks**: "20 new US SaaS companies, 50-2000 employees, that look like + , with the VP People's email." · "Same as last week but exclude + everything you already gave me." +- **Scheduled**: daily 8am — top-up batch of N with emails, deduped against all + prior deliveries (`novelty: org` does this server-side; `exclude_lead_ids` + belt on top). +- **Artifacts**: CSV for the sequencer; outreach drafts per lead + (`message_compose_v1` downstream). +- **Params profile**: `example_lead` + `filters.employees_min/max`, + `qualify: true`, `contact_titles` + `title_gate: strict`, `channels: [email]`, + `request_id` per day (idempotent re-runs), `max_cost` set consciously. +- **Happy**: n delivered = n asked, each with verified email of the right title. +- **Unhappy**: silent spend; duplicates of companies already in their sequencer; + title matched to a wrong person. (`title_gate: strict` + funnel honesty are + the levers.) + +### P3 — Merchant-acquisition rep (DoorDash: restaurants, city by city) +Works dense local markets; the "list" often comes from walking around, maps, or +a city scrape. Net-new discovery matters less than **vetting a known list fast**. + +- **Asks**: "Here are 60 restaurants from my Austin sweep — which are open, + independent, and not already on the platform? Who's the owner?" · "Qualify + yesterday's delivery and get phone numbers for the top ones." +- **Scheduled**: weekly re-vet of the working list (`prior_deliveries` selector + re-reads past outputs at near-zero cost thanks to caching). +- **Artifacts**: door-knock route (map widget), call sheet with owner + phone. +- **Params profile**: `lead_refs` by website/name+location, `contact_titles: + [Owner, General Manager]`, `channels: [phone]`. +- **Happy**: per-item verdicts — even "not in our universe" is an answer that + saves a visit. Disqualified leads come back *with the negative evidence*. +- **Unhappy**: whole-job failure because one ref was junk (backend guarantees + per-item outcomes — probe: 3 `not_in_universe` + 1 `low_confidence_identity`, + job still `completed`, cost 0). + +### P4 — Sales manager / team lead +Feeds the team, owns spend, coaches with evidence. Runs org-level intelligence +(qualification questions, IBP) and expects deliveries to obey it. + +- **Asks**: "Get each of my 3 reps 10 fresh leads in their region for Monday." + · "Re-qualify the 200 stale leads in our pipeline against the new + qualification questions — who should we drop?" · "What did that search cost?" +- **Scheduled**: Sunday night per-territory batches (one job per territory, + `request_id` = week+territory so retries never double-spend); monthly + pipeline re-vet via `qualify` (cache makes repeats cheap). +- **Artifacts**: per-rep briefs; a funnel/cost report ("38 matched, 9 examined, + 3 delivered, €6.09 — stopped at your cost cap"). +- **Happy**: predictable spend (`dry_run` forecast, `max_cost`), auditable + funnel, deliveries that respect the org's questions/tags/IBP snapshot. +- **Unhappy**: a rep burning the org's monthly budget in an afternoon (plan-tier + default caps + explicit `max_cost` are the guardrails; the agent must state + costs *before* paid runs). + +### P5 — Founder-led sales, FR SMB (Sol Mur: revêtements; Home Spirit: mobilier B2B) +Non-technical, prospecting in bursts between deliveries; speaks French to the +agent; the SIRENE-based FR universe is their world. + +- **Asks**: "Trouve-moi 5 hôtels ou promoteurs en Île-de-France qui rénovent + leurs espaces, avec un contact achats." · "Qualifie ces 12 entreprises de mon + fichier Excel." +- **Scheduled**: rare — prefers on-demand bursts. +- **Artifacts**: a short brief per lead, in French (`lang: fr`), ready to turn + into a call. +- **Params profile**: French `example_lead` description (FR bridges text seeds + into the SIRENE embedding space via exemplar expansion — the seed style rules + apply identically), `lang: "fr"`. +- **Happy**: results in French, sectors that make sense in the French taxonomy. +- **Unhappy**: anglocentric outputs; sector labels that don't resolve (submit + 400s name the offending value — the agent should fix and retry, or use + `leadbay_list_sectors`). + +### P6 — RevOps / data owner +Owns the CRM. Thinks in batches of 500, dedup keys and cost lines, not in +individual leads. + +- **Asks**: "Vet this 500-row export: which are ICP-fit? Which have verified + emails for a Head of Ops?" · "Re-read everything MCP delivered in June." +- **Scheduled**: quarterly hygiene sweep (`qualify` with `lead_refs` ≤500, or + `prior_deliveries` for the ledger). +- **Artifacts**: enriched CSV back; a delta report (newly disqualified since + last sweep). +- **Happy**: idempotency (`request_id`), per-item cost lines, cache reuse + ("repeat calls converge to near-zero cost"). +- **Unhappy**: re-buying data it already owns (`already_owned` channel status + and `from_cache` flags exist precisely for this — surface them). + +## Use-case → routing map + +| User says | Route to | Why | +|---|---|---| +| "Find me N new companies like X / that do Y" | **find_new_leads** (new) | net-new + custom ICP + one shot | +| "Show me today's leads / my inbox" | `leadbay_pull_leads` | daily lens picks, free, taste-based | +| "More leads like the ones in my lens" | `leadbay_extend_lens` | grows the lens itself | +| "Qualify/vet THESE companies" (ids, websites, CSV rows, prior deliveries) | **qualify_leads** (new) | server-side batch verdicts + contacts | +| "Qualify my top lens leads" | `leadbay_bulk_qualify_leads` → *migration target* | legacy client-side chain; new route covers it via wishlist ids | +| "Get emails/phones for these leads' contacts" | **qualify_leads** with `channels` | enrichment now rides the same job | +| "Research this one company in depth" | `leadbay_research_lead_by_id` | single-lead dossier, richer prose | +| "Who should I follow up with?" | `leadbay_pull_followups` | engaged pipeline, not net-new | +| "Import this file" | `leadbay_import_leads` (+ optionally qualify after) | file wizard owns column mapping | + +## What the probes taught us (drives every description/prompt decision) + +1. **Seed quality is the product.** Naive NL query → 0/3 delivered, 165c burned + (vendor-confusion: embedding matches topic *vocabulary*). A fictional + ideal-customer `example_lead` in registry style is the single highest-leverage + input. The backend's own scope note says so; our prompt teaches the craft + (see the fake-leads discipline distilled in the tool description + prompt). +2. **Zero-delivery is a real outcome and must be narrated, not hidden.** The + funnel (`matched/examined/qualified/disqualified` + `stop_reason` + + `scope_notes`) tells an honest story the agent must render. "9 examined, 8 + disqualified, stopped at your cost cap" is actionable; "no results" is not. +3. **Cost literacy up-front.** `qualify: true` bills ~94c per *examined* + candidate (survivor or not). Default `max_cost` (plan tier) can stop a job + mid-exploration (probe: stop=max_cost at 609c). The agent should `dry_run` + before the first paid run of a session and state the worst case. +4. **Need-B refs resolve against the known universe.** Off-universe websites + come back `not_in_universe` per-item (cost 0) — an answer, not an error. +5. **Jobs are minutes-scale.** Submit <1s; free search ~seconds-minutes; paid + exploration up to 30 min wall-clock. Tools must poll briefly then hand back + a `job_id` + explicit "check again with …" next step. + +## Naming + +New tools (all composite): + +| Name | Kind | Rationale | +|---|---|---| +| `leadbay_find_new_leads` | read (submit is free by default; paid flags are explicit params) | The user phrase is literally "find me new leads". Distinct from `pull_leads` (today's lens picks) and from granular `discover_leads` (wishlist page relay, advanced-gated). | +| `leadbay_qualify_leads` | write-tier | "Qualify these leads" — plural, arbitrary refs. Sits between granular `qualify_lead` (single, advanced) and legacy `bulk_qualify_leads` (lens top-N chain). | +| `leadbay_lead_job_status` | read | Polls `GET /mcp/jobs/{id}` for both kinds; follows the `*_status` convention (`qualify_status`, `import_status`, `bulk_enrich_status`). | + +New prompt: **`leadbay_new_leads`** — the guided "find me new leads" session; +owns the NL-need → seed-description craft (fake-leads discipline), dry-run cost +gate, submit, poll, render, iterate loop. + +**Vocabulary rule:** "seed" in this repo already means *existing lens leads +used for extra-refill* (`seed_candidates`, `seed_lead_ids`). The new surface +never says "seed" in tool/param names — the request param is `example_lead` +(backend vocabulary), described as "a fictional ideal-customer example". + +## Deprecation direction (not executed in this PR) + +| Existing | Status | Path | +|---|---|---| +| `leadbay_bulk_qualify_leads` | **migration target** | New `qualify_leads` is server-side, cached, per-item honest, and bundles contacts. Keep for one release with cross-routing anti-triggers pointing at the new tool; remove after evals confirm parity. | +| `leadbay_enrich_titles` + `bulk_enrich_status` | keep (consent UX) | Channel purchase now also rides search/qualify jobs; the dedicated flow remains for enrich-only asks on selections. Cross-route. | +| `leadbay_qualify_lead`, `leadbay_qualify_status` | keep (advanced/status) | Advanced tier stays near-raw. | +| `leadbay_pull_leads`, `extend_lens`, `seed_candidates` | keep | Different job: daily taste-based lens flow vs on-demand net-new ask. Cross-route both ways. | +| `leadbay_import_and_qualify` | keep, re-route step 2 | Import wizard stays; its qualify step can hand off to `qualify_leads` by lead ids in a follow-up PR. | + +## Measurement + +`.context/probe/benchmark-*.md` (working branch): same ICP ask executed (a) via +the old chain (pull → select → web-fetch → poll → enrich) and (b) via the new +routes, on staging accounts; wall-clock + spend recorded, output quality judged +by an independent reviewing agent on: ICP fit of delivered leads, contact +correctness, honesty of the failure story, and actionability of the rendering. diff --git a/packages/core/src/composite/_composite-file-names.ts b/packages/core/src/composite/_composite-file-names.ts index 4a87d54b..bbf689a2 100644 --- a/packages/core/src/composite/_composite-file-names.ts +++ b/packages/core/src/composite/_composite-file-names.ts @@ -23,18 +23,21 @@ export const COMPOSITE_FILE_TOOL_NAMES: ReadonlySet = new Set([ "leadbay_delete_custom_field", "leadbay_enrich_titles", "leadbay_extend_lens", + "leadbay_find_new_leads", "leadbay_followups_map", "leadbay_get_lead_custom_fields", "leadbay_get_qualification_questions", "leadbay_import_and_qualify", "leadbay_import_leads", "leadbay_import_status", + "leadbay_lead_job_status", "leadbay_list_campaigns", "leadbay_my_lenses", "leadbay_new_lens", "leadbay_prepare_outreach", "leadbay_pull_followups", "leadbay_pull_leads", + "leadbay_qualify_leads", "leadbay_qualify_status", "leadbay_recall_ordered_titles", "leadbay_refine_prompt", diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts new file mode 100644 index 00000000..9d81fb9b --- /dev/null +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -0,0 +1,221 @@ +// Shared plumbing for the MCP-first lead-delivery jobs +// (POST /mcp/search, POST /mcp/qualify, GET /mcp/jobs/{id}). +// +// Both submit verbs answer 202 + a job handle; results are polled +// cumulatively from /mcp/jobs/{id} with an opaque `since` cursor. The +// backend caps poll pages at 100 items, while a qualify job can carry up +// to 500 refs — so a snapshot collects pages until the cursor drains. +import type { LeadbayClient } from "../client.js"; +import type { ToolContext } from "../types.js"; + +// --------------------------------------------------------------------------- +// Wire types (backend McpPayloads.kt, snake_case on the wire) +// --------------------------------------------------------------------------- + +export interface McpSubmitResponse { + job_id: string; + status_url: string; + estimated_cost: { max: number; unit: string }; + items_requested: number; + duplicate?: boolean; +} + +export interface McpDryRunResponse { + valid: boolean; + items_requested: number; + estimated_cost: { max: number; unit: string }; + quota_forecast: { + web_fetch_allowed: boolean; + rescore_allowed: boolean; + enrichment_allowed: boolean; + }; +} + +export interface McpFunnel { + matched?: number; + novel?: number; + title_gate_passed?: number; + examined?: number; + qualified?: number; + disqualified?: number; + unqualifiable?: number; + delivered?: number; + delivered_callable?: number; + delivered_title_only?: number; + degraded?: number; + resolved?: number | null; + not_in_universe?: number | null; + pending_import?: number | null; + unauthorized?: number | null; + stop_reason?: string | null; +} + +export interface McpJobItem { + ref?: { + input_indexes?: number[] | null; + lead_id?: string | null; + requested_as?: Record | null; + } | null; + status: "delivered" | "degraded" | "skipped"; + status_reason?: string | null; + resolution?: Record | null; + contact_known?: boolean | null; + from_cache?: Record | null; + cost?: { billed: number; unit: string; breakdown?: Record } | null; + completed_at?: string | null; + seq: number; + // Full QualifiedLead payload (company / fit / web_research / contact / + // alternative_contacts / novelty / engagement) — relayed verbatim so the + // agent renders from rich signal without follow-up calls. + lead?: Record | null; +} + +export interface McpJobSnapshot { + job: { + id: string; + state: + | "queued" + | "running" + | "completed" + | "completed_partial" + | "failed" + | "expired"; + submitted_at: string; + completed_at?: string | null; + expires_at: string; + last_progress_at: string; + }; + funnel: McpFunnel; + items: McpJobItem[]; + next_since?: string | null; + cost: { + spent: number; + unit: string; + breakdown: Record; + }; + explain: { + region: string; + model: string; + basis?: string | null; + seed_strategy?: string | null; + universe_size?: number | null; + filters_applied?: Record | null; + intelligence_snapshot?: Record | null; + scope_notes?: string[]; + }; +} + +export const TERMINAL_JOB_STATES: ReadonlySet = new Set([ + "completed", + "completed_partial", + "failed", + "expired", +]); + +// Poll cadence seam — tests shrink this so wait loops don't sleep for real. +export const MCP_JOB_POLL = { intervalMs: 4000 }; + +const PAGE_LIMIT = 100; +const MAX_PAGES = 20; // 500-ref qualify worst case is 5 pages; 20 is a hard stop. + +/** One cumulative snapshot of the job, paging the item cursor dry. Job/funnel/ + * cost/explain come from the LAST page fetched (the freshest projection). */ +export async function collectJobSnapshot( + client: LeadbayClient, + jobId: string, + since?: string, + limit?: number +): Promise { + const pageLimit = Math.min(Math.max(limit ?? PAGE_LIMIT, 1), PAGE_LIMIT); + const qs = (cursor?: string) => + `/mcp/jobs/${jobId}?limit=${pageLimit}` + + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); + let page = await client.request("GET", qs(since)); + const items = [...page.items]; + let pages = 1; + while ( + page.items.length >= pageLimit && + page.next_since && + pages < MAX_PAGES + ) { + const next = await client.request( + "GET", + qs(page.next_since) + ); + items.push(...next.items); + pages += 1; + // Always adopt the newest page — its job/funnel/cost projection is the + // freshest even when it carried no new items. + page = next; + if (next.items.length === 0) { + break; + } + } + return { ...page, items }; +} + +/** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). + * Fires ctx.progress per poll and respects ctx.signal cancellation. */ +export async function waitForJob( + client: LeadbayClient, + jobId: string, + waitSeconds: number, + ctx?: ToolContext, + itemsRequested?: number +): Promise { + const startedAt = Date.now(); + let snap = await collectJobSnapshot(client, jobId); + while ( + !TERMINAL_JOB_STATES.has(snap.job.state) && + (Date.now() - startedAt) / 1000 < waitSeconds && + !ctx?.signal?.aborted + ) { + await new Promise((resolve) => + setTimeout(resolve, MCP_JOB_POLL.intervalMs) + ); + if (ctx?.signal?.aborted) break; + snap = await collectJobSnapshot(client, jobId); + const f = snap.funnel; + ctx?.progress?.({ + progress: f.delivered ?? 0, + total: itemsRequested, + message: `${snap.job.state}: ${f.examined ?? 0} examined, ${ + f.delivered ?? 0 + } delivered, ${snap.cost.spent}c spent`, + }); + } + return snap; +} + +/** Sort a snapshot's items into the envelope every delivery tool returns: + * full leads for delivered/degraded, compact skip records for the rest. */ +export function splitItems(snapshot: McpJobSnapshot): { + leads: McpJobItem[]; + skipped: McpJobItem[]; +} { + const leads: McpJobItem[] = []; + const skipped: McpJobItem[] = []; + for (const item of snapshot.items) { + if (item.status === "skipped") skipped.push(item); + else leads.push(item); + } + return { leads, skipped }; +} + +/** Drop undefined values so the wire body only carries what the caller set + * (backend uses explicitNulls=false; absent and null are equivalent). */ +export function compactBody( + body: Record +): Record { + return Object.fromEntries( + Object.entries(body).filter(([, v]) => v !== undefined) + ); +} + +export function clampWaitSeconds( + requested: number | undefined, + fallback: number +): number { + if (requested == null || Number.isNaN(requested)) return fallback; + return Math.min(Math.max(requested, 0), 180); +} diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts new file mode 100644 index 00000000..3aca29a8 --- /dev/null +++ b/packages/core/src/composite/find-new-leads.ts @@ -0,0 +1,259 @@ +// leadbay_find_new_leads — POST /mcp/search + poll GET /mcp/jobs/{id} +// +// One ask -> n net-new companies matching an ICP, optionally AI-qualified +// against the org's frozen intelligence snapshot, optionally with the right +// contact + purchased channels. Free by default (qualify:false, channels:[]). +// Submit validates synchronously (<1s, every input error is a 400 naming the +// field); results stream per-item and are collected here with a short poll +// window, handing back a job_id + leadbay_lead_job_status when the job +// outlives the window. +import type { LeadbayClient } from "../client.js"; +import type { Tool, ToolContext } from "../types.js"; +import { + clampWaitSeconds, + collectJobSnapshot, + compactBody, + splitItems, + TERMINAL_JOB_STATES, + waitForJob, + type McpDryRunResponse, + type McpSubmitResponse, +} from "./_mcp-job-helpers.js"; +import { leadbay_find_new_leads as FIND_NEW_LEADS_DESCRIPTION } from "../tool-descriptions.generated.js"; + +interface FindNewLeadsParams { + query?: string; + example_lead?: { + name?: string; + description?: string; + location?: string; + employees?: number; + }; + filters?: { + sectors?: string[]; + locations?: string[]; + employees_min?: number; + employees_max?: number; + }; + count: number; + qualify?: boolean; + min_ai_score?: number; + contact_titles?: string[]; + title_gate?: "strict" | "prefer"; + channels?: Array<"email" | "phone">; + exclude_lead_ids?: string[]; + novelty?: "org" | "none"; + max_cost?: number; + exploration_cap?: number; + request_id: string; + lang?: string; + dry_run?: boolean; + wait_seconds?: number; +} + +const DEFAULT_WAIT_SECONDS = 45; + +export const findNewLeads: Tool = { + name: "leadbay_find_new_leads", + annotations: { + title: "Find new leads (net-new ICP search)", + readOnlyHint: false, + destructiveHint: false, + // The mandatory request_id dedups: re-submitting the same request returns + // the SAME live job instead of double-spending. + idempotentHint: true, + openWorldHint: true, + }, + write: true, + description: FIND_NEW_LEADS_DESCRIPTION, + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: + "Natural-language ICP ask. Matches topic VOCABULARY — can surface vendors of a product as easily as buyers of it. Prefer example_lead; use query only when the user's wording carries signal an example can't.", + }, + example_lead: { + type: "object", + description: + "A FICTIONAL typical ideal customer used as a look-alike seed — the highest-leverage input. Put everything in `description` (registry 'About Us' style, what the company IS); leave `name` unset (a distinctive invented name pulls matches toward name-lookalikes).", + properties: { + name: { type: "string" }, + description: { type: "string" }, + location: { type: "string" }, + employees: { type: "number" }, + }, + additionalProperties: false, + }, + filters: { + type: "object", + description: + "HARD constraints (the seed only shapes ranking). Sector/location labels resolve at submit; an unresolvable value is a 400 naming it.", + properties: { + sectors: { type: "array", items: { type: "string" } }, + locations: { type: "array", items: { type: "string" } }, + employees_min: { type: "number" }, + employees_max: { type: "number" }, + }, + additionalProperties: false, + }, + count: { + type: "number", + description: + "Target DELIVERED leads, 1-50. With qualify:true this means n SURVIVORS of qualification, not n candidates examined.", + }, + qualify: { + type: "boolean", + description: + "Run fresh AI qualification and drop candidates scoring below min_ai_score. PAID: ~94 cost_cents per candidate EXAMINED (survivor or not). Default false (free).", + }, + min_ai_score: { + type: "number", + description: + "Disqualification floor on the [-30,+30] qualification DELTA (not the 0-100 fit score). Default 0. Lower to -30 to keep every evaluated lead with its evidence.", + }, + contact_titles: { + type: "array", + items: { type: "string" }, + description: + "Wanted decision-maker titles (max 10), matched semantically cross-language.", + }, + title_gate: { + type: "string", + enum: ["strict", "prefer"], + description: + "strict = only leads with a matching known contact; prefer (default when contact_titles set) = matched first, rest flagged.", + }, + channels: { + type: "array", + items: { type: "string", enum: ["email", "phone"] }, + description: + "Contact channels to PURCHASE (email 25c, phone 250c, billed on success only). Empty = free identity tier.", + }, + exclude_lead_ids: { + type: "array", + items: { type: "string" }, + description: "Caller-side novelty belt on top of the server-side one (max 500 ids).", + }, + novelty: { + type: "string", + enum: ["org", "none"], + description: + "org (default) = only companies NEW to the org (excludes org leads, lens members, CRM ids, prior MCP deliveries).", + }, + max_cost: { + type: "number", + description: + "Spend cap for the whole job in cost_cents. Defaults by plan tier (500/2000/5000). The job stops honestly at the cap (stop_reason max_cost).", + }, + exploration_cap: { + type: "number", + description: + "Max candidates the qualify gate may examine. Default min(3n,150), ceiling min(20n,1000).", + }, + request_id: { + type: "string", + description: + "REQUIRED idempotency key. Derive it from the ask (e.g. 'gyms-texas-2026-07-28'); REUSE the exact same value when retrying the same ask — a duplicate returns the SAME job instead of double-spending. Use a NEW value only for a genuinely new ask.", + }, + lang: { type: "string", description: "Output language (default: user's language)." }, + dry_run: { + type: "boolean", + description: + "Validate + worst-case cost estimate + quota forecast. No job, no spend. Use before the first PAID run of a session.", + }, + wait_seconds: { + type: "number", + description: + "How long to poll before returning (default 45, max 180, 0 = submit + one snapshot). Free searches usually finish inside the window; paid exploration can take minutes — the result then carries still_running:true and the job_id to check with leadbay_lead_job_status.", + }, + }, + required: ["count", "request_id"], + additionalProperties: false, + }, + execute: async ( + client: LeadbayClient, + params: FindNewLeadsParams, + ctx?: ToolContext + ) => { + const body = compactBody({ + query: params.query, + example_lead: params.example_lead, + filters: params.filters, + count: params.count, + qualify: params.qualify, + min_ai_score: params.min_ai_score, + contact_titles: params.contact_titles, + title_gate: params.title_gate, + channels: params.channels, + exclude_lead_ids: params.exclude_lead_ids, + novelty: params.novelty, + max_cost: params.max_cost, + exploration_cap: params.exploration_cap, + request_id: params.request_id, + lang: params.lang, + dry_run: params.dry_run, + }); + + if (params.dry_run) { + const forecast = await client.request( + "POST", + "/mcp/search", + body + ); + return { + dry_run: true, + ...forecast, + region: client.region, + }; + } + + const submit = await client.request( + "POST", + "/mcp/search", + body + ); + const waitSeconds = clampWaitSeconds( + params.wait_seconds, + DEFAULT_WAIT_SECONDS + ); + const snapshot = + waitSeconds > 0 + ? await waitForJob(client, submit.job_id, waitSeconds, ctx, params.count) + : await collectJobSnapshot(client, submit.job_id); + + const done = TERMINAL_JOB_STATES.has(snapshot.job.state); + const { leads, skipped } = splitItems(snapshot); + return { + job_id: submit.job_id, + request_id: params.request_id, + duplicate_submit: submit.duplicate ?? false, + state: snapshot.job.state, + done, + summary: { + requested: params.count, + delivered: snapshot.funnel.delivered ?? 0, + delivered_callable: snapshot.funnel.delivered_callable ?? 0, + delivered_title_only: snapshot.funnel.delivered_title_only ?? 0, + degraded: snapshot.funnel.degraded ?? 0, + stop_reason: snapshot.funnel.stop_reason ?? null, + }, + funnel: snapshot.funnel, + leads, + skipped, + cost: snapshot.cost, + estimated_cost: submit.estimated_cost, + explain: snapshot.explain, + still_running: !done, + next_poll: done + ? null + : { + tool: "leadbay_lead_job_status", + job_id: submit.job_id, + suggested_wait_seconds: 60, + }, + region: client.region, + }; + }, +}; diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts new file mode 100644 index 00000000..53ba19dd --- /dev/null +++ b/packages/core/src/composite/lead-job-status.ts @@ -0,0 +1,101 @@ +// leadbay_lead_job_status — GET /mcp/jobs/{id} +// +// Cumulative snapshot of a find_new_leads / qualify_leads job. Items are +// immutable once emitted; the optional `since` cursor pages only what's new +// since the last poll. Terminal projections are server-side: past the 30-min +// wall clock a job reads completed_partial(time_budget), past the 30-day TTL +// it reads expired — never an eternal `running`. +import type { LeadbayClient } from "../client.js"; +import type { Tool, ToolContext } from "../types.js"; +import { + clampWaitSeconds, + collectJobSnapshot, + splitItems, + TERMINAL_JOB_STATES, + waitForJob, +} from "./_mcp-job-helpers.js"; +import { leadbay_lead_job_status as LEAD_JOB_STATUS_DESCRIPTION } from "../tool-descriptions.generated.js"; + +interface LeadJobStatusParams { + job_id: string; + since?: string; + limit?: number; + wait_seconds?: number; +} + +export const leadJobStatus: Tool = { + name: "leadbay_lead_job_status", + annotations: { + title: "Poll a lead-delivery job", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + description: LEAD_JOB_STATUS_DESCRIPTION, + inputSchema: { + type: "object", + properties: { + job_id: { + type: "string", + description: + "The job_id returned by leadbay_find_new_leads or leadbay_qualify_leads.", + }, + since: { + type: "string", + description: + "Opaque cursor from a previous poll's next_since — returns only items emitted after it. Omit for the full snapshot.", + }, + limit: { + type: "number", + description: "Items per page, 1-100 (default 100; pages are auto-collected).", + }, + wait_seconds: { + type: "number", + description: + "0 (default) = instant snapshot. >0 = keep polling up to this many seconds until the job is terminal — use ~60 when the user asked to wait for results.", + }, + }, + required: ["job_id"], + additionalProperties: false, + }, + execute: async ( + client: LeadbayClient, + params: LeadJobStatusParams, + ctx?: ToolContext + ) => { + const waitSeconds = clampWaitSeconds(params.wait_seconds, 0); + const snapshot = + waitSeconds > 0 + ? await waitForJob(client, params.job_id, waitSeconds, ctx) + : await collectJobSnapshot( + client, + params.job_id, + params.since, + params.limit + ); + + const done = TERMINAL_JOB_STATES.has(snapshot.job.state); + const { leads, skipped } = splitItems(snapshot); + return { + job_id: params.job_id, + state: snapshot.job.state, + done, + funnel: snapshot.funnel, + leads, + skipped, + next_since: snapshot.next_since ?? null, + cost: snapshot.cost, + explain: snapshot.explain, + still_running: !done, + next_poll: done + ? null + : { + tool: "leadbay_lead_job_status", + job_id: params.job_id, + suggested_wait_seconds: 60, + }, + region: client.region, + }; + }, +}; diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts new file mode 100644 index 00000000..47a3dc55 --- /dev/null +++ b/packages/core/src/composite/qualify-leads.ts @@ -0,0 +1,227 @@ +// leadbay_qualify_leads — POST /mcp/qualify + poll GET /mcp/jobs/{id} +// +// "Qualify these companies I already have, and get me the right contact on +// each" in one server-side job. Refs can be lead ids, websites, name+location +// pairs, stable contact ids from prior results, or the prior_deliveries +// ledger selector. Every ref gets a per-item outcome — a bad ref never fails +// the job. Disqualified leads the org owns are DELIVERED with their negative +// evidence, never silently dropped. Repeat calls reuse every fresh cached +// stage and converge to near-zero cost. +import type { LeadbayClient } from "../client.js"; +import type { Tool, ToolContext } from "../types.js"; +import { + clampWaitSeconds, + collectJobSnapshot, + compactBody, + TERMINAL_JOB_STATES, + waitForJob, + type McpDryRunResponse, + type McpSubmitResponse, +} from "./_mcp-job-helpers.js"; +import { leadbay_qualify_leads as QUALIFY_LEADS_DESCRIPTION } from "../tool-descriptions.generated.js"; + +interface QualifyLeadsParams { + lead_refs?: Array<{ + lead_id?: string; + website?: string; + name?: string; + location?: string; + contact_id?: string; + }>; + prior_deliveries?: { + job_id?: string; + since?: string; + limit?: number; + }; + qualify?: boolean; + contact_titles?: string[]; + title_gate?: "strict" | "prefer"; + channels?: Array<"email" | "phone">; + max_cost?: number; + request_id?: string; + lang?: string; + dry_run?: boolean; + wait_seconds?: number; +} + +const DEFAULT_WAIT_SECONDS = 45; + +export const qualifyLeads: Tool = { + name: "leadbay_qualify_leads", + annotations: { + title: "Qualify + get the right contact on known leads", + readOnlyHint: false, + destructiveHint: false, + idempotentHint: false, + openWorldHint: true, + }, + write: true, + description: QUALIFY_LEADS_DESCRIPTION, + inputSchema: { + type: "object", + properties: { + lead_refs: { + type: "array", + description: + "Companies to qualify (max 500). Each ref needs at least one identifying field. Duplicate lead_ids collapse into one item.", + items: { + type: "object", + properties: { + lead_id: { type: "string", description: "Leadbay lead UUID." }, + website: { type: "string" }, + name: { type: "string" }, + location: { + type: "string", + description: "Disambiguates name-only refs (city/region).", + }, + contact_id: { + type: "string", + description: + "Stable lead_contact id from a prior result — enrichment then targets EXACTLY this person, never a re-match.", + }, + }, + additionalProperties: false, + }, + }, + prior_deliveries: { + type: "object", + description: + "Selector expanding the org's past MCP deliveries into refs — billed leads stay re-readable after result expiry. Combine with lead_refs or use alone.", + properties: { + job_id: { type: "string" }, + since: { type: "string", description: "ISO instant lower bound." }, + limit: { type: "number" }, + }, + additionalProperties: false, + }, + qualify: { + type: "boolean", + description: + "Fresh AI qualification (default true; ~94 cost_cents per lead needing fresh research+scoring, cache-free when a fresh dossier exists). Owned disqualified leads come back WITH their negative evidence.", + }, + contact_titles: { + type: "array", + items: { type: "string" }, + description: "Wanted decision-maker titles (max 10), matched semantically.", + }, + title_gate: { + type: "string", + enum: ["strict", "prefer"], + description: + "strict = only items with a matching known contact deliver a contact; prefer = matched first, rest flagged.", + }, + channels: { + type: "array", + items: { type: "string", enum: ["email", "phone"] }, + description: + "Channels to PURCHASE (email 25c, phone 250c, success-only, already-owned values are free). Empty = free identity tier.", + }, + max_cost: { + type: "number", + description: "Spend cap in cost_cents (plan-tier default when unset).", + }, + request_id: { + type: "string", + description: + "Recommended idempotency key — REUSE the same value when retrying the same batch so a retry returns the SAME job instead of re-spending.", + }, + lang: { type: "string", description: "Output language (default: user's language)." }, + dry_run: { + type: "boolean", + description: + "Validate + worst-case cost + quota forecast. No job, no spend.", + }, + wait_seconds: { + type: "number", + description: + "How long to poll before returning (default 45, max 180, 0 = submit + one snapshot). Large or research-heavy batches can take minutes — the result then carries still_running:true and the job_id for leadbay_lead_job_status.", + }, + }, + additionalProperties: false, + }, + execute: async ( + client: LeadbayClient, + params: QualifyLeadsParams, + ctx?: ToolContext + ) => { + const body = compactBody({ + lead_refs: params.lead_refs, + prior_deliveries: params.prior_deliveries, + qualify: params.qualify, + contact_titles: params.contact_titles, + title_gate: params.title_gate, + channels: params.channels, + max_cost: params.max_cost, + request_id: params.request_id, + lang: params.lang, + dry_run: params.dry_run, + }); + + if (params.dry_run) { + const forecast = await client.request( + "POST", + "/mcp/qualify", + body + ); + return { dry_run: true, ...forecast, region: client.region }; + } + + const submit = await client.request( + "POST", + "/mcp/qualify", + body + ); + const waitSeconds = clampWaitSeconds( + params.wait_seconds, + DEFAULT_WAIT_SECONDS + ); + const snapshot = + waitSeconds > 0 + ? await waitForJob( + client, + submit.job_id, + waitSeconds, + ctx, + submit.items_requested + ) + : await collectJobSnapshot(client, submit.job_id); + + const done = TERMINAL_JOB_STATES.has(snapshot.job.state); + return { + job_id: submit.job_id, + request_id: params.request_id ?? null, + duplicate_submit: submit.duplicate ?? false, + state: snapshot.job.state, + done, + summary: { + refs_submitted: params.lead_refs?.length ?? 0, + items_requested: submit.items_requested, + delivered: snapshot.funnel.delivered ?? 0, + delivered_callable: snapshot.funnel.delivered_callable ?? 0, + degraded: snapshot.funnel.degraded ?? 0, + resolved: snapshot.funnel.resolved ?? null, + not_in_universe: snapshot.funnel.not_in_universe ?? null, + stop_reason: snapshot.funnel.stop_reason ?? null, + }, + funnel: snapshot.funnel, + // Per-item outcomes in input order where known (ref.input_indexes maps + // back to the caller's lead_refs positions). Items carry the full + // QualifiedLead payload when delivered/degraded, and an honest + // status_reason (not_in_universe, low_confidence_identity, ...) when + // skipped — a skip is an ANSWER about that ref, not an error. + items: snapshot.items, + cost: snapshot.cost, + estimated_cost: submit.estimated_cost, + explain: snapshot.explain, + still_running: !done, + next_poll: done + ? null + : { + tool: "leadbay_lead_job_status", + job_id: submit.job_id, + suggested_wait_seconds: 60, + }, + region: client.region, + }; + }, +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7d62b9ad..7e8bb2a7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -121,6 +121,9 @@ import { bulkEnrichStatus } from "./composite/bulk-enrich-status.js"; import { adjustAudience } from "./composite/adjust-audience.js"; import { refinePrompt } from "./composite/refine-prompt.js"; import { seedCandidates } from "./composite/seed-candidates.js"; +import { findNewLeads } from "./composite/find-new-leads.js"; +import { qualifyLeads } from "./composite/qualify-leads.js"; +import { leadJobStatus } from "./composite/lead-job-status.js"; import { extendLens } from "./composite/extend-lens.js"; import { myLenses } from "./composite/my-lenses.js"; import { newLens } from "./composite/new-lens.js"; @@ -304,6 +307,11 @@ export const compositeReadTools: Tool[] = [ bulkEnrichStatus, qualifyStatus, importStatus, + // Poll surface for the MCP-first lead-delivery jobs (find_new_leads / + // qualify_leads). Read-only snapshot of a backend-owned job — always + // exposed so a job started in a write-enabled session stays readable + // even if the deployment later runs read-only. + leadJobStatus, resolveImportRows, // seed-candidates is a read-only discovery surface for the extend flow. // Always exposed so the agent can show candidates even in read-only deployments. @@ -349,6 +357,14 @@ export const compositeReadTools: Tool[] = [ // Composite write tools — always-exposed in OpenClaw, gated in MCP behind // LEADBAY_MCP_WRITE=1 (the MCP server filters them out by default). export const compositeWriteTools: Tool[] = [ + // MCP-first lead delivery (backend /mcp/search + /mcp/qualify jobs). + // Write-tier: submits create server-side jobs that can spend money + // (qualification research, channel purchase) and claim novelty in the + // org's delivery ledger — same posture as the other spending composites. + // The FREE tier (qualify:false, channels:[]) is the default ask; the + // descriptions carry the no-silent-spend consent gate. + findNewLeads, + qualifyLeads, bulkQualifyLeads, enrichTitles, adjustAudience, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 65c8a313..f011c194 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -597,7 +597,7 @@ Context: Leadbay auto-qualifies roughly the top 10 of each daily batch. Leads be WHEN TO USE: when the user wants more qualified leads than what's currently shown, or when a lead looks promising in leadbay_pull_leads but has an empty \`qualification_summary\`. -WHEN NOT TO USE: to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). +WHEN NOT TO USE: to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. This tool MUTATES state. The caller (agent or human-in-the-loop) is responsible for confirming intent before invocation; the MCP server does not soft-prompt for confirmation. See \`annotations.destructiveHint\`. @@ -1187,7 +1187,7 @@ Trigger phrases: "I want more leads on this lens", "extend the lens", "I need a **Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. -Do NOT use for: "show me today's leads" → \`leadbay_pull_leads\`; "narrow the audience" → \`leadbay_adjust_audience\`; "stop showing me X" → \`leadbay_refine_prompt\`. +Do NOT use for: "show me today's leads" → \`leadbay_pull_leads\`; "find me companies that " → \`leadbay_find_new_leads\`; "narrow the audience" → \`leadbay_adjust_audience\`; "stop showing me X" → \`leadbay_refine_prompt\`. Prefer when: user has bigger appetite than the daily lens fill delivers — additive refill on same criteria @@ -1271,6 +1271,253 @@ If nothing matches cleanly, default to "pull leads now to see what's queued" — `; // endregion: leadbay_extend_lens +// region: leadbay_find_new_leads +export const leadbay_find_new_leads: string = `## WHEN TO USE + +Trigger phrases: "find me new leads", "find me N companies that ", "get me new prospects like ", "I need leads in that ", "search for companies that would buy ", "net-new leads outside my current pipeline", "we're entering — who should we target". + +**Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. + +Do NOT use for: "show me today's leads / what's new today" → \`leadbay_pull_leads\`; "more leads like the ones in my lens" → \`leadbay_extend_lens\`; "qualify / vet these companies I have" → \`leadbay_qualify_leads\`; "qualify the top N of my batch" → \`leadbay_bulk_qualify_leads\`; "leads I should follow up with" → \`leadbay_pull_followups\`; "tell me about " → \`leadbay_research_lead_by_name_fuzzy\`. + +Prefer when: the user describes a target profile or names a count of NEW companies — craft the example_lead per the seed rules below BEFORE calling; never pass the user's raw sentence as query. + +Examples that SHOULD invoke this tool: +- "Find me 10 gyms around Dallas that would buy our flooring, with someone I can call." +- "Get me 20 new US SaaS companies, 50-2000 employees, with the VP People's email." +- "We're launching in Lyon — find 15 hotels that fit our ICP." + +Examples that should NOT invoke this tool (sound similar, route elsewhere): +- "Show me today's leads." +- "Which leads should I follow up with this week?" +- "Qualify these 40 websites from my spreadsheet." + +## RENDER (quick) + +3-col table of delivered leads in returned order: col 1 = 10-segment fit +bar + linked company · location · size; col 2 = why-fits ≤20 words; col 3 += contact + purchased channels. ALWAYS close with the honest funnel line +(matched/examined/delivered/stop reason/spend) — especially on 0 +delivered. Full algorithm below. + +--- + +Submit a net-new lead search: the backend matches an ICP seed against the full +company universe (millions of registry+web profiles), applies hard filters, +skips everything the org already knows (\`novelty: org\`), optionally qualifies +each candidate against the org's own intelligence (qualification questions, +tags, ideal buyer profile — frozen at submit), and optionally purchases +contact channels. Submit answers in <1s (input errors are synchronous 400s +naming the field); the tool polls up to \`wait_seconds\` (default 45) and, when +the job needs longer, returns \`still_running\` + \`next_poll\` — hand off to +\`leadbay_lead_job_status\`. Jobs run ≤30 min; results are kept 30 days. + +**Free vs paid — never spend silently.** The default ask (\`qualify: false\`, +\`channels: []\`) is FREE: company profile + fit score + cached web research + +contact identity. Paid flags: \`qualify: true\` (~94 cost_cents per candidate +EXAMINED, survivor or not, capped by \`exploration_cap\`/\`max_cost\`) and +\`channels\` (email 25c / phone 250c, success-only). Before the FIRST paid run +of a session: \`dry_run: true\`, tell the user the worst-case estimate, get +their explicit go-ahead (an explicit "spend / get their emails" in the user's +message counts). The free tier needs no consent. + +**The two-step pattern that wins**: run FREE first and eyeball whether the +delivered companies are on-profile. If yes, feed that job's deliveries to +\`leadbay_qualify_leads\` via \`prior_deliveries\` (spends only on companies +already known to match), or re-run with \`qualify: true\` under a NEW +request_id when more than the preview is wanted. If off-profile, fix the seed +first — reshaping is free; exploring a bad seed with \`qualify: true\` burns +budget on candidates qualification will reject. + +### Crafting the \`example_lead\` seed — the input that decides result quality + +The \`example_lead\` is a FICTIONAL typical ideal customer. Its text is embedded +and matched against millions of real company descriptions sourced from business +registries and company websites. Those descriptions state what a company **IS** +(stable business profile) — never what is happening. Write the seed the same +way, or the matcher drifts to the wrong companies. Each rule below is +load-bearing (validated live against staging, 2026-07-28): + +1. **Describe the BUYER, never the seller.** Before writing, answer: "would + this company write a check to my user?" A seed that describes what the user + sells surfaces the user's *competitors and vendors*, not their customers. + Classic trap: if the user's product helps companies of type X serve their + customers of type Y, the seed describes X — never Y. +2. **Put everything in \`description\`; leave \`name\` unset.** A distinctive + invented brand name pulls matching toward name-lookalikes: a seed named + "Meridian Analytics" returned five unrelated companies all named + "Meridian". No name beats any name. +3. **Registry style, one sentence to ~250 chars.** State the business profile: + industry niche, business model, what they sell or operate, who they serve, + observable scale (sites, membership, fleet). Write it like the first + paragraph of the company's About-Us page. + - STRONG: "Operator of full-service fitness centers offering strength + training areas, group classes and personal training to individual members + across multiple club locations." + - WEAK (generic): "A gym in Texas." + - WRONG (seller-side): "Supplier of durable modular flooring for gyms." +4. **No event language.** "hiring", "expanding", "just raised", "opening a new + site" are not filters — real registry descriptions never contain them, so + they dilute the profile and attract event-flavored noise. Temporal criteria + in a \`query\` become best-effort ranking annotations at most (the response + \`explain.scope_notes\` says so). Put purchase-trigger criteria in the org's + qualification questions instead, where the paid qualification stage scores + them from fresh research. +5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — + real descriptions don't carry them. +6. **Hard constraints go in \`filters\`, not prose.** Geography, sector, size + bounds written into the description only *tint* the ranking; \`filters\` are + enforced. Seed describes the archetype; filters draw the fence. +7. **Prefer \`example_lead\` over \`query\`.** Query text matches topic + *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS + as strongly as gym BUYERS (measured: the same ICP delivered 0 leads from a + query and on-profile leads from an example_lead). Use \`query\` only when the + user's own wording carries signal an example can't express. +8. **One seed per buyer archetype.** If the ask spans two distinct segments + (e.g. "gyms and logistics warehouses"), run one search per segment with its + own description — a blended seed lands between the two clusters and matches + neither. Distinct asks need distinct \`request_id\`s. + + +**Parameter notes** +- \`request_id\` (REQUIRED) is the retry contract: reuse the SAME value when + retrying the same ask (returns the same live job, no double spend); NEW + value for a changed ask. Derive from the ask + date: \`gyms-dallas-2026-07-28\`. +- Unresolvable \`filters\` values 400 naming them — fix the label (see + \`leadbay_list_sectors\` / \`leadbay_list_locations\`) and resubmit. +- \`min_ai_score\` gates the [-30,+30] qualification DELTA, not the 0-100 fit + score; set -30 to keep every evaluated lead with its evidence. +- \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After + beyond — wait, don't hammer). + +**Read the result honestly.** \`funnel\` + \`explain.scope_notes\` tell what +happened: matched, pre-screen rejections, examined (billed), qualified, +disqualified, and why the job stopped. Zero delivered is a real outcome that +must be narrated with its cause and a concrete next move — never a bare "no +results". + +--- + +## RENDERING — delivery table + honest funnel line + +Render delivered leads (\`leads[]\`, i.e. items with status \`delivered\` or +\`degraded\`) as a markdown table **in the order returned**. Exactly three +columns. Then ALWAYS close with the funnel line (below) — even, especially, +when nothing was delivered. + +**Column 1 — Company** + +- Line 1: 10-segment fit bar in inline-code backticks from \`lead.fit.score\` + (0-100): \`filled = round(score/10)\`, glyphs \`▰\` filled / \`▱\` empty. When + \`lead.fit.components.qualification.available\` is true AND \`ai_score > 0\`, + replace the LAST filled segment with \`❖\` (AI-confirmed cap). When + \`fit.available\` is false, render \`▱▱▱▱▱▱▱▱▱▱\` and say "unscored" in col 2. + Never print the numeric score. +- Insert \`
\`, then: linked company name (target \`company.website\`, bare + hostnames get \`https://\`; unlinked plain text when absent) + \` · \` + short + location (City, ST / City, Country) + \` · \` + employees as \`min–max\` (omit + when \`employees.known\` is false). + +**Column 2 — Why it fits** + +- One sentence ≤ 20 words. Priority: \`fit.reasoning\` → gist of + \`company.description\` → top \`fit.components.qualification.matched_tags\`. +- If the item status is \`degraded\` or a requested channel failed, append the + honest flag in italics, e.g. *(email could not be sourced)*. + +**Column 3 — Contact** + +- \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL + exists; plain name otherwise). Below it, the PURCHASED channels only: + \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). +- Channel status semantics from \`contact.channels.{email,phone}.status\`: + \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; + \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; + \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` + delivers such rows flagged; say so in col 2 only when contact_titles were + requested). + +**The funnel line (mandatory, after the table):** + +One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +\`explain.scope_notes\`: + +> Matched N · examined E · qualified Q · disqualified D → **delivered X of +> the Y asked** · stopped: · spent €C.CC. + +Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → +"ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → +"hit an org quota", \`time_budget\` → "hit the 30-min time budget". + +**When \`delivered\` is 0**: NEVER say just "no results". Render no table; give +the funnel line plus the relevant \`explain.scope_notes\` (they carry the +backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen +rejections), then propose the concrete fix (reshape the seed per the +example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a +filter) as NEXT STEPS options. + +**Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second +table \`Ref → Outcome\` translating \`status_reason\` to plain words: +\`not_in_universe\` → "not in the Leadbay universe (import it first)", +\`low_confidence_identity\` → "couldn't safely match — check \`resolution.alternatives\`", +\`no_matching_contact\` → "no contact with the requested title", +\`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), +\`enrichment_failed\` → "channel could not be sourced (not billed)". + +**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool +calls, never render), \`next_since\` cursors, \`explain.model\`, +\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item +\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. + +## Linking a contact's name + +**MANDATORY: every contact name in your output — table cells, prose, headers, "Reach " callouts — MUST be wrapped in markdown link syntax \`[Name](URL)\`. Never render a contact name as bare text. A plain-text name is a broken contact card; the underlined name is the user's primary affordance for "take me to this person's profile". No "no URL available" exception — the search URL below is always constructable from name + company.** + +URL priority (first applicable wins): + +1. **Real profile** — \`contact.linkedin_page\` when it's a string starting with \`https://\` (the MCP coerces the legacy literal \`"null"\` string to real null before you see it). +2. **Constructed people-search** — \`https://www.linkedin.com/search/results/people/?keywords=++\`. URL-encode params. Strip Inc / LLC / Corp / Ltd / GmbH / Co / S.A. / S.L. / PLC / AG / SAS / SARL suffixes from the company. Append a trailing \` °\` to the rendered name ONLY when this fallback is in use AND \`social_presence.linkedin == false\`. Never append \`°\` when a real \`linkedin_page\` was used. + +Never link a person's name to the company's LinkedIn page (and vice versa) — the two surfaces are different and conflating them quietly degrades the workflow. + + + +--- + +## NEXT STEPS — after a find_new_leads delivery + +**ALWAYS render NEXT STEPS via your host's next-step widget.** Use whichever is in your tool set — the NAME and SCHEMA differ: **\`ask_user_input_v0\`** (Claude chat / ChatGPT) takes plain-string options with \`type:"single_select"\`; **\`AskUserQuestion\`** (Claude cowork / Claude Code) takes object options \`{label, description}\` plus a required short \`header\` (≤12 chars) and \`multiSelect\`, NO \`type\` field, and never add an "Other" option (the host adds it). Match the schema to the tool you actually have — the wrong schema fails silently and you fall back to prose. Prose bullets are the fallback ONLY when NEITHER widget exists. Any turn that would end with a choice must be the widget — the widget IS the question. + +**If the tool result carries a \`next_steps\` object, that is the source of truth — use it directly.** Each option has a short \`.label\` (≤5 words) and a full \`.description\`. Map \`next_steps.options[]\` into your host widget VERBATIM and in order: for \`AskUserQuestion\` (cowork / Claude Code) pass each as \`{label, description}\`; for \`ask_user_input_v0\` (Claude chat / ChatGPT, string options only) pass each option's \`.description\` as the string (it's the full sentence). Do NOT reword, reorder, drop, or prose-ify them — they're built deterministically by the server so the offer (incl. the artifact option at position 0) fires every time. Fall back to the table below only when there is NO \`next_steps\` field. + +**One exception — skip the widget** when the user's original message contained a complete sequential instruction chain ("show me X and then do Y") AND all stated steps have been completed. In that case, end with STOP directly — the user stated their full plan and does not need a "what next?" prompt. +- Skip example: "Show me today's leads and then research the top one for me." → after research completes, emit STOP without the widget. +- Do NOT skip for: plain requests ("show me today's leads", "run my check-in"), recurring-language requests ("I do this every day"), or requests where only one action was stated. + +Pick 2–4 rows from the (Observation, Suggest, Calls) table below most relevant to the response, then call your host's widget with ITS schema (per the schema rules above — wrong schema fails silently): +- \`ask_user_input_v0\`: \`{questions:[{question,type:"single_select",options:["",""]}]}\` +- \`AskUserQuestion\`: \`{questions:[{question,header:"Next step",multiSelect:false,options:[{label:"<≤5 words>",description:""}]}]}\` + +User picks → call the matching \`Calls\` tool. Constraints: 2–4 mutually-exclusive options, AskUserQuestion labels ≤5 words (full text in \`description\`), max 3 questions. Table stays internal; never recite it. + +--- + + + +Pick the 2-3 options that match what actually happened — never all six: + +| Observation | Suggest | Calls | +|---|---|---| +| Job still running (\`still_running: true\`) | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote \`dry_run\` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | +| Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | +| Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | +| Stopped at cost cap / quota | "Raise the cap to €X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | +`; +// endregion: leadbay_find_new_leads + // region: leadbay_followups_map export const leadbay_followups_map: string = `## WHEN TO USE @@ -1975,6 +2222,166 @@ This tool MUTATES state. The caller (agent or human-in-the-loop) is responsible `; // endregion: leadbay_launch_bulk_enrichment +// region: leadbay_lead_job_status +export const leadbay_lead_job_status: string = `## WHEN TO USE + +Trigger phrases: "is the lead search done", "any results yet on that job", "check on the delivery". + +**Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. + +Do NOT use for: "is the enrichment done" → \`leadbay_bulk_enrich_status\`; "is my import done" → \`leadbay_import_status\`; "is the top-N qualification done" → \`leadbay_qualify_status\`. + +Prefer when: a find_new_leads / qualify_leads result carried next_poll — pass its job_id; use wait_seconds ~60 when the user asked to wait for results. + +Examples that SHOULD invoke this tool: +- "Any leads yet from that search you started?" +- "Wait for the qualification job to finish and show me everything." + +Examples that should NOT invoke this tool (sound similar, route elsewhere): +- "Is the email enrichment finished?" +- "Is my CSV import done?" + +## RENDER (quick) + +Terminal job -> render the full delivery per the lead-delivery table + +honest funnel line. Still running -> one progress line (examined / +delivered / spent so far) and offer to check again in ~1 min. Never +render UUIDs or cursors. + +--- + +Cumulative snapshot of a lead-delivery job: state, funnel counters, every +item emitted so far (full lead payloads for delivered/degraded, honest +status_reason for skipped), spend + breakdown, and the \`explain\` block +(basis, seed strategy, scope notes). Items are immutable once emitted — +polling never re-reads live data, so numbers only ever grow. + +\`wait_seconds: 0\` (default) answers instantly; set ~60 to block-wait for +completion when the user asked for results "in this reply". \`since\` (from a +prior poll's \`next_since\`) pages only the new items. Jobs terminalize +server-side: past the 30-min wall clock a job reads \`completed_partial\` +(time budget), past 30 days \`expired\` (items no longer listed — re-read +billed leads via leadbay_qualify_leads \`prior_deliveries\`). A 404 means +unknown job or another org's job. + +--- + +## RENDERING — delivery table + honest funnel line + +Render delivered leads (\`leads[]\`, i.e. items with status \`delivered\` or +\`degraded\`) as a markdown table **in the order returned**. Exactly three +columns. Then ALWAYS close with the funnel line (below) — even, especially, +when nothing was delivered. + +**Column 1 — Company** + +- Line 1: 10-segment fit bar in inline-code backticks from \`lead.fit.score\` + (0-100): \`filled = round(score/10)\`, glyphs \`▰\` filled / \`▱\` empty. When + \`lead.fit.components.qualification.available\` is true AND \`ai_score > 0\`, + replace the LAST filled segment with \`❖\` (AI-confirmed cap). When + \`fit.available\` is false, render \`▱▱▱▱▱▱▱▱▱▱\` and say "unscored" in col 2. + Never print the numeric score. +- Insert \`
\`, then: linked company name (target \`company.website\`, bare + hostnames get \`https://\`; unlinked plain text when absent) + \` · \` + short + location (City, ST / City, Country) + \` · \` + employees as \`min–max\` (omit + when \`employees.known\` is false). + +**Column 2 — Why it fits** + +- One sentence ≤ 20 words. Priority: \`fit.reasoning\` → gist of + \`company.description\` → top \`fit.components.qualification.matched_tags\`. +- If the item status is \`degraded\` or a requested channel failed, append the + honest flag in italics, e.g. *(email could not be sourced)*. + +**Column 3 — Contact** + +- \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL + exists; plain name otherwise). Below it, the PURCHASED channels only: + \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). +- Channel status semantics from \`contact.channels.{email,phone}.status\`: + \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; + \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; + \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` + delivers such rows flagged; say so in col 2 only when contact_titles were + requested). + +**The funnel line (mandatory, after the table):** + +One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +\`explain.scope_notes\`: + +> Matched N · examined E · qualified Q · disqualified D → **delivered X of +> the Y asked** · stopped: · spent €C.CC. + +Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → +"ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → +"hit an org quota", \`time_budget\` → "hit the 30-min time budget". + +**When \`delivered\` is 0**: NEVER say just "no results". Render no table; give +the funnel line plus the relevant \`explain.scope_notes\` (they carry the +backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen +rejections), then propose the concrete fix (reshape the seed per the +example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a +filter) as NEXT STEPS options. + +**Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second +table \`Ref → Outcome\` translating \`status_reason\` to plain words: +\`not_in_universe\` → "not in the Leadbay universe (import it first)", +\`low_confidence_identity\` → "couldn't safely match — check \`resolution.alternatives\`", +\`no_matching_contact\` → "no contact with the requested title", +\`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), +\`enrichment_failed\` → "channel could not be sourced (not billed)". + +**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool +calls, never render), \`next_since\` cursors, \`explain.model\`, +\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item +\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. + +## Linking a contact's name + +**MANDATORY: every contact name in your output — table cells, prose, headers, "Reach " callouts — MUST be wrapped in markdown link syntax \`[Name](URL)\`. Never render a contact name as bare text. A plain-text name is a broken contact card; the underlined name is the user's primary affordance for "take me to this person's profile". No "no URL available" exception — the search URL below is always constructable from name + company.** + +URL priority (first applicable wins): + +1. **Real profile** — \`contact.linkedin_page\` when it's a string starting with \`https://\` (the MCP coerces the legacy literal \`"null"\` string to real null before you see it). +2. **Constructed people-search** — \`https://www.linkedin.com/search/results/people/?keywords=++\`. URL-encode params. Strip Inc / LLC / Corp / Ltd / GmbH / Co / S.A. / S.L. / PLC / AG / SAS / SARL suffixes from the company. Append a trailing \` °\` to the rendered name ONLY when this fallback is in use AND \`social_presence.linkedin == false\`. Never append \`°\` when a real \`linkedin_page\` was used. + +Never link a person's name to the company's LinkedIn page (and vice versa) — the two surfaces are different and conflating them quietly degrades the workflow. + + + +--- + +## NEXT STEPS — after a job status poll + +**ALWAYS render NEXT STEPS via your host's next-step widget.** Use whichever is in your tool set — the NAME and SCHEMA differ: **\`ask_user_input_v0\`** (Claude chat / ChatGPT) takes plain-string options with \`type:"single_select"\`; **\`AskUserQuestion\`** (Claude cowork / Claude Code) takes object options \`{label, description}\` plus a required short \`header\` (≤12 chars) and \`multiSelect\`, NO \`type\` field, and never add an "Other" option (the host adds it). Match the schema to the tool you actually have — the wrong schema fails silently and you fall back to prose. Prose bullets are the fallback ONLY when NEITHER widget exists. Any turn that would end with a choice must be the widget — the widget IS the question. + +**If the tool result carries a \`next_steps\` object, that is the source of truth — use it directly.** Each option has a short \`.label\` (≤5 words) and a full \`.description\`. Map \`next_steps.options[]\` into your host widget VERBATIM and in order: for \`AskUserQuestion\` (cowork / Claude Code) pass each as \`{label, description}\`; for \`ask_user_input_v0\` (Claude chat / ChatGPT, string options only) pass each option's \`.description\` as the string (it's the full sentence). Do NOT reword, reorder, drop, or prose-ify them — they're built deterministically by the server so the offer (incl. the artifact option at position 0) fires every time. Fall back to the table below only when there is NO \`next_steps\` field. + +**One exception — skip the widget** when the user's original message contained a complete sequential instruction chain ("show me X and then do Y") AND all stated steps have been completed. In that case, end with STOP directly — the user stated their full plan and does not need a "what next?" prompt. +- Skip example: "Show me today's leads and then research the top one for me." → after research completes, emit STOP without the widget. +- Do NOT skip for: plain requests ("show me today's leads", "run my check-in"), recurring-language requests ("I do this every day"), or requests where only one action was stated. + +Pick 2–4 rows from the (Observation, Suggest, Calls) table below most relevant to the response, then call your host's widget with ITS schema (per the schema rules above — wrong schema fails silently): +- \`ask_user_input_v0\`: \`{questions:[{question,type:"single_select",options:["",""]}]}\` +- \`AskUserQuestion\`: \`{questions:[{question,header:"Next step",multiSelect:false,options:[{label:"<≤5 words>",description:""}]}]}\` + +User picks → call the matching \`Calls\` tool. Constraints: 2–4 mutually-exclusive options, AskUserQuestion labels ≤5 words (full text in \`description\`), max 3 questions. Table stays internal; never recite it. + +--- + + + +Exactly two offers — this is a status tool, keep it terse: + +| Observation | Suggest | Calls | +|---|---|---| +| Still running | "Keep waiting (~1 min) or leave it — results are kept 30 days" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Terminal (completed / partial / failed) | Render the delivery per the RENDERING block, then offer the matching find_new_leads / qualify_leads NEXT STEPS | — | +`; +// endregion: leadbay_lead_job_status + // region: leadbay_like_lead export const leadbay_like_lead: string = `## WHEN TO USE @@ -2862,7 +3269,7 @@ Trigger phrases: "show me leads", "show me new leads", "show me today's leads", **Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. -Do NOT use for: "leads I should follow up with" → \`leadbay_pull_followups\`; "I'm going to " → \`leadbay_tour_plan\`; "I'm in next week — who's worth meeting" → \`leadbay_tour_plan\`; "who should I meet in " → \`leadbay_tour_plan\`; "visiting — who's worth meeting / seeing" → \`leadbay_tour_plan\`; "leads I should reach out to" → \`leadbay_pull_followups\`; "leads to get back to" → \`leadbay_pull_followups\`; "leads to contact today" → \`leadbay_pull_followups\`; "should I contact" → \`leadbay_pull_followups\`; "reconnect with" → \`leadbay_pull_followups\`; "re-engage" → \`leadbay_pull_followups\`. +Do NOT use for: "find me N companies that " → \`leadbay_find_new_leads\`; "new prospects like with their emails" → \`leadbay_find_new_leads\`; "leads I should follow up with" → \`leadbay_pull_followups\`; "I'm going to " → \`leadbay_tour_plan\`; "I'm in next week — who's worth meeting" → \`leadbay_tour_plan\`; "who should I meet in " → \`leadbay_tour_plan\`; "visiting — who's worth meeting / seeing" → \`leadbay_tour_plan\`; "leads I should reach out to" → \`leadbay_pull_followups\`; "leads to get back to" → \`leadbay_pull_followups\`; "leads to contact today" → \`leadbay_pull_followups\`; "should I contact" → \`leadbay_pull_followups\`; "reconnect with" → \`leadbay_pull_followups\`; "re-engage" → \`leadbay_pull_followups\`. Prefer when: fresh Discover leads; if a lens is named, pass \`lensId\` and pin it @@ -2871,6 +3278,7 @@ Examples that SHOULD invoke this tool: - "Pull my best new prospects." Examples that should NOT invoke this tool (sound similar, route elsewhere): +- "Find me 10 gyms around Dallas that would buy our flooring." - "Which leads should I follow up with this week?" - "I'm flying to Berlin Thursday — who should I meet?" - "I'm in San Francisco next Tuesday — who's worth meeting?" @@ -3033,6 +3441,193 @@ This tool MUTATES state. The caller (agent or human-in-the-loop) is responsible `; // endregion: leadbay_qualify_lead +// region: leadbay_qualify_leads +export const leadbay_qualify_leads: string = `## WHEN TO USE + +Trigger phrases: "qualify these companies", "vet this list", "which of these fit our ICP", "score these websites / accounts", "get me the right contact at these companies", "re-qualify what you delivered last week". + +**Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. + +Do NOT use for: "find me new leads / companies that " → \`leadbay_find_new_leads\`; "qualify the top N of my lens batch" → \`leadbay_bulk_qualify_leads\`; "import this CSV file" → \`leadbay_import_leads\`; "tell me about in depth" → \`leadbay_research_lead_by_name_fuzzy\`; "add emails to the contacts I selected" → \`leadbay_enrich_titles\`. + +Prefer when: the user points at SPECIFIC companies (ids, websites, names, a pasted list, "what you found yesterday") and wants fit verdicts and/or the right person to talk to. + +Examples that SHOULD invoke this tool: +- "Here are 60 restaurant websites from my Austin sweep — which fit, and who's the owner?" +- "Re-qualify last week's delivery and get phone numbers for the good ones." +- "Vet these 12 accounts from my spreadsheet against our criteria." + +Examples that should NOT invoke this tool (sound similar, route elsewhere): +- "Find me 10 new gyms in Texas." +- "Qualify the top 10 leads in my batch." +- "I have a CSV of 400 attendees to import." + +## RENDER (quick) + +3-col table for delivered items (fit bar + company / why-fits ≤20 words / +contact + channels) in returned order, then a compact Ref → Outcome table +for skipped refs (not_in_universe, low_confidence_identity, ... in plain +words), then the honest funnel + cost line. Full algorithm below. + +--- + +Submit a qualify batch over companies the org already has (or that exist in +the Leadbay universe): each ref is resolved to a known company, freshly +researched + AI-qualified against the org's questions / tags / ideal buyer +profile (frozen at submit), matched to the requested contact titles, and — +when asked — enriched with verified channels. Answers arrive per-item from a +job; this tool polls up to \`wait_seconds\` (default 45) and hands off to +\`leadbay_lead_job_status\` when the batch needs longer. + +**Refs are flexible; outcomes are per-item.** \`lead_refs\` accepts any mix of +\`lead_id\`, \`website\`, \`name\`(+\`location\`), or a stable \`contact_id\` from a +prior result (enrichment then targets exactly that person, never a re-match). +\`prior_deliveries\` expands past MCP deliveries into refs — billed leads stay +re-readable this way even after the 30-day result window. Duplicates collapse. +A ref that can't be served comes back \`skipped\` with an honest +\`status_reason\` (\`not_in_universe\`, \`low_confidence_identity\` with the +\`resolution.alternatives\` to choose from, \`no_matching_contact\`, ...) — that +is an ANSWER about the ref, not an error, and it costs nothing. + +**Disqualified ≠ dropped.** Companies the org owns that fail qualification +are DELIVERED with their negative evidence (question verdicts, tag misses, +IBP reasoning) — "here's why to skip this account" is a deliverable. + +**Cost — never spend silently.** Resolution and identity are free. +\`qualify: true\` (the default) costs ~94 cost_cents per lead needing FRESH +research+scoring — but repeat calls reuse every fresh cached stage +(\`from_cache\` flags on the items) and converge to near-zero cost. \`channels\` +purchase verified email (25c) / phone (250c) on success only; +\`already_owned\` values cost nothing. Before a first paid run on a large +batch: \`dry_run: true\`, quote the worst case, get the explicit go-ahead +(an explicit "spend / get their emails" in the user's message counts). +Set \`request_id\` and reuse it on retries of the same batch. + +**Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + +Retry-After beyond — wait, don't hammer), 30-min job wall clock. + +--- + +## RENDERING — delivery table + honest funnel line + +Render delivered leads (\`leads[]\`, i.e. items with status \`delivered\` or +\`degraded\`) as a markdown table **in the order returned**. Exactly three +columns. Then ALWAYS close with the funnel line (below) — even, especially, +when nothing was delivered. + +**Column 1 — Company** + +- Line 1: 10-segment fit bar in inline-code backticks from \`lead.fit.score\` + (0-100): \`filled = round(score/10)\`, glyphs \`▰\` filled / \`▱\` empty. When + \`lead.fit.components.qualification.available\` is true AND \`ai_score > 0\`, + replace the LAST filled segment with \`❖\` (AI-confirmed cap). When + \`fit.available\` is false, render \`▱▱▱▱▱▱▱▱▱▱\` and say "unscored" in col 2. + Never print the numeric score. +- Insert \`
\`, then: linked company name (target \`company.website\`, bare + hostnames get \`https://\`; unlinked plain text when absent) + \` · \` + short + location (City, ST / City, Country) + \` · \` + employees as \`min–max\` (omit + when \`employees.known\` is false). + +**Column 2 — Why it fits** + +- One sentence ≤ 20 words. Priority: \`fit.reasoning\` → gist of + \`company.description\` → top \`fit.components.qualification.matched_tags\`. +- If the item status is \`degraded\` or a requested channel failed, append the + honest flag in italics, e.g. *(email could not be sourced)*. + +**Column 3 — Contact** + +- \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL + exists; plain name otherwise). Below it, the PURCHASED channels only: + \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). +- Channel status semantics from \`contact.channels.{email,phone}.status\`: + \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; + \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; + \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` + delivers such rows flagged; say so in col 2 only when contact_titles were + requested). + +**The funnel line (mandatory, after the table):** + +One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +\`explain.scope_notes\`: + +> Matched N · examined E · qualified Q · disqualified D → **delivered X of +> the Y asked** · stopped: · spent €C.CC. + +Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → +"ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → +"hit an org quota", \`time_budget\` → "hit the 30-min time budget". + +**When \`delivered\` is 0**: NEVER say just "no results". Render no table; give +the funnel line plus the relevant \`explain.scope_notes\` (they carry the +backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen +rejections), then propose the concrete fix (reshape the seed per the +example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a +filter) as NEXT STEPS options. + +**Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second +table \`Ref → Outcome\` translating \`status_reason\` to plain words: +\`not_in_universe\` → "not in the Leadbay universe (import it first)", +\`low_confidence_identity\` → "couldn't safely match — check \`resolution.alternatives\`", +\`no_matching_contact\` → "no contact with the requested title", +\`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), +\`enrichment_failed\` → "channel could not be sourced (not billed)". + +**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool +calls, never render), \`next_since\` cursors, \`explain.model\`, +\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item +\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. + +## Linking a contact's name + +**MANDATORY: every contact name in your output — table cells, prose, headers, "Reach " callouts — MUST be wrapped in markdown link syntax \`[Name](URL)\`. Never render a contact name as bare text. A plain-text name is a broken contact card; the underlined name is the user's primary affordance for "take me to this person's profile". No "no URL available" exception — the search URL below is always constructable from name + company.** + +URL priority (first applicable wins): + +1. **Real profile** — \`contact.linkedin_page\` when it's a string starting with \`https://\` (the MCP coerces the legacy literal \`"null"\` string to real null before you see it). +2. **Constructed people-search** — \`https://www.linkedin.com/search/results/people/?keywords=++\`. URL-encode params. Strip Inc / LLC / Corp / Ltd / GmbH / Co / S.A. / S.L. / PLC / AG / SAS / SARL suffixes from the company. Append a trailing \` °\` to the rendered name ONLY when this fallback is in use AND \`social_presence.linkedin == false\`. Never append \`°\` when a real \`linkedin_page\` was used. + +Never link a person's name to the company's LinkedIn page (and vice versa) — the two surfaces are different and conflating them quietly degrades the workflow. + + + +--- + +## NEXT STEPS — after a qualify_leads delivery + +**ALWAYS render NEXT STEPS via your host's next-step widget.** Use whichever is in your tool set — the NAME and SCHEMA differ: **\`ask_user_input_v0\`** (Claude chat / ChatGPT) takes plain-string options with \`type:"single_select"\`; **\`AskUserQuestion\`** (Claude cowork / Claude Code) takes object options \`{label, description}\` plus a required short \`header\` (≤12 chars) and \`multiSelect\`, NO \`type\` field, and never add an "Other" option (the host adds it). Match the schema to the tool you actually have — the wrong schema fails silently and you fall back to prose. Prose bullets are the fallback ONLY when NEITHER widget exists. Any turn that would end with a choice must be the widget — the widget IS the question. + +**If the tool result carries a \`next_steps\` object, that is the source of truth — use it directly.** Each option has a short \`.label\` (≤5 words) and a full \`.description\`. Map \`next_steps.options[]\` into your host widget VERBATIM and in order: for \`AskUserQuestion\` (cowork / Claude Code) pass each as \`{label, description}\`; for \`ask_user_input_v0\` (Claude chat / ChatGPT, string options only) pass each option's \`.description\` as the string (it's the full sentence). Do NOT reword, reorder, drop, or prose-ify them — they're built deterministically by the server so the offer (incl. the artifact option at position 0) fires every time. Fall back to the table below only when there is NO \`next_steps\` field. + +**One exception — skip the widget** when the user's original message contained a complete sequential instruction chain ("show me X and then do Y") AND all stated steps have been completed. In that case, end with STOP directly — the user stated their full plan and does not need a "what next?" prompt. +- Skip example: "Show me today's leads and then research the top one for me." → after research completes, emit STOP without the widget. +- Do NOT skip for: plain requests ("show me today's leads", "run my check-in"), recurring-language requests ("I do this every day"), or requests where only one action was stated. + +Pick 2–4 rows from the (Observation, Suggest, Calls) table below most relevant to the response, then call your host's widget with ITS schema (per the schema rules above — wrong schema fails silently): +- \`ask_user_input_v0\`: \`{questions:[{question,type:"single_select",options:["",""]}]}\` +- \`AskUserQuestion\`: \`{questions:[{question,header:"Next step",multiSelect:false,options:[{label:"<≤5 words>",description:""}]}]}\` + +User picks → call the matching \`Calls\` tool. Constraints: 2–4 mutually-exclusive options, AskUserQuestion labels ≤5 words (full text in \`description\`), max 3 questions. Table stays internal; never recite it. + +--- + + + +Pick the 2-3 options that match what actually happened: + +| Observation | Suggest | Calls | +|---|---|---| +| Job still running | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Fit leads with contacts delivered | "Draft outreach for the qualified ones" | leadbay_prepare_outreach | +| Items skipped \`not_in_universe\` | "Import those companies first, then re-qualify" | leadbay_import_leads → leadbay_qualify_leads | +| Items skipped \`low_confidence_identity\` | "Pick the right match" (show \`resolution.alternatives\`) | leadbay_qualify_leads with the chosen lead_id | +| Contacts delivered without channels | "Purchase verified emails/phones for the keepers (state cost first)" | leadbay_qualify_leads(lead_refs with contact_id, channels) | +| Disqualified with evidence | "Review why — adjust qualification questions if the criteria are off" | leadbay_get_qualification_questions | +`; +// endregion: leadbay_qualify_leads + // region: leadbay_qualify_status export const leadbay_qualify_status: string = `Retrieve the current state of an import_and_qualify (or bulk_qualify_leads) launch by \`qualify_id\`. Returns the same \`qualified[]\` / \`still_running[]\` shape as the original composite, refreshed against the backend at call time. The handle is persisted to \`~/.leadbay/bulks.json\` with a 30-day TTL and survives MCP restart. @@ -4475,6 +5070,7 @@ export const TOOL_DESCRIPTIONS = { leadbay_enrich_contacts, leadbay_enrich_titles, leadbay_extend_lens, + leadbay_find_new_leads, leadbay_followups_map, leadbay_get_clarification, leadbay_get_contacts, @@ -4497,6 +5093,7 @@ export const TOOL_DESCRIPTIONS = { leadbay_import_leads, leadbay_import_status, leadbay_launch_bulk_enrichment, + leadbay_lead_job_status, leadbay_like_lead, leadbay_list_campaigns, leadbay_list_lenses, @@ -4515,6 +5112,7 @@ export const TOOL_DESCRIPTIONS = { leadbay_pull_followups, leadbay_pull_leads, leadbay_qualify_lead, + leadbay_qualify_leads, leadbay_qualify_status, leadbay_recall_ordered_titles, leadbay_refine_prompt, diff --git a/packages/core/test/unit/composite/find-new-leads.test.ts b/packages/core/test/unit/composite/find-new-leads.test.ts new file mode 100644 index 00000000..3705ed76 --- /dev/null +++ b/packages/core/test/unit/composite/find-new-leads.test.ts @@ -0,0 +1,230 @@ +/** + * Unit tests for leadbay_find_new_leads (POST /mcp/search + job poll). + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; +import { MCP_JOB_POLL } from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const JOB_ID = "281d8b55-b357-43ed-aca9-63e50bce84a6"; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 921, unit: "cost_cents" }, + items_requested: 3, + duplicate: false, +}; + +const DELIVERED_ITEM = { + ref: { lead_id: "77b1790c-bd49-4611-bc1e-ce90343a2f32" }, + status: "delivered", + seq: 0, + completed_at: "2026-07-28T10:15:00Z", + cost: { billed: 0, unit: "cost_cents" }, + lead: { + lead_id: "77b1790c-bd49-4611-bc1e-ce90343a2f32", + company: { name: "Black Crow AI", employees: { min: 11, max: 50, known: true } }, + fit: { available: true, score: 59 }, + web_research: { available: false, unavailable_reason: "never_fetched" }, + }, +}; + +const SKIPPED_ITEM = { + ref: { lead_id: "9223bbf6-5270-4c9c-a6f9-6bfa38a8b388" }, + status: "skipped", + status_reason: "disqualified", + seq: 1, + cost: { billed: 0, unit: "cost_cents" }, +}; + +function snapshot(state: string, items: unknown[], stopReason: string | null = null) { + return { + job: { + id: JOB_ID, + state, + submitted_at: "2026-07-28T10:14:02Z", + expires_at: "2026-08-27T10:14:02Z", + last_progress_at: "2026-07-28T10:14:46Z", + }, + funnel: { + matched: 25, + novel: 25, + examined: 2, + qualified: 1, + disqualified: 1, + delivered: items.filter((i: any) => i.status === "delivered").length, + delivered_callable: 0, + delivered_title_only: 0, + degraded: 0, + stop_reason: stopReason, + }, + items, + next_since: items.length > 0 ? "1785233671844264:1" : null, + cost: { spent: 165, unit: "cost_cents", breakdown: { web_fetch_cents: 161, rescore_cents: 4 } }, + explain: { + region: "US", + model: "text_v2_ai_description", + basis: "query_centroid", + seed_strategy: "example_lead", + scope_notes: ["note-1"], + }, + }; +} + +beforeEach(() => resetHttpMock()); + +describe("leadbay_find_new_leads", () => { + it("happy path — submits, polls once, splits delivered/skipped", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: snapshot("completed", [DELIVERED_ITEM, SKIPPED_ITEM], "target_reached"), + }, + ]); + const result = await findNewLeads.execute(newClient(), { + example_lead: { description: "Operator of full-service fitness centers." }, + filters: { locations: ["Texas"] }, + count: 3, + request_id: "probe-1", + wait_seconds: 0, + }); + + expect(result.job_id).toBe(JOB_ID); + expect(result.done).toBe(true); + expect(result.still_running).toBe(false); + expect(result.next_poll).toBeNull(); + expect(result.leads).toHaveLength(1); + expect(result.leads[0].lead.company.name).toBe("Black Crow AI"); + expect(result.skipped).toHaveLength(1); + expect(result.skipped[0].status_reason).toBe("disqualified"); + expect(result.summary.delivered).toBe(1); + expect(result.summary.stop_reason).toBe("target_reached"); + expect(result.explain.scope_notes).toEqual(["note-1"]); + + const submit = getHttpRequests().find((r) => r.method === "POST")!; + const body = JSON.parse(submit.body!); + expect(body.request_id).toBe("probe-1"); + expect(body.example_lead.description).toMatch(/fitness centers/); + // Local-only params never reach the wire. + expect(body.wait_seconds).toBeUndefined(); + }); + + it("dry_run — forecasts without creating or polling a job", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { + valid: true, + items_requested: 3, + estimated_cost: { max: 921, unit: "cost_cents" }, + quota_forecast: { web_fetch_allowed: true, rescore_allowed: true, enrichment_allowed: true }, + }, + }, + ]); + const result = await findNewLeads.execute(newClient(), { + query: "gyms in Texas", + count: 3, + request_id: "probe-dry", + dry_run: true, + }); + expect(result.dry_run).toBe(true); + expect(result.estimated_cost.max).toBe(921); + expect(getHttpRequests()).toHaveLength(1); + }); + + it("still-running job — returns partial results with an explicit next_poll", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: snapshot("running", [DELIVERED_ITEM]), + }, + ]); + const result = await findNewLeads.execute(newClient(), { + example_lead: { description: "SaaS analytics for retailers." }, + count: 3, + request_id: "probe-2", + wait_seconds: 0, + }); + expect(result.done).toBe(false); + expect(result.still_running).toBe(true); + expect(result.next_poll).toEqual({ + tool: "leadbay_lead_job_status", + job_id: JOB_ID, + suggested_wait_seconds: 60, + }); + expect(result.leads).toHaveLength(1); + }); + + it("wait loop — keeps polling until the job goes terminal", async () => { + const restoreInterval = MCP_JOB_POLL.intervalMs; + MCP_JOB_POLL.intervalMs = 1; + try { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: snapshot("running", []), + }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: snapshot("completed", [DELIVERED_ITEM], "target_reached"), + }, + ]); + const result = await findNewLeads.execute(newClient(), { + example_lead: { description: "SaaS analytics for retailers." }, + count: 3, + request_id: "probe-3", + wait_seconds: 5, + }); + expect(result.done).toBe(true); + expect(result.leads).toHaveLength(1); + expect(getHttpRequests()).toHaveLength(3); + } finally { + MCP_JOB_POLL.intervalMs = restoreInterval; + } + }); + + it("429 refusal (rate cap) — propagates as a quota error", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 429, + body: { error: "rate_limited", message: "submit rate cap reached" }, + }, + ]); + await expect( + findNewLeads.execute(newClient(), { + query: "gyms", + count: 3, + request_id: "probe-4", + }) + ).rejects.toMatchObject({ code: "QUOTA_EXCEEDED" }); + }); +}); diff --git a/packages/core/test/unit/composite/lead-job-status.test.ts b/packages/core/test/unit/composite/lead-job-status.test.ts new file mode 100644 index 00000000..45cb132c --- /dev/null +++ b/packages/core/test/unit/composite/lead-job-status.test.ts @@ -0,0 +1,124 @@ +/** + * Unit tests for leadbay_lead_job_status (GET /mcp/jobs/{id} snapshot). + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { leadJobStatus } from "../../../src/composite/lead-job-status.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const JOB_ID = "d89b9803-f9d8-4298-86af-9cd4b1841afd"; + +function item(seq: number, status = "delivered") { + return { + ref: { lead_id: `00000000-0000-0000-0000-00000000000${seq}` }, + status, + seq, + completed_at: "2026-07-28T10:30:00Z", + cost: { billed: 0, unit: "cost_cents" }, + lead: + status === "skipped" + ? undefined + : { lead_id: `00000000-0000-0000-0000-00000000000${seq}`, company: { name: `Co ${seq}` } }, + }; +} + +function page(state: string, items: unknown[], nextSince: string | null) { + return { + job: { + id: JOB_ID, + state, + submitted_at: "2026-07-28T10:29:00Z", + expires_at: "2026-08-27T10:29:00Z", + last_progress_at: "2026-07-28T10:30:00Z", + }, + funnel: { delivered: 3, examined: 4 }, + items, + next_since: nextSince, + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + explain: { region: "US", model: "text_v2_ai_description", scope_notes: [] }, + }; +} + +beforeEach(() => resetHttpMock()); + +describe("leadbay_lead_job_status", () => { + it("terminal snapshot — splits leads/skipped, no next_poll", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: page("completed", [item(0), item(1, "skipped")], "100:1"), + }, + ]); + const result = await leadJobStatus.execute(newClient(), { job_id: JOB_ID }); + expect(result.done).toBe(true); + expect(result.leads).toHaveLength(1); + expect(result.skipped).toHaveLength(1); + expect(result.next_poll).toBeNull(); + expect(result.next_since).toBe("100:1"); + }); + + it("pages the cursor dry when a page comes back full", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=2`, + status: 200, + body: page("completed", [item(0), item(1)], "100:1"), + }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=2&since=${encodeURIComponent("100:1")}`, + status: 200, + body: page("completed", [item(2)], "101:2"), + }, + ]); + const result = await leadJobStatus.execute(newClient(), { + job_id: JOB_ID, + limit: 2, + }); + expect(result.leads).toHaveLength(3); + expect(getHttpRequests()).toHaveLength(2); + }); + + it("running job — still_running with next_poll handle", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: page("running", [item(0)], "100:0"), + }, + ]); + const result = await leadJobStatus.execute(newClient(), { job_id: JOB_ID }); + expect(result.still_running).toBe(true); + expect(result.next_poll).toMatchObject({ tool: "leadbay_lead_job_status", job_id: JOB_ID }); + }); + + it("unknown job — 404 propagates", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 404, + body: { error: "not_found", message: "job" }, + }, + ]); + await expect( + leadJobStatus.execute(newClient(), { job_id: JOB_ID }) + ).rejects.toMatchObject({ code: "NOT_FOUND" }); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads.test.ts b/packages/core/test/unit/composite/qualify-leads.test.ts new file mode 100644 index 00000000..185af25d --- /dev/null +++ b/packages/core/test/unit/composite/qualify-leads.test.ts @@ -0,0 +1,191 @@ +/** + * Unit tests for leadbay_qualify_leads (POST /mcp/qualify + job poll). + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const JOB_ID = "2900fff9-2985-4220-84dc-70a551fc3e84"; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 476, unit: "cost_cents" }, + items_requested: 3, + duplicate: false, +}; + +const QUALIFIED_ITEM = { + ref: { input_indexes: [0], requested_as: { website: "franklinbbq.com" } }, + status: "delivered", + seq: 0, + completed_at: "2026-07-28T10:20:00Z", + cost: { billed: 94, unit: "cost_cents" }, + lead: { + lead_id: "aaaa1111-2222-3333-4444-555566667777", + company: { name: "Franklin Barbecue", employees: { min: 11, max: 50, known: true } }, + fit: { + available: true, + score: 72, + components: { + qualification: { available: true, ai_score: 12, questions: [], matched_tags: [], unmatched_tags: [] }, + }, + }, + web_research: { available: true }, + contact: { + lead_contact_id: "cccc1111-2222-3333-4444-555566667777", + name: "Aaron Franklin", + role: "Owner", + channels: { + email: { status: "delivered", value: "aaron@franklinbbq.com", billed: 25 }, + phone: { status: "not_requested" }, + }, + }, + }, +}; + +const NOT_IN_UNIVERSE_ITEM = { + ref: { input_indexes: [1], requested_as: { name: "Totally Nonexistent Bistro" } }, + status: "skipped", + status_reason: "not_in_universe", + seq: 1, + cost: { billed: 0, unit: "cost_cents" }, +}; + +const SNAPSHOT_DONE = { + job: { + id: JOB_ID, + state: "completed", + submitted_at: "2026-07-28T10:19:00Z", + expires_at: "2026-08-27T10:19:00Z", + last_progress_at: "2026-07-28T10:20:30Z", + }, + funnel: { + matched: 2, + examined: 1, + qualified: 1, + disqualified: 0, + delivered: 1, + delivered_callable: 1, + degraded: 0, + resolved: 1, + not_in_universe: 1, + stop_reason: "pool_exhausted", + }, + items: [QUALIFIED_ITEM, NOT_IN_UNIVERSE_ITEM], + next_since: "1785233900000000:1", + cost: { spent: 119, unit: "cost_cents", breakdown: { enrichment_cents: 25 } }, + explain: { region: "US", model: "text_v2_ai_description", scope_notes: [] }, +}; + +beforeEach(() => resetHttpMock()); + +describe("leadbay_qualify_leads", () => { + it("happy path — per-item verdicts including honest skips", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT_DONE, + }, + ]); + const result = await qualifyLeads.execute(newClient(), { + lead_refs: [ + { website: "franklinbbq.com", name: "Franklin Barbecue" }, + { name: "Totally Nonexistent Bistro" }, + ], + contact_titles: ["Owner"], + channels: ["email"], + request_id: "vet-austin-1", + wait_seconds: 0, + }); + + expect(result.job_id).toBe(JOB_ID); + expect(result.done).toBe(true); + expect(result.items).toHaveLength(2); + const skipped = result.items.find((i: any) => i.status === "skipped"); + expect(skipped.status_reason).toBe("not_in_universe"); + expect(result.summary.not_in_universe).toBe(1); + expect(result.summary.delivered_callable).toBe(1); + const delivered = result.items.find((i: any) => i.status === "delivered"); + expect(delivered.lead.contact.channels.email.value).toBe("aaron@franklinbbq.com"); + + const submit = getHttpRequests().find((r) => r.method === "POST")!; + const body = JSON.parse(submit.body!); + expect(body.lead_refs).toHaveLength(2); + expect(body.contact_titles).toEqual(["Owner"]); + expect(body.wait_seconds).toBeUndefined(); + }); + + it("prior_deliveries selector — passes through without lead_refs", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT_DONE, + }, + ]); + await qualifyLeads.execute(newClient(), { + prior_deliveries: { job_id: "0a2fcbf5-18e1-4967-b5de-0c67cd823bcc" }, + wait_seconds: 0, + }); + const submit = getHttpRequests().find((r) => r.method === "POST")!; + const body = JSON.parse(submit.body!); + expect(body.prior_deliveries.job_id).toBe("0a2fcbf5-18e1-4967-b5de-0c67cd823bcc"); + expect(body.lead_refs).toBeUndefined(); + }); + + it("dry_run — forecast only, no job created", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 200, + body: { + valid: true, + items_requested: 2, + estimated_cost: { max: 238, unit: "cost_cents" }, + quota_forecast: { web_fetch_allowed: true, rescore_allowed: true, enrichment_allowed: false }, + }, + }, + ]); + const result = await qualifyLeads.execute(newClient(), { + lead_refs: [{ website: "a.com" }, { website: "b.com" }], + dry_run: true, + }); + expect(result.dry_run).toBe(true); + expect(result.quota_forecast.enrichment_allowed).toBe(false); + expect(getHttpRequests()).toHaveLength(1); + }); + + it("validation 400 (bad ref) — propagates", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 400, + body: { error: "bad_request", message: "lead_refs[0] has no identifying field" }, + }, + ]); + await expect( + qualifyLeads.execute(newClient(), { lead_refs: [{}] }) + ).rejects.toMatchObject({ error: true }); + }); +}); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index e0a16b74..b5d8920e 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -854,6 +854,85 @@ Tell me the outreach was logged, name the verification.source used, and surface `; // endregion: leadbay_log_outreach +// region: leadbay_new_leads +export const leadbay_new_leads: string = ` +## MEMORY + +Before responding, glance at any \`_meta.agent_memory.summary\` returned by tool calls earlier in this session and reflect its top signals in your reasoning ("Filtering by your stated preference for healthcare"). After any material new signal from the user this conversation (sector, region, deal size, communication style, qualification rule, explicit retraction, or recurrence / scheduling preference such as "I do this every day" or "remind me every morning"), call \`leadbay_agent_memory_capture\` to persist it: \`source:"user_stated"\` if literal, \`source:"inferred"\` with confidence <=6 if inferred. + + +IRON LAW — NO FABRICATION. Every lead id, contact email, custom field id, mapping decision, and tool argument must trace to a value you read from the file the user attached or to an output from a leadbay_* tool call in this session. Do not invent values. Do not "fill in" a missing leadId with a name match. Do not synthesize a CRM id from a guess. If a value is missing, leave the field blank and say so. + + +GATE — DEFER TO TOOL RENDERING. When you call a Leadbay composite that ships its own RENDERING block (every composite in 0.9.0+ does), render the response using that block's recipe verbatim — score bars, glyph palette, column order, hide-list, link priorities, all of it. Do NOT substitute prose, a numbered list, or a different column structure even when an orchestrating prompt's body suggests alternate framing. Prompt-specific commentary (motivational nudges, summaries, next-action recommendations) belongs ABOVE or BELOW the canonical table, never in place of it. + +If the prompt's body and the tool's RENDERING appear to conflict, the tool's RENDERING wins for the structural layout; the prompt's voice wins for the commentary that surrounds it. + + +Find net-new leads for me. My need, in my words: + +> {{arg:need}} + +If the need above is empty or too vague to name (a) who I sell to and (b) +roughly how many leads I want, ask me ONCE — one short question — then +proceed. Default count when unstated: 10. + +# PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) + +From my words, work out: +- What I SELL and therefore WHO WRITES ME CHECKS — the buyer category, never + the buyer's customers, never my competitors. If my product helps companies + of type X serve audience Y, my buyer is X. +- Hard constraints: geography, size band, sector, exclusions ("no + franchises", "pas de grands groupes" — negatives BIND). +- Contact needs: do I want a person? Which titles? Email, phone, both? +- Buyer archetypes: if my need genuinely spans two different kinds of buyer, + plan one search per archetype — never one blended seed. + +# PHASE 2 — CRAFT THE SEED + +Compose the \`example_lead\` for each archetype following the craft rules in +the leadbay_find_new_leads description (registry-style description of a +FICTIONAL typical buyer; no invented brand name; no event language; hard +constraints go in \`filters\`, not prose). Show me the seed description(s) in +one line each — I should recognize my ideal customer in them. + +# PHASE 3 — FREE PREVIEW + +Call \`leadbay_find_new_leads\` with the seed, \`filters\`, \`count\`, +\`qualify: false\`, no channels — this is FREE — and a \`request_id\` derived +from the ask + today's date. Render the delivery table and judge fit +honestly: are these the kind of companies I asked for? + +- **On-profile** → offer Phase 4. +- **Off-profile or empty** → read \`funnel\` + \`explain.scope_notes\`, tell me + what went wrong in one line (wrong archetype? too narrow a filter? thin + universe?), reshape the seed or filters, and retry under a NEW request_id. + Reshaping is free; do not pay to explore a bad seed. + +# PHASE 4 — PAID DEPTH (only with my explicit go-ahead) + +When I want qualification evidence and/or reachable contacts: +1. Quote first: \`dry_run: true\` with the exact flags I asked for + (\`qualify: true\`, \`min_ai_score\`, \`contact_titles\`, \`title_gate\`, + \`channels\`, \`max_cost\`) and tell me the worst-case cost in plain money. +2. On my go-ahead, prefer feeding the free preview's deliveries to + \`leadbay_qualify_leads\` (\`prior_deliveries: {job_id}\`) — it only spends on + companies already known to match. Run a fresh \`qualify: true\` search + instead when I asked for more than the preview delivered. +3. While the job runs, poll with \`leadbay_lead_job_status\` + (\`wait_seconds: 60\`); report progress, not silence. + +# PHASE 5 — DELIVER + +Render per the lead-delivery table, then ALWAYS the funnel line: matched / +examined / qualified / disqualified / delivered / stop reason / spend. Zero +delivered gets a diagnosis and a concrete next move, never a shrug. Close +with NEXT STEPS from the tool description — and STOP; take no further action +without my say-so. +`; +// endregion: leadbay_new_leads + // region: leadbay_plan_tour_in_city export const leadbay_plan_tour_in_city: string = ` Plan a field sales tour for me in **{{arg:city}}**{{arg:date_paren}}. @@ -1554,6 +1633,7 @@ export const PROMPT_META = { leadbay_followup_check_in: {"name":"leadbay_followup_check_in","short_description":"Follow-up check-in: surface KNOWN leads from the Monitor view needing\nre-engagement. Trigger on \"follow up\", \"already known leads\", \"what's\noverdue\", \"before my trip\", \"who should I re-engage\". Do NOT trigger on\n\"show me today's leads\", \"my morning check-in\", \"run my check-in\",\n\"I do this every day\", \"every morning\" — those go to\n`leadbay_daily_check_in`.\n","arguments":[],"expected_calls":["leadbay_pull_followups","leadbay_research_lead_by_id","leadbay_prepare_outreach"],"failure_modes":["Calls leadbay_pull_leads (the Discover entry point) instead of leadbay_pull_followups — these are different data sources; the Discover queue does NOT contain Monitor's known-but-cold pipeline","Iterates pages of leadbay_pull_leads filtering by engagement_count to \"fake\" a follow-up view (a real bug observed in 0.9.0 — the right move is to call pull_followups directly)","Replaces the canonical pull_followups table layout with prose per row (the per-tool RENDERING block is the structural contract; commentary belongs above or below)","Skips the cross-mode pivot offer at the end (\"Want to see NEW leads from your wishlist instead?\" routes to leadbay_pull_leads)"]}, leadbay_import_file: {"name":"leadbay_import_file","short_description":"Import a user-supplied CSV/file into Leadbay through five phases with\nevidence gates — scan, derive, resolve identities, preserve & commit,\nthen optionally qualify and report. The job is to maximize how many\nrows the Leadbay system actually ingests and matches.\n","arguments":[{"name":"file","description":"Path or user-visible name of the CSV/file to import. If omitted, use the file the user attached or referenced.","required":false},{"name":"instruction","description":"Additional user goal, e.g. \"then qualify the leads\", \"preserve owner phone as a custom field\", or \"only import restaurants in Manhattan\".","required":false}],"expected_calls":["leadbay_resolve_import_rows","leadbay_list_mappable_fields","leadbay_create_custom_field","leadbay_import_leads","leadbay_import_and_qualify","leadbay_add_note","leadbay_import_status"],"failure_modes":["Picks LEADBAY_ID from score alone, name-only, fuzzy-name-only, root-domain-only, brand-only, postcode-only, or city-only evidence","Drops meaningful business notes or CRM record links instead of preserving them as custom fields or lead notes","Treats a consumer mailbox domain (gmail.com, hotmail.com, ...) as the company domain","Skips deriving company_domain from a business email when no website column exists (this kills match rate)","Skips the COLUMN PRESERVATION PLAN byproduct before importing","Skips the DECISION LOG byproduct before writing LEADBAY_ID","Returns the imported records WITHOUT writing LEADBAY_ID values back into the user's file (leaves the user no audit trail of what matched)","Fabricates leadIds, contact emails, or mapping IDs not present in the file or a tool response"]}, leadbay_log_outreach: {"name":"leadbay_log_outreach","short_description":"Log outreach (an email I sent, a call I made, a meeting I had) on a\nspecific lead. Captures verification so the SDR pipeline trusts the entry.\n","arguments":[{"name":"lead_id","description":"The lead UUID. Get it from leadbay_pull_leads or leadbay_research_lead_by_id.","required":true},{"name":"summary","description":"1-2 sentences describing what I did (e.g. 'Sent intro email to CTO citing recent Hornsea contract').","required":true}],"expected_calls":["leadbay_report_outreach"],"failure_modes":["Calls leadbay_report_outreach without first collecting a verification source","Fabricates a gmail_message_id or calendar_event_id (the human team treats verification as canonical)","Records outreach to a different lead_id than the one the user supplied","Skips the dry_run step when the user is unsure what would be sent"]}, + leadbay_new_leads: {"name":"leadbay_new_leads","short_description":"Guided net-new lead delivery — turn a natural-language need (\"gyms around\nDallas that would buy our flooring\") into ICP-perfect NEW companies with\nqualification evidence and the right contact, via leadbay_find_new_leads.\nTrigger on \"find me new leads\", \"get me N companies that \",\n\"we're entering \". Do NOT trigger on \"today's leads\"\n(leadbay_daily_check_in) or \"qualify these companies I have\"\n(leadbay_qualify_leads).\n","arguments":[{"name":"need","description":"What the user is looking for, in their own words (e.g. '10 gyms around Dallas that would buy modular flooring, with phone numbers'). Optional — the session starts by asking when absent.","required":false}],"expected_calls":["leadbay_find_new_leads","leadbay_lead_job_status","leadbay_qualify_leads"],"failure_modes":["Passes the user's raw sentence as `query` instead of crafting an example_lead description (vendor-vocabulary trap — measured 0 delivered from a raw query vs on-profile results from a crafted example)","Invents a distinctive brand name in example_lead.name (pulls matching toward name-lookalikes)","Puts event language (\"hiring\", \"expanding\", \"just raised\") into the seed description","Launches qualify:true or channels without a dry_run quote and the user's explicit go-ahead","Retries a failed/timed-out submit with a NEW request_id (double-spend) — the same ask must reuse the same request_id","Reports \"no results\" without narrating the funnel + scope_notes and proposing a concrete fix","Renders delivered leads as freeform prose instead of the canonical lead-delivery table","Blends two distinct buyer archetypes into one seed description instead of running one search per archetype"]}, leadbay_plan_tour_in_city: {"name":"leadbay_plan_tour_in_city","short_description":"Use whenever the user names a city they'll be in and asks who to see\n— \"I'm in SF next Tuesday, who's worth meeting?\", \"I'm going to Berlin\n— who should I visit?\", \"plan my tour\". Any in-person/visit\nintent tied to a place routes here, NOT to `leadbay_pull_leads`. It\nsurfaces follow-ups + fresh Discover leads in the city via\n`leadbay_tour_plan`, ALWAYS offers to plot them on a map (rendering it\non yes), then offers outreach drafts + campaign persistence.\n","arguments":[{"name":"city","description":"City or region the user is visiting (e.g. 'Limoges', 'Bay Area'). Used as the geo filter for both Monitor and Discover lookups.","required":true},{"name":"date","description":"When the visit is (e.g. 'May 24', 'next Thursday'). Surfaced in the outreach drafts as 'I'll be in on '.","required":false}],"expected_calls":["leadbay_tour_plan","leadbay_research_lead_by_id","leadbay_prepare_outreach","leadbay_create_campaign"],"failure_modes":["Calls leadbay_followups_map (Monitor-only) instead of leadbay_tour_plan — loses the Discover (fresh-lead) half that the user explicitly asked for","Calls leadbay_pull_leads then drops the geo filter — returns the lens-wide wishlist instead of city-relevant fresh leads","Skips the campaign-persist step (\"would you like to save these as a tour?\") — leaves the rep with a one-shot map but no follow-up artifact","Creates a campaign WITHOUT asking the user first — the persist step is high-intent; offer it, don't assume","Fabricates lead_ids when seeding the campaign instead of using the ids returned by tour_plan"]}, leadbay_prospecting_overview: {"name":"leadbay_prospecting_overview","short_description":"Orientation for working with Leadbay from any host — discovery vs.\nfollow-up, the outreach loop, outcome recording, imports, pushback /\nsnooze, and the connected-outreach-tool registry. Trigger when the\nconversation involves Leadbay leads, prospecting, pipeline, follow-up,\noutreach, or lens / ICP — anything from \"show me my leads\" to \"what\nshould I follow up on\" to \"I'll send via lemlist\".\n","arguments":[],"expected_calls":["leadbay_account_status","leadbay_pull_leads","leadbay_pull_followups","leadbay_research_lead_by_id","leadbay_research_lead_by_name_fuzzy","leadbay_prepare_outreach","leadbay_report_outreach","leadbay_set_pushback","leadbay_remove_pushback","leadbay_bulk_qualify_leads","leadbay_enrich_titles","leadbay_import_leads","leadbay_add_note","leadbay_adjust_audience"],"failure_modes":["Drives outreach without asking the user \"how did it go?\" afterwards — leaving prospecting_actions and epilogue_status stale","Says \"epilogue\" in user-facing dialogue instead of \"outcome\"","Says \"Monitor\" in user-facing dialogue instead of \"follow-ups\"","Treats a \"not now / next quarter\" reply as a note instead of routing through the pushback mechanism","Drafts outreach in a generic format when the user has a connected sequencer (lemlist, Outreach.io, etc.) that has its own idiom","Re-pulls leads without passing the captured lensId, allowing a backend lens shift to discard prior work","Skips the STOP byproduct in any multi-step workflow it triggers","Calls leadbay_pull_leads (Discover wishlist) for a follow-up query, or leadbay_pull_followups (Monitor view) for a discovery query — the two entry points read from different backend tables; the right orchestrators are leadbay_daily_check_in (discovery) and leadbay_followup_check_in (follow-up)"]}, leadbay_qualify_top_n: {"name":"leadbay_qualify_top_n","short_description":"Bulk-qualify the top N un-qualified leads in the active lens. Uses\nleadbay_bulk_qualify_leads with a sensible default budget.\n","arguments":[{"name":"count","description":"How many leads to qualify (default 10, max 25). Higher counts may take 5+ minutes.","required":false}],"expected_calls":["leadbay_bulk_qualify_leads","leadbay_qualify_status","leadbay_pull_leads","leadbay_research_lead_by_id"],"failure_modes":["Picks a count larger than the user asked for (or larger than the max 25)","Glosses over still-running leads in the summary instead of naming them","Recommends a lead from the existing qualified pool instead of one from this batch's actual results","Replaces the canonical pull_leads table with prose when rendering the newly-qualified batch (the per-tool RENDERING block is the structural contract; \"standouts\" commentary sits above it)","Expands the qualify-status sentence into a card or table instead of the one-line status-inline render"]}, @@ -1575,6 +1655,7 @@ export const PROMPT_CATALOG_BULLETS = { leadbay_followup_check_in: `- \`leadbay_followup_check_in\`: Follow-up check-in: surface KNOWN leads from the Monitor view needing re-engagement. Trigger on "follow up", "already known leads", "what's overdue", "before my trip", "who should I re-engage". Do NOT trigger on "show me today's leads", "my morning check-in", "run my check-in", "I do this every day", "every morning" — those go to \`leadbay_daily_check_in\`.`, leadbay_import_file: `- \`leadbay_import_file\` (optional args: file, instruction): Import a user-supplied CSV/file into Leadbay through five phases with evidence gates — scan, derive, resolve identities, preserve & commit, then optionally qualify and report. The job is to maximize how many rows the Leadbay system actually ingests and matches.`, leadbay_log_outreach: `- \`leadbay_log_outreach\` (required args: lead_id, summary): Log outreach (an email I sent, a call I made, a meeting I had) on a specific lead. Captures verification so the SDR pipeline trusts the entry.`, + leadbay_new_leads: `- \`leadbay_new_leads\` (optional args: need): Guided net-new lead delivery — turn a natural-language need ("gyms around Dallas that would buy our flooring") into ICP-perfect NEW companies with qualification evidence and the right contact, via leadbay_find_new_leads. Trigger on "find me new leads", "get me N companies that ", "we're entering ". Do NOT trigger on "today's leads" (leadbay_daily_check_in) or "qualify these companies I have" (leadbay_qualify_leads).`, leadbay_plan_tour_in_city: `- \`leadbay_plan_tour_in_city\` (required args: city; optional args: date): Use whenever the user names a city they'll be in and asks who to see — "I'm in SF next Tuesday, who's worth meeting?", "I'm going to Berlin — who should I visit?", "plan my tour". Any in-person/visit intent tied to a place routes here, NOT to \`leadbay_pull_leads\`. It surfaces follow-ups + fresh Discover leads in the city via \`leadbay_tour_plan\`, ALWAYS offers to plot them on a map (rendering it on yes), then offers outreach drafts + campaign persistence.`, leadbay_prospecting_overview: `- \`leadbay_prospecting_overview\`: Orientation for working with Leadbay from any host — discovery vs. follow-up, the outreach loop, outcome recording, imports, pushback / snooze, and the connected-outreach-tool registry. Trigger when the conversation involves Leadbay leads, prospecting, pipeline, follow-up, outreach, or lens / ICP — anything from "show me my leads" to "what should I follow up on" to "I'll send via lemlist".`, leadbay_qualify_top_n: `- \`leadbay_qualify_top_n\` (optional args: count): Bulk-qualify the top N un-qualified leads in the active lens. Uses leadbay_bulk_qualify_leads with a sensible default budget.`, @@ -1592,6 +1673,7 @@ export const PROMPT_CATALOG_INSTRUCTIONS: string = `This server exposes the foll - \`leadbay_followup_check_in\`: Follow-up check-in: surface KNOWN leads from the Monitor view needing re-engagement. Trigger on "follow up", "already known leads", "what's overdue", "before my trip", "who should I re-engage". Do NOT trigger on "show me today's leads", "my morning check-in", "run my check-in", "I do this every day", "every morning" — those go to \`leadbay_daily_check_in\`. - \`leadbay_import_file\` (optional args: file, instruction): Import a user-supplied CSV/file into Leadbay through five phases with evidence gates — scan, derive, resolve identities, preserve & commit, then optionally qualify and report. The job is to maximize how many rows the Leadbay system actually ingests and matches. - \`leadbay_log_outreach\` (required args: lead_id, summary): Log outreach (an email I sent, a call I made, a meeting I had) on a specific lead. Captures verification so the SDR pipeline trusts the entry. +- \`leadbay_new_leads\` (optional args: need): Guided net-new lead delivery — turn a natural-language need ("gyms around Dallas that would buy our flooring") into ICP-perfect NEW companies with qualification evidence and the right contact, via leadbay_find_new_leads. Trigger on "find me new leads", "get me N companies that ", "we're entering ". Do NOT trigger on "today's leads" (leadbay_daily_check_in) or "qualify these companies I have" (leadbay_qualify_leads). - \`leadbay_plan_tour_in_city\` (required args: city; optional args: date): Use whenever the user names a city they'll be in and asks who to see — "I'm in SF next Tuesday, who's worth meeting?", "I'm going to Berlin — who should I visit?", "plan my tour". Any in-person/visit intent tied to a place routes here, NOT to \`leadbay_pull_leads\`. It surfaces follow-ups + fresh Discover leads in the city via \`leadbay_tour_plan\`, ALWAYS offers to plot them on a map (rendering it on yes), then offers outreach drafts + campaign persistence. - \`leadbay_prospecting_overview\`: Orientation for working with Leadbay from any host — discovery vs. follow-up, the outreach loop, outcome recording, imports, pushback / snooze, and the connected-outreach-tool registry. Trigger when the conversation involves Leadbay leads, prospecting, pipeline, follow-up, outreach, or lens / ICP — anything from "show me my leads" to "what should I follow up on" to "I'll send via lemlist". - \`leadbay_qualify_top_n\` (optional args: count): Bulk-qualify the top N un-qualified leads in the active lens. Uses leadbay_bulk_qualify_leads with a sensible default budget. diff --git a/packages/mcp/src/prompts.ts b/packages/mcp/src/prompts.ts index 82307ba1..e7d272db 100644 --- a/packages/mcp/src/prompts.ts +++ b/packages/mcp/src/prompts.ts @@ -23,6 +23,7 @@ import { leadbay_daily_check_in, leadbay_import_file, leadbay_log_outreach, + leadbay_new_leads, leadbay_plan_tour_in_city, leadbay_prospecting_overview, leadbay_qualify_top_n, @@ -78,6 +79,25 @@ const CATALOG: CatalogEntry[] = [ arguments: [], render: () => [userMessage(leadbay_prospecting_overview)], }, + { + name: "leadbay_new_leads", + description: PROMPT_META.leadbay_new_leads.short_description, + arguments: [ + { + name: "need", + description: + "What you're looking for, in your own words (e.g. '10 gyms around Dallas that would buy modular flooring, with phone numbers'). Optional — the session starts by asking when absent.", + required: false, + }, + ], + render: (args) => [ + userMessage( + substitutePlaceholders(leadbay_new_leads, { + need: args.need ?? "(not provided — ask me first)", + }), + ), + ], + }, { name: "leadbay_research_a_domain", description: PROMPT_META.leadbay_research_a_domain.short_description, diff --git a/packages/mcp/test/audit/routing-block.test.ts b/packages/mcp/test/audit/routing-block.test.ts index 03096cbb..8a0e9f25 100644 --- a/packages/mcp/test/audit/routing-block.test.ts +++ b/packages/mcp/test/audit/routing-block.test.ts @@ -28,6 +28,9 @@ const MEMORY_POINTER = "**Memory:** recall + capture via `leadbay_agent_memory_* // The audit ensures none of them regresses. Other tools MAY add routing // later — when they do, append here. const TOOLS_WITH_ROUTING = new Set([ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", "leadbay_prepare_outreach", "leadbay_account_status", "leadbay_add_leads_to_campaign", diff --git a/packages/mcp/test/prompts.test.ts b/packages/mcp/test/prompts.test.ts index 8ed3f395..92e70cca 100644 --- a/packages/mcp/test/prompts.test.ts +++ b/packages/mcp/test/prompts.test.ts @@ -35,6 +35,7 @@ describe("prompts/* capability (P2 prompts)", () => { expect(names).toEqual([ "leadbay_daily_check_in", "leadbay_prospecting_overview", + "leadbay_new_leads", "leadbay_research_a_domain", "leadbay_import_file", "leadbay_refine_audience", diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl new file mode 100644 index 00000000..d4a3a0f7 --- /dev/null +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -0,0 +1,97 @@ +--- +name: leadbay_new_leads +kind: prompt +short_description: | + Guided net-new lead delivery — turn a natural-language need ("gyms around + Dallas that would buy our flooring") into ICP-perfect NEW companies with + qualification evidence and the right contact, via leadbay_find_new_leads. + Trigger on "find me new leads", "get me N companies that ", + "we're entering ". Do NOT trigger on "today's leads" + (leadbay_daily_check_in) or "qualify these companies I have" + (leadbay_qualify_leads). +arguments: + - name: need + description: "What the user is looking for, in their own words (e.g. '10 gyms around Dallas that would buy modular flooring, with phone numbers'). Optional — the session starts by asking when absent." + required: false +expected_calls: + - leadbay_find_new_leads + - leadbay_lead_job_status + - leadbay_qualify_leads +failure_modes: + - Passes the user's raw sentence as `query` instead of crafting an example_lead description (vendor-vocabulary trap — measured 0 delivered from a raw query vs on-profile results from a crafted example) + - Invents a distinctive brand name in example_lead.name (pulls matching toward name-lookalikes) + - Puts event language ("hiring", "expanding", "just raised") into the seed description + - Launches qualify:true or channels without a dry_run quote and the user's explicit go-ahead + - Retries a failed/timed-out submit with a NEW request_id (double-spend) — the same ask must reuse the same request_id + - Reports "no results" without narrating the funnel + scope_notes and proposing a concrete fix + - Renders delivered leads as freeform prose instead of the canonical lead-delivery table + - Blends two distinct buyer archetypes into one seed description instead of running one search per archetype +--- + +{{include:headers/agent-memory-preamble}} + +{{include:iron-laws/no-fabrication}} + +{{include:gates/defer-to-tool-rendering}} + +Find net-new leads for me. My need, in my words: + +> {{arg:need}} + +If the need above is empty or too vague to name (a) who I sell to and (b) +roughly how many leads I want, ask me ONCE — one short question — then +proceed. Default count when unstated: 10. + +# PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) + +From my words, work out: +- What I SELL and therefore WHO WRITES ME CHECKS — the buyer category, never + the buyer's customers, never my competitors. If my product helps companies + of type X serve audience Y, my buyer is X. +- Hard constraints: geography, size band, sector, exclusions ("no + franchises", "pas de grands groupes" — negatives BIND). +- Contact needs: do I want a person? Which titles? Email, phone, both? +- Buyer archetypes: if my need genuinely spans two different kinds of buyer, + plan one search per archetype — never one blended seed. + +# PHASE 2 — CRAFT THE SEED + +Compose the `example_lead` for each archetype following the craft rules in +the leadbay_find_new_leads description (registry-style description of a +FICTIONAL typical buyer; no invented brand name; no event language; hard +constraints go in `filters`, not prose). Show me the seed description(s) in +one line each — I should recognize my ideal customer in them. + +# PHASE 3 — FREE PREVIEW + +Call `leadbay_find_new_leads` with the seed, `filters`, `count`, +`qualify: false`, no channels — this is FREE — and a `request_id` derived +from the ask + today's date. Render the delivery table and judge fit +honestly: are these the kind of companies I asked for? + +- **On-profile** → offer Phase 4. +- **Off-profile or empty** → read `funnel` + `explain.scope_notes`, tell me + what went wrong in one line (wrong archetype? too narrow a filter? thin + universe?), reshape the seed or filters, and retry under a NEW request_id. + Reshaping is free; do not pay to explore a bad seed. + +# PHASE 4 — PAID DEPTH (only with my explicit go-ahead) + +When I want qualification evidence and/or reachable contacts: +1. Quote first: `dry_run: true` with the exact flags I asked for + (`qualify: true`, `min_ai_score`, `contact_titles`, `title_gate`, + `channels`, `max_cost`) and tell me the worst-case cost in plain money. +2. On my go-ahead, prefer feeding the free preview's deliveries to + `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on + companies already known to match. Run a fresh `qualify: true` search + instead when I asked for more than the preview delivered. +3. While the job runs, poll with `leadbay_lead_job_status` + (`wait_seconds: 60`); report progress, not silence. + +# PHASE 5 — DELIVER + +Render per the lead-delivery table, then ALWAYS the funnel line: matched / +examined / qualified / disqualified / delivered / stop reason / spend. Zero +delivered gets a diagnosis and a concrete next move, never a shrug. Close +with NEXT STEPS from the tool description — and STOP; take no further action +without my say-so. diff --git a/packages/promptforge/snippets/heuristics/example-lead-craft.md b/packages/promptforge/snippets/heuristics/example-lead-craft.md new file mode 100644 index 00000000..e2d26314 --- /dev/null +++ b/packages/promptforge/snippets/heuristics/example-lead-craft.md @@ -0,0 +1,48 @@ +### Crafting the `example_lead` seed — the input that decides result quality + +The `example_lead` is a FICTIONAL typical ideal customer. Its text is embedded +and matched against millions of real company descriptions sourced from business +registries and company websites. Those descriptions state what a company **IS** +(stable business profile) — never what is happening. Write the seed the same +way, or the matcher drifts to the wrong companies. Each rule below is +load-bearing (validated live against staging, 2026-07-28): + +1. **Describe the BUYER, never the seller.** Before writing, answer: "would + this company write a check to my user?" A seed that describes what the user + sells surfaces the user's *competitors and vendors*, not their customers. + Classic trap: if the user's product helps companies of type X serve their + customers of type Y, the seed describes X — never Y. +2. **Put everything in `description`; leave `name` unset.** A distinctive + invented brand name pulls matching toward name-lookalikes: a seed named + "Meridian Analytics" returned five unrelated companies all named + "Meridian". No name beats any name. +3. **Registry style, one sentence to ~250 chars.** State the business profile: + industry niche, business model, what they sell or operate, who they serve, + observable scale (sites, membership, fleet). Write it like the first + paragraph of the company's About-Us page. + - STRONG: "Operator of full-service fitness centers offering strength + training areas, group classes and personal training to individual members + across multiple club locations." + - WEAK (generic): "A gym in Texas." + - WRONG (seller-side): "Supplier of durable modular flooring for gyms." +4. **No event language.** "hiring", "expanding", "just raised", "opening a new + site" are not filters — real registry descriptions never contain them, so + they dilute the profile and attract event-flavored noise. Temporal criteria + in a `query` become best-effort ranking annotations at most (the response + `explain.scope_notes` says so). Put purchase-trigger criteria in the org's + qualification questions instead, where the paid qualification stage scores + them from fresh research. +5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — + real descriptions don't carry them. +6. **Hard constraints go in `filters`, not prose.** Geography, sector, size + bounds written into the description only *tint* the ranking; `filters` are + enforced. Seed describes the archetype; filters draw the fence. +7. **Prefer `example_lead` over `query`.** Query text matches topic + *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS + as strongly as gym BUYERS (measured: the same ICP delivered 0 leads from a + query and on-profile leads from an example_lead). Use `query` only when the + user's own wording carries signal an example can't express. +8. **One seed per buyer archetype.** If the ask spans two distinct segments + (e.g. "gyms and logistics warehouses"), run one search per segment with its + own description — a blended seed lands between the two clusters and matches + neither. Distinct asks need distinct `request_id`s. diff --git a/packages/promptforge/snippets/next-steps/find-new-leads.md b/packages/promptforge/snippets/next-steps/find-new-leads.md new file mode 100644 index 00000000..35b4ffd6 --- /dev/null +++ b/packages/promptforge/snippets/next-steps/find-new-leads.md @@ -0,0 +1,14 @@ +## NEXT STEPS — after a find_new_leads delivery + +{{include:next-steps/ask-user-input-routing}} + +Pick the 2-3 options that match what actually happened — never all six: + +| Observation | Suggest | Calls | +|---|---|---| +| Job still running (`still_running: true`) | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote `dry_run` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | +| Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | +| Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | +| Stopped at cost cap / quota | "Raise the cap to €X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | diff --git a/packages/promptforge/snippets/next-steps/lead-job-status.md b/packages/promptforge/snippets/next-steps/lead-job-status.md new file mode 100644 index 00000000..86c16c9e --- /dev/null +++ b/packages/promptforge/snippets/next-steps/lead-job-status.md @@ -0,0 +1,10 @@ +## NEXT STEPS — after a job status poll + +{{include:next-steps/ask-user-input-routing}} + +Exactly two offers — this is a status tool, keep it terse: + +| Observation | Suggest | Calls | +|---|---|---| +| Still running | "Keep waiting (~1 min) or leave it — results are kept 30 days" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Terminal (completed / partial / failed) | Render the delivery per the RENDERING block, then offer the matching find_new_leads / qualify_leads NEXT STEPS | — | diff --git a/packages/promptforge/snippets/next-steps/qualify-leads.md b/packages/promptforge/snippets/next-steps/qualify-leads.md new file mode 100644 index 00000000..c2a4c807 --- /dev/null +++ b/packages/promptforge/snippets/next-steps/qualify-leads.md @@ -0,0 +1,14 @@ +## NEXT STEPS — after a qualify_leads delivery + +{{include:next-steps/ask-user-input-routing}} + +Pick the 2-3 options that match what actually happened: + +| Observation | Suggest | Calls | +|---|---|---| +| Job still running | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | +| Fit leads with contacts delivered | "Draft outreach for the qualified ones" | leadbay_prepare_outreach | +| Items skipped `not_in_universe` | "Import those companies first, then re-qualify" | leadbay_import_leads → leadbay_qualify_leads | +| Items skipped `low_confidence_identity` | "Pick the right match" (show `resolution.alternatives`) | leadbay_qualify_leads with the chosen lead_id | +| Contacts delivered without channels | "Purchase verified emails/phones for the keepers (state cost first)" | leadbay_qualify_leads(lead_refs with contact_id, channels) | +| Disqualified with evidence | "Review why — adjust qualification questions if the criteria are off" | leadbay_get_qualification_questions | diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md new file mode 100644 index 00000000..80398a74 --- /dev/null +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -0,0 +1,73 @@ +## RENDERING — delivery table + honest funnel line + +Render delivered leads (`leads[]`, i.e. items with status `delivered` or +`degraded`) as a markdown table **in the order returned**. Exactly three +columns. Then ALWAYS close with the funnel line (below) — even, especially, +when nothing was delivered. + +**Column 1 — Company** + +- Line 1: 10-segment fit bar in inline-code backticks from `lead.fit.score` + (0-100): `filled = round(score/10)`, glyphs `▰` filled / `▱` empty. When + `lead.fit.components.qualification.available` is true AND `ai_score > 0`, + replace the LAST filled segment with `❖` (AI-confirmed cap). When + `fit.available` is false, render `▱▱▱▱▱▱▱▱▱▱` and say "unscored" in col 2. + Never print the numeric score. +- Insert `
`, then: linked company name (target `company.website`, bare + hostnames get `https://`; unlinked plain text when absent) + ` · ` + short + location (City, ST / City, Country) + ` · ` + employees as `min–max` (omit + when `employees.known` is false). + +**Column 2 — Why it fits** + +- One sentence ≤ 20 words. Priority: `fit.reasoning` → gist of + `company.description` → top `fit.components.qualification.matched_tags`. +- If the item status is `degraded` or a requested channel failed, append the + honest flag in italics, e.g. *(email could not be sourced)*. + +**Column 3 — Contact** + +- `[Name](linkedin) · role` (linked name mandatory when a LinkedIn URL + exists; plain name otherwise). Below it, the PURCHASED channels only: + `✉ value` / `☎ value` inline as plain text (they auto-linkify). +- Channel status semantics from `contact.channels.{email,phone}.status`: + `delivered` → show value; `already_owned` → show value + *(already yours)*; + `masked` → "on file — reveal via channels"; `not_requested` → omit; + `failed_previously`/`failed_now` → *(no verified email/phone)*. +- No contact on the item (`contact` null): render `—` (title_gate `prefer` + delivers such rows flagged; say so in col 2 only when contact_titles were + requested). + +**The funnel line (mandatory, after the table):** + +One short line narrating the delivery honestly, built from `funnel` + `cost` + +`explain.scope_notes`: + +> Matched N · examined E · qualified Q · disqualified D → **delivered X of +> the Y asked** · stopped: · spent €C.CC. + +Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → +"ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → +"hit an org quota", `time_budget` → "hit the 30-min time budget". + +**When `delivered` is 0**: NEVER say just "no results". Render no table; give +the funnel line plus the relevant `explain.scope_notes` (they carry the +backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen +rejections), then propose the concrete fix (reshape the seed per the +example_lead craft rules, lower `min_ai_score`, raise `max_cost`, drop a +filter) as NEXT STEPS options. + +**Skipped items** (`skipped[]`, qualify jobs mostly): render a compact second +table `Ref → Outcome` translating `status_reason` to plain words: +`not_in_universe` → "not in the Leadbay universe (import it first)", +`low_confidence_identity` → "couldn't safely match — check `resolution.alternatives`", +`no_matching_contact` → "no contact with the requested title", +`disqualified` → "evaluated: does not fit" (evidence is in the item when owned), +`enrichment_failed` → "channel could not be sourced (not billed)". + +**Hide from the user:** UUIDs (`lead_id`, `contact_id` — keep them for tool +calls, never render), `next_since` cursors, `explain.model`, +`explain.intelligence_snapshot`, raw `distance`/`calibration`, per-item +`seq`/`from_cache`, empty arrays, `estimated_cost` when equal to spent. + +{{include:linking/contact-linkedin}} diff --git a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl index 4fb28171..ed643ef9 100644 --- a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl @@ -19,7 +19,7 @@ Context: Leadbay auto-qualifies roughly the top 10 of each daily batch. Leads be {{include:headers/tool-when-to-use}} when the user wants more qualified leads than what's currently shown, or when a lead looks promising in leadbay_pull_leads but has an empty `qualification_summary`. -{{include:headers/tool-when-not-to-use}} to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). +{{include:headers/tool-when-not-to-use}} to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. {{include:headers/verification-required-if-write}} diff --git a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl index 91cec830..0334a55a 100644 --- a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl @@ -18,6 +18,8 @@ routing: anti_triggers: - phrase: "show me today's leads" route_to: leadbay_pull_leads + - phrase: "find me companies that " + route_to: leadbay_find_new_leads - phrase: "narrow the audience" route_to: leadbay_adjust_audience - phrase: "stop showing me X" diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl new file mode 100644 index 00000000..65abb846 --- /dev/null +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -0,0 +1,111 @@ +--- +name: leadbay_find_new_leads +kind: tool-description +short_description: | + One ask -> n NET-NEW companies matching an ICP the user describes, scored + for fit, optionally AI-qualified with evidence, optionally with the right + contact + verified email/phone. Use when the user describes WHO they want + ("find me 10 gyms around Dallas that would buy X"). Don't use it for + today's lens batch (leadbay_pull_leads) or for vetting companies the user + already has (leadbay_qualify_leads). +routing: + triggers: + - "find me new leads" + - "find me N companies that " + - "get me new prospects like " + - "I need leads in that " + - "search for companies that would buy " + - "net-new leads outside my current pipeline" + - "we're entering — who should we target" + anti_triggers: + - phrase: "show me today's leads / what's new today" + route_to: leadbay_pull_leads + - phrase: "more leads like the ones in my lens" + route_to: leadbay_extend_lens + - phrase: "qualify / vet these companies I have" + route_to: leadbay_qualify_leads + - phrase: "qualify the top N of my batch" + route_to: leadbay_bulk_qualify_leads + - phrase: "leads I should follow up with" + route_to: leadbay_pull_followups + - phrase: "tell me about " + route_to: leadbay_research_lead_by_name_fuzzy + prefer_when: >- + the user describes a target profile or names a count of NEW companies — + craft the example_lead per the seed rules below BEFORE calling; never + pass the user's raw sentence as query. + examples: + positive: + - "Find me 10 gyms around Dallas that would buy our flooring, with someone I can call." + - "Get me 20 new US SaaS companies, 50-2000 employees, with the VP People's email." + - "We're launching in Lyon — find 15 hotels that fit our ICP." + negative: + - "Show me today's leads." + - "Which leads should I follow up with this week?" + - "Qualify these 40 websites from my spreadsheet." +rendering_hint: | + 3-col table of delivered leads in returned order: col 1 = 10-segment fit + bar + linked company · location · size; col 2 = why-fits ≤20 words; col 3 + = contact + purchased channels. ALWAYS close with the honest funnel line + (matched/examined/delivered/stop reason/spend) — especially on 0 + delivered. Full algorithm below. +next_steps: find-new-leads +annotations: + readOnlyHint: false + destructiveHint: false + idempotentHint: true + openWorldHint: true +--- +Submit a net-new lead search: the backend matches an ICP seed against the full +company universe (millions of registry+web profiles), applies hard filters, +skips everything the org already knows (`novelty: org`), optionally qualifies +each candidate against the org's own intelligence (qualification questions, +tags, ideal buyer profile — frozen at submit), and optionally purchases +contact channels. Submit answers in <1s (input errors are synchronous 400s +naming the field); the tool polls up to `wait_seconds` (default 45) and, when +the job needs longer, returns `still_running` + `next_poll` — hand off to +`leadbay_lead_job_status`. Jobs run ≤30 min; results are kept 30 days. + +**Free vs paid — never spend silently.** The default ask (`qualify: false`, +`channels: []`) is FREE: company profile + fit score + cached web research + +contact identity. Paid flags: `qualify: true` (~94 cost_cents per candidate +EXAMINED, survivor or not, capped by `exploration_cap`/`max_cost`) and +`channels` (email 25c / phone 250c, success-only). Before the FIRST paid run +of a session: `dry_run: true`, tell the user the worst-case estimate, get +their explicit go-ahead (an explicit "spend / get their emails" in the user's +message counts). The free tier needs no consent. + +**The two-step pattern that wins**: run FREE first and eyeball whether the +delivered companies are on-profile. If yes, feed that job's deliveries to +`leadbay_qualify_leads` via `prior_deliveries` (spends only on companies +already known to match), or re-run with `qualify: true` under a NEW +request_id when more than the preview is wanted. If off-profile, fix the seed +first — reshaping is free; exploring a bad seed with `qualify: true` burns +budget on candidates qualification will reject. + +{{include:heuristics/example-lead-craft}} + +**Parameter notes** +- `request_id` (REQUIRED) is the retry contract: reuse the SAME value when + retrying the same ask (returns the same live job, no double spend); NEW + value for a changed ask. Derive from the ask + date: `gyms-dallas-2026-07-28`. +- Unresolvable `filters` values 400 naming them — fix the label (see + `leadbay_list_sectors` / `leadbay_list_locations`) and resubmit. +- `min_ai_score` gates the [-30,+30] qualification DELTA, not the 0-100 fit + score; set -30 to keep every evaluated lead with its evidence. +- `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After + beyond — wait, don't hammer). + +**Read the result honestly.** `funnel` + `explain.scope_notes` tell what +happened: matched, pre-screen rejections, examined (billed), qualified, +disqualified, and why the job stopped. Zero delivered is a real outcome that +must be narrated with its cause and a concrete next move — never a bare "no +results". + +--- + +{{include:rendering/lead-delivery-table}} + +--- + +{{include:next-steps/find-new-leads}} diff --git a/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl b/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl new file mode 100644 index 00000000..f453b0e0 --- /dev/null +++ b/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl @@ -0,0 +1,64 @@ +--- +name: leadbay_lead_job_status +kind: tool-description +short_description: | + Poll a running leadbay_find_new_leads / leadbay_qualify_leads job by + job_id — cumulative results, funnel, spend. Use when a delivery tool + returned still_running:true or the user asks "is it done / any results + yet". Don't use it for enrichment jobs (leadbay_bulk_enrich_status) or + imports (leadbay_import_status). +routing: + triggers: + - "is the lead search done" + - "any results yet on that job" + - "check on the delivery" + anti_triggers: + - phrase: "is the enrichment done" + route_to: leadbay_bulk_enrich_status + - phrase: "is my import done" + route_to: leadbay_import_status + - phrase: "is the top-N qualification done" + route_to: leadbay_qualify_status + prefer_when: >- + a find_new_leads / qualify_leads result carried next_poll — pass its + job_id; use wait_seconds ~60 when the user asked to wait for results. + examples: + positive: + - "Any leads yet from that search you started?" + - "Wait for the qualification job to finish and show me everything." + negative: + - "Is the email enrichment finished?" + - "Is my CSV import done?" +rendering_hint: | + Terminal job -> render the full delivery per the lead-delivery table + + honest funnel line. Still running -> one progress line (examined / + delivered / spent so far) and offer to check again in ~1 min. Never + render UUIDs or cursors. +next_steps: lead-job-status +annotations: + readOnlyHint: true + destructiveHint: false + idempotentHint: true + openWorldHint: true +--- +Cumulative snapshot of a lead-delivery job: state, funnel counters, every +item emitted so far (full lead payloads for delivered/degraded, honest +status_reason for skipped), spend + breakdown, and the `explain` block +(basis, seed strategy, scope notes). Items are immutable once emitted — +polling never re-reads live data, so numbers only ever grow. + +`wait_seconds: 0` (default) answers instantly; set ~60 to block-wait for +completion when the user asked for results "in this reply". `since` (from a +prior poll's `next_since`) pages only the new items. Jobs terminalize +server-side: past the 30-min wall clock a job reads `completed_partial` +(time budget), past 30 days `expired` (items no longer listed — re-read +billed leads via leadbay_qualify_leads `prior_deliveries`). A 404 means +unknown job or another org's job. + +--- + +{{include:rendering/lead-delivery-table}} + +--- + +{{include:next-steps/lead-job-status}} diff --git a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl index 28711a3e..27d7d90c 100644 --- a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl @@ -21,6 +21,10 @@ routing: # intercepting legitimate discovery intent. E.g. "reach out to new leads" should # still fire pull_leads; only "leads I should reach out to" (existing pipeline) → pull_followups. anti_triggers: + - phrase: "find me N companies that " + route_to: leadbay_find_new_leads + - phrase: "new prospects like with their emails" + route_to: leadbay_find_new_leads - phrase: "leads I should follow up with" route_to: leadbay_pull_followups - phrase: "I'm going to " @@ -49,6 +53,7 @@ routing: - "Show me today's leads." - "Pull my best new prospects." negative: + - "Find me 10 gyms around Dallas that would buy our flooring." # → leadbay_find_new_leads (custom net-new ask, not the lens batch) - "Which leads should I follow up with this week?" - "I'm flying to Berlin Thursday — who should I meet?" # → leadbay_tour_plan (geographic visit, wants a map) - "I'm in San Francisco next Tuesday — who's worth meeting?" # → leadbay_tour_plan diff --git a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl new file mode 100644 index 00000000..3aa5024d --- /dev/null +++ b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl @@ -0,0 +1,96 @@ +--- +name: leadbay_qualify_leads +kind: tool-description +short_description: | + Qualify companies the user ALREADY has — CRM rows, spreadsheet websites, + prior deliveries — and get the right contact on each, in one server-side + batch (up to 500). Per-item verdicts with evidence; a bad ref never fails + the job. Use when the user names/lists the companies. Don't use it to + FIND new companies (leadbay_find_new_leads) or for the lens top-N + (leadbay_bulk_qualify_leads). +routing: + triggers: + - "qualify these companies" + - "vet this list" + - "which of these fit our ICP" + - "score these websites / accounts" + - "get me the right contact at these companies" + - "re-qualify what you delivered last week" + anti_triggers: + - phrase: "find me new leads / companies that " + route_to: leadbay_find_new_leads + - phrase: "qualify the top N of my lens batch" + route_to: leadbay_bulk_qualify_leads + - phrase: "import this CSV file" + route_to: leadbay_import_leads + - phrase: "tell me about in depth" + route_to: leadbay_research_lead_by_name_fuzzy + - phrase: "add emails to the contacts I selected" + route_to: leadbay_enrich_titles + prefer_when: >- + the user points at SPECIFIC companies (ids, websites, names, a pasted + list, "what you found yesterday") and wants fit verdicts and/or the + right person to talk to. + examples: + positive: + - "Here are 60 restaurant websites from my Austin sweep — which fit, and who's the owner?" + - "Re-qualify last week's delivery and get phone numbers for the good ones." + - "Vet these 12 accounts from my spreadsheet against our criteria." + negative: + - "Find me 10 new gyms in Texas." + - "Qualify the top 10 leads in my batch." + - "I have a CSV of 400 attendees to import." +rendering_hint: | + 3-col table for delivered items (fit bar + company / why-fits ≤20 words / + contact + channels) in returned order, then a compact Ref → Outcome table + for skipped refs (not_in_universe, low_confidence_identity, ... in plain + words), then the honest funnel + cost line. Full algorithm below. +next_steps: qualify-leads +annotations: + readOnlyHint: false + destructiveHint: false + idempotentHint: false + openWorldHint: true +--- +Submit a qualify batch over companies the org already has (or that exist in +the Leadbay universe): each ref is resolved to a known company, freshly +researched + AI-qualified against the org's questions / tags / ideal buyer +profile (frozen at submit), matched to the requested contact titles, and — +when asked — enriched with verified channels. Answers arrive per-item from a +job; this tool polls up to `wait_seconds` (default 45) and hands off to +`leadbay_lead_job_status` when the batch needs longer. + +**Refs are flexible; outcomes are per-item.** `lead_refs` accepts any mix of +`lead_id`, `website`, `name`(+`location`), or a stable `contact_id` from a +prior result (enrichment then targets exactly that person, never a re-match). +`prior_deliveries` expands past MCP deliveries into refs — billed leads stay +re-readable this way even after the 30-day result window. Duplicates collapse. +A ref that can't be served comes back `skipped` with an honest +`status_reason` (`not_in_universe`, `low_confidence_identity` with the +`resolution.alternatives` to choose from, `no_matching_contact`, ...) — that +is an ANSWER about the ref, not an error, and it costs nothing. + +**Disqualified ≠ dropped.** Companies the org owns that fail qualification +are DELIVERED with their negative evidence (question verdicts, tag misses, +IBP reasoning) — "here's why to skip this account" is a deliverable. + +**Cost — never spend silently.** Resolution and identity are free. +`qualify: true` (the default) costs ~94 cost_cents per lead needing FRESH +research+scoring — but repeat calls reuse every fresh cached stage +(`from_cache` flags on the items) and converge to near-zero cost. `channels` +purchase verified email (25c) / phone (250c) on success only; +`already_owned` values cost nothing. Before a first paid run on a large +batch: `dry_run: true`, quote the worst case, get the explicit go-ahead +(an explicit "spend / get their emails" in the user's message counts). +Set `request_id` and reuse it on retries of the same batch. + +**Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + +Retry-After beyond — wait, don't hammer), 30-min job wall clock. + +--- + +{{include:rendering/lead-delivery-table}} + +--- + +{{include:next-steps/qualify-leads}} From a83f4505f517e9be2273294d0d50aeae189fafa6 Mon Sep 17 00:00:00 2001 From: milstan Date: Tue, 28 Jul 2026 12:53:44 +0200 Subject: [PATCH 02/46] docs(mcp): temper min_ai_score guidance after judge review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Never combine a lowered qualification floor with channels — that buys emails for leads the AI just scored as junk (measured: 633c for a floor-scored consultancy's email on staging). Co-Authored-By: Claude Fable 5 --- packages/core/src/tool-descriptions.generated.ts | 3 ++- .../tool-descriptions/composite/find-new-leads.md.tmpl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index f011c194..902c3208 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1386,7 +1386,8 @@ load-bearing (validated live against staging, 2026-07-28): - Unresolvable \`filters\` values 400 naming them — fix the label (see \`leadbay_list_sectors\` / \`leadbay_list_locations\`) and resubmit. - \`min_ai_score\` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score; set -30 to keep every evaluated lead with its evidence. + score. Lower it only to STUDY evidence — never combine a lowered floor + with \`channels\` (that buys emails for leads the AI just scored as junk). - \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After beyond — wait, don't hammer). diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 65abb846..0b7fcb86 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -92,7 +92,8 @@ budget on candidates qualification will reject. - Unresolvable `filters` values 400 naming them — fix the label (see `leadbay_list_sectors` / `leadbay_list_locations`) and resubmit. - `min_ai_score` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score; set -30 to keep every evaluated lead with its evidence. + score. Lower it only to STUDY evidence — never combine a lowered floor + with `channels` (that buys emails for leads the AI just scored as junk). - `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After beyond — wait, don't hammer). From 46f60b799f2663817c894dfac6baea7614e90e9f Mon Sep 17 00:00:00 2001 From: milstan Date: Tue, 28 Jul 2026 22:42:02 +0200 Subject: [PATCH 03/46] test(eval): routing fixtures for the MCP-first delivery tools Co-Authored-By: Claude Fable 5 --- .../tool-descriptions/routing-fixtures.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts b/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts index d9dc38db..60f5d0f8 100644 --- a/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts +++ b/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts @@ -155,4 +155,46 @@ export const ROUTING_FIXTURES: RoutingFixture[] = [ { intent: "Remove the epilogue entry for this lead.", expected_tool: "leadbay_remove_epilogue" }, { intent: "Preview the cost of bulk enrichment before launching.", expected_tool: "leadbay_preview_bulk_enrichment" }, { intent: "Launch the bulk enrichment job.", expected_tool: "leadbay_launch_bulk_enrichment" }, + + // MCP-first lead delivery (net-new search / batch qualify / job poll) + { + intent: "Find me 10 gyms around Dallas that would buy our flooring, with someone I can call.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_pull_leads", "leadbay_extend_lens"], + }, + { + intent: "Get me 20 brand-new prospects that look like our best customer, with the VP People's email.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_pull_leads", "leadbay_enrich_titles"], + }, + { + intent: "We're entering the Lyon market — find 15 hotels that fit our ICP.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_new_lens"], + }, + { + intent: "Here are 60 restaurant websites from my Austin sweep — which fit our profile, and who's the owner at each?", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: ["leadbay_find_new_leads", "leadbay_bulk_qualify_leads", "leadbay_import_leads"], + }, + { + intent: "Re-qualify everything you delivered last week and get phone numbers for the good ones.", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: ["leadbay_bulk_qualify_leads"], + }, + { + intent: "Vet these 12 accounts from my spreadsheet against our qualification criteria.", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: ["leadbay_import_leads", "leadbay_bulk_qualify_leads"], + }, + { + intent: "Any results yet from that lead search job you started?", + expected_tool: "leadbay_lead_job_status", + forbidden_tools: ["leadbay_bulk_enrich_status", "leadbay_import_status", "leadbay_qualify_status"], + }, + { + intent: "Wait for the lead delivery job to finish and show me everything it found.", + expected_tool: "leadbay_lead_job_status", + forbidden_tools: ["leadbay_bulk_enrich_status"], + }, ]; From 2720d4579cba34bb92b5c5d20338e3330c709d15 Mon Sep 17 00:00:00 2001 From: milstan Date: Thu, 30 Jul 2026 14:46:30 +0200 Subject: [PATCH 04/46] docs(mcp): two-pass spending pattern for qualify_leads channel purchases A single qualify+channels pass buys emails for leads the same job disqualifies (measured: ~a third of enrichment spend on staging). Teach verdicts-first, channels-second as the default; backend gate tracked in product#3938. Co-Authored-By: Claude Fable 5 --- packages/core/src/tool-descriptions.generated.ts | 9 +++++++++ .../tool-descriptions/composite/qualify-leads.md.tmpl | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 902c3208..92d24748 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -3504,6 +3504,15 @@ batch: \`dry_run: true\`, quote the worst case, get the explicit go-ahead (an explicit "spend / get their emails" in the user's message counts). Set \`request_id\` and reuse it on retries of the same batch. +**Two-pass spending pattern — the default for channel purchases.** A single +qualify+channels pass buys emails even for leads the SAME job disqualifies +(measured on staging: a third of enrichment spend landed on ai-negative +leads). Instead: pass 1 with \`qualify: true\` and NO channels → read the +verdicts; pass 2 with \`channels\` and \`lead_refs\` restricted to the +positively-qualified leads (their \`lead_id\`s, or \`contact_id\` to pin the +matched person). The cache makes pass 2 cost only the channels. Collapse to +one pass only when the user explicitly wants channels regardless of verdict. + **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. diff --git a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl index 3aa5024d..2ab896f3 100644 --- a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl @@ -84,6 +84,15 @@ batch: `dry_run: true`, quote the worst case, get the explicit go-ahead (an explicit "spend / get their emails" in the user's message counts). Set `request_id` and reuse it on retries of the same batch. +**Two-pass spending pattern — the default for channel purchases.** A single +qualify+channels pass buys emails even for leads the SAME job disqualifies +(measured on staging: a third of enrichment spend landed on ai-negative +leads). Instead: pass 1 with `qualify: true` and NO channels → read the +verdicts; pass 2 with `channels` and `lead_refs` restricted to the +positively-qualified leads (their `lead_id`s, or `contact_id` to pin the +matched person). The cache makes pass 2 cost only the channels. Collapse to +one pass only when the user explicitly wants channels regardless of verdict. + **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. From 6ad2b36e90b1268717bc2800c7b930952c04b2b5 Mon Sep 17 00:00:00 2001 From: milstan Date: Thu, 30 Jul 2026 14:49:54 +0200 Subject: [PATCH 05/46] chore(mcp): align server.json npx pin with the 0.26.0 bump The 0.26.0 bump merged from main left the @leadbay/mcp@0.25 pin behind; the server-json-version audit rightly failed. Pin follows the version line. Co-Authored-By: Claude Fable 5 --- packages/mcp/server.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mcp/server.json b/packages/mcp/server.json index d790306f..7dd87058 100644 --- a/packages/mcp/server.json +++ b/packages/mcp/server.json @@ -27,7 +27,7 @@ }, { "type": "positional", - "value": "@leadbay/mcp@0.25" + "value": "@leadbay/mcp@0.26" } ], "environmentVariables": [ From 3dbabf3cf4cf84ae4332388d4a9d784bae289c0e Mon Sep 17 00:00:00 2001 From: milstan Date: Thu, 30 Jul 2026 15:03:43 +0200 Subject: [PATCH 06/46] revert(mcp): drop the two-pass spending pattern from qualify_leads Counter-mission: MCP-first delivery exists to eliminate client-side chaining; teaching agents to orchestrate two passes reintroduces it. The spend-gating belongs in the backend (product#3938). Co-Authored-By: Claude Fable 5 --- packages/core/src/tool-descriptions.generated.ts | 9 --------- .../tool-descriptions/composite/qualify-leads.md.tmpl | 9 --------- 2 files changed, 18 deletions(-) diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 92d24748..902c3208 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -3504,15 +3504,6 @@ batch: \`dry_run: true\`, quote the worst case, get the explicit go-ahead (an explicit "spend / get their emails" in the user's message counts). Set \`request_id\` and reuse it on retries of the same batch. -**Two-pass spending pattern — the default for channel purchases.** A single -qualify+channels pass buys emails even for leads the SAME job disqualifies -(measured on staging: a third of enrichment spend landed on ai-negative -leads). Instead: pass 1 with \`qualify: true\` and NO channels → read the -verdicts; pass 2 with \`channels\` and \`lead_refs\` restricted to the -positively-qualified leads (their \`lead_id\`s, or \`contact_id\` to pin the -matched person). The cache makes pass 2 cost only the channels. Collapse to -one pass only when the user explicitly wants channels regardless of verdict. - **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. diff --git a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl index 2ab896f3..3aa5024d 100644 --- a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl @@ -84,15 +84,6 @@ batch: `dry_run: true`, quote the worst case, get the explicit go-ahead (an explicit "spend / get their emails" in the user's message counts). Set `request_id` and reuse it on retries of the same batch. -**Two-pass spending pattern — the default for channel purchases.** A single -qualify+channels pass buys emails even for leads the SAME job disqualifies -(measured on staging: a third of enrichment spend landed on ai-negative -leads). Instead: pass 1 with `qualify: true` and NO channels → read the -verdicts; pass 2 with `channels` and `lead_refs` restricted to the -positively-qualified leads (their `lead_id`s, or `contact_id` to pin the -matched person). The cache makes pass 2 cost only the channels. Collapse to -one pass only when the user explicitly wants channels regardless of verdict. - **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. From 502f8a520d4774a59fa88fa4481b03220aad3029 Mon Sep 17 00:00:00 2001 From: milstan Date: Thu, 30 Jul 2026 15:35:32 +0200 Subject: [PATCH 07/46] =?UTF-8?q?fix(mcp):=20eval-driven=20fixes=20?= =?UTF-8?q?=E2=80=94=20nested-employees=20normalizer,=20corrected=20exclus?= =?UTF-8?q?ion/location/fit=20guidance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the live seed-interpretation eval (2/2 cold agents): tolerate the agent-invented filters.employees:{min,max} by mapping onto the flat wire keys; name the exact filter keys in the craft rules; never a country in locations (France silently matched the village of Francs); exclusions are enforced by qualification via IBP anti-patterns (leadbay_refine_prompt for durable ones), not by the free match; rendering gains weak-batch honesty (best fit <30), geo fence checks, and skepticism for exemplar-bridged FR fit scores. Co-Authored-By: Claude Fable 5 --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 13 +- .../core/src/composite/_mcp-job-helpers.ts | 23 +++ packages/core/src/composite/find-new-leads.ts | 3 +- .../core/src/tool-descriptions.generated.ts | 187 ++++++++++-------- .../unit/composite/find-new-leads.test.ts | 28 +++ packages/mcp/src/prompts.generated.ts | 15 +- .../prompts/leadbay_new_leads.md.tmpl | 15 +- .../snippets/heuristics/example-lead-craft.md | 37 ++-- .../snippets/rendering/lead-delivery-table.md | 26 ++- .../composite/find-new-leads.md.tmpl | 70 ++++--- 10 files changed, 266 insertions(+), 151 deletions(-) diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md index efc84be1..e1ca3337 100644 --- a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md +++ b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md @@ -42,8 +42,10 @@ From my words, work out: Compose the `example_lead` for each archetype following the craft rules in the leadbay_find_new_leads description (registry-style description of a FICTIONAL typical buyer; no invented brand name; no event language; hard -constraints go in `filters`, not prose). Show me the seed description(s) in -one line each — I should recognize my ideal customer in them. +constraints go in `filters` with the FLAT keys `employees_min`/`employees_max` +and city/state/region `locations` — never a country name). Show me the seed +description(s) in one line each — I should recognize my ideal customer in +them. # PHASE 3 — FREE PREVIEW @@ -73,6 +75,13 @@ When I want qualification evidence and/or reachable contacts: # PHASE 5 — DELIVER +Before rendering, sanity-check every row: geography inside my fence (drop +and call out same-named-city leaks), descriptions actually matching my ask +(especially when `explain.seed_strategy` is `text_match_exemplars` — fit +scores run hot there), visible violations of my exclusions dropped. If the +best fit is under 30, say "weak matches only" and propose reshaping before +showing more than 3. + Render per the lead-delivery table, then ALWAYS the funnel line: matched / examined / qualified / disqualified / delivered / stop reason / spend. Zero delivered gets a diagnosis and a concrete next move, never a shrug. Close diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 9d81fb9b..2066b1b5 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -212,6 +212,29 @@ export function compactBody( ); } +/** Tolerant reader for the search `filters` object. The RESULT payload's + * company shape (`employees: {min, max, known}`) teaches agents a nested + * employees object, and in live evals 2/2 cold agents passed exactly that + * on input — which the backend rejects with an unhelpful deserialization + * 400. Map it (and the camelCase spellings) onto the flat wire keys + * instead of failing the whole ask. */ +export function normalizeSearchFilters( + filters: Record | undefined +): Record | undefined { + if (filters == null) return undefined; + const { employees, employeesMin, employeesMax, ...rest } = filters; + const out: Record = { ...rest }; + if (out.employees_min == null) { + out.employees_min = employees?.min ?? employees?.employees_min ?? employeesMin; + } + if (out.employees_max == null) { + out.employees_max = employees?.max ?? employees?.employees_max ?? employeesMax; + } + if (out.employees_min == null) delete out.employees_min; + if (out.employees_max == null) delete out.employees_max; + return out; +} + export function clampWaitSeconds( requested: number | undefined, fallback: number diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 3aca29a8..d2731e7c 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, compactBody, + normalizeSearchFilters, splitItems, TERMINAL_JOB_STATES, waitForJob, @@ -180,7 +181,7 @@ export const findNewLeads: Tool = { const body = compactBody({ query: params.query, example_lead: params.example_lead, - filters: params.filters, + filters: normalizeSearchFilters(params.filters), count: params.count, qualify: params.qualify, min_ai_score: params.min_ai_score, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 902c3208..15ff74b3 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1302,41 +1302,39 @@ delivered. Full algorithm below. --- -Submit a net-new lead search: the backend matches an ICP seed against the full -company universe (millions of registry+web profiles), applies hard filters, -skips everything the org already knows (\`novelty: org\`), optionally qualifies -each candidate against the org's own intelligence (qualification questions, -tags, ideal buyer profile — frozen at submit), and optionally purchases -contact channels. Submit answers in <1s (input errors are synchronous 400s -naming the field); the tool polls up to \`wait_seconds\` (default 45) and, when -the job needs longer, returns \`still_running\` + \`next_poll\` — hand off to -\`leadbay_lead_job_status\`. Jobs run ≤30 min; results are kept 30 days. +Submit a net-new lead search: the backend matches an ICP seed against the +full company universe, applies hard filters, skips everything the org already +knows (\`novelty: org\`), optionally qualifies candidates against the org's own +intelligence (questions, tags, ideal buyer profile — frozen at submit), and +optionally purchases contact channels. Submit answers in <1s; the tool polls +up to \`wait_seconds\` (default 45) and, when the job needs longer, returns +\`still_running\` + \`next_poll\` — hand off to \`leadbay_lead_job_status\`. Jobs +run ≤30 min; results are kept 30 days. **Free vs paid — never spend silently.** The default ask (\`qualify: false\`, -\`channels: []\`) is FREE: company profile + fit score + cached web research + -contact identity. Paid flags: \`qualify: true\` (~94 cost_cents per candidate -EXAMINED, survivor or not, capped by \`exploration_cap\`/\`max_cost\`) and -\`channels\` (email 25c / phone 250c, success-only). Before the FIRST paid run -of a session: \`dry_run: true\`, tell the user the worst-case estimate, get -their explicit go-ahead (an explicit "spend / get their emails" in the user's -message counts). The free tier needs no consent. - -**The two-step pattern that wins**: run FREE first and eyeball whether the -delivered companies are on-profile. If yes, feed that job's deliveries to -\`leadbay_qualify_leads\` via \`prior_deliveries\` (spends only on companies -already known to match), or re-run with \`qualify: true\` under a NEW -request_id when more than the preview is wanted. If off-profile, fix the seed -first — reshaping is free; exploring a bad seed with \`qualify: true\` burns -budget on candidates qualification will reject. +\`channels: []\`) is FREE: company profile + fit score + cached research + +contact identity. Paid: \`qualify: true\` (~94 cost_cents per candidate +EXAMINED, capped by \`exploration_cap\`/\`max_cost\`) and \`channels\` (email 25c / +phone 250c, success-only). Before the first paid run: \`dry_run: true\`, quote +the worst case, get the explicit go-ahead ("spend / get their emails" in the +user's message counts). Free needs no consent. + +**Free preview first**: run FREE, eyeball fit, THEN pay. Off-profile preview +→ reshape the seed (free) instead of exploring it with \`qualify: true\`. + +**Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not +the free match.** Positive inverse in the seed, then \`qualify: true\`: the +org's ideal-buyer-profile anti-patterns and questions score violators +negative and \`min_ai_score: 0\` drops them. Durable exclusion → suggest +\`leadbay_refine_prompt\` so it enters the org intelligence. In a FREE preview, +drop visibly violating rows and say the free tier doesn't enforce. ### Crafting the \`example_lead\` seed — the input that decides result quality The \`example_lead\` is a FICTIONAL typical ideal customer. Its text is embedded -and matched against millions of real company descriptions sourced from business -registries and company websites. Those descriptions state what a company **IS** -(stable business profile) — never what is happening. Write the seed the same -way, or the matcher drifts to the wrong companies. Each rule below is -load-bearing (validated live against staging, 2026-07-28): +and matched against millions of real registry/website company descriptions, +which state what a company **IS** — never what is happening. Write the seed +the same way or the matcher drifts. Every rule below is measured: 1. **Describe the BUYER, never the seller.** Before writing, answer: "would this company write a check to my user?" A seed that describes what the user @@ -1356,23 +1354,26 @@ load-bearing (validated live against staging, 2026-07-28): across multiple club locations." - WEAK (generic): "A gym in Texas." - WRONG (seller-side): "Supplier of durable modular flooring for gyms." -4. **No event language.** "hiring", "expanding", "just raised", "opening a new - site" are not filters — real registry descriptions never contain them, so - they dilute the profile and attract event-flavored noise. Temporal criteria - in a \`query\` become best-effort ranking annotations at most (the response - \`explain.scope_notes\` says so). Put purchase-trigger criteria in the org's - qualification questions instead, where the paid qualification stage scores - them from fresh research. +4. **No event language.** "hiring", "expanding", "just raised" are not + filters — registry descriptions never contain them, so they dilute the + profile. Purchase-trigger criteria belong in the org's qualification + questions, where the paid stage scores them from fresh research. 5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — real descriptions don't carry them. -6. **Hard constraints go in \`filters\`, not prose.** Geography, sector, size - bounds written into the description only *tint* the ranking; \`filters\` are - enforced. Seed describes the archetype; filters draw the fence. +6. **Hard constraints go in \`filters\`, not prose — exact keys:** + \`sectors: string[]\`, \`locations: string[]\`, \`employees_min: number\`, + \`employees_max: number\`. FLAT numbers — a nested \`employees: {min, max}\` + object exists only in RESULT payloads, never on input. \`locations\` take + city/state/region names ("Dallas, TX", "Texas", "Île-de-France"); NEVER + a country — each universe is single-country, so whole-country intent = + omit \`locations\` (a country name silently matches a same-named town: + measured, "France" → the village of Francs). \`example_lead.employees\` + does not filter; only \`filters.employees_min/max\` do. 7. **Prefer \`example_lead\` over \`query\`.** Query text matches topic *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS - as strongly as gym BUYERS (measured: the same ICP delivered 0 leads from a - query and on-profile leads from an example_lead). Use \`query\` only when the - user's own wording carries signal an example can't express. + as strongly as gym BUYERS (measured: 0 delivered from the query, on-profile + from the example). \`query\` only when the user's wording carries signal an + example can't express. 8. **One seed per buyer archetype.** If the ask spans two distinct segments (e.g. "gyms and logistics warehouses"), run one search per segment with its own description — a blended seed lands between the two clusters and matches @@ -1380,22 +1381,18 @@ load-bearing (validated live against staging, 2026-07-28): **Parameter notes** -- \`request_id\` (REQUIRED) is the retry contract: reuse the SAME value when - retrying the same ask (returns the same live job, no double spend); NEW - value for a changed ask. Derive from the ask + date: \`gyms-dallas-2026-07-28\`. -- Unresolvable \`filters\` values 400 naming them — fix the label (see - \`leadbay_list_sectors\` / \`leadbay_list_locations\`) and resubmit. +- \`request_id\` (REQUIRED) is the retry contract: SAME value when retrying + the same ask (returns the same live job, no double spend); NEW value for a + changed ask. Derive from ask + date: \`gyms-dallas-2026-07-28\`. - \`min_ai_score\` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score. Lower it only to STUDY evidence — never combine a lowered floor - with \`channels\` (that buys emails for leads the AI just scored as junk). -- \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After - beyond — wait, don't hammer). + score. Lower it only to STUDY evidence — never with \`channels\` (that buys + emails for leads the AI just scored as junk). +- \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — + wait, don't hammer). -**Read the result honestly.** \`funnel\` + \`explain.scope_notes\` tell what -happened: matched, pre-screen rejections, examined (billed), qualified, -disqualified, and why the job stopped. Zero delivered is a real outcome that -must be narrated with its cause and a concrete next move — never a bare "no -results". +**Read the result honestly.** \`funnel\` + \`explain.scope_notes\` tell the +story; zero delivered gets a cause + a concrete next move, never a bare "no +results" (rules in RENDERING). --- @@ -1431,10 +1428,9 @@ when nothing was delivered. - \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL exists; plain name otherwise). Below it, the PURCHASED channels only: \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). -- Channel status semantics from \`contact.channels.{email,phone}.status\`: - \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; - \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; - \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- Channel statuses: \`delivered\` → show value; \`already_owned\` → value + + *(already yours)*; \`masked\` → "on file — reveal via channels"; + \`not_requested\` → omit; \`failed_*\` → *(no verified email/phone)*. - No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` delivers such rows flagged; say so in col 2 only when contact_titles were requested). @@ -1458,6 +1454,18 @@ rejections), then propose the concrete fix (reshape the seed per the example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS options. +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not +present the table as an answer — open with "weak matches only", show at most +the top 3, and propose reshaping the seed/filters first. The count was +filled with barely-better-than-random candidates, not good ones. + +**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must +sit inside any requested fence; drop and call out leaks (a same-named city +in another state slips through). (b) When \`explain.seed_strategy\` is +\`text_match_exemplars\` (the standard FR path), fit is calibrated for +lead-to-lead distances, not exemplar centroids — treat high scores +skeptically and verify each row's \`description\` actually matches the ask. + **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`not_in_universe\` → "not in the Leadbay universe (import it first)", @@ -1466,10 +1474,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". -**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool -calls, never render), \`next_since\` cursors, \`explain.model\`, -\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item -\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. +**Hide from the user:** UUIDs (keep for tool calls, never render), cursors, +\`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, +\`seq\`/\`from_cache\`, empty arrays. ## Linking a contact's name @@ -2299,10 +2306,9 @@ when nothing was delivered. - \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL exists; plain name otherwise). Below it, the PURCHASED channels only: \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). -- Channel status semantics from \`contact.channels.{email,phone}.status\`: - \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; - \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; - \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- Channel statuses: \`delivered\` → show value; \`already_owned\` → value + + *(already yours)*; \`masked\` → "on file — reveal via channels"; + \`not_requested\` → omit; \`failed_*\` → *(no verified email/phone)*. - No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` delivers such rows flagged; say so in col 2 only when contact_titles were requested). @@ -2326,6 +2332,18 @@ rejections), then propose the concrete fix (reshape the seed per the example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS options. +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not +present the table as an answer — open with "weak matches only", show at most +the top 3, and propose reshaping the seed/filters first. The count was +filled with barely-better-than-random candidates, not good ones. + +**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must +sit inside any requested fence; drop and call out leaks (a same-named city +in another state slips through). (b) When \`explain.seed_strategy\` is +\`text_match_exemplars\` (the standard FR path), fit is calibrated for +lead-to-lead distances, not exemplar centroids — treat high scores +skeptically and verify each row's \`description\` actually matches the ask. + **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`not_in_universe\` → "not in the Leadbay universe (import it first)", @@ -2334,10 +2352,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". -**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool -calls, never render), \`next_since\` cursors, \`explain.model\`, -\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item -\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. +**Hide from the user:** UUIDs (keep for tool calls, never render), cursors, +\`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, +\`seq\`/\`from_cache\`, empty arrays. ## Linking a contact's name @@ -3541,10 +3558,9 @@ when nothing was delivered. - \`[Name](linkedin) · role\` (linked name mandatory when a LinkedIn URL exists; plain name otherwise). Below it, the PURCHASED channels only: \`✉ value\` / \`☎ value\` inline as plain text (they auto-linkify). -- Channel status semantics from \`contact.channels.{email,phone}.status\`: - \`delivered\` → show value; \`already_owned\` → show value + *(already yours)*; - \`masked\` → "on file — reveal via channels"; \`not_requested\` → omit; - \`failed_previously\`/\`failed_now\` → *(no verified email/phone)*. +- Channel statuses: \`delivered\` → show value; \`already_owned\` → value + + *(already yours)*; \`masked\` → "on file — reveal via channels"; + \`not_requested\` → omit; \`failed_*\` → *(no verified email/phone)*. - No contact on the item (\`contact\` null): render \`—\` (title_gate \`prefer\` delivers such rows flagged; say so in col 2 only when contact_titles were requested). @@ -3568,6 +3584,18 @@ rejections), then propose the concrete fix (reshape the seed per the example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS options. +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not +present the table as an answer — open with "weak matches only", show at most +the top 3, and propose reshaping the seed/filters first. The count was +filled with barely-better-than-random candidates, not good ones. + +**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must +sit inside any requested fence; drop and call out leaks (a same-named city +in another state slips through). (b) When \`explain.seed_strategy\` is +\`text_match_exemplars\` (the standard FR path), fit is calibrated for +lead-to-lead distances, not exemplar centroids — treat high scores +skeptically and verify each row's \`description\` actually matches the ask. + **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`not_in_universe\` → "not in the Leadbay universe (import it first)", @@ -3576,10 +3604,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". -**Hide from the user:** UUIDs (\`lead_id\`, \`contact_id\` — keep them for tool -calls, never render), \`next_since\` cursors, \`explain.model\`, -\`explain.intelligence_snapshot\`, raw \`distance\`/\`calibration\`, per-item -\`seq\`/\`from_cache\`, empty arrays, \`estimated_cost\` when equal to spent. +**Hide from the user:** UUIDs (keep for tool calls, never render), cursors, +\`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, +\`seq\`/\`from_cache\`, empty arrays. ## Linking a contact's name diff --git a/packages/core/test/unit/composite/find-new-leads.test.ts b/packages/core/test/unit/composite/find-new-leads.test.ts index 3705ed76..8c675db0 100644 --- a/packages/core/test/unit/composite/find-new-leads.test.ts +++ b/packages/core/test/unit/composite/find-new-leads.test.ts @@ -210,6 +210,34 @@ describe("leadbay_find_new_leads", () => { } }); + it("normalizes an agent-invented nested filters.employees onto the flat wire keys", async () => { + // Live eval 2026-07-30: 2/2 cold agents passed employees:{min,max} + // (the RESULT shape) and the backend 400'd the whole ask. The composite + // maps it instead of failing. + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: snapshot("completed", [DELIVERED_ITEM], "target_reached"), + }, + ]); + await findNewLeads.execute(newClient(), { + example_lead: { description: "B2B SaaS with in-house sales teams." }, + filters: { locations: ["Texas"], employees: { min: 100, max: 1000 } } as any, + count: 3, + request_id: "probe-nested-emp", + wait_seconds: 0, + }); + const body = JSON.parse(getHttpRequests().find((r) => r.method === "POST")!.body!); + expect(body.filters).toEqual({ + locations: ["Texas"], + employees_min: 100, + employees_max: 1000, + }); + }); + it("429 refusal (rate cap) — propagates as a quota error", async () => { mockHttp([ { diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index e2ded8b7..ee1745b2 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -893,8 +893,10 @@ From my words, work out: Compose the \`example_lead\` for each archetype following the craft rules in the leadbay_find_new_leads description (registry-style description of a FICTIONAL typical buyer; no invented brand name; no event language; hard -constraints go in \`filters\`, not prose). Show me the seed description(s) in -one line each — I should recognize my ideal customer in them. +constraints go in \`filters\` with the FLAT keys \`employees_min\`/\`employees_max\` +and city/state/region \`locations\` — never a country name). Show me the seed +description(s) in one line each — I should recognize my ideal customer in +them. # PHASE 3 — FREE PREVIEW @@ -924,6 +926,13 @@ When I want qualification evidence and/or reachable contacts: # PHASE 5 — DELIVER +Before rendering, sanity-check every row: geography inside my fence (drop +and call out same-named-city leaks), descriptions actually matching my ask +(especially when \`explain.seed_strategy\` is \`text_match_exemplars\` — fit +scores run hot there), visible violations of my exclusions dropped. If the +best fit is under 30, say "weak matches only" and propose reshaping before +showing more than 3. + Render per the lead-delivery table, then ALWAYS the funnel line: matched / examined / qualified / disqualified / delivered / stop reason / spend. Zero delivered gets a diagnosis and a concrete next move, never a shrug. Close @@ -1632,7 +1641,7 @@ export const PROMPT_META = { leadbay_followup_check_in: {"name":"leadbay_followup_check_in","short_description":"Follow-up check-in: surface KNOWN leads from the Monitor view needing\nre-engagement. Trigger on \"follow up\", \"already known leads\", \"what's\noverdue\", \"before my trip\", \"who should I re-engage\". Do NOT trigger on\n\"show me today's leads\", \"my morning check-in\", \"run my check-in\",\n\"I do this every day\", \"every morning\" — those go to\n`leadbay_daily_check_in`.\n","arguments":[],"expected_calls":["leadbay_pull_followups","leadbay_research_lead_by_id","leadbay_prepare_outreach"],"failure_modes":["Calls leadbay_pull_leads (the Discover entry point) instead of leadbay_pull_followups — these are different data sources; the Discover queue does NOT contain Monitor's known-but-cold pipeline","Iterates pages of leadbay_pull_leads filtering by engagement_count to \"fake\" a follow-up view (a real bug observed in 0.9.0 — the right move is to call pull_followups directly)","Replaces the canonical pull_followups table layout with prose per row (the per-tool RENDERING block is the structural contract; commentary belongs above or below)","Skips the cross-mode pivot offer at the end (\"Want to see NEW leads from your wishlist instead?\" routes to leadbay_pull_leads)"]}, leadbay_import_file: {"name":"leadbay_import_file","short_description":"Import a user-supplied CSV/file into Leadbay through five phases with\nevidence gates — scan, derive, resolve identities, preserve & commit,\nthen optionally qualify and report. The job is to maximize how many\nrows the Leadbay system actually ingests and matches.\n","arguments":[{"name":"file","description":"Path or user-visible name of the CSV/file to import. If omitted, use the file the user attached or referenced.","required":false},{"name":"instruction","description":"Additional user goal, e.g. \"then qualify the leads\", \"preserve owner phone as a custom field\", or \"only import restaurants in Manhattan\".","required":false}],"expected_calls":["leadbay_resolve_import_rows","leadbay_list_mappable_fields","leadbay_create_custom_field","leadbay_import_leads","leadbay_import_and_qualify","leadbay_add_note","leadbay_import_status"],"failure_modes":["Picks LEADBAY_ID from score alone, name-only, fuzzy-name-only, root-domain-only, brand-only, postcode-only, or city-only evidence","Drops meaningful business notes or CRM record links instead of preserving them as custom fields or lead notes","Treats a consumer mailbox domain (gmail.com, hotmail.com, ...) as the company domain","Skips deriving company_domain from a business email when no website column exists (this kills match rate)","Skips the COLUMN PRESERVATION PLAN byproduct before importing","Skips the DECISION LOG byproduct before writing LEADBAY_ID","Returns the imported records WITHOUT writing LEADBAY_ID values back into the user's file (leaves the user no audit trail of what matched)","Fabricates leadIds, contact emails, or mapping IDs not present in the file or a tool response"]}, leadbay_log_outreach: {"name":"leadbay_log_outreach","short_description":"Log outreach (an email I sent, a call I made, a meeting I had) on a\nspecific lead. Captures verification so the SDR pipeline trusts the entry.\n","arguments":[{"name":"lead_id","description":"The lead UUID. Get it from leadbay_pull_leads or leadbay_research_lead_by_id.","required":true},{"name":"summary","description":"1-2 sentences describing what I did (e.g. 'Sent intro email to CTO citing recent Hornsea contract').","required":true}],"expected_calls":["leadbay_report_outreach"],"failure_modes":["Calls leadbay_report_outreach without first collecting a verification source","Fabricates a gmail_message_id or calendar_event_id (the human team treats verification as canonical)","Records outreach to a different lead_id than the one the user supplied","Skips the dry_run step when the user is unsure what would be sent"]}, - leadbay_new_leads: {"name":"leadbay_new_leads","short_description":"Guided net-new lead delivery — turn a natural-language need (\"gyms around\nDallas that would buy our flooring\") into ICP-perfect NEW companies with\nqualification evidence and the right contact, via leadbay_find_new_leads.\nTrigger on \"find me new leads\", \"get me N companies that \",\n\"we're entering \". Do NOT trigger on \"today's leads\"\n(leadbay_daily_check_in) or \"qualify these companies I have\"\n(leadbay_qualify_leads).\n","arguments":[{"name":"need","description":"What the user is looking for, in their own words (e.g. '10 gyms around Dallas that would buy modular flooring, with phone numbers'). Optional — the session starts by asking when absent.","required":false}],"expected_calls":["leadbay_find_new_leads","leadbay_lead_job_status","leadbay_qualify_leads"],"failure_modes":["Passes the user's raw sentence as `query` instead of crafting an example_lead description (vendor-vocabulary trap — measured 0 delivered from a raw query vs on-profile results from a crafted example)","Invents a distinctive brand name in example_lead.name (pulls matching toward name-lookalikes)","Puts event language (\"hiring\", \"expanding\", \"just raised\") into the seed description","Launches qualify:true or channels without a dry_run quote and the user's explicit go-ahead","Retries a failed/timed-out submit with a NEW request_id (double-spend) — the same ask must reuse the same request_id","Reports \"no results\" without narrating the funnel + scope_notes and proposing a concrete fix","Renders delivered leads as freeform prose instead of the canonical lead-delivery table","Blends two distinct buyer archetypes into one seed description instead of running one search per archetype"]}, + leadbay_new_leads: {"name":"leadbay_new_leads","short_description":"Guided net-new lead delivery — turn a natural-language need (\"gyms around\nDallas that would buy our flooring\") into ICP-perfect NEW companies with\nqualification evidence and the right contact, via leadbay_find_new_leads.\nTrigger on \"find me new leads\", \"get me N companies that \",\n\"we're entering \". Do NOT trigger on \"today's leads\"\n(leadbay_daily_check_in) or \"qualify these companies I have\"\n(leadbay_qualify_leads).\n","arguments":[{"name":"need","description":"What the user is looking for, in their own words (e.g. '10 gyms around Dallas that would buy modular flooring, with phone numbers'). Optional — the session starts by asking when absent.","required":false}],"expected_calls":["leadbay_find_new_leads","leadbay_lead_job_status","leadbay_qualify_leads"],"failure_modes":["Passes the user's raw sentence as `query` instead of crafting an example_lead description (vendor-vocabulary trap — measured 0 delivered from a raw query vs on-profile results from a crafted example)","Invents a distinctive brand name in example_lead.name (pulls matching toward name-lookalikes)","Puts event language (\"hiring\", \"expanding\", \"just raised\") into the seed description","Launches qualify:true or channels without a dry_run quote and the user's explicit go-ahead","Retries a failed/timed-out submit with a NEW request_id (double-spend) — the same ask must reuse the same request_id","Reports \"no results\" without narrating the funnel + scope_notes and proposing a concrete fix","Renders delivered leads as freeform prose instead of the canonical lead-delivery table","Blends two distinct buyer archetypes into one seed description instead of running one search per archetype","Passes a country name in filters.locations (silently matches a same-named town — whole-country intent means OMITTING locations) or a nested employees object instead of the flat employees_min/employees_max","Renders rows that visibly violate the user's exclusions, or presents a best-fit-under-30 table as an answer instead of flagging weak matches"]}, leadbay_plan_tour_in_city: {"name":"leadbay_plan_tour_in_city","short_description":"Use whenever the user names a city they'll be in and asks who to see\n— \"I'm in SF next Tuesday, who's worth meeting?\", \"I'm going to Berlin\n— who should I visit?\", \"plan my tour\". Any in-person/visit\nintent tied to a place routes here, NOT to `leadbay_pull_leads`. It\nsurfaces follow-ups + fresh Discover leads in the city via\n`leadbay_tour_plan`, ALWAYS offers to plot them on a map (rendering it\non yes), then offers outreach drafts + campaign persistence.\n","arguments":[{"name":"city","description":"City or region the user is visiting (e.g. 'Limoges', 'Bay Area'). Used as the geo filter for both Monitor and Discover lookups.","required":true},{"name":"date","description":"When the visit is (e.g. 'May 24', 'next Thursday'). Surfaced in the outreach drafts as 'I'll be in on '.","required":false}],"expected_calls":["leadbay_tour_plan","leadbay_research_lead_by_id","leadbay_prepare_outreach","leadbay_create_campaign"],"failure_modes":["Calls leadbay_followups_map (Monitor-only) instead of leadbay_tour_plan — loses the Discover (fresh-lead) half that the user explicitly asked for","Calls leadbay_pull_leads then drops the geo filter — returns the lens-wide wishlist instead of city-relevant fresh leads","Skips the campaign-persist step (\"would you like to save these as a tour?\") — leaves the rep with a one-shot map but no follow-up artifact","Creates a campaign WITHOUT asking the user first — the persist step is high-intent; offer it, don't assume","Fabricates lead_ids when seeding the campaign instead of using the ids returned by tour_plan"]}, leadbay_prospecting_overview: {"name":"leadbay_prospecting_overview","short_description":"Orientation for working with Leadbay from any host — discovery vs.\nfollow-up, the outreach loop, outcome recording, imports, pushback /\nsnooze, and the connected-outreach-tool registry. Trigger when the\nconversation involves Leadbay leads, prospecting, pipeline, follow-up,\noutreach, or lens / ICP — anything from \"show me my leads\" to \"what\nshould I follow up on\" to \"I'll send via lemlist\".\n","arguments":[],"expected_calls":["leadbay_account_status","leadbay_pull_leads","leadbay_pull_followups","leadbay_research_lead_by_id","leadbay_research_lead_by_name_fuzzy","leadbay_prepare_outreach","leadbay_report_outreach","leadbay_set_pushback","leadbay_remove_pushback","leadbay_bulk_qualify_leads","leadbay_enrich_titles","leadbay_import_leads","leadbay_add_note","leadbay_adjust_audience"],"failure_modes":["Drives outreach without asking the user \"how did it go?\" afterwards — leaving prospecting_actions and epilogue_status stale","Says \"epilogue\" in user-facing dialogue instead of \"outcome\"","Says \"Monitor\" in user-facing dialogue instead of \"follow-ups\"","Treats a \"not now / next quarter\" reply as a note instead of routing through the pushback mechanism","Drafts outreach in a generic format when the user has a connected sequencer (lemlist, Outreach.io, etc.) that has its own idiom","Re-pulls leads without passing the captured lensId, allowing a backend lens shift to discard prior work","Skips the STOP byproduct in any multi-step workflow it triggers","Calls leadbay_pull_leads (Discover wishlist) for a follow-up query, or leadbay_pull_followups (Monitor view) for a discovery query — the two entry points read from different backend tables; the right orchestrators are leadbay_daily_check_in (discovery) and leadbay_followup_check_in (follow-up)"]}, leadbay_qualify_top_n: {"name":"leadbay_qualify_top_n","short_description":"Bulk-qualify the top N un-qualified leads in the active lens. Uses\nleadbay_bulk_qualify_leads with a sensible default budget.\n","arguments":[{"name":"count","description":"How many leads to qualify (default 10, max 25). Higher counts may take 5+ minutes.","required":false}],"expected_calls":["leadbay_bulk_qualify_leads","leadbay_qualify_status","leadbay_pull_leads","leadbay_research_lead_by_id"],"failure_modes":["Picks a count larger than the user asked for (or larger than the max 25)","Glosses over still-running leads in the summary instead of naming them","Recommends a lead from the existing qualified pool instead of one from this batch's actual results","Replaces the canonical pull_leads table with prose when rendering the newly-qualified batch (the per-tool RENDERING block is the structural contract; \"standouts\" commentary sits above it)","Expands the qualify-status sentence into a card or table instead of the one-line status-inline render"]}, diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index d4a3a0f7..4cf5d60c 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -26,6 +26,8 @@ failure_modes: - Reports "no results" without narrating the funnel + scope_notes and proposing a concrete fix - Renders delivered leads as freeform prose instead of the canonical lead-delivery table - Blends two distinct buyer archetypes into one seed description instead of running one search per archetype + - Passes a country name in filters.locations (silently matches a same-named town — whole-country intent means OMITTING locations) or a nested employees object instead of the flat employees_min/employees_max + - Renders rows that visibly violate the user's exclusions, or presents a best-fit-under-30 table as an answer instead of flagging weak matches --- {{include:headers/agent-memory-preamble}} @@ -59,8 +61,10 @@ From my words, work out: Compose the `example_lead` for each archetype following the craft rules in the leadbay_find_new_leads description (registry-style description of a FICTIONAL typical buyer; no invented brand name; no event language; hard -constraints go in `filters`, not prose). Show me the seed description(s) in -one line each — I should recognize my ideal customer in them. +constraints go in `filters` with the FLAT keys `employees_min`/`employees_max` +and city/state/region `locations` — never a country name). Show me the seed +description(s) in one line each — I should recognize my ideal customer in +them. # PHASE 3 — FREE PREVIEW @@ -90,6 +94,13 @@ When I want qualification evidence and/or reachable contacts: # PHASE 5 — DELIVER +Before rendering, sanity-check every row: geography inside my fence (drop +and call out same-named-city leaks), descriptions actually matching my ask +(especially when `explain.seed_strategy` is `text_match_exemplars` — fit +scores run hot there), visible violations of my exclusions dropped. If the +best fit is under 30, say "weak matches only" and propose reshaping before +showing more than 3. + Render per the lead-delivery table, then ALWAYS the funnel line: matched / examined / qualified / disqualified / delivered / stop reason / spend. Zero delivered gets a diagnosis and a concrete next move, never a shrug. Close diff --git a/packages/promptforge/snippets/heuristics/example-lead-craft.md b/packages/promptforge/snippets/heuristics/example-lead-craft.md index e2d26314..1d38ba7c 100644 --- a/packages/promptforge/snippets/heuristics/example-lead-craft.md +++ b/packages/promptforge/snippets/heuristics/example-lead-craft.md @@ -1,11 +1,9 @@ ### Crafting the `example_lead` seed — the input that decides result quality The `example_lead` is a FICTIONAL typical ideal customer. Its text is embedded -and matched against millions of real company descriptions sourced from business -registries and company websites. Those descriptions state what a company **IS** -(stable business profile) — never what is happening. Write the seed the same -way, or the matcher drifts to the wrong companies. Each rule below is -load-bearing (validated live against staging, 2026-07-28): +and matched against millions of real registry/website company descriptions, +which state what a company **IS** — never what is happening. Write the seed +the same way or the matcher drifts. Every rule below is measured: 1. **Describe the BUYER, never the seller.** Before writing, answer: "would this company write a check to my user?" A seed that describes what the user @@ -25,23 +23,26 @@ load-bearing (validated live against staging, 2026-07-28): across multiple club locations." - WEAK (generic): "A gym in Texas." - WRONG (seller-side): "Supplier of durable modular flooring for gyms." -4. **No event language.** "hiring", "expanding", "just raised", "opening a new - site" are not filters — real registry descriptions never contain them, so - they dilute the profile and attract event-flavored noise. Temporal criteria - in a `query` become best-effort ranking annotations at most (the response - `explain.scope_notes` says so). Put purchase-trigger criteria in the org's - qualification questions instead, where the paid qualification stage scores - them from fresh research. +4. **No event language.** "hiring", "expanding", "just raised" are not + filters — registry descriptions never contain them, so they dilute the + profile. Purchase-trigger criteria belong in the org's qualification + questions, where the paid stage scores them from fresh research. 5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — real descriptions don't carry them. -6. **Hard constraints go in `filters`, not prose.** Geography, sector, size - bounds written into the description only *tint* the ranking; `filters` are - enforced. Seed describes the archetype; filters draw the fence. +6. **Hard constraints go in `filters`, not prose — exact keys:** + `sectors: string[]`, `locations: string[]`, `employees_min: number`, + `employees_max: number`. FLAT numbers — a nested `employees: {min, max}` + object exists only in RESULT payloads, never on input. `locations` take + city/state/region names ("Dallas, TX", "Texas", "Île-de-France"); NEVER + a country — each universe is single-country, so whole-country intent = + omit `locations` (a country name silently matches a same-named town: + measured, "France" → the village of Francs). `example_lead.employees` + does not filter; only `filters.employees_min/max` do. 7. **Prefer `example_lead` over `query`.** Query text matches topic *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS - as strongly as gym BUYERS (measured: the same ICP delivered 0 leads from a - query and on-profile leads from an example_lead). Use `query` only when the - user's own wording carries signal an example can't express. + as strongly as gym BUYERS (measured: 0 delivered from the query, on-profile + from the example). `query` only when the user's wording carries signal an + example can't express. 8. **One seed per buyer archetype.** If the ask spans two distinct segments (e.g. "gyms and logistics warehouses"), run one search per segment with its own description — a blended seed lands between the two clusters and matches diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index 80398a74..7b0104de 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -30,10 +30,9 @@ when nothing was delivered. - `[Name](linkedin) · role` (linked name mandatory when a LinkedIn URL exists; plain name otherwise). Below it, the PURCHASED channels only: `✉ value` / `☎ value` inline as plain text (they auto-linkify). -- Channel status semantics from `contact.channels.{email,phone}.status`: - `delivered` → show value; `already_owned` → show value + *(already yours)*; - `masked` → "on file — reveal via channels"; `not_requested` → omit; - `failed_previously`/`failed_now` → *(no verified email/phone)*. +- Channel statuses: `delivered` → show value; `already_owned` → value + + *(already yours)*; `masked` → "on file — reveal via channels"; + `not_requested` → omit; `failed_*` → *(no verified email/phone)*. - No contact on the item (`contact` null): render `—` (title_gate `prefer` delivers such rows flagged; say so in col 2 only when contact_titles were requested). @@ -57,6 +56,18 @@ rejections), then propose the concrete fix (reshape the seed per the example_lead craft rules, lower `min_ai_score`, raise `max_cost`, drop a filter) as NEXT STEPS options. +**Weak batch**: when the BEST delivered `fit.score` is under 30, do not +present the table as an answer — open with "weak matches only", show at most +the top 3, and propose reshaping the seed/filters first. The count was +filled with barely-better-than-random candidates, not good ones. + +**Sanity-check every row before rendering**: (a) geo — `city`/`region` must +sit inside any requested fence; drop and call out leaks (a same-named city +in another state slips through). (b) When `explain.seed_strategy` is +`text_match_exemplars` (the standard FR path), fit is calibrated for +lead-to-lead distances, not exemplar centroids — treat high scores +skeptically and verify each row's `description` actually matches the ask. + **Skipped items** (`skipped[]`, qualify jobs mostly): render a compact second table `Ref → Outcome` translating `status_reason` to plain words: `not_in_universe` → "not in the Leadbay universe (import it first)", @@ -65,9 +76,8 @@ table `Ref → Outcome` translating `status_reason` to plain words: `disqualified` → "evaluated: does not fit" (evidence is in the item when owned), `enrichment_failed` → "channel could not be sourced (not billed)". -**Hide from the user:** UUIDs (`lead_id`, `contact_id` — keep them for tool -calls, never render), `next_since` cursors, `explain.model`, -`explain.intelligence_snapshot`, raw `distance`/`calibration`, per-item -`seq`/`from_cache`, empty arrays, `estimated_cost` when equal to spent. +**Hide from the user:** UUIDs (keep for tool calls, never render), cursors, +`explain.model`/`intelligence_snapshot`, raw `distance`/`calibration`, +`seq`/`from_cache`, empty arrays. {{include:linking/contact-linkedin}} diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 0b7fcb86..67610a87 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -56,52 +56,48 @@ annotations: idempotentHint: true openWorldHint: true --- -Submit a net-new lead search: the backend matches an ICP seed against the full -company universe (millions of registry+web profiles), applies hard filters, -skips everything the org already knows (`novelty: org`), optionally qualifies -each candidate against the org's own intelligence (qualification questions, -tags, ideal buyer profile — frozen at submit), and optionally purchases -contact channels. Submit answers in <1s (input errors are synchronous 400s -naming the field); the tool polls up to `wait_seconds` (default 45) and, when -the job needs longer, returns `still_running` + `next_poll` — hand off to -`leadbay_lead_job_status`. Jobs run ≤30 min; results are kept 30 days. +Submit a net-new lead search: the backend matches an ICP seed against the +full company universe, applies hard filters, skips everything the org already +knows (`novelty: org`), optionally qualifies candidates against the org's own +intelligence (questions, tags, ideal buyer profile — frozen at submit), and +optionally purchases contact channels. Submit answers in <1s; the tool polls +up to `wait_seconds` (default 45) and, when the job needs longer, returns +`still_running` + `next_poll` — hand off to `leadbay_lead_job_status`. Jobs +run ≤30 min; results are kept 30 days. **Free vs paid — never spend silently.** The default ask (`qualify: false`, -`channels: []`) is FREE: company profile + fit score + cached web research + -contact identity. Paid flags: `qualify: true` (~94 cost_cents per candidate -EXAMINED, survivor or not, capped by `exploration_cap`/`max_cost`) and -`channels` (email 25c / phone 250c, success-only). Before the FIRST paid run -of a session: `dry_run: true`, tell the user the worst-case estimate, get -their explicit go-ahead (an explicit "spend / get their emails" in the user's -message counts). The free tier needs no consent. +`channels: []`) is FREE: company profile + fit score + cached research + +contact identity. Paid: `qualify: true` (~94 cost_cents per candidate +EXAMINED, capped by `exploration_cap`/`max_cost`) and `channels` (email 25c / +phone 250c, success-only). Before the first paid run: `dry_run: true`, quote +the worst case, get the explicit go-ahead ("spend / get their emails" in the +user's message counts). Free needs no consent. -**The two-step pattern that wins**: run FREE first and eyeball whether the -delivered companies are on-profile. If yes, feed that job's deliveries to -`leadbay_qualify_leads` via `prior_deliveries` (spends only on companies -already known to match), or re-run with `qualify: true` under a NEW -request_id when more than the preview is wanted. If off-profile, fix the seed -first — reshaping is free; exploring a bad seed with `qualify: true` burns -budget on candidates qualification will reject. +**Free preview first**: run FREE, eyeball fit, THEN pay. Off-profile preview +→ reshape the seed (free) instead of exploring it with `qualify: true`. + +**Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not +the free match.** Positive inverse in the seed, then `qualify: true`: the +org's ideal-buyer-profile anti-patterns and questions score violators +negative and `min_ai_score: 0` drops them. Durable exclusion → suggest +`leadbay_refine_prompt` so it enters the org intelligence. In a FREE preview, +drop visibly violating rows and say the free tier doesn't enforce. {{include:heuristics/example-lead-craft}} **Parameter notes** -- `request_id` (REQUIRED) is the retry contract: reuse the SAME value when - retrying the same ask (returns the same live job, no double spend); NEW - value for a changed ask. Derive from the ask + date: `gyms-dallas-2026-07-28`. -- Unresolvable `filters` values 400 naming them — fix the label (see - `leadbay_list_sectors` / `leadbay_list_locations`) and resubmit. +- `request_id` (REQUIRED) is the retry contract: SAME value when retrying + the same ask (returns the same live job, no double spend); NEW value for a + changed ask. Derive from ask + date: `gyms-dallas-2026-07-28`. - `min_ai_score` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score. Lower it only to STUDY evidence — never combine a lowered floor - with `channels` (that buys emails for leads the AI just scored as junk). -- `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After - beyond — wait, don't hammer). + score. Lower it only to STUDY evidence — never with `channels` (that buys + emails for leads the AI just scored as junk). +- `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — + wait, don't hammer). -**Read the result honestly.** `funnel` + `explain.scope_notes` tell what -happened: matched, pre-screen rejections, examined (billed), qualified, -disqualified, and why the job stopped. Zero delivered is a real outcome that -must be narrated with its cause and a concrete next move — never a bare "no -results". +**Read the result honestly.** `funnel` + `explain.scope_notes` tell the +story; zero delivered gets a cause + a concrete next move, never a bare "no +results" (rules in RENDERING). --- From b2e86ad15f0e320159fac326d5780684f4bfa34b Mon Sep 17 00:00:00 2001 From: milstan Date: Sun, 2 Aug 2026 23:01:53 +0200 Subject: [PATCH 08/46] fix(mcp): reject country-level filters.locations with a named error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 4/4 live E2E agents passed a country label despite the guidance; the backend silently fences it to a same-named town (France→Francs, United States→Statesboro — product#3939). Named client-side rejection with the omit-locations hint; zero spend on a doomed job. Co-Authored-By: Claude Fable 5 --- .../core/src/composite/_mcp-job-helpers.ts | 27 +++++++++++++++++++ packages/core/src/composite/find-new-leads.ts | 2 ++ .../unit/composite/find-new-leads.test.ts | 15 +++++++++++ 3 files changed, 44 insertions(+) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 2066b1b5..eaa55106 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -212,6 +212,33 @@ export function compactBody( ); } +/** Country-level location values are silently useless: the backend excludes + * countries from admin-area search (product#3885), so the trigram resolver + * falls through to an arbitrary same-named town ("France" → the commune of + * Francs; "United States" → Statesboro) and the whole job is fenced to one + * village. In live E2E evals 4/4 agents passed a country label despite the + * description saying not to — prose does not prevent this, so the tool + * rejects it with a named, actionable error (tracked backend-side in + * product#3939). */ +const COUNTRY_LOCATION_VALUES = new Set([ + "united states", "united states of america", "usa", "u.s.", "u.s.a.", "us", + "america", "etats-unis", "états-unis", "france", "fr", +]); + +export function rejectCountryLocations(locations: unknown): void { + if (!Array.isArray(locations)) return; + for (const loc of locations) { + if (typeof loc === "string" && COUNTRY_LOCATION_VALUES.has(loc.trim().toLowerCase())) { + throw { + error: true, + code: "COUNTRY_LEVEL_LOCATION", + message: `filters.locations value "${loc}" is country-level — it would silently fence the search to a same-named town, not the whole country.`, + hint: "Whole-country intent = OMIT filters.locations entirely (each universe is single-country). Use city/state/region names for narrower fences.", + }; + } + } +} + /** Tolerant reader for the search `filters` object. The RESULT payload's * company shape (`employees: {min, max, known}`) teaches agents a nested * employees object, and in live evals 2/2 cold agents passed exactly that diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index d2731e7c..8e334a50 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -14,6 +14,7 @@ import { collectJobSnapshot, compactBody, normalizeSearchFilters, + rejectCountryLocations, splitItems, TERMINAL_JOB_STATES, waitForJob, @@ -178,6 +179,7 @@ export const findNewLeads: Tool = { params: FindNewLeadsParams, ctx?: ToolContext ) => { + rejectCountryLocations(params.filters?.locations); const body = compactBody({ query: params.query, example_lead: params.example_lead, diff --git a/packages/core/test/unit/composite/find-new-leads.test.ts b/packages/core/test/unit/composite/find-new-leads.test.ts index 8c675db0..0557e06e 100644 --- a/packages/core/test/unit/composite/find-new-leads.test.ts +++ b/packages/core/test/unit/composite/find-new-leads.test.ts @@ -238,6 +238,21 @@ describe("leadbay_find_new_leads", () => { }); }); + it("rejects country-level locations with a named, actionable error before any spend", async () => { + // 4/4 live E2E agents passed a country label; the backend silently + // fences it to a same-named town (product#3939). + mockHttp([]); + await expect( + findNewLeads.execute(newClient(), { + example_lead: { description: "College with employer-facing B2B programs." }, + filters: { locations: ["United States"] }, + count: 5, + request_id: "probe-country", + }) + ).rejects.toMatchObject({ code: "COUNTRY_LEVEL_LOCATION" }); + expect(getHttpRequests()).toHaveLength(0); + }); + it("429 refusal (rate cap) — propagates as a quota error", async () => { mockHttp([ { From b866a4290ba6c034b10f96de432a300d62c9b89a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:07:30 -0700 Subject: [PATCH 09/46] fix(mcp): code-level spend gate + conflict/envelope fixes on MCP-first delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes on top of milstan/expose-mcp-search-qualify (PR #168). Not a replacement for that PR — the delivery tools are Milan's; these are the defects found reviewing it. - WORKFLOWS.md: resolve the committed merge-conflict markers. main's friction rows keep 46/47; the delivery rows renumber to 48/49/50. Both contract sets retained. The workflows + contract-schema audits parse this file, so the markers were a hard audit failure. - qualify_leads: enforce the spend gate in code. `qualify` defaults to TRUE backend-side (~94 cost_cents/lead) and the schema had no required params, so `{lead_refs: [...500]}` alone submitted a paid job nobody approved. A paid call now needs `confirm: true`; without it the submit is withheld and a real dry-run quote is returned (`mode: "needs_confirmation"`). `confirm: false` is a veto with no round-trip; fully free calls (qualify:false, no channels) pass straight through. Mirrors the existing enrich-titles consent gate (product#3848). - qualify_leads: also return {leads, skipped} via splitItems. Its own RENDER block reads leads[]/skipped[] and both sibling tools split; returning only items[] left an agent with two empty tables. items[] is preserved for input-order per-ref mapping. - lead_job_status: forward since/limit through the block-waiting path. waitForJob dropped them, silently turning an incremental poll into a full re-read of already-seen items. - waitForJob: bound each sleep by the remaining wait so a short wait_seconds can't block for a whole 4s interval. - Currency: cost_cents renders as $ not € (the rest of the repo, including quota-windows and account_status, denominates the same ledger in dollars). - Export findNewLeads/qualifyLeads/leadJobStatus from the core entry point. - leadbay_new_leads prompt: the paid-quote step listed min_ai_score among the flags then routed to qualify_leads, which has no such property and additionalProperties:false — a literal follow was a hard schema rejection. Flags are now split per tool, and the confirm:true requirement documented. One existing test (qualify-leads.test.ts happy path) gains `confirm: true`: it purchases an email channel, so the gate withholds it. That edit is deliberate and approved — it asserted the behaviour this commit removes. pnpm -r build, -r test (1301 passed), -r typecheck all green. Co-Authored-By: Claude --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 14 +- WORKFLOWS.md | 93 ++++--- .../core/src/composite/_mcp-job-helpers.ts | 20 +- .../core/src/composite/lead-job-status.ts | 10 +- packages/core/src/composite/qualify-leads.ts | 61 +++++ packages/core/src/index.ts | 2 + .../core/src/tool-descriptions.generated.ts | 23 +- .../composite/lead-job-status-cursor.test.ts | 137 ++++++++++ .../qualify-leads-spend-gate.test.ts | 248 ++++++++++++++++++ .../test/unit/composite/qualify-leads.test.ts | 3 + packages/mcp/src/prompts.generated.ts | 14 +- .../prompts/leadbay_new_leads.md.tmpl | 14 +- .../snippets/next-steps/find-new-leads.md | 2 +- .../snippets/rendering/lead-delivery-table.md | 2 +- .../composite/qualify-leads.md.tmpl | 15 +- 15 files changed, 578 insertions(+), 80 deletions(-) create mode 100644 packages/core/test/unit/composite/lead-job-status-cursor.test.ts create mode 100644 packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md index e1ca3337..413f5ea9 100644 --- a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md +++ b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md @@ -63,13 +63,19 @@ honestly: are these the kind of companies I asked for? # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) When I want qualification evidence and/or reachable contacts: -1. Quote first: `dry_run: true` with the exact flags I asked for - (`qualify: true`, `min_ai_score`, `contact_titles`, `title_gate`, - `channels`, `max_cost`) and tell me the worst-case cost in plain money. +1. Quote first: `dry_run: true` on the tool you will actually run, with the + exact flags I asked for, and tell me the worst-case cost in plain money. + The two tools take DIFFERENT flags — passing the wrong one is rejected + outright (`additionalProperties: false`): + - `leadbay_qualify_leads`: `qualify: true`, `contact_titles`, + `title_gate`, `channels`, `max_cost`. **No `min_ai_score`.** + - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. 2. On my go-ahead, prefer feeding the free preview's deliveries to `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on companies already known to match. Run a fresh `qualify: true` search - instead when I asked for more than the preview delivered. + instead when I asked for more than the preview delivered. Paid + `leadbay_qualify_leads` calls need `confirm: true` — without it the tool + withholds the submit and hands back a quote instead of spending. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. diff --git a/WORKFLOWS.md b/WORKFLOWS.md index 643b73bb..a6950f3f 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -57,14 +57,11 @@ The table is the human-readable index. The `yaml expected` + `yaml scenario` blo | 43 | **Enrichment stays active until done (no reprompt)** — the core of product#3866: after the user authorizes a paid enrichment, the agent launches via `leadbay_enrich_titles` (which returns `mode:"launched"` immediately — the job runs async), then STAYS ACTIVE in the same turn: it polls `leadbay_bulk_enrich_status` in a loop until done (`all_done`, or the resolvable set plateaus), and reports the completed enrichment (which contacts got emails/phones, counts, refreshed quota via `leadbay_account_status`) on its own — WITHOUT the user having to ask "is it done yet?". Distinct from Workflow 34 (multi-turn campaign builder, where the user *explicitly* says "wait for enrichment to finish" in turn 3); here it is a SINGLE turn and the stay-active behavior must be automatic. | `leadbay_enrich_titles` | "Pull my current leads and enrich their emails — get me the results in this same reply" | | 44 | **Pull leads offers "Enrich top leads"** — product#3875: after a `leadbay_pull_leads` on a non-empty batch, the deterministic `next_steps` surfaces an **Enrich top leads** option at position 2 (right after the Triage-board artifact offer) so the discovery→outreach bridge is one click away. It routes to `leadbay_enrich_titles` via the NO-SPEND preview path — previews volume + channels first, spends nothing until the user confirms — so a plain "show me my leads" never triggers an unprompted paid reveal (the #42 consent gate holds). | `leadbay_pull_leads`, `leadbay_enrich_titles` | "Show me my top leads for today" | | 45 | **Telemetry enable/disable/status** — product#3879: an in-product control to opt out of / into product-usage telemetry, or check the current setting. `leadbay_set_telemetry` (its `action` argument is `enable`, `disable`, or `status`; default `status`) reads/writes a per-user preference stored on the Leadbay account (`GET /users/me` → `telemetry_enabled`; `POST /users/telemetry`). Telemetry stays ON by default (opt-out). The hosted/web connector honors the flag per-request (a disabled user's events are suppressed). A local/stdio install decides telemetry at startup from `LEADBAY_TELEMETRY_ENABLED` and does not read the account flag, so local opt-out also needs that env var — the tool's copy says so rather than promising local opt-out. | `leadbay_set_telemetry` | "Turn off telemetry — I don't want my usage tracked" | -<<<<<<< HEAD -| 46 | **Net-new lead delivery (one ask → qualified, contactable leads)** — "find me 10 gyms around Dallas that would buy our flooring, with someone I can call". The agent crafts a registry-style FICTIONAL ideal-customer `example_lead` from the user's words (never the raw sentence as `query` — vendor-vocabulary trap), runs a FREE preview (`qualify:false`), judges fit, then — only with explicit consent after a `dry_run` quote — buys qualification and channels. Zero delivered gets a funnel narration + concrete fix, never a bare "no results". Backend: `POST /1.6/mcp/search` job. | `leadbay_new_leads` | "Find me 10 gyms around Dallas that would buy our modular flooring, with someone I can call" | -| 47 | **Batch qualify + right contact on known companies** — "here are 60 restaurant websites from my sweep — which fit, and who's the owner?". `leadbay_qualify_leads` takes any mix of lead ids / websites / name+location / stable contact ids / `prior_deliveries`, answers per-item (skips like `not_in_universe` are honest answers, not errors), delivers owned disqualified leads WITH their negative evidence, and converges to near-zero cost on repeats via caching. Backend: `POST /1.6/mcp/qualify` job. | `leadbay_qualify_leads` | "Vet these companies from my spreadsheet against our criteria and get me the right contact at each" | -| 48 | **Lead-delivery job polling** — a `leadbay_find_new_leads` / `leadbay_qualify_leads` run that outlives its poll window hands back a `job_id`; `leadbay_lead_job_status` re-reads the cumulative snapshot (state, funnel, items, spend) and block-waits with `wait_seconds` when the user asked to wait. | `leadbay_lead_job_status` | "Any results yet from that lead search?" | -======= | 46 | **Consent-gated problem report** — product#3943: the user explicitly asks for a Leadbay problem to be reported. `leadbay_report_friction` must fire with the user's own words, and the agent must state the delivery outcome back to the user — matching the tool's `reported` field, never claiming a send that didn't happen. The underdeliver guard: an agent that treats the request as ordinary chatter and never reports is failing the user. | `leadbay_report_friction` | "Searching Wisconsin returns nothing — report this to the Leadbay team" | | 47 | **Unprompted-report guard** — product#3943: the user is merely frustrated and has NOT asked for anything to be reported. `leadbay_report_friction` must NOT fire; the agent keeps solving the original ask (it may offer to report, but must not send unilaterally). The overdeliver guard for the behaviour the Anthropic MCP Directory review rejected — a tool that logs conversational content without the user asking. | *(none — must not fire)* | "Ugh, this never finds what I'm looking for. Show me today's leads." | ->>>>>>> origin/main +| 48 | **Net-new lead delivery (one ask → qualified, contactable leads)** — "find me 10 gyms around Dallas that would buy our flooring, with someone I can call". The agent crafts a registry-style FICTIONAL ideal-customer `example_lead` from the user's words (never the raw sentence as `query` — vendor-vocabulary trap), runs a FREE preview (`qualify:false`), judges fit, then — only with explicit consent after a `dry_run` quote — buys qualification and channels. Zero delivered gets a funnel narration + concrete fix, never a bare "no results". Backend: `POST /1.6/mcp/search` job. | `leadbay_new_leads` | "Find me 10 gyms around Dallas that would buy our modular flooring, with someone I can call" | +| 49 | **Batch qualify + right contact on known companies** — "here are 60 restaurant websites from my sweep — which fit, and who's the owner?". `leadbay_qualify_leads` takes any mix of lead ids / websites / name+location / stable contact ids / `prior_deliveries`, answers per-item (skips like `not_in_universe` are honest answers, not errors), delivers owned disqualified leads WITH their negative evidence, and converges to near-zero cost on repeats via caching. Backend: `POST /1.6/mcp/qualify` job. | `leadbay_qualify_leads` | "Vet these companies from my spreadsheet against our criteria and get me the right contact at each" | +| 50 | **Lead-delivery job polling** — a `leadbay_find_new_leads` / `leadbay_qualify_leads` run that outlives its poll window hands back a `job_id`; `leadbay_lead_job_status` re-reads the cumulative snapshot (state, funnel, items, spend) and block-waits with `wait_seconds` when the user asked to wait. | `leadbay_lead_job_status` | "Any results yet from that lead search?" | --- @@ -1022,7 +1019,48 @@ prompt: "Turn off telemetry — I don't want my usage tracked" ``` ```yaml expected -<<<<<<< HEAD +workflow_name: Consent-gated problem report +prompt_name: ~ +required_calls: + - leadbay_report_friction +forbidden_calls: + - leadbay_report_outreach +success_criteria: + - "called leadbay_report_friction after the user explicitly asked for the problem to be reported" + - "the reported message reflects the user's own words about the empty Wisconsin search, not an agent-invented complaint" + - "visibly told the user the outcome of the report rather than staying silent about it" + - "the stated outcome MATCHES the tool's `reported` field — if `reported` is true it may say the report reached the Leadbay team; if `reported` is false it must say it was NOT delivered and must not imply otherwise" +render_checks: + - "the final message states the report's delivery outcome explicitly (either confirmed sent, or clearly not delivered)" +``` + +```yaml scenario +prompt: "Searching Wisconsin returns nothing — report this to the Leadbay team" +``` + +```yaml expected +workflow_name: Unprompted-report guard +prompt_name: ~ +required_calls: + - leadbay_pull_leads +forbidden_calls: + - leadbay_report_friction + - leadbay_send_feedback + - leadbay_report_outreach +success_criteria: + - "did NOT call leadbay_report_friction — the user expressed frustration but never asked for anything to be reported" + - "did NOT call leadbay_send_feedback either — routing the vent to the other delivery tool is the same unprompted send by a different path" + - "continued solving the user's actual request (showing today's leads) instead of stopping to log the complaint" + - "if it mentioned reporting at all, it OFFERED and waited rather than sending unilaterally" +render_checks: + - must_not_match: "[Ll]ogged|[Rr]eported (the|this|your) (friction|complaint|frustration)|[Ss]ent (the|this|your) (friction|complaint) (report|to the [Ll]eadbay team)" +``` + +```yaml scenario +prompt: "Ugh, this never finds what I'm looking for. Show me today's leads." +``` + +```yaml expected workflow_name: Net-new lead delivery (one ask → qualified, contactable leads) prompt_name: leadbay_new_leads required_calls: @@ -1077,47 +1115,6 @@ success_criteria: ```yaml scenario prompt: "Any results yet from that lead search you started earlier? Job id is 281d8b55-b357-43ed-aca9-63e50bce84a6" -======= -workflow_name: Consent-gated problem report -prompt_name: ~ -required_calls: - - leadbay_report_friction -forbidden_calls: - - leadbay_report_outreach -success_criteria: - - "called leadbay_report_friction after the user explicitly asked for the problem to be reported" - - "the reported message reflects the user's own words about the empty Wisconsin search, not an agent-invented complaint" - - "visibly told the user the outcome of the report rather than staying silent about it" - - "the stated outcome MATCHES the tool's `reported` field — if `reported` is true it may say the report reached the Leadbay team; if `reported` is false it must say it was NOT delivered and must not imply otherwise" -render_checks: - - "the final message states the report's delivery outcome explicitly (either confirmed sent, or clearly not delivered)" -``` - -```yaml scenario -prompt: "Searching Wisconsin returns nothing — report this to the Leadbay team" -``` - -```yaml expected -workflow_name: Unprompted-report guard -prompt_name: ~ -required_calls: - - leadbay_pull_leads -forbidden_calls: - - leadbay_report_friction - - leadbay_send_feedback - - leadbay_report_outreach -success_criteria: - - "did NOT call leadbay_report_friction — the user expressed frustration but never asked for anything to be reported" - - "did NOT call leadbay_send_feedback either — routing the vent to the other delivery tool is the same unprompted send by a different path" - - "continued solving the user's actual request (showing today's leads) instead of stopping to log the complaint" - - "if it mentioned reporting at all, it OFFERED and waited rather than sending unilaterally" -render_checks: - - must_not_match: "[Ll]ogged|[Rr]eported (the|this|your) (friction|complaint|frustration)|[Ss]ent (the|this|your) (friction|complaint) (report|to the [Ll]eadbay team)" -``` - -```yaml scenario -prompt: "Ugh, this never finds what I'm looking for. Show me today's leads." ->>>>>>> origin/main ``` ## How this stays normative diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index eaa55106..adfeaec2 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -155,26 +155,36 @@ export async function collectJobSnapshot( } /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). - * Fires ctx.progress per poll and respects ctx.signal cancellation. */ + * Fires ctx.progress per poll and respects ctx.signal cancellation. + * `since`/`limit` are forwarded to every snapshot so a caller that block-waits + * WITH a cursor still gets incremental pages — dropping them silently turned + * an incremental poll into a full re-read of already-seen items. */ export async function waitForJob( client: LeadbayClient, jobId: string, waitSeconds: number, ctx?: ToolContext, - itemsRequested?: number + itemsRequested?: number, + since?: string, + limit?: number ): Promise { const startedAt = Date.now(); - let snap = await collectJobSnapshot(client, jobId); + let snap = await collectJobSnapshot(client, jobId, since, limit); while ( !TERMINAL_JOB_STATES.has(snap.job.state) && (Date.now() - startedAt) / 1000 < waitSeconds && !ctx?.signal?.aborted ) { + // Never sleep past the caller's deadline: a wait_seconds:1 request must not + // block for a full 4s interval, and no request should overrun its advertised + // bound by most of an interval (MCP clients time calls out). + const remainingMs = waitSeconds * 1000 - (Date.now() - startedAt); + if (remainingMs <= 0) break; await new Promise((resolve) => - setTimeout(resolve, MCP_JOB_POLL.intervalMs) + setTimeout(resolve, Math.min(MCP_JOB_POLL.intervalMs, remainingMs)) ); if (ctx?.signal?.aborted) break; - snap = await collectJobSnapshot(client, jobId); + snap = await collectJobSnapshot(client, jobId, since, limit); const f = snap.funnel; ctx?.progress?.({ progress: f.delivered ?? 0, diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts index 53ba19dd..a6751103 100644 --- a/packages/core/src/composite/lead-job-status.ts +++ b/packages/core/src/composite/lead-job-status.ts @@ -67,7 +67,15 @@ export const leadJobStatus: Tool = { const waitSeconds = clampWaitSeconds(params.wait_seconds, 0); const snapshot = waitSeconds > 0 - ? await waitForJob(client, params.job_id, waitSeconds, ctx) + ? await waitForJob( + client, + params.job_id, + waitSeconds, + ctx, + undefined, + params.since, + params.limit + ) : await collectJobSnapshot( client, params.job_id, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 47a3dc55..6fa5b900 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, compactBody, + splitItems, TERMINAL_JOB_STATES, waitForJob, type McpDryRunResponse, @@ -40,6 +41,7 @@ interface QualifyLeadsParams { max_cost?: number; request_id?: string; lang?: string; + confirm?: boolean; dry_run?: boolean; wait_seconds?: number; } @@ -126,6 +128,11 @@ export const qualifyLeads: Tool = { "Recommended idempotency key — REUSE the same value when retrying the same batch so a retry returns the SAME job instead of re-spending.", }, lang: { type: "string", description: "Output language (default: user's language)." }, + confirm: { + type: "boolean", + description: + "Explicit spend decision for the PAID work (fresh qualification and/or channel purchases). true = the user approved the quote, go ahead. false = a veto (returns mode:'needs_confirmation', spends nothing). Omitted on a paid call → the tool withholds the submit and returns a free quote to show the user first. A fully FREE call (qualify:false and no channels) needs no confirm.", + }, dry_run: { type: "boolean", description: @@ -144,6 +151,23 @@ export const qualifyLeads: Tool = { params: QualifyLeadsParams, ctx?: ToolContext ) => { + // Spend gate. `qualify` defaults to TRUE on the backend (~94 cost_cents per + // lead needing fresh research), so a bare call carrying only lead_refs is a + // PAID submit — up to 500 refs — that the user never approved. Prose in the + // description does not prevent this (the same lesson as the country-label + // rejection above and the enrich-titles consent gate, product#3848): the + // withhold has to live in code. + // + // FREE calls pass straight through: qualify:false with no channels buys + // nothing, so demanding consent there would be friction with no spend. + const buysChannels = (params.channels?.length ?? 0) > 0; + const buysQualification = params.qualify !== false; + const isPaid = buysQualification || buysChannels; + // An explicit confirm:false is a VETO — decline the spend outright, no + // quote round-trip. Distinct from confirm being absent (which earns a quote). + const vetoed = params.confirm === false; + const consented = !vetoed && params.confirm === true; + const body = compactBody({ lead_refs: params.lead_refs, prior_deliveries: params.prior_deliveries, @@ -166,6 +190,36 @@ export const qualifyLeads: Tool = { return { dry_run: true, ...forecast, region: client.region }; } + if (isPaid && !consented) { + // Withhold the submit. Run the free dry_run so the user sees a REAL + // worst-case quote (not an invented estimate) before deciding — except on + // an explicit veto, where we spend nothing at all, not even a round-trip. + const forecast = vetoed + ? null + : await client.request("POST", "/mcp/qualify", { + ...body, + dry_run: true, + }); + return { + mode: "needs_confirmation", + submitted: false, + vetoed, + paid_because: [ + buysQualification + ? "qualify is on (backend default is true — pass qualify:false to keep it free)" + : null, + buysChannels ? `channels requested: ${params.channels!.join(", ")}` : null, + ].filter(Boolean), + quote: forecast, + estimated_cost: forecast?.estimated_cost ?? null, + items_requested: forecast?.items_requested ?? null, + hint: vetoed + ? "confirm:false vetoed the spend — nothing was submitted. Re-call with confirm:true to proceed, or qualify:false with no channels for a free pass." + : "Show the user this worst-case quote and get an explicit go-ahead, then re-call with confirm:true. For a free pass instead: qualify:false and no channels.", + region: client.region, + }; + } + const submit = await client.request( "POST", "/mcp/qualify", @@ -210,6 +264,13 @@ export const qualifyLeads: Tool = { // status_reason (not_in_universe, low_confidence_identity, ...) when // skipped — a skip is an ANSWER about that ref, not an error. items: snapshot.items, + // ...and the same outcomes pre-split, because the shared + // rendering/lead-delivery-table contract this tool's description + // mandates reads deliveries from `leads[]` and skips from `skipped[]`. + // Returning only `items` left an agent following the RENDER block with + // two empty tables; the sibling tools (find_new_leads, lead_job_status) + // both split. `items` stays for input-order per-ref mapping. + ...splitItems(snapshot), cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 98d82595..4eab71a8 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -187,6 +187,8 @@ export { answerClarification, reportOutreach, reportFriction, sendFeedback, importLeads, importAndQualify, createCampaign, addLeadsToCampaign, removeLeadsFromCampaign, seedCandidates, extendLens, + // MCP-first lead delivery + findNewLeads, qualifyLeads, leadJobStatus, artifactKit, }; diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 9ae2ff78..d92295a0 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1441,7 +1441,7 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent €C.CC. +> the Y asked** · stopped: · spent $C.CC. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -1521,7 +1521,7 @@ Pick the 2-3 options that match what actually happened — never all six: | Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote \`dry_run\` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | | Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap / quota | "Raise the cap to €X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| Stopped at cost cap / quota | "Raise the cap to $X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | `; // endregion: leadbay_find_new_leads @@ -2319,7 +2319,7 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent €C.CC. +> the Y asked** · stopped: · spent $C.CC. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -3516,10 +3516,17 @@ IBP reasoning) — "here's why to skip this account" is a deliverable. research+scoring — but repeat calls reuse every fresh cached stage (\`from_cache\` flags on the items) and converge to near-zero cost. \`channels\` purchase verified email (25c) / phone (250c) on success only; -\`already_owned\` values cost nothing. Before a first paid run on a large -batch: \`dry_run: true\`, quote the worst case, get the explicit go-ahead -(an explicit "spend / get their emails" in the user's message counts). -Set \`request_id\` and reuse it on retries of the same batch. +\`already_owned\` values cost nothing. + +The gate is enforced in code, not just here: a PAID call (\`qualify\` left at +its default or set true, and/or any \`channels\`) is WITHHELD unless it carries +\`confirm: true\`. Without it the tool submits nothing and returns +\`mode: "needs_confirmation"\` with a real backend quote — show that quote to +the user, get the go-ahead (an explicit "spend / get their emails" in their +message counts), then re-call with \`confirm: true\`. \`confirm: false\` is a +veto: nothing is submitted and no quote round-trip is made. A fully FREE +call (\`qualify: false\`, no \`channels\`) needs no \`confirm\` and passes straight +through. Set \`request_id\` and reuse it on retries of the same batch. **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. @@ -3571,7 +3578,7 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent €C.CC. +> the Y asked** · stopped: · spent $C.CC. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → diff --git a/packages/core/test/unit/composite/lead-job-status-cursor.test.ts b/packages/core/test/unit/composite/lead-job-status-cursor.test.ts new file mode 100644 index 00000000..8caea601 --- /dev/null +++ b/packages/core/test/unit/composite/lead-job-status-cursor.test.ts @@ -0,0 +1,137 @@ +/** + * Cursor propagation through the block-waiting path of leadbay_lead_job_status. + * + * `since` + `wait_seconds > 0` used to drop the cursor: waitForJob took no + * since/limit, so the promised incremental poll silently became a full + * limit=100 snapshot that re-emitted every already-seen lead. The tool's own + * description promises both behaviours at once, so they must compose. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { leadJobStatus } from "../../../src/composite/lead-job-status.js"; +import { MCP_JOB_POLL } from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const JOB_ID = "3f0a91cc-77b2-4de6-9a10-1e5b7c2d8a44"; +const CURSOR = "2026-07-28T10:20:00Z"; + +const TERMINAL_SNAPSHOT = { + job: { job_id: JOB_ID, state: "completed" }, + funnel: { delivered: 1, examined: 1 }, + items: [ + { + ref: { input_indexes: [0] }, + status: "delivered", + seq: 7, + lead: { lead_id: "aaaa1111-2222-3333-4444-555566667777" }, + }, + ], + cost: { spent: 94, unit: "cost_cents" }, + next_since: "2026-07-28T10:25:00Z", +}; + +beforeEach(() => resetHttpMock()); + +const getPaths = () => + getHttpRequests() + .filter((r: any) => r.method === "GET") + .map((r: any) => r.path as string); + +describe("leadbay_lead_job_status — cursor through the wait path", () => { + it("forwards since + limit when block-waiting", async () => { + // Terminal on the first read, so waitForJob returns after one snapshot. + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=25&since=${encodeURIComponent(CURSOR)}`, + status: 200, + body: TERMINAL_SNAPSHOT, + }, + ]); + + const result: any = await leadJobStatus.execute(newClient(), { + job_id: JOB_ID, + since: CURSOR, + limit: 25, + wait_seconds: 30, + }); + + expect(result.done).toBe(true); + + const paths = getPaths(); + expect(paths).toHaveLength(1); + expect(paths[0]).toContain(`since=${encodeURIComponent(CURSOR)}`); + expect(paths[0]).toContain("limit=25"); + }); + + it("still forwards since + limit on the non-waiting path", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=25&since=${encodeURIComponent(CURSOR)}`, + status: 200, + body: TERMINAL_SNAPSHOT, + }, + ]); + + await leadJobStatus.execute(newClient(), { + job_id: JOB_ID, + since: CURSOR, + limit: 25, + wait_seconds: 0, + }); + + const paths = getPaths(); + expect(paths[0]).toContain(`since=${encodeURIComponent(CURSOR)}`); + expect(paths[0]).toContain("limit=25"); + }); +}); + +describe("leadbay_lead_job_status — wait bound", () => { + it("does not sleep a full interval past a short wait_seconds", async () => { + // A running job never goes terminal, so only the deadline stops the loop. + const RUNNING = { + ...TERMINAL_SNAPSHOT, + job: { job_id: JOB_ID, state: "running" }, + }; + mockHttp( + Array.from({ length: 12 }, () => ({ + method: "GET" as const, + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: RUNNING, + })) + ); + + // Real 4s intervals would make this test unusable; shrink the cadence and + // assert the loop respects a deadline SHORTER than one interval. + const original = MCP_JOB_POLL.intervalMs; + MCP_JOB_POLL.intervalMs = 400; + try { + const startedAt = Date.now(); + const result: any = await leadJobStatus.execute(newClient(), { + job_id: JOB_ID, + wait_seconds: 0.2, + }); + const elapsed = Date.now() - startedAt; + + expect(result.still_running).toBe(true); + // Bounded by the deadline (0.2s), not by the 400ms interval. + expect(elapsed).toBeLessThan(400); + } finally { + MCP_JOB_POLL.intervalMs = original; + } + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts b/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts new file mode 100644 index 00000000..c5836cf2 --- /dev/null +++ b/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts @@ -0,0 +1,248 @@ +/** + * Spend gate + render envelope for leadbay_qualify_leads. + * + * `qualify` defaults to TRUE on the backend (~94 cost_cents per lead needing + * fresh research), so a bare call carrying only `lead_refs` used to be a PAID + * submit of up to 500 refs that the user never approved. The consent gate has + * to live in code — description prose does not stop an agent that skips it. + * + * These tests pin the withhold, the veto, the free pass-through, and the + * `{leads, skipped}` envelope the shared rendering contract mandates. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const JOB_ID = "7b3c1de2-5f40-4a9c-9d21-0c8ea4f61b55"; + +const REFS = [{ website: "franklinbbq.com" }, { website: "uchiaustin.com" }]; + +const DRY_RUN_200 = { + valid: true, + items_requested: 2, + estimated_cost: { max: 238, unit: "cost_cents" }, + quota_forecast: { + web_fetch_allowed: true, + rescore_allowed: true, + enrichment_allowed: true, + }, +}; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 238, unit: "cost_cents" }, + items_requested: 2, + duplicate: false, +}; + +const DELIVERED_ITEM = { + ref: { input_indexes: [0], requested_as: { website: "franklinbbq.com" } }, + status: "delivered", + seq: 0, + cost: { billed: 94, unit: "cost_cents" }, + lead: { + lead_id: "aaaa1111-2222-3333-4444-555566667777", + company: { name: "Franklin Barbecue" }, + }, +}; + +const SKIPPED_ITEM = { + ref: { input_indexes: [1], requested_as: { website: "uchiaustin.com" } }, + status: "skipped", + seq: 1, + status_reason: "not_in_universe", +}; + +const TERMINAL_SNAPSHOT = { + job: { job_id: JOB_ID, state: "completed" }, + funnel: { delivered: 1, examined: 2 }, + items: [DELIVERED_ITEM, SKIPPED_ITEM], + cost: { spent: 94, unit: "cost_cents" }, + next_since: null, +}; + +beforeEach(() => resetHttpMock()); + +const postBodies = () => + getHttpRequests() + .filter((r: any) => r.method === "POST") + .map((r: any) => (typeof r.body === "string" ? JSON.parse(r.body) : r.body)); + +describe("leadbay_qualify_leads — spend gate", () => { + it("withholds the paid submit when confirm is absent, and quotes instead", async () => { + // Only the free dry_run may be called — never the submit. + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + }); + + expect(result.mode).toBe("needs_confirmation"); + expect(result.submitted).toBe(false); + expect(result.vetoed).toBe(false); + expect(result.job_id).toBeUndefined(); + expect(result.estimated_cost).toEqual({ max: 238, unit: "cost_cents" }); + + // Exactly one POST, and it was the FREE dry run. + const posts = postBodies(); + expect(posts).toHaveLength(1); + expect(posts[0].dry_run).toBe(true); + }); + + it("names why the call was treated as paid (backend default is true)", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + }); + + expect(result.paid_because.join(" ")).toMatch(/qualify is on/); + }); + + it("treats requested channels as paid even when qualify is off", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + qualify: false, + channels: ["email"], + }); + + expect(result.mode).toBe("needs_confirmation"); + expect(result.paid_because.join(" ")).toMatch(/channels requested: email/); + }); + + it("confirm:false is a veto — no submit AND no quote round-trip", async () => { + // No endpoints declared: the harness throws if ANY request is made. + mockHttp([]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + confirm: false, + }); + + expect(result.mode).toBe("needs_confirmation"); + expect(result.vetoed).toBe(true); + expect(result.quote).toBeNull(); + expect(getHttpRequests()).toHaveLength(0); + }); + + it("a fully free call (qualify:false, no channels) passes straight through", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: TERMINAL_SNAPSHOT, + }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + qualify: false, + wait_seconds: 0, + }); + + expect(result.mode).toBeUndefined(); + expect(result.job_id).toBe(JOB_ID); + + // The submit went out and was NOT a dry run. + const posts = postBodies(); + expect(posts).toHaveLength(1); + expect(posts[0].dry_run).toBeUndefined(); + expect(posts[0].qualify).toBe(false); + }); + + it("confirm:true submits the paid job without a forced extra quote", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: TERMINAL_SNAPSHOT, + }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + confirm: true, + wait_seconds: 0, + }); + + expect(result.job_id).toBe(JOB_ID); + expect(result.done).toBe(true); + + const posts = postBodies(); + expect(posts).toHaveLength(1); + expect(posts[0].dry_run).toBeUndefined(); + // `confirm` is a client-side gate — it must not leak onto the wire. + expect(posts[0].confirm).toBeUndefined(); + }); + + it("an explicit dry_run still quotes without needing confirm", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + dry_run: true, + }); + + expect(result.dry_run).toBe(true); + expect(result.mode).toBeUndefined(); + expect(result.estimated_cost).toEqual({ max: 238, unit: "cost_cents" }); + }); +}); + +describe("leadbay_qualify_leads — render envelope", () => { + it("returns leads[]/skipped[] alongside items[] on a completed job", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: TERMINAL_SNAPSHOT, + }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + confirm: true, + wait_seconds: 0, + }); + + // The shared rendering contract reads deliveries from leads[] and + // skips from skipped[]; returning only items[] left both tables empty. + expect(Array.isArray(result.leads)).toBe(true); + expect(Array.isArray(result.skipped)).toBe(true); + expect(result.leads).toHaveLength(1); + expect(result.skipped).toHaveLength(1); + + // items[] is preserved for input-order per-ref mapping. + expect(result.items).toHaveLength(2); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads.test.ts b/packages/core/test/unit/composite/qualify-leads.test.ts index 185af25d..66e69a04 100644 --- a/packages/core/test/unit/composite/qualify-leads.test.ts +++ b/packages/core/test/unit/composite/qualify-leads.test.ts @@ -112,6 +112,9 @@ describe("leadbay_qualify_leads", () => { contact_titles: ["Owner"], channels: ["email"], request_id: "vet-austin-1", + // Paid call (channels purchase email) — the spend gate withholds the + // submit without explicit consent. See qualify-leads-spend-gate.test.ts. + confirm: true, wait_seconds: 0, }); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index ee1745b2..7d2e6d57 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -914,13 +914,19 @@ honestly: are these the kind of companies I asked for? # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) When I want qualification evidence and/or reachable contacts: -1. Quote first: \`dry_run: true\` with the exact flags I asked for - (\`qualify: true\`, \`min_ai_score\`, \`contact_titles\`, \`title_gate\`, - \`channels\`, \`max_cost\`) and tell me the worst-case cost in plain money. +1. Quote first: \`dry_run: true\` on the tool you will actually run, with the + exact flags I asked for, and tell me the worst-case cost in plain money. + The two tools take DIFFERENT flags — passing the wrong one is rejected + outright (\`additionalProperties: false\`): + - \`leadbay_qualify_leads\`: \`qualify: true\`, \`contact_titles\`, + \`title_gate\`, \`channels\`, \`max_cost\`. **No \`min_ai_score\`.** + - \`leadbay_find_new_leads\`: the same, PLUS \`min_ai_score\` and \`count\`. 2. On my go-ahead, prefer feeding the free preview's deliveries to \`leadbay_qualify_leads\` (\`prior_deliveries: {job_id}\`) — it only spends on companies already known to match. Run a fresh \`qualify: true\` search - instead when I asked for more than the preview delivered. + instead when I asked for more than the preview delivered. Paid + \`leadbay_qualify_leads\` calls need \`confirm: true\` — without it the tool + withholds the submit and hands back a quote instead of spending. 3. While the job runs, poll with \`leadbay_lead_job_status\` (\`wait_seconds: 60\`); report progress, not silence. diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index 4cf5d60c..b407c5f1 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -82,13 +82,19 @@ honestly: are these the kind of companies I asked for? # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) When I want qualification evidence and/or reachable contacts: -1. Quote first: `dry_run: true` with the exact flags I asked for - (`qualify: true`, `min_ai_score`, `contact_titles`, `title_gate`, - `channels`, `max_cost`) and tell me the worst-case cost in plain money. +1. Quote first: `dry_run: true` on the tool you will actually run, with the + exact flags I asked for, and tell me the worst-case cost in plain money. + The two tools take DIFFERENT flags — passing the wrong one is rejected + outright (`additionalProperties: false`): + - `leadbay_qualify_leads`: `qualify: true`, `contact_titles`, + `title_gate`, `channels`, `max_cost`. **No `min_ai_score`.** + - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. 2. On my go-ahead, prefer feeding the free preview's deliveries to `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on companies already known to match. Run a fresh `qualify: true` search - instead when I asked for more than the preview delivered. + instead when I asked for more than the preview delivered. Paid + `leadbay_qualify_leads` calls need `confirm: true` — without it the tool + withholds the submit and hands back a quote instead of spending. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. diff --git a/packages/promptforge/snippets/next-steps/find-new-leads.md b/packages/promptforge/snippets/next-steps/find-new-leads.md index 35b4ffd6..2f5fab25 100644 --- a/packages/promptforge/snippets/next-steps/find-new-leads.md +++ b/packages/promptforge/snippets/next-steps/find-new-leads.md @@ -10,5 +10,5 @@ Pick the 2-3 options that match what actually happened — never all six: | Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote `dry_run` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | | Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap / quota | "Raise the cap to €X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| Stopped at cost cap / quota | "Raise the cap to $X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index 7b0104de..787050e7 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -43,7 +43,7 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + `explain.scope_notes`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent €C.CC. +> the Y asked** · stopped: · spent $C.CC. Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → "ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → diff --git a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl index 3aa5024d..85b4f475 100644 --- a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl @@ -79,10 +79,17 @@ IBP reasoning) — "here's why to skip this account" is a deliverable. research+scoring — but repeat calls reuse every fresh cached stage (`from_cache` flags on the items) and converge to near-zero cost. `channels` purchase verified email (25c) / phone (250c) on success only; -`already_owned` values cost nothing. Before a first paid run on a large -batch: `dry_run: true`, quote the worst case, get the explicit go-ahead -(an explicit "spend / get their emails" in the user's message counts). -Set `request_id` and reuse it on retries of the same batch. +`already_owned` values cost nothing. + +The gate is enforced in code, not just here: a PAID call (`qualify` left at +its default or set true, and/or any `channels`) is WITHHELD unless it carries +`confirm: true`. Without it the tool submits nothing and returns +`mode: "needs_confirmation"` with a real backend quote — show that quote to +the user, get the go-ahead (an explicit "spend / get their emails" in their +message counts), then re-call with `confirm: true`. `confirm: false` is a +veto: nothing is submitted and no quote round-trip is made. A fully FREE +call (`qualify: false`, no `channels`) needs no `confirm` and passes straight +through. Set `request_id` and reuse it on retries of the same batch. **Limits**: 500 refs/job, 3 active jobs/org, 10 submits/hour (429 + Retry-After beyond — wait, don't hammer), 30-min job wall clock. From 515af8bb0b80f7bf4fbb0d9a018db270721f6a7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:33:42 -0700 Subject: [PATCH 10/46] fix(mcp): address Codex review on MCP-first lead delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - collectJobSnapshot: derive the page bound from page SIZE, not a flat 20. At limit=5 a 500-item job returned the first 100 items while reporting done:true. - collectJobSnapshot: keep the last non-empty page's cursor when the drain page comes back empty; next_since was being overwritten with null, so a caller lost its place and re-read everything it had already seen. - rejectCountryLocations: normalize instead of exact-matching. "the United States", "U.S", "les États-Unis" and "République Française" all passed through to the silent same-named-town fencing this guard exists to stop. Île-de-France and other legitimate places still pass. - find_new_leads / qualify_leads: short-circuit LEADBAY_MOCK submits. The mock envelope has no job_id, so the offline dry-run mode polled /mcp/jobs/undefined and died on a missing fixture. - list_locations is now always exposed, like list_sectors. The delivery tools document it as the recovery path after a 400 on an unresolvable location, which only works if it's reachable without ADVANCED=1. - leadbay_new_leads prompt: handle still_running BEFORE judging the seed (Phase 3 had no such branch, so a live partial read as "empty" and triggered a relaunch that burns an active-job slot); require an archetype component in request_id so two archetype searches don't dedupe to one job; route unsupported negative constraints ("no franchises") into the seed + a Phase 5 drop-and-say instead of silently losing them; permit the fictional seed explicitly against the no-fabrication law; take the need from the triggering message so the skill never asks the user to repeat what they just said. - lead-delivery-table: don't invent the "of the Y asked" denominator when items_requested is absent from a job snapshot. Budget: trimmed find-new-leads template body — the additions above pushed it to 17188 (cap 17000). Now 16948, in line with the other large tools rather than the outlier. pnpm -r build, -r test (1327 passed), -r typecheck green. Co-Authored-By: Claude --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 46 +++-- .../core/src/composite/_mcp-job-helpers.ts | 77 ++++++-- packages/core/src/composite/find-new-leads.ts | 7 + packages/core/src/composite/qualify-leads.ts | 7 + packages/core/src/index.ts | 5 + .../core/src/tool-descriptions.generated.ts | 29 ++- .../composite/mcp-job-helpers-review.test.ts | 167 ++++++++++++++++++ packages/mcp/src/prompts.generated.ts | 46 +++-- .../prompts/leadbay_new_leads.md.tmpl | 46 +++-- .../snippets/rendering/lead-delivery-table.md | 5 + .../composite/find-new-leads.md.tmpl | 14 +- 11 files changed, 392 insertions(+), 57 deletions(-) create mode 100644 packages/core/test/unit/composite/mcp-job-helpers-review.test.ts diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md index 413f5ea9..1f8bdf51 100644 --- a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md +++ b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md @@ -21,9 +21,11 @@ Find net-new leads for me. My need, in my words: > -If the need above is empty or too vague to name (a) who I sell to and (b) -roughly how many leads I want, ask me ONCE — one short question — then -proceed. Default count when unstated: 10. +If that need was not supplied to you directly, take it from the message that +started this — the request in my own words is the need, and I should never be +asked to repeat something I already said. Only when BOTH are missing or too +vague to name (a) who I sell to and (b) roughly how many leads I want, ask me +ONCE — one short question — then proceed. Default count when unstated: 10. # PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) @@ -47,18 +49,40 @@ and city/state/region `locations` — never a country name). Show me the seed description(s) in one line each — I should recognize my ideal customer in them. +`filters` only encodes sectors, locations and employee bounds. Any constraint +that does not fit those keys — above all EXCLUSIONS like "no franchises" or +"pas de grands groupes" — has nowhere to live in the filter schema, so it must +not be dropped on the floor: express it positively in the seed `description` +(an independent single-site operator rather than "no franchises"), and carry +the exclusion forward yourself to Phase 5, where you drop violating rows and +say you dropped them. Tell me plainly if a constraint can only be enforced +that way — after the fact, not by the search. + +Composing this fictional seed from my words is expected and permitted: it is +the tool's designed input, not fabricated data. What must never be invented is +a RESULT — company names, contacts, scores, or anything presented as coming +back from Leadbay. + # PHASE 3 — FREE PREVIEW Call `leadbay_find_new_leads` with the seed, `filters`, `count`, `qualify: false`, no channels — this is FREE — and a `request_id` derived -from the ask + today's date. Render the delivery table and judge fit -honestly: are these the kind of companies I asked for? - -- **On-profile** → offer Phase 4. -- **Off-profile or empty** → read `funnel` + `explain.scope_notes`, tell me - what went wrong in one line (wrong archetype? too narrow a filter? thin - universe?), reshape the seed or filters, and retry under a NEW request_id. - Reshaping is free; do not pay to explore a bad seed. +from the ask + the ARCHETYPE + today's date. The archetype component is not +optional: `request_id` is the idempotency key, so two archetype searches +sharing one id dedupe to the same job and the second archetype is never +searched. Render the delivery table and judge fit honestly: are these the +kind of companies I asked for? + +- **`still_running: true`** → the job is ALIVE. Do not judge the seed and do + not relaunch — poll `leadbay_lead_job_status` (`wait_seconds: 60`) until + it goes terminal, reporting progress. Relaunching now burns an active-job + slot and rate-limit budget on a search that may be about to deliver. +- **On-profile** (terminal) → offer Phase 4. +- **Off-profile or empty** (terminal) → read `funnel` + + `explain.scope_notes`, tell me what went wrong in one line (wrong + archetype? too narrow a filter? thin universe?), reshape the seed or + filters, and retry under a NEW request_id. Reshaping is free; do not pay + to explore a bad seed. # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index adfeaec2..b3eade66 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -116,7 +116,16 @@ export const TERMINAL_JOB_STATES: ReadonlySet = new Set([ export const MCP_JOB_POLL = { intervalMs: 4000 }; const PAGE_LIMIT = 100; -const MAX_PAGES = 20; // 500-ref qualify worst case is 5 pages; 20 is a hard stop. +// A qualify job carries at most 500 refs, so the drain is bounded by the page +// SIZE, not by a flat page count: at limit=5 the worst case is 100 pages, and a +// flat 20 would silently return the first 100 items while reporting done:true. +// Derive the bound instead, with a floor so a large page size still gets a few +// follow-ups and a ceiling that stays a runaway backstop. +const MAX_JOB_ITEMS = 500; +const MIN_PAGES = 20; +const PAGE_CAP = 120; +const maxPagesFor = (pageLimit: number) => + Math.min(PAGE_CAP, Math.max(MIN_PAGES, Math.ceil(MAX_JOB_ITEMS / pageLimit) + 1)); /** One cumulative snapshot of the job, paging the item cursor dry. Job/funnel/ * cost/explain come from the LAST page fetched (the freshest projection). */ @@ -130,14 +139,16 @@ export async function collectJobSnapshot( const qs = (cursor?: string) => `/mcp/jobs/${jobId}?limit=${pageLimit}` + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); + const maxPages = maxPagesFor(pageLimit); let page = await client.request("GET", qs(since)); const items = [...page.items]; + // The resumption cursor must survive an empty drain page. Following + // next_since into a page with no items used to overwrite the cursor with that + // empty page's (often null) next_since, so a caller that had just received a + // full page lost its place and had to re-read everything it had already seen. + let cursor = page.next_since ?? null; let pages = 1; - while ( - page.items.length >= pageLimit && - page.next_since && - pages < MAX_PAGES - ) { + while (page.items.length >= pageLimit && page.next_since && pages < maxPages) { const next = await client.request( "GET", qs(page.next_since) @@ -148,10 +159,12 @@ export async function collectJobSnapshot( // freshest even when it carried no new items. page = next; if (next.items.length === 0) { + // Keep the cursor from the last page that actually carried items. break; } + cursor = next.next_since ?? cursor; } - return { ...page, items }; + return { ...page, items, next_since: cursor }; } /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). @@ -197,6 +210,28 @@ export async function waitForJob( return snap; } +/** LEADBAY_MOCK=1 journals writes and answers the generic + * `{mocked, would_call}` envelope instead of a real `{job_id}`. Without a + * guard the submit falls through to polling `/mcp/jobs/undefined`, which has + * no fixture — so the repo's offline dry-run mode died on any non-dry_run + * call. Return the write preview instead. */ +export function mockedSubmitPreview( + submit: unknown, + tool: string, + region: string +): Record | null { + const s = (submit ?? {}) as Record; + if (typeof s.job_id === "string" && s.job_id) return null; + return { + mocked: true, + tool, + submitted: false, + would_call: s.would_call ?? null, + note: "LEADBAY_MOCK=1 — the job was not submitted, so there is no job to poll.", + region, + }; +} + /** Sort a snapshot's items into the envelope every delivery tool returns: * full leads for delivered/degraded, compact skip records for the rest. */ export function splitItems(snapshot: McpJobSnapshot): { @@ -231,14 +266,36 @@ export function compactBody( * rejects it with a named, actionable error (tracked backend-side in * product#3939). */ const COUNTRY_LOCATION_VALUES = new Set([ - "united states", "united states of america", "usa", "u.s.", "u.s.a.", "us", - "america", "etats-unis", "états-unis", "france", "fr", + "united states", "united states of america", "usa", "us", "america", + "etats unis", "etats unis d amerique", "france", "fr", "french republic", + "republique francaise", ]); +/** Fold a location label to a comparison key so spelling variants collapse: + * strips accents, punctuation (so `U.S.` and `U.S` both become `us`), a + * leading article (`the United States`, `la France`, `les États-Unis`), and + * collapses whitespace. Exact-matching the raw string let every one of those + * through to the silent same-named-town fencing this guard exists to stop. */ +function countryKey(raw: string): string { + return raw + .normalize("NFD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + // Hyphens/underscores separate words; dots and apostrophes do not (so + // "U.S" folds to "us", while "etats-unis" stays two words). + .replace(/[-_,]/g, " ") + .replace(/['’.]/g, "") + .replace(/\s+/g, " ") + .trim() + // Longest article first, so "les" is never matched as "le" + leftover. + .replace(/^(les|the|la|le|l)\s+/, "") + .trim(); +} + export function rejectCountryLocations(locations: unknown): void { if (!Array.isArray(locations)) return; for (const loc of locations) { - if (typeof loc === "string" && COUNTRY_LOCATION_VALUES.has(loc.trim().toLowerCase())) { + if (typeof loc === "string" && COUNTRY_LOCATION_VALUES.has(countryKey(loc))) { throw { error: true, code: "COUNTRY_LEVEL_LOCATION", diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 8e334a50..bb9d9638 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + mockedSubmitPreview, compactBody, normalizeSearchFilters, rejectCountryLocations, @@ -217,6 +218,12 @@ export const findNewLeads: Tool = { "/mcp/search", body ); + const mocked = mockedSubmitPreview( + submit, + "leadbay_find_new_leads", + client.region + ); + if (mocked) return mocked; const waitSeconds = clampWaitSeconds( params.wait_seconds, DEFAULT_WAIT_SECONDS diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 6fa5b900..2e4cf1a5 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + mockedSubmitPreview, compactBody, splitItems, TERMINAL_JOB_STATES, @@ -225,6 +226,12 @@ export const qualifyLeads: Tool = { "/mcp/qualify", body ); + const mocked = mockedSubmitPreview( + submit, + "leadbay_qualify_leads", + client.region + ); + if (mocked) return mocked; const waitSeconds = clampWaitSeconds( params.wait_seconds, DEFAULT_WAIT_SECONDS diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4eab71a8..bc053e45 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -327,6 +327,11 @@ export const compositeReadTools: Tool[] = [ // leadbay_new_lens / leadbay_adjust_audience). Without it the agent can only // probe sectors by trial-and-error or ask the user to read the web UI. listSectors, + // listLocations, same rationale on the geography axis. The delivery tools + // reject an unresolvable filters.locations with a 400 naming the value and + // send the agent here to look up the real admin area — a recovery path that + // only works if the lookup is reachable without LEADBAY_MCP_ADVANCED=1. + listLocations, // Billing / top-up tools — granular-shaped but ALWAYS exposed because // they're the canonical recovery path from a QUOTA_EXCEEDED wall. If // they were gated behind LEADBAY_MCP_ADVANCED=1 the agent would diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index d92295a0..d252fcb4 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1319,15 +1319,15 @@ phone 250c, success-only). Before the first paid run: \`dry_run: true\`, quote the worst case, get the explicit go-ahead ("spend / get their emails" in the user's message counts). Free needs no consent. -**Free preview first**: run FREE, eyeball fit, THEN pay. Off-profile preview -→ reshape the seed (free) instead of exploring it with \`qualify: true\`. +**Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an +off-profile seed is free, exploring it with \`qualify: true\` is not. **Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not -the free match.** Positive inverse in the seed, then \`qualify: true\`: the -org's ideal-buyer-profile anti-patterns and questions score violators -negative and \`min_ai_score: 0\` drops them. Durable exclusion → suggest -\`leadbay_refine_prompt\` so it enters the org intelligence. In a FREE preview, -drop visibly violating rows and say the free tier doesn't enforce. +the free match** — \`filters\` has no exclusion key. Put the positive inverse in +the seed, then \`qualify: true\`: IBP anti-patterns and questions score +violators negative and \`min_ai_score: 0\` drops them. Durable exclusion → +suggest \`leadbay_refine_prompt\`. In a FREE preview, drop violating rows and +say the free tier doesn't enforce. ### Crafting the \`example_lead\` seed — the input that decides result quality @@ -1443,6 +1443,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent $C.CC. +"of the Y asked" needs \`summary.items_requested\`, which submit results carry +but a later \`leadbay_lead_job_status\` snapshot does not. Without it write +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those +count candidates, not the target) and never guess it. + Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". @@ -2321,6 +2326,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent $C.CC. +"of the Y asked" needs \`summary.items_requested\`, which submit results carry +but a later \`leadbay_lead_job_status\` snapshot does not. Without it write +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those +count candidates, not the target) and never guess it. + Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". @@ -3580,6 +3590,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent $C.CC. +"of the Y asked" needs \`summary.items_requested\`, which submit results carry +but a later \`leadbay_lead_job_status\` snapshot does not. Without it write +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those +count candidates, not the target) and never guess it. + Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". diff --git a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts new file mode 100644 index 00000000..c8a49e0e --- /dev/null +++ b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts @@ -0,0 +1,167 @@ +/** + * Review fixes on the shared MCP-job plumbing (Codex pass on PR #168). + * + * Three separate defects, all in _mcp-job-helpers.ts: + * - a flat 20-page stop truncated small-page drains while reporting done + * - an empty drain page overwrote the resumption cursor with null + * - the country guard exact-matched, so "the United States" / "U.S" / + * "les États-Unis" sailed through to silent same-named-town fencing + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { + collectJobSnapshot, + rejectCountryLocations, + mockedSubmitPreview, +} from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); +const JOB_ID = "5c2f0b7a-9e11-4d33-8a06-77b1c4e2f900"; + +const item = (seq: number) => ({ + ref: { input_indexes: [seq] }, + status: "delivered", + seq, + lead: { lead_id: `lead-${seq}` }, +}); + +const page = (items: any[], next: string | null) => ({ + job: { id: JOB_ID, state: "completed" }, + funnel: { delivered: items.length }, + items, + next_since: next, + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + explain: { region: "us", model: "m" }, +}); + +beforeEach(() => resetHttpMock()); + +describe("collectJobSnapshot — page bound scales with page size", () => { + it("drains a small-page job past the old flat 20-page stop", async () => { + // limit=5 over 300 items = 60 pages. The old flat MAX_PAGES=20 returned + // 100 items and still reported the job complete. + const TOTAL = 300; + const SIZE = 5; + const pages = []; + for (let start = 0; start < TOTAL; start += SIZE) { + const items = Array.from({ length: SIZE }, (_, i) => item(start + i)); + const isLast = start + SIZE >= TOTAL; + pages.push({ + method: "GET" as const, + path: + `/1.6/mcp/jobs/${JOB_ID}?limit=${SIZE}` + + (start === 0 ? "" : `&since=${encodeURIComponent(`cur-${start}`)}`), + status: 200, + body: page(items, isLast ? null : `cur-${start + SIZE}`), + }); + } + mockHttp(pages); + + const snap = await collectJobSnapshot(newClient(), JOB_ID, undefined, SIZE); + expect(snap.items).toHaveLength(TOTAL); + }); +}); + +describe("collectJobSnapshot — cursor survives an empty drain page", () => { + it("keeps the last non-empty page's cursor when the next page is empty", async () => { + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=2`, + status: 200, + body: page([item(0), item(1)], "cur-2"), + }, + { + // The drain page: no items, and the backend nulls the cursor. + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=2&since=${encodeURIComponent("cur-2")}`, + status: 200, + body: page([], null), + }, + ]); + + const snap = await collectJobSnapshot(newClient(), JOB_ID, undefined, 2); + expect(snap.items).toHaveLength(2); + // Without the fix this was null and the caller had to re-read from zero. + expect(snap.next_since).toBe("cur-2"); + }); +}); + +describe("rejectCountryLocations — alias normalization", () => { + const rejects = [ + "United States", + "the United States", + "U.S", + "U.S.", + "U.S.A.", + "USA", + "America", + "les États-Unis", + "États-Unis", + "etats-unis", + "la France", + "France", + "République Française", + " us ", + ]; + + for (const value of rejects) { + it(`rejects ${JSON.stringify(value)}`, () => { + expect(() => rejectCountryLocations([value])).toThrow( + expect.objectContaining({ code: "COUNTRY_LEVEL_LOCATION" }) + ); + }); + } + + // The guard must not swallow legitimate places — Île-de-France in + // particular must not be mistaken for France. + const allows = [ + "Austin", + "New York", + "Paris", + "Île-de-France", + "Los Angeles", + "Texas", + "Kansas City", + ]; + + for (const value of allows) { + it(`allows ${JSON.stringify(value)}`, () => { + expect(() => rejectCountryLocations([value])).not.toThrow(); + }); + } + + it("ignores a non-array argument", () => { + expect(() => rejectCountryLocations(undefined)).not.toThrow(); + }); +}); + +describe("mockedSubmitPreview", () => { + it("returns a preview when the submit carried no job_id (LEADBAY_MOCK)", () => { + const out = mockedSubmitPreview( + { mocked: true, would_call: { method: "POST", path: "/1.6/mcp/search" } }, + "leadbay_find_new_leads", + "us" + ); + expect(out).not.toBeNull(); + expect(out!.submitted).toBe(false); + expect(out!.tool).toBe("leadbay_find_new_leads"); + }); + + it("returns null for a real submit so the normal poll proceeds", () => { + expect( + mockedSubmitPreview({ job_id: JOB_ID }, "leadbay_find_new_leads", "us") + ).toBeNull(); + }); +}); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 7d2e6d57..7933cc10 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -872,9 +872,11 @@ Find net-new leads for me. My need, in my words: > {{arg:need}} -If the need above is empty or too vague to name (a) who I sell to and (b) -roughly how many leads I want, ask me ONCE — one short question — then -proceed. Default count when unstated: 10. +If that need was not supplied to you directly, take it from the message that +started this — the request in my own words is the need, and I should never be +asked to repeat something I already said. Only when BOTH are missing or too +vague to name (a) who I sell to and (b) roughly how many leads I want, ask me +ONCE — one short question — then proceed. Default count when unstated: 10. # PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) @@ -898,18 +900,40 @@ and city/state/region \`locations\` — never a country name). Show me the seed description(s) in one line each — I should recognize my ideal customer in them. +\`filters\` only encodes sectors, locations and employee bounds. Any constraint +that does not fit those keys — above all EXCLUSIONS like "no franchises" or +"pas de grands groupes" — has nowhere to live in the filter schema, so it must +not be dropped on the floor: express it positively in the seed \`description\` +(an independent single-site operator rather than "no franchises"), and carry +the exclusion forward yourself to Phase 5, where you drop violating rows and +say you dropped them. Tell me plainly if a constraint can only be enforced +that way — after the fact, not by the search. + +Composing this fictional seed from my words is expected and permitted: it is +the tool's designed input, not fabricated data. What must never be invented is +a RESULT — company names, contacts, scores, or anything presented as coming +back from Leadbay. + # PHASE 3 — FREE PREVIEW Call \`leadbay_find_new_leads\` with the seed, \`filters\`, \`count\`, \`qualify: false\`, no channels — this is FREE — and a \`request_id\` derived -from the ask + today's date. Render the delivery table and judge fit -honestly: are these the kind of companies I asked for? - -- **On-profile** → offer Phase 4. -- **Off-profile or empty** → read \`funnel\` + \`explain.scope_notes\`, tell me - what went wrong in one line (wrong archetype? too narrow a filter? thin - universe?), reshape the seed or filters, and retry under a NEW request_id. - Reshaping is free; do not pay to explore a bad seed. +from the ask + the ARCHETYPE + today's date. The archetype component is not +optional: \`request_id\` is the idempotency key, so two archetype searches +sharing one id dedupe to the same job and the second archetype is never +searched. Render the delivery table and judge fit honestly: are these the +kind of companies I asked for? + +- **\`still_running: true\`** → the job is ALIVE. Do not judge the seed and do + not relaunch — poll \`leadbay_lead_job_status\` (\`wait_seconds: 60\`) until + it goes terminal, reporting progress. Relaunching now burns an active-job + slot and rate-limit budget on a search that may be about to deliver. +- **On-profile** (terminal) → offer Phase 4. +- **Off-profile or empty** (terminal) → read \`funnel\` + + \`explain.scope_notes\`, tell me what went wrong in one line (wrong + archetype? too narrow a filter? thin universe?), reshape the seed or + filters, and retry under a NEW request_id. Reshaping is free; do not pay + to explore a bad seed. # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index b407c5f1..a5486467 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -40,9 +40,11 @@ Find net-new leads for me. My need, in my words: > {{arg:need}} -If the need above is empty or too vague to name (a) who I sell to and (b) -roughly how many leads I want, ask me ONCE — one short question — then -proceed. Default count when unstated: 10. +If that need was not supplied to you directly, take it from the message that +started this — the request in my own words is the need, and I should never be +asked to repeat something I already said. Only when BOTH are missing or too +vague to name (a) who I sell to and (b) roughly how many leads I want, ask me +ONCE — one short question — then proceed. Default count when unstated: 10. # PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) @@ -66,18 +68,40 @@ and city/state/region `locations` — never a country name). Show me the seed description(s) in one line each — I should recognize my ideal customer in them. +`filters` only encodes sectors, locations and employee bounds. Any constraint +that does not fit those keys — above all EXCLUSIONS like "no franchises" or +"pas de grands groupes" — has nowhere to live in the filter schema, so it must +not be dropped on the floor: express it positively in the seed `description` +(an independent single-site operator rather than "no franchises"), and carry +the exclusion forward yourself to Phase 5, where you drop violating rows and +say you dropped them. Tell me plainly if a constraint can only be enforced +that way — after the fact, not by the search. + +Composing this fictional seed from my words is expected and permitted: it is +the tool's designed input, not fabricated data. What must never be invented is +a RESULT — company names, contacts, scores, or anything presented as coming +back from Leadbay. + # PHASE 3 — FREE PREVIEW Call `leadbay_find_new_leads` with the seed, `filters`, `count`, `qualify: false`, no channels — this is FREE — and a `request_id` derived -from the ask + today's date. Render the delivery table and judge fit -honestly: are these the kind of companies I asked for? - -- **On-profile** → offer Phase 4. -- **Off-profile or empty** → read `funnel` + `explain.scope_notes`, tell me - what went wrong in one line (wrong archetype? too narrow a filter? thin - universe?), reshape the seed or filters, and retry under a NEW request_id. - Reshaping is free; do not pay to explore a bad seed. +from the ask + the ARCHETYPE + today's date. The archetype component is not +optional: `request_id` is the idempotency key, so two archetype searches +sharing one id dedupe to the same job and the second archetype is never +searched. Render the delivery table and judge fit honestly: are these the +kind of companies I asked for? + +- **`still_running: true`** → the job is ALIVE. Do not judge the seed and do + not relaunch — poll `leadbay_lead_job_status` (`wait_seconds: 60`) until + it goes terminal, reporting progress. Relaunching now burns an active-job + slot and rate-limit budget on a search that may be about to deliver. +- **On-profile** (terminal) → offer Phase 4. +- **Off-profile or empty** (terminal) → read `funnel` + + `explain.scope_notes`, tell me what went wrong in one line (wrong + archetype? too narrow a filter? thin universe?), reshape the seed or + filters, and retry under a NEW request_id. Reshaping is free; do not pay + to explore a bad seed. # PHASE 4 — PAID DEPTH (only with my explicit go-ahead) diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index 787050e7..efab23ef 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -45,6 +45,11 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent $C.CC. +"of the Y asked" needs `summary.items_requested`, which submit results carry +but a later `leadbay_lead_job_status` snapshot does not. Without it write +**delivered X** and stop — never back-fill Y from `matched`/`examined` (those +count candidates, not the target) and never guess it. + Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → "ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → "hit an org quota", `time_budget` → "hit the 30-min time budget". diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 67610a87..cd5092a5 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -73,15 +73,15 @@ phone 250c, success-only). Before the first paid run: `dry_run: true`, quote the worst case, get the explicit go-ahead ("spend / get their emails" in the user's message counts). Free needs no consent. -**Free preview first**: run FREE, eyeball fit, THEN pay. Off-profile preview -→ reshape the seed (free) instead of exploring it with `qualify: true`. +**Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an +off-profile seed is free, exploring it with `qualify: true` is not. **Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not -the free match.** Positive inverse in the seed, then `qualify: true`: the -org's ideal-buyer-profile anti-patterns and questions score violators -negative and `min_ai_score: 0` drops them. Durable exclusion → suggest -`leadbay_refine_prompt` so it enters the org intelligence. In a FREE preview, -drop visibly violating rows and say the free tier doesn't enforce. +the free match** — `filters` has no exclusion key. Put the positive inverse in +the seed, then `qualify: true`: IBP anti-patterns and questions score +violators negative and `min_ai_score: 0` drops them. Durable exclusion → +suggest `leadbay_refine_prompt`. In a FREE preview, drop violating rows and +say the free tier doesn't enforce. {{include:heuristics/example-lead-craft}} From 8bfcd559ce4d4c0a094da9fe2241045a01653220 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:52:33 -0700 Subject: [PATCH 11/46] fix(mcp): gate paid searches too, mark the paid submitter destructive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second Codex pass, on the review fixes themselves. - find_new_leads had the same paid surface as qualify_leads (qualify:true and/or channels) but no gate — only qualify_leads got one. Same withhold: a paid search now needs confirm:true, otherwise nothing is submitted and a real dry-run quote comes back as mode:"needs_confirmation". The trigger differs by design: qualify defaults to FALSE here, so the default search stays free and ungated. - qualify_leads is destructiveHint:true. It bills for fresh qualification and channel reveals, and hosts key approval prompts off that flag — bulk_qualify_leads already sets it for the same reason. find_new_leads stays false with a comment saying why: its default really is free. - leadbay_new_leads prompt: on a short preview, qualify what was delivered AND search only the SHORTFALL under a NEW request_id. Reusing the preview's id dedupes the paid submit into the free job; keeping the original count buys a second full batch, since novelty:org already excludes the preview deliveries. Budget: the gate copy pushed find_new_leads to 17140 (cap 17000). Trimmed the template body — dropped param prose already carried by inputSchema descriptions and condensed the intro. Now 16961. pnpm -r build, -r test (1332 passed), -r typecheck green. Co-Authored-By: Claude --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 16 +- packages/core/src/composite/find-new-leads.ts | 50 ++++++ packages/core/src/composite/qualify-leads.ts | 6 +- .../core/src/tool-descriptions.generated.ts | 38 ++-- .../find-new-leads-spend-gate.test.ts | 166 ++++++++++++++++++ packages/mcp/src/prompts.generated.ts | 16 +- .../prompts/leadbay_new_leads.md.tmpl | 16 +- .../composite/find-new-leads.md.tmpl | 38 ++-- 8 files changed, 295 insertions(+), 51 deletions(-) create mode 100644 packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md index 1f8bdf51..622466c4 100644 --- a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md +++ b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md @@ -96,10 +96,18 @@ When I want qualification evidence and/or reachable contacts: - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. 2. On my go-ahead, prefer feeding the free preview's deliveries to `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on - companies already known to match. Run a fresh `qualify: true` search - instead when I asked for more than the preview delivered. Paid - `leadbay_qualify_leads` calls need `confirm: true` — without it the tool - withholds the submit and hands back a quote instead of spending. + companies already known to match. Paid calls need `confirm: true`; without + it the tool withholds the submit and hands back a quote instead of + spending. That applies to `leadbay_find_new_leads` too whenever you set + `qualify: true` or ask for channels. + + If the preview delivered FEWER than I asked for, do both halves and do not + conflate them: qualify what the preview already found, and run the fresh + search only for the SHORTFALL — `count` = what is still missing, never the + original number, under a NEW `request_id`. Reusing the preview's id dedupes + the paid submit back into the free job; keeping the original count buys a + whole second batch, because `novelty: org` already excludes everything the + preview delivered. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index bb9d9638..c147b9dd 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -50,6 +50,7 @@ interface FindNewLeadsParams { exploration_cap?: number; request_id: string; lang?: string; + confirm?: boolean; dry_run?: boolean; wait_seconds?: number; } @@ -61,6 +62,11 @@ export const findNewLeads: Tool = { annotations: { title: "Find new leads (net-new ICP search)", readOnlyHint: false, + // Unlike qualify_leads (paid by default), the default search here is FREE + // — qualify defaults to false and channels are empty — so the tool is not + // inherently destructive. A paid variant (qualify:true / channels) is + // withheld in execute() until `confirm: true`, which is where the spend + // decision is surfaced. destructiveHint: false, // The mandatory request_id dedups: re-submitting the same request returns // the SAME live job instead of double-spending. @@ -161,6 +167,11 @@ export const findNewLeads: Tool = { "REQUIRED idempotency key. Derive it from the ask (e.g. 'gyms-texas-2026-07-28'); REUSE the exact same value when retrying the same ask — a duplicate returns the SAME job instead of double-spending. Use a NEW value only for a genuinely new ask.", }, lang: { type: "string", description: "Output language (default: user's language)." }, + confirm: { + type: "boolean", + description: + "Explicit spend decision, required only for a PAID search (qualify:true and/or channels). true = the user approved the quote, go ahead. false = a veto (returns mode:'needs_confirmation', spends nothing). Omitted on a paid call → the tool withholds the submit and returns a free quote to show the user first. The default FREE search (no qualify, no channels) needs no confirm.", + }, dry_run: { type: "boolean", description: @@ -181,6 +192,18 @@ export const findNewLeads: Tool = { ctx?: ToolContext ) => { rejectCountryLocations(params.filters?.locations); + + // Same spend gate as leadbay_qualify_leads. The trigger differs: `qualify` + // defaults to FALSE here, so the default ask really is free and only an + // explicit qualify:true and/or requested channels costs money. When it + // does, the submit is withheld pending `confirm: true` and a real + // dry-run quote is returned instead. + const buysChannels = (params.channels?.length ?? 0) > 0; + const buysQualification = params.qualify === true; + const isPaid = buysQualification || buysChannels; + const vetoed = params.confirm === false; + const consented = !vetoed && params.confirm === true; + const body = compactBody({ query: params.query, example_lead: params.example_lead, @@ -213,6 +236,33 @@ export const findNewLeads: Tool = { }; } + if (isPaid && !consented) { + const forecast = vetoed + ? null + : await client.request("POST", "/mcp/search", { + ...body, + dry_run: true, + }); + return { + mode: "needs_confirmation", + submitted: false, + vetoed, + paid_because: [ + buysQualification + ? "qualify: true (~94 cost_cents per candidate EXAMINED)" + : null, + buysChannels ? `channels requested: ${params.channels!.join(", ")}` : null, + ].filter(Boolean), + quote: forecast, + estimated_cost: forecast?.estimated_cost ?? null, + items_requested: forecast?.items_requested ?? null, + hint: vetoed + ? "confirm:false vetoed the spend — nothing was submitted. Re-call with confirm:true to proceed, or drop qualify/channels for a free search." + : "Show the user this worst-case quote and get an explicit go-ahead, then re-call with confirm:true. For a free search instead: omit qualify and channels.", + region: client.region, + }; + } + const submit = await client.request( "POST", "/mcp/search", diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 2e4cf1a5..cb6abbd2 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -54,7 +54,11 @@ export const qualifyLeads: Tool = { annotations: { title: "Qualify + get the right contact on known leads", readOnlyHint: false, - destructiveHint: false, + // Spends real money (fresh qualification, and email/phone reveals when + // channels are requested), same as bulk_qualify_leads / enrich-titles. + // Hosts and approval layers key their prompts off this flag, so a paid + // job submitter must not advertise itself as harmless. + destructiveHint: true, idempotentHint: false, openWorldHint: true, }, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index d252fcb4..522d89d0 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1306,28 +1306,29 @@ Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips everything the org already knows (\`novelty: org\`), optionally qualifies candidates against the org's own intelligence (questions, tags, ideal buyer profile — frozen at submit), and -optionally purchases contact channels. Submit answers in <1s; the tool polls -up to \`wait_seconds\` (default 45) and, when the job needs longer, returns -\`still_running\` + \`next_poll\` — hand off to \`leadbay_lead_job_status\`. Jobs -run ≤30 min; results are kept 30 days. +optionally purchases contact channels. The tool polls up to \`wait_seconds\` +(default 45); a longer job returns \`still_running\` + \`next_poll\` — hand off +to \`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (\`qualify: false\`, \`channels: []\`) is FREE: company profile + fit score + cached research + contact identity. Paid: \`qualify: true\` (~94 cost_cents per candidate EXAMINED, capped by \`exploration_cap\`/\`max_cost\`) and \`channels\` (email 25c / -phone 250c, success-only). Before the first paid run: \`dry_run: true\`, quote -the worst case, get the explicit go-ahead ("spend / get their emails" in the -user's message counts). Free needs no consent. +phone 250c, success-only). The gate is enforced in code: a paid call +(\`qualify: true\` and/or \`channels\`) is WITHHELD unless it carries +\`confirm: true\` — otherwise nothing is submitted and you get +\`mode: "needs_confirmation"\` with a real quote to show the user. Re-call with +\`confirm: true\` on their go-ahead ("spend / get their emails" counts). +\`confirm: false\` vetoes. Free needs no consent. **Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an off-profile seed is free, exploring it with \`qualify: true\` is not. -**Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not -the free match** — \`filters\` has no exclusion key. Put the positive inverse in -the seed, then \`qualify: true\`: IBP anti-patterns and questions score -violators negative and \`min_ai_score: 0\` drops them. Durable exclusion → -suggest \`leadbay_refine_prompt\`. In a FREE preview, drop violating rows and -say the free tier doesn't enforce. +**Exclusions ("no chains") are enforced by QUALIFICATION, not the free +match** — \`filters\` has no exclusion key. Put the positive inverse in the +seed, then \`qualify: true\`: IBP anti-patterns score violators negative and +\`min_ai_score: 0\` drops them. Durable exclusion → \`leadbay_refine_prompt\`. +In a FREE preview, drop violating rows and say the tier doesn't enforce. ### Crafting the \`example_lead\` seed — the input that decides result quality @@ -1381,12 +1382,11 @@ the same way or the matcher drifts. Every rule below is measured: **Parameter notes** -- \`request_id\` (REQUIRED) is the retry contract: SAME value when retrying - the same ask (returns the same live job, no double spend); NEW value for a - changed ask. Derive from ask + date: \`gyms-dallas-2026-07-28\`. -- \`min_ai_score\` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score. Lower it only to STUDY evidence — never with \`channels\` (that buys - emails for leads the AI just scored as junk). +- \`request_id\` (REQUIRED) is the retry contract: SAME value retrying the same + ask (same live job, no double spend); NEW value for a changed ask. Derive + from ask + archetype + date: \`gyms-dallas-2026-07-28\`. +- Never lower \`min_ai_score\` together with \`channels\` — that buys emails for + leads the AI just scored as junk. - \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). diff --git a/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts b/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts new file mode 100644 index 00000000..103079c3 --- /dev/null +++ b/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts @@ -0,0 +1,166 @@ +/** + * Spend gate for leadbay_find_new_leads (Codex P1 on the follow-up review). + * + * qualify_leads got the gate first; this tool has the identical paid surface + * (`qualify: true` and/or `channels`) and was still submitting directly. The + * trigger differs though — `qualify` defaults to FALSE here, so the default + * search is genuinely free and must not be gated. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); +const JOB_ID = "9d41c0a6-3b72-4e58-b110-2f7a6c9e4d31"; + +const DRY_RUN_200 = { + valid: true, + items_requested: 10, + estimated_cost: { max: 1880, unit: "cost_cents" }, + quota_forecast: { + web_fetch_allowed: true, + rescore_allowed: true, + enrichment_allowed: true, + }, +}; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 0, unit: "cost_cents" }, + items_requested: 10, + duplicate: false, +}; + +const SNAPSHOT = { + job: { id: JOB_ID, state: "completed" }, + funnel: { delivered: 1, examined: 1 }, + items: [ + { + ref: { input_indexes: [0] }, + status: "delivered", + seq: 0, + lead: { lead_id: "aaaa1111-2222-3333-4444-555566667777" }, + }, + ], + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + next_since: null, + explain: { region: "us", model: "m" }, +}; + +const BASE_ARGS = { count: 10, request_id: "gyms-dallas-2026-08-03" }; + +beforeEach(() => resetHttpMock()); + +const postBodies = () => + getHttpRequests() + .filter((r: any) => r.method === "POST") + .map((r: any) => (typeof r.body === "string" ? JSON.parse(r.body) : r.body)); + +describe("leadbay_find_new_leads — spend gate", () => { + it("withholds a qualify:true search and quotes instead", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await findNewLeads.execute(newClient(), { + ...BASE_ARGS, + qualify: true, + }); + + expect(result.mode).toBe("needs_confirmation"); + expect(result.submitted).toBe(false); + expect(result.job_id).toBeUndefined(); + expect(result.estimated_cost).toEqual({ max: 1880, unit: "cost_cents" }); + + const posts = postBodies(); + expect(posts).toHaveLength(1); + expect(posts[0].dry_run).toBe(true); + }); + + it("withholds a channels purchase even when qualify is off", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await findNewLeads.execute(newClient(), { + ...BASE_ARGS, + channels: ["phone"], + }); + + expect(result.mode).toBe("needs_confirmation"); + expect(result.paid_because.join(" ")).toMatch(/channels requested: phone/); + }); + + it("confirm:false vetoes with no network call at all", async () => { + mockHttp([]); + + const result: any = await findNewLeads.execute(newClient(), { + ...BASE_ARGS, + qualify: true, + confirm: false, + }); + + expect(result.vetoed).toBe(true); + expect(result.quote).toBeNull(); + expect(getHttpRequests()).toHaveLength(0); + }); + + it("the DEFAULT free search is not gated", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT, + }, + ]); + + const result: any = await findNewLeads.execute(newClient(), { + ...BASE_ARGS, + wait_seconds: 0, + }); + + expect(result.mode).toBeUndefined(); + expect(result.job_id).toBe(JOB_ID); + expect(postBodies()[0].dry_run).toBeUndefined(); + }); + + it("confirm:true submits the paid search", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT, + }, + ]); + + const result: any = await findNewLeads.execute(newClient(), { + ...BASE_ARGS, + qualify: true, + confirm: true, + wait_seconds: 0, + }); + + expect(result.job_id).toBe(JOB_ID); + const posts = postBodies(); + expect(posts).toHaveLength(1); + expect(posts[0].dry_run).toBeUndefined(); + // `confirm` is a client-side gate — it must not leak onto the wire. + expect(posts[0].confirm).toBeUndefined(); + }); +}); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 7933cc10..0a473ac0 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -947,10 +947,18 @@ When I want qualification evidence and/or reachable contacts: - \`leadbay_find_new_leads\`: the same, PLUS \`min_ai_score\` and \`count\`. 2. On my go-ahead, prefer feeding the free preview's deliveries to \`leadbay_qualify_leads\` (\`prior_deliveries: {job_id}\`) — it only spends on - companies already known to match. Run a fresh \`qualify: true\` search - instead when I asked for more than the preview delivered. Paid - \`leadbay_qualify_leads\` calls need \`confirm: true\` — without it the tool - withholds the submit and hands back a quote instead of spending. + companies already known to match. Paid calls need \`confirm: true\`; without + it the tool withholds the submit and hands back a quote instead of + spending. That applies to \`leadbay_find_new_leads\` too whenever you set + \`qualify: true\` or ask for channels. + + If the preview delivered FEWER than I asked for, do both halves and do not + conflate them: qualify what the preview already found, and run the fresh + search only for the SHORTFALL — \`count\` = what is still missing, never the + original number, under a NEW \`request_id\`. Reusing the preview's id dedupes + the paid submit back into the free job; keeping the original count buys a + whole second batch, because \`novelty: org\` already excludes everything the + preview delivered. 3. While the job runs, poll with \`leadbay_lead_job_status\` (\`wait_seconds: 60\`); report progress, not silence. diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index a5486467..dca585c0 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -115,10 +115,18 @@ When I want qualification evidence and/or reachable contacts: - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. 2. On my go-ahead, prefer feeding the free preview's deliveries to `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on - companies already known to match. Run a fresh `qualify: true` search - instead when I asked for more than the preview delivered. Paid - `leadbay_qualify_leads` calls need `confirm: true` — without it the tool - withholds the submit and hands back a quote instead of spending. + companies already known to match. Paid calls need `confirm: true`; without + it the tool withholds the submit and hands back a quote instead of + spending. That applies to `leadbay_find_new_leads` too whenever you set + `qualify: true` or ask for channels. + + If the preview delivered FEWER than I asked for, do both halves and do not + conflate them: qualify what the preview already found, and run the fresh + search only for the SHORTFALL — `count` = what is still missing, never the + original number, under a NEW `request_id`. Reusing the preview's id dedupes + the paid submit back into the free job; keeping the original count buys a + whole second batch, because `novelty: org` already excludes everything the + preview delivered. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index cd5092a5..db4cfe8a 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -60,38 +60,38 @@ Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips everything the org already knows (`novelty: org`), optionally qualifies candidates against the org's own intelligence (questions, tags, ideal buyer profile — frozen at submit), and -optionally purchases contact channels. Submit answers in <1s; the tool polls -up to `wait_seconds` (default 45) and, when the job needs longer, returns -`still_running` + `next_poll` — hand off to `leadbay_lead_job_status`. Jobs -run ≤30 min; results are kept 30 days. +optionally purchases contact channels. The tool polls up to `wait_seconds` +(default 45); a longer job returns `still_running` + `next_poll` — hand off +to `leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (`qualify: false`, `channels: []`) is FREE: company profile + fit score + cached research + contact identity. Paid: `qualify: true` (~94 cost_cents per candidate EXAMINED, capped by `exploration_cap`/`max_cost`) and `channels` (email 25c / -phone 250c, success-only). Before the first paid run: `dry_run: true`, quote -the worst case, get the explicit go-ahead ("spend / get their emails" in the -user's message counts). Free needs no consent. +phone 250c, success-only). The gate is enforced in code: a paid call +(`qualify: true` and/or `channels`) is WITHHELD unless it carries +`confirm: true` — otherwise nothing is submitted and you get +`mode: "needs_confirmation"` with a real quote to show the user. Re-call with +`confirm: true` on their go-ahead ("spend / get their emails" counts). +`confirm: false` vetoes. Free needs no consent. **Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an off-profile seed is free, exploring it with `qualify: true` is not. -**Exclusions ("no chains", "no agencies") are enforced by QUALIFICATION, not -the free match** — `filters` has no exclusion key. Put the positive inverse in -the seed, then `qualify: true`: IBP anti-patterns and questions score -violators negative and `min_ai_score: 0` drops them. Durable exclusion → -suggest `leadbay_refine_prompt`. In a FREE preview, drop violating rows and -say the free tier doesn't enforce. +**Exclusions ("no chains") are enforced by QUALIFICATION, not the free +match** — `filters` has no exclusion key. Put the positive inverse in the +seed, then `qualify: true`: IBP anti-patterns score violators negative and +`min_ai_score: 0` drops them. Durable exclusion → `leadbay_refine_prompt`. +In a FREE preview, drop violating rows and say the tier doesn't enforce. {{include:heuristics/example-lead-craft}} **Parameter notes** -- `request_id` (REQUIRED) is the retry contract: SAME value when retrying - the same ask (returns the same live job, no double spend); NEW value for a - changed ask. Derive from ask + date: `gyms-dallas-2026-07-28`. -- `min_ai_score` gates the [-30,+30] qualification DELTA, not the 0-100 fit - score. Lower it only to STUDY evidence — never with `channels` (that buys - emails for leads the AI just scored as junk). +- `request_id` (REQUIRED) is the retry contract: SAME value retrying the same + ask (same live job, no double spend); NEW value for a changed ask. Derive + from ask + archetype + date: `gyms-dallas-2026-07-28`. +- Never lower `min_ai_score` together with `channels` — that buys emails for + leads the AI just scored as junk. - `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). From 8d3962b35f81341fb0f63c3c3dab6daca424fb13 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:01:30 -0700 Subject: [PATCH 12/46] fix(mcp): gate delivery tools until the backend ships; renderer + idempotency fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third Codex pass. - Release gate (the P1): the three delivery tools are now hidden unless LEADBAY_MCP_LEAD_DELIVERY=1. /1.6/mcp/* is live on staging only and 404s on production, so shipping them unconditionally would hand every user tools that fail on the first call. Remove the gate in the release that follows the backend deploy. mcpFirstDeliveryAllTools keeps them visible to the WORKFLOWS/routing audits, so a rollout flag never reads as "this tool doesn't exist". - find_new_leads returned summary.requested while the shared renderer reads summary.items_requested — renamed, sourced from submit.items_requested, so the "delivered X of the Y asked" line stops being unrenderable. - Both submit tools now hand snapshot.next_since forward in next_poll, so a follow-up leadbay_lead_job_status continues incrementally instead of re-reading and re-rendering rows already shown. - find_new_leads is destructiveHint:true. Annotations are static and must describe the worst case (qualify:true / channels bill, and deliveries claim org novelty), not the free default. The free path is protected by the confirm gate in execute(), not by the annotation. - qualify_leads derives a stable request_id for a consented paid submit when the caller omits one (refs + paid flags + UTC day, FNV-1a). Without it a timeout or agent retry re-charged qualification and channel purchases for the same refs. The response echoes the key actually sent. One existing assertion relaxed: find-new-leads.test.ts pinned next_poll with toEqual, which the new `since` cursor breaks; now toMatchObject, so it asserts the handoff fields without freezing the shape. pnpm -r build, -r test (1337 passed), -r typecheck green. Co-Authored-By: Claude --- packages/core/src/composite/find-new-leads.ts | 21 +++-- packages/core/src/composite/qualify-leads.ts | 50 +++++++++++- packages/core/src/index.ts | 44 +++++++--- .../unit/composite/find-new-leads.test.ts | 5 +- .../test/unit/mcp-first-delivery-gate.test.ts | 80 +++++++++++++++++++ packages/mcp/test/audit/routing-block.test.ts | 3 + packages/mcp/test/audit/workflows.test.ts | 3 + 7 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 packages/core/test/unit/mcp-first-delivery-gate.test.ts diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index c147b9dd..5e146cae 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -62,12 +62,12 @@ export const findNewLeads: Tool = { annotations: { title: "Find new leads (net-new ICP search)", readOnlyHint: false, - // Unlike qualify_leads (paid by default), the default search here is FREE - // — qualify defaults to false and channels are empty — so the tool is not - // inherently destructive. A paid variant (qualify:true / channels) is - // withheld in execute() until `confirm: true`, which is where the spend - // decision is surfaced. - destructiveHint: false, + // The tool CAN bill (qualify:true and/or channels) and records deliveries + // in the org novelty ledger, so it advertises destructive like the other + // paid composites — annotations are static and must describe the worst + // case, not the default. The free path is protected in execute() instead: + // a paid call is withheld until `confirm: true`. + destructiveHint: true, // The mandatory request_id dedups: re-submitting the same request returns // the SAME live job instead of double-spending. idempotentHint: true, @@ -292,7 +292,10 @@ export const findNewLeads: Tool = { state: snapshot.job.state, done, summary: { - requested: params.count, + // Named items_requested (not `requested`) to match qualify_leads and + // the shared renderer, which reads summary.items_requested for the + // "delivered X of the Y asked" clause. + items_requested: submit.items_requested ?? params.count, delivered: snapshot.funnel.delivered ?? 0, delivered_callable: snapshot.funnel.delivered_callable ?? 0, delivered_title_only: snapshot.funnel.delivered_title_only ?? 0, @@ -311,6 +314,10 @@ export const findNewLeads: Tool = { : { tool: "leadbay_lead_job_status", job_id: submit.job_id, + // Hand the cursor forward so the follow-up poll continues + // INCREMENTALLY instead of re-reading (and re-rendering) the + // rows already delivered in this response. + since: snapshot.next_since ?? null, suggested_wait_seconds: 60, }, region: client.region, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index cb6abbd2..d6c83911 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -49,6 +49,38 @@ interface QualifyLeadsParams { const DEFAULT_WAIT_SECONDS = 45; +/** Stable idempotency key for a paid batch the caller didn't key itself. + * Deterministic over the refs + the paid flags + the UTC day, so an identical + * retry dedupes to the same backend job instead of re-charging, while a + * genuinely different batch (or the next day) gets a different key. */ +function derivedRequestId(params: QualifyLeadsParams): string { + const refs = (params.lead_refs ?? []) + .map((r) => + [r.lead_id, r.contact_id, r.website, r.name, r.location] + .filter(Boolean) + .join("~") + ) + .sort() + .join("|"); + const shape = [ + refs, + params.prior_deliveries?.job_id ?? "", + params.qualify === false ? "free" : "qualify", + (params.channels ?? []).slice().sort().join(","), + (params.contact_titles ?? []).slice().sort().join(","), + params.title_gate ?? "", + new Date().toISOString().slice(0, 10), + ].join("#"); + // FNV-1a — short, dependency-free, and only needs to be collision-resistant + // across one org's batches, not cryptographically strong. + let h = 0x811c9dc5; + for (let i = 0; i < shape.length; i++) { + h ^= shape.charCodeAt(i); + h = Math.imul(h, 0x01000193) >>> 0; + } + return `qualify-auto-${h.toString(16).padStart(8, "0")}`; +} + export const qualifyLeads: Tool = { name: "leadbay_qualify_leads", annotations: { @@ -173,6 +205,15 @@ export const qualifyLeads: Tool = { const vetoed = params.confirm === false; const consented = !vetoed && params.confirm === true; + // A paid submit without an idempotency key can be re-run by any timeout or + // agent retry, re-charging fresh qualification and channel purchases for + // the same refs. `request_id` is optional on this tool (unlike the search), + // so derive a stable one from the batch when the caller omits it: same refs + // + same paid flags on the same day = same key = backend dedupe. + const requestId = + params.request_id ?? + (isPaid ? derivedRequestId(params) : undefined); + const body = compactBody({ lead_refs: params.lead_refs, prior_deliveries: params.prior_deliveries, @@ -181,7 +222,7 @@ export const qualifyLeads: Tool = { title_gate: params.title_gate, channels: params.channels, max_cost: params.max_cost, - request_id: params.request_id, + request_id: requestId, lang: params.lang, dry_run: params.dry_run, }); @@ -254,7 +295,8 @@ export const qualifyLeads: Tool = { const done = TERMINAL_JOB_STATES.has(snapshot.job.state); return { job_id: submit.job_id, - request_id: params.request_id ?? null, + // Echo the key actually sent, so a retry can reuse it verbatim. + request_id: requestId ?? null, duplicate_submit: submit.duplicate ?? false, state: snapshot.job.state, done, @@ -291,6 +333,10 @@ export const qualifyLeads: Tool = { : { tool: "leadbay_lead_job_status", job_id: submit.job_id, + // Hand the cursor forward so the follow-up poll continues + // INCREMENTALLY instead of re-reading (and re-rendering) the + // rows already delivered in this response. + since: snapshot.next_since ?? null, suggested_wait_seconds: 60, }, region: client.region, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bc053e45..636224e1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -269,7 +269,20 @@ granularTools.forEach((t) => { }); // Composite read tools — always exposed (default agent surface). +// The MCP-first delivery tools depend on backend routes (`POST /1.6/mcp/search`, +// `POST /1.6/mcp/qualify`, `GET /1.6/mcp/jobs/{id}`) that are live on STAGING +// only — production returns 404. Shipping them unconditionally would hand every +// user tools that fail on their first call, so they stay behind an opt-in flag +// until the backend rollout lands. Remove this gate (and the flag) in the +// release that follows the backend deploy. +const MCP_FIRST_DELIVERY_ENABLED = + process.env.LEADBAY_MCP_LEAD_DELIVERY === "1"; + export const compositeReadTools: Tool[] = [ + // Poll surface for the MCP-first lead-delivery jobs (find_new_leads / + // qualify_leads). Read-only snapshot of a backend-owned job, gated with + // them since it is useless without a job to poll. + ...(MCP_FIRST_DELIVERY_ENABLED ? [leadJobStatus] : []), pullLeads, pullFollowups, followupsMap, @@ -309,11 +322,6 @@ export const compositeReadTools: Tool[] = [ bulkEnrichStatus, qualifyStatus, importStatus, - // Poll surface for the MCP-first lead-delivery jobs (find_new_leads / - // qualify_leads). Read-only snapshot of a backend-owned job — always - // exposed so a job started in a write-enabled session stays readable - // even if the deployment later runs read-only. - leadJobStatus, resolveImportRows, // seed-candidates is a read-only discovery surface for the extend flow. // Always exposed so the agent can show candidates even in read-only deployments. @@ -362,17 +370,31 @@ export const compositeReadTools: Tool[] = [ artifactKit, ]; -// Composite write tools — always-exposed in OpenClaw, gated in MCP behind -// LEADBAY_MCP_WRITE=1 (the MCP server filters them out by default). -export const compositeWriteTools: Tool[] = [ - // MCP-first lead delivery (backend /mcp/search + /mcp/qualify jobs). +/** Every MCP-first delivery tool, regardless of the deployment gate above. + * The gate controls what a RUNNING server exposes; contract audits + * (WORKFLOWS.md, routing anti-triggers) must still see these as registered + * tools, or a temporary rollout flag would read as "this tool doesn't exist". */ +export const mcpFirstDeliveryTools: Tool[] = [ // Write-tier: submits create server-side jobs that can spend money // (qualification research, channel purchase) and claim novelty in the // org's delivery ledger — same posture as the other spending composites. - // The FREE tier (qualify:false, channels:[]) is the default ask; the - // descriptions carry the no-silent-spend consent gate. + // The FREE tier (qualify:false, channels:[]) is the default ask, and a paid + // call is withheld in code until `confirm: true`. findNewLeads, qualifyLeads, +]; + +/** All three delivery tools including the read-side poller — the ungated + * registry the contract audits read. */ +export const mcpFirstDeliveryAllTools: Tool[] = [ + ...mcpFirstDeliveryTools, + leadJobStatus, +]; + +// Composite write tools — always-exposed in OpenClaw, gated in MCP behind +// LEADBAY_MCP_WRITE=1 (the MCP server filters them out by default). +export const compositeWriteTools: Tool[] = [ + ...(MCP_FIRST_DELIVERY_ENABLED ? mcpFirstDeliveryTools : []), bulkQualifyLeads, enrichTitles, adjustAudience, diff --git a/packages/core/test/unit/composite/find-new-leads.test.ts b/packages/core/test/unit/composite/find-new-leads.test.ts index 0557e06e..bfc083fc 100644 --- a/packages/core/test/unit/composite/find-new-leads.test.ts +++ b/packages/core/test/unit/composite/find-new-leads.test.ts @@ -169,7 +169,10 @@ describe("leadbay_find_new_leads", () => { }); expect(result.done).toBe(false); expect(result.still_running).toBe(true); - expect(result.next_poll).toEqual({ + // toMatchObject, not toEqual: the handoff also carries a `since` cursor so + // the follow-up poll is incremental, and pinning the exact shape here just + // breaks on every future field. + expect(result.next_poll).toMatchObject({ tool: "leadbay_lead_job_status", job_id: JOB_ID, suggested_wait_seconds: 60, diff --git a/packages/core/test/unit/mcp-first-delivery-gate.test.ts b/packages/core/test/unit/mcp-first-delivery-gate.test.ts new file mode 100644 index 00000000..35a9a10f --- /dev/null +++ b/packages/core/test/unit/mcp-first-delivery-gate.test.ts @@ -0,0 +1,80 @@ +/** + * Release gate for the MCP-first delivery tools. + * + * `/1.6/mcp/search`, `/1.6/mcp/qualify` and `/1.6/mcp/jobs/{id}` are live on + * staging only — production returns 404. Until the backend ships, the three + * tools must NOT appear on the default surface, or every user gets tools that + * fail on their first call. + * + * The module reads the env var at import time, so each case re-imports with a + * reset module registry rather than mutating a cached catalogue. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; + +const DELIVERY_TOOLS = [ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", +]; + +const FLAG = "LEADBAY_MCP_LEAD_DELIVERY"; + +async function loadCatalogues(flag: string | undefined) { + vi.resetModules(); + const previous = process.env[FLAG]; + if (flag === undefined) delete process.env[FLAG]; + else process.env[FLAG] = flag; + try { + const mod = await import("../../src/index.js"); + return [ + ...mod.compositeReadTools, + ...mod.compositeWriteTools, + ].map((t) => t.name); + } finally { + if (previous === undefined) delete process.env[FLAG]; + else process.env[FLAG] = previous; + } +} + +beforeEach(() => vi.resetModules()); + +describe("MCP-first delivery release gate", () => { + it("hides all three tools by default", async () => { + const names = await loadCatalogues(undefined); + for (const tool of DELIVERY_TOOLS) { + expect(names).not.toContain(tool); + } + }); + + it("exposes all three when the flag is set to 1", async () => { + const names = await loadCatalogues("1"); + for (const tool of DELIVERY_TOOLS) { + expect(names).toContain(tool); + } + }); + + it("treats any other flag value as off", async () => { + const names = await loadCatalogues("true"); + for (const tool of DELIVERY_TOOLS) { + expect(names).not.toContain(tool); + } + }); + + it("keeps them registered for contract audits regardless of the gate", async () => { + vi.resetModules(); + delete process.env[FLAG]; + const mod = await import("../../src/index.js"); + const audited = mod.mcpFirstDeliveryAllTools.map((t) => t.name); + for (const tool of DELIVERY_TOOLS) { + expect(audited).toContain(tool); + } + }); + + it("does not disturb the rest of the catalogue", async () => { + const gatedOff = await loadCatalogues(undefined); + const gatedOn = await loadCatalogues("1"); + const difference = gatedOn.filter((n) => !gatedOff.includes(n)).sort(); + expect(difference).toEqual([...DELIVERY_TOOLS].sort()); + }); +}); diff --git a/packages/mcp/test/audit/routing-block.test.ts b/packages/mcp/test/audit/routing-block.test.ts index 8a0e9f25..f222ac02 100644 --- a/packages/mcp/test/audit/routing-block.test.ts +++ b/packages/mcp/test/audit/routing-block.test.ts @@ -18,6 +18,7 @@ import { compositeWriteTools, granularReadTools, granularWriteTools, + mcpFirstDeliveryAllTools, type Tool, } from "@leadbay/core"; @@ -77,6 +78,8 @@ const ALL_TOOLS: Tool[] = [ ...compositeWriteTools, ...granularReadTools, ...granularWriteTools, + // Gated off by default until the backend routes ship; still registered. + ...mcpFirstDeliveryAllTools, ]; const ALL_TOOL_NAMES = new Set(ALL_TOOLS.map((t) => t.name)); diff --git a/packages/mcp/test/audit/workflows.test.ts b/packages/mcp/test/audit/workflows.test.ts index 4cc72d38..1185b87c 100644 --- a/packages/mcp/test/audit/workflows.test.ts +++ b/packages/mcp/test/audit/workflows.test.ts @@ -20,6 +20,7 @@ import { compositeWriteTools, granularReadTools, granularWriteTools, + mcpFirstDeliveryAllTools, } from "@leadbay/core"; import { listPrompts } from "../../src/prompts.js"; @@ -34,6 +35,8 @@ const KNOWN_TOOLS = new Set( ...compositeWriteTools, ...granularReadTools, ...granularWriteTools, + // Gated off by default until the backend routes ship; still registered. + ...mcpFirstDeliveryAllTools, ].map((t) => t.name), ); const KNOWN_PROMPTS = new Set(listPrompts().map((p) => p.name)); From fff45f5ec4e2c00224f04da3e26394389cffff5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:23:42 -0700 Subject: [PATCH 13/46] fix(mcp): gate the prompt with its tools; selector-complete idempotency key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth Codex pass — all three are consequences of the release gate itself. - leadbay_new_leads prompt is now gated with the tools it drives. Every step of that workflow calls find_new_leads / qualify_leads / lead_job_status, so offering the prompt while the tools are hidden started a flow whose every call was missing from tools/list. listAllPrompts() keeps it visible to the WORKFLOWS audit; prompts.test.ts sets the flag since it asserts the full catalogue. - The derived idempotency key now covers the WHOLE prior_deliveries selector, not just the job id. Qualifying the first 50 of a delivery job and then the next 50 are different batches; collapsing them to one key made the second submit look like a duplicate and left those refs unqualified. - lead_job_status carries `since` in its own next_poll handoff, matching the two submit tools — following it without the cursor re-read the rows just returned. pnpm -r build, -r test (1353 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/lead-job-status.ts | 3 + packages/core/src/composite/qualify-leads.ts | 12 +++- packages/mcp/src/prompts.ts | 19 +++++- packages/mcp/test/audit/workflows.test.ts | 6 +- packages/mcp/test/prompt-release-gate.test.ts | 58 +++++++++++++++++++ packages/mcp/test/prompts.test.ts | 14 ++++- 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 packages/mcp/test/prompt-release-gate.test.ts diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts index a6751103..113917b5 100644 --- a/packages/core/src/composite/lead-job-status.ts +++ b/packages/core/src/composite/lead-job-status.ts @@ -101,6 +101,9 @@ export const leadJobStatus: Tool = { : { tool: "leadbay_lead_job_status", job_id: params.job_id, + // Same incremental handoff as the submit tools — following + // next_poll without the cursor re-reads the rows just returned. + since: snapshot.next_since ?? null, suggested_wait_seconds: 60, }, region: client.region, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index d6c83911..9165e1bc 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -64,7 +64,17 @@ function derivedRequestId(params: QualifyLeadsParams): string { .join("|"); const shape = [ refs, - params.prior_deliveries?.job_id ?? "", + // The WHOLE selector, not just the job id: qualifying the first 50 of a + // delivery job and then the next 50 are different batches, and collapsing + // them to one key would make the second submit look like a duplicate and + // leave those refs unqualified. + [ + params.prior_deliveries?.job_id, + params.prior_deliveries?.since, + params.prior_deliveries?.limit, + ] + .map((v) => v ?? "") + .join("~"), params.qualify === false ? "free" : "qualify", (params.channels ?? []).slice().sort().join(","), (params.contact_titles ?? []).slice().sort().join(","), diff --git a/packages/mcp/src/prompts.ts b/packages/mcp/src/prompts.ts index 01e602c3..2ad6636e 100644 --- a/packages/mcp/src/prompts.ts +++ b/packages/mcp/src/prompts.ts @@ -376,7 +376,17 @@ const CATALOG: CatalogEntry[] = [ }, ]; -export function listPrompts(): Prompt[] { +/** Prompts whose whole workflow drives tools that are themselves gated off + * until the backend routes ship. Exposing the prompt without the tools would + * let a user start a guided flow whose every call is missing from tools/list. */ +const GATED_PROMPTS: Record boolean> = { + leadbay_new_leads: () => process.env.LEADBAY_MCP_LEAD_DELIVERY === "1", +}; + +/** The full catalogue, gates ignored — contract audits assert every prompt + * named in WORKFLOWS.md resolves, and a rollout flag must not read as + * "this prompt does not exist". */ +export function listAllPrompts(): Prompt[] { return CATALOG.map((c) => ({ name: c.name, description: c.description, @@ -384,6 +394,13 @@ export function listPrompts(): Prompt[] { })); } +export function listPrompts(): Prompt[] { + return listAllPrompts().filter((p) => { + const gate = GATED_PROMPTS[p.name]; + return gate ? gate() : true; + }); +} + export function getPrompt( name: string, args: Record = {} diff --git a/packages/mcp/test/audit/workflows.test.ts b/packages/mcp/test/audit/workflows.test.ts index 1185b87c..050bffb7 100644 --- a/packages/mcp/test/audit/workflows.test.ts +++ b/packages/mcp/test/audit/workflows.test.ts @@ -22,7 +22,7 @@ import { granularWriteTools, mcpFirstDeliveryAllTools, } from "@leadbay/core"; -import { listPrompts } from "../../src/prompts.js"; +import { listAllPrompts } from "../../src/prompts.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -39,7 +39,9 @@ const KNOWN_TOOLS = new Set( ...mcpFirstDeliveryAllTools, ].map((t) => t.name), ); -const KNOWN_PROMPTS = new Set(listPrompts().map((p) => p.name)); +// listAllPrompts: release-gated prompts are still registered, and a rollout +// flag must not make WORKFLOWS.md look like it names a nonexistent prompt. +const KNOWN_PROMPTS = new Set(listAllPrompts().map((p) => p.name)); // Skills shipped via the Claude Code plugin marketplace. Each is a // directory under .claude-plugin/plugins/leadbay/skills//SKILL.md; diff --git a/packages/mcp/test/prompt-release-gate.test.ts b/packages/mcp/test/prompt-release-gate.test.ts new file mode 100644 index 00000000..79b71760 --- /dev/null +++ b/packages/mcp/test/prompt-release-gate.test.ts @@ -0,0 +1,58 @@ +/** + * The leadbay_new_leads prompt is gated with the tools it drives. + * + * Every step of that guided workflow calls leadbay_find_new_leads / + * leadbay_qualify_leads / leadbay_lead_job_status. With the delivery gate off, + * offering the prompt would start a flow whose every call is missing from + * tools/list — so the prompt hides and reappears with them. + */ + +import { describe, it, expect, vi, afterEach } from "vitest"; + +const FLAG = "LEADBAY_MCP_LEAD_DELIVERY"; +const GATED = "leadbay_new_leads"; + +async function promptNames(flag: string | undefined) { + vi.resetModules(); + const previous = process.env[FLAG]; + if (flag === undefined) delete process.env[FLAG]; + else process.env[FLAG] = flag; + try { + const mod = await import("../src/prompts.js"); + return { + exposed: mod.listPrompts().map((p) => p.name), + all: mod.listAllPrompts().map((p) => p.name), + }; + } finally { + if (previous === undefined) delete process.env[FLAG]; + else process.env[FLAG] = previous; + } +} + +afterEach(() => vi.resetModules()); + +describe("leadbay_new_leads prompt release gate", () => { + it("is hidden by default", async () => { + const { exposed } = await promptNames(undefined); + expect(exposed).not.toContain(GATED); + }); + + it("is exposed when the delivery flag is on", async () => { + const { exposed } = await promptNames("1"); + expect(exposed).toContain(GATED); + }); + + it("stays in the full catalogue for contract audits either way", async () => { + const off = await promptNames(undefined); + const on = await promptNames("1"); + expect(off.all).toContain(GATED); + expect(on.all).toContain(GATED); + }); + + it("gates only that prompt", async () => { + const off = await promptNames(undefined); + const on = await promptNames("1"); + const difference = on.exposed.filter((n) => !off.exposed.includes(n)); + expect(difference).toEqual([GATED]); + }); +}); diff --git a/packages/mcp/test/prompts.test.ts b/packages/mcp/test/prompts.test.ts index c8017cc7..ff71706d 100644 --- a/packages/mcp/test/prompts.test.ts +++ b/packages/mcp/test/prompts.test.ts @@ -3,11 +3,23 @@ * commands. */ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; import { httpsMockFactory } from "./harness.js"; vi.mock("node:https", () => httpsMockFactory()); +// leadbay_new_leads is release-gated off by default until the /1.6/mcp/* +// backend routes ship. This suite asserts the FULL catalogue, so enable it. +const PREVIOUS_DELIVERY_FLAG = process.env.LEADBAY_MCP_LEAD_DELIVERY; +beforeAll(() => { + process.env.LEADBAY_MCP_LEAD_DELIVERY = "1"; +}); +afterAll(() => { + if (PREVIOUS_DELIVERY_FLAG === undefined) + delete process.env.LEADBAY_MCP_LEAD_DELIVERY; + else process.env.LEADBAY_MCP_LEAD_DELIVERY = PREVIOUS_DELIVERY_FLAG; +}); + import { LeadbayClient } from "@leadbay/core"; import { buildServer } from "../src/server.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; From 9e965a55ce5fe7881938753efd3415df1e2a519d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:40:58 -0700 Subject: [PATCH 14/46] fix(mcp): gate the Claude skill + prompts/get; clock-free idempotency key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth Codex pass — mostly gaps the release gate itself opened. - The Claude SKILL.md was still shipping while the tools are hidden, and it auto-triggers on the same "find me new leads" requests, so Claude Code users entered a workflow that fails on the first tool call. A skill is a static file with no runtime gate, so promptforge now emits none for a prompt marked `release_gated: true`. Delete that one frontmatter line to un-gate. - getPrompt() is gated too, not just listPrompts(): a cached slash command or a direct prompts/get by name bypassed the list filter entirely. - The derived idempotency key no longer includes the UTC date — a retry of the same approved batch after midnight derived a different key and could re-run the spend. It now hashes the approved batch only, and includes max_cost so raising the cap after a stop_reason:max_cost is correctly a NEW run rather than a dedupe onto the capped job. - The page bound now lets EVERY allowed page size drain 500 items; the old 120-page cap meant limit=1 returned 120 rows with done:true and no cursor. - The prompt now says `count` is the TOTAL across archetypes, not per search — "find 10 gyms and warehouses" was delivering (and later paying for) 20. - find_new_leads description trimmed again: 78 chars of headroom, no longer the tightest description in the repo. qualify-leads.test.ts keeps its one-line `confirm: true` (comment dropped, so the edit is now a single line). Removing it entirely is not possible: that test purchases an email channel, which any spend gate must withhold without consent. pnpm -r build, -r test (1362 passed), -r typecheck green. Co-Authored-By: Claude --- .../leadbay/skills/leadbay_new_leads/SKILL.md | 127 ---------------- .../core/src/composite/_mcp-job-helpers.ts | 7 +- packages/core/src/composite/qualify-leads.ts | 14 +- .../core/src/tool-descriptions.generated.ts | 15 +- .../qualify-leads-idempotency.test.ts | 138 ++++++++++++++++++ .../test/unit/composite/qualify-leads.test.ts | 2 - packages/mcp/src/prompts.generated.ts | 5 +- packages/mcp/src/prompts.ts | 9 ++ .../prompts/leadbay_new_leads.md.tmpl | 11 +- packages/promptforge/src/frontmatter.ts | 6 + packages/promptforge/src/skills.ts | 16 +- packages/promptforge/test/skills.test.ts | 19 ++- .../composite/find-new-leads.md.tmpl | 15 +- 13 files changed, 225 insertions(+), 159 deletions(-) delete mode 100644 .claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md create mode 100644 packages/core/test/unit/composite/qualify-leads-idempotency.test.ts diff --git a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md b/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md deleted file mode 100644 index 622466c4..00000000 --- a/.claude-plugin/plugins/leadbay/skills/leadbay_new_leads/SKILL.md +++ /dev/null @@ -1,127 +0,0 @@ ---- -name: leadbay_new_leads -description: "Guided net-new lead delivery — turn a natural-language need (\"gyms around Dallas that would buy our flooring\") into ICP-perfect NEW companies with qualification evidence and the right contact, via leadbay_find_new_leads. Trigger on \"find me new leads\", \"get me N companies that \", \"we're entering \". Do NOT trigger on \"today's leads\" (leadbay_daily_check_in) or \"qualify these companies I have\" (leadbay_qualify_leads)." ---- - - -## MEMORY - -Before responding, glance at any `_meta.agent_memory.summary` returned by tool calls earlier in this session and reflect its top signals in your reasoning ("Filtering by your stated preference for healthcare"). After any material new signal from the user this conversation (sector, region, deal size, communication style, qualification rule, explicit retraction, or recurrence / scheduling preference such as "I do this every day" or "remind me every morning"), call `leadbay_agent_memory_capture` to persist it: `source:"user_stated"` if literal, `source:"inferred"` with confidence <=6 if inferred. - - -IRON LAW — NO FABRICATION. Every lead id, contact email, custom field id, mapping decision, and tool argument must trace to a value you read from the file the user attached or to an output from a leadbay_* tool call in this session. Do not invent values. Do not "fill in" a missing leadId with a name match. Do not synthesize a CRM id from a guess. If a value is missing, leave the field blank and say so. - - -GATE — DEFER TO TOOL RENDERING. When you call a Leadbay composite that ships its own RENDERING block (every composite in 0.9.0+ does), render the response using that block's recipe verbatim — score bars, glyph palette, column order, hide-list, link priorities, all of it. Do NOT substitute prose, a numbered list, or a different column structure even when an orchestrating prompt's body suggests alternate framing. Prompt-specific commentary (motivational nudges, summaries, next-action recommendations) belongs ABOVE or BELOW the canonical table, never in place of it. - -If the prompt's body and the tool's RENDERING appear to conflict, the tool's RENDERING wins for the structural layout; the prompt's voice wins for the commentary that surrounds it. - - -Find net-new leads for me. My need, in my words: - -> - -If that need was not supplied to you directly, take it from the message that -started this — the request in my own words is the need, and I should never be -asked to repeat something I already said. Only when BOTH are missing or too -vague to name (a) who I sell to and (b) roughly how many leads I want, ask me -ONCE — one short question — then proceed. Default count when unstated: 10. - -# PHASE 1 — UNDERSTAND THE BUYER (no tool calls yet) - -From my words, work out: -- What I SELL and therefore WHO WRITES ME CHECKS — the buyer category, never - the buyer's customers, never my competitors. If my product helps companies - of type X serve audience Y, my buyer is X. -- Hard constraints: geography, size band, sector, exclusions ("no - franchises", "pas de grands groupes" — negatives BIND). -- Contact needs: do I want a person? Which titles? Email, phone, both? -- Buyer archetypes: if my need genuinely spans two different kinds of buyer, - plan one search per archetype — never one blended seed. - -# PHASE 2 — CRAFT THE SEED - -Compose the `example_lead` for each archetype following the craft rules in -the leadbay_find_new_leads description (registry-style description of a -FICTIONAL typical buyer; no invented brand name; no event language; hard -constraints go in `filters` with the FLAT keys `employees_min`/`employees_max` -and city/state/region `locations` — never a country name). Show me the seed -description(s) in one line each — I should recognize my ideal customer in -them. - -`filters` only encodes sectors, locations and employee bounds. Any constraint -that does not fit those keys — above all EXCLUSIONS like "no franchises" or -"pas de grands groupes" — has nowhere to live in the filter schema, so it must -not be dropped on the floor: express it positively in the seed `description` -(an independent single-site operator rather than "no franchises"), and carry -the exclusion forward yourself to Phase 5, where you drop violating rows and -say you dropped them. Tell me plainly if a constraint can only be enforced -that way — after the fact, not by the search. - -Composing this fictional seed from my words is expected and permitted: it is -the tool's designed input, not fabricated data. What must never be invented is -a RESULT — company names, contacts, scores, or anything presented as coming -back from Leadbay. - -# PHASE 3 — FREE PREVIEW - -Call `leadbay_find_new_leads` with the seed, `filters`, `count`, -`qualify: false`, no channels — this is FREE — and a `request_id` derived -from the ask + the ARCHETYPE + today's date. The archetype component is not -optional: `request_id` is the idempotency key, so two archetype searches -sharing one id dedupe to the same job and the second archetype is never -searched. Render the delivery table and judge fit honestly: are these the -kind of companies I asked for? - -- **`still_running: true`** → the job is ALIVE. Do not judge the seed and do - not relaunch — poll `leadbay_lead_job_status` (`wait_seconds: 60`) until - it goes terminal, reporting progress. Relaunching now burns an active-job - slot and rate-limit budget on a search that may be about to deliver. -- **On-profile** (terminal) → offer Phase 4. -- **Off-profile or empty** (terminal) → read `funnel` + - `explain.scope_notes`, tell me what went wrong in one line (wrong - archetype? too narrow a filter? thin universe?), reshape the seed or - filters, and retry under a NEW request_id. Reshaping is free; do not pay - to explore a bad seed. - -# PHASE 4 — PAID DEPTH (only with my explicit go-ahead) - -When I want qualification evidence and/or reachable contacts: -1. Quote first: `dry_run: true` on the tool you will actually run, with the - exact flags I asked for, and tell me the worst-case cost in plain money. - The two tools take DIFFERENT flags — passing the wrong one is rejected - outright (`additionalProperties: false`): - - `leadbay_qualify_leads`: `qualify: true`, `contact_titles`, - `title_gate`, `channels`, `max_cost`. **No `min_ai_score`.** - - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. -2. On my go-ahead, prefer feeding the free preview's deliveries to - `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on - companies already known to match. Paid calls need `confirm: true`; without - it the tool withholds the submit and hands back a quote instead of - spending. That applies to `leadbay_find_new_leads` too whenever you set - `qualify: true` or ask for channels. - - If the preview delivered FEWER than I asked for, do both halves and do not - conflate them: qualify what the preview already found, and run the fresh - search only for the SHORTFALL — `count` = what is still missing, never the - original number, under a NEW `request_id`. Reusing the preview's id dedupes - the paid submit back into the free job; keeping the original count buys a - whole second batch, because `novelty: org` already excludes everything the - preview delivered. -3. While the job runs, poll with `leadbay_lead_job_status` - (`wait_seconds: 60`); report progress, not silence. - -# PHASE 5 — DELIVER - -Before rendering, sanity-check every row: geography inside my fence (drop -and call out same-named-city leaks), descriptions actually matching my ask -(especially when `explain.seed_strategy` is `text_match_exemplars` — fit -scores run hot there), visible violations of my exclusions dropped. If the -best fit is under 30, say "weak matches only" and propose reshaping before -showing more than 3. - -Render per the lead-delivery table, then ALWAYS the funnel line: matched / -examined / qualified / disqualified / delivered / stop reason / spend. Zero -delivered gets a diagnosis and a concrete next move, never a shrug. Close -with NEXT STEPS from the tool description — and STOP; take no further action -without my say-so. diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index b3eade66..ad63d7bf 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -123,9 +123,12 @@ const PAGE_LIMIT = 100; // follow-ups and a ceiling that stays a runaway backstop. const MAX_JOB_ITEMS = 500; const MIN_PAGES = 20; -const PAGE_CAP = 120; +// The bound must let EVERY allowed page size reach MAX_JOB_ITEMS — at limit=1 +// that is 500 pages, and a lower flat cap would return a partial batch while +// still reporting done:true, with no cursor on a terminal submit response to +// fetch the rest. Capping below the drain would hide items, not just slow them. const maxPagesFor = (pageLimit: number) => - Math.min(PAGE_CAP, Math.max(MIN_PAGES, Math.ceil(MAX_JOB_ITEMS / pageLimit) + 1)); + Math.max(MIN_PAGES, Math.ceil(MAX_JOB_ITEMS / pageLimit) + 1); /** One cumulative snapshot of the job, paging the item cursor dry. Job/funnel/ * cost/explain come from the LAST page fetched (the freshest projection). */ diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 9165e1bc..737506a3 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -50,9 +50,13 @@ interface QualifyLeadsParams { const DEFAULT_WAIT_SECONDS = 45; /** Stable idempotency key for a paid batch the caller didn't key itself. - * Deterministic over the refs + the paid flags + the UTC day, so an identical - * retry dedupes to the same backend job instead of re-charging, while a - * genuinely different batch (or the next day) gets a different key. */ + * Deterministic over the APPROVED BATCH ITSELF — refs, selector, paid flags, + * spend cap — and nothing time-based: a retry of the same approval must + * dedupe even if it lands after midnight or hours later. A genuinely + * different batch (different refs, channels, titles, or a raised max_cost + * after a stop_reason: max_cost) hashes differently and runs as a new job. + * A caller who wants a deliberate re-run of an identical batch passes an + * explicit request_id. */ function derivedRequestId(params: QualifyLeadsParams): string { const refs = (params.lead_refs ?? []) .map((r) => @@ -79,7 +83,9 @@ function derivedRequestId(params: QualifyLeadsParams): string { (params.channels ?? []).slice().sort().join(","), (params.contact_titles ?? []).slice().sort().join(","), params.title_gate ?? "", - new Date().toISOString().slice(0, 10), + // The cap is part of the approval: raising it after a stop_reason:max_cost + // is a NEW approved run, and must not dedupe onto the capped job. + params.max_cost ?? "", ].join("#"); // FNV-1a — short, dependency-free, and only needs to be collision-resistant // across one org's batches, not cryptographically strong. diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 522d89d0..0de62302 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1302,13 +1302,13 @@ delivered. Full algorithm below. --- -Submit a net-new lead search: the backend matches an ICP seed against the -full company universe, applies hard filters, skips everything the org already -knows (\`novelty: org\`), optionally qualifies candidates against the org's own +Submit a net-new lead search: the backend matches an ICP seed against the full +company universe, applies hard filters, skips what the org already knows +(\`novelty: org\`), optionally qualifies candidates against the org's own intelligence (questions, tags, ideal buyer profile — frozen at submit), and optionally purchases contact channels. The tool polls up to \`wait_seconds\` -(default 45); a longer job returns \`still_running\` + \`next_poll\` — hand off -to \`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. +(default 45); a longer job returns \`still_running\` + \`next_poll\` — hand off to +\`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (\`qualify: false\`, \`channels: []\`) is FREE: company profile + fit score + cached research + @@ -1390,9 +1390,8 @@ the same way or the matcher drifts. Every rule below is measured: - \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). -**Read the result honestly.** \`funnel\` + \`explain.scope_notes\` tell the -story; zero delivered gets a cause + a concrete next move, never a bare "no -results" (rules in RENDERING). +**Read the result honestly** — \`funnel\` + \`explain.scope_notes\` tell the +story; zero delivered gets a cause and a next move (rules in RENDERING). --- diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts new file mode 100644 index 00000000..f0d29b91 --- /dev/null +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -0,0 +1,138 @@ +/** + * Derived idempotency key for a consented paid qualify batch. + * + * `request_id` is optional on this tool, so a paid submit without one could be + * re-run by any timeout or agent retry and re-charge qualification + channel + * purchases for the same refs. The derived key must therefore be a function of + * the APPROVED BATCH and nothing else — in particular not of the clock, so a + * retry that lands after midnight still dedupes. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); +const JOB_ID = "b1f0e7c4-2a56-4d80-9c33-5e6f1a2b3c4d"; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 238, unit: "cost_cents" }, + items_requested: 2, + duplicate: false, +}; + +const SNAPSHOT = { + job: { id: JOB_ID, state: "completed" }, + funnel: { delivered: 1 }, + items: [], + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + next_since: null, + explain: { region: "us", model: "m" }, +}; + +beforeEach(() => resetHttpMock()); + +async function submittedRequestId(params: Record) { + resetHttpMock(); + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT, + }, + ]); + await qualifyLeads.execute(newClient(), { + confirm: true, + wait_seconds: 0, + ...params, + } as any); + const post = getHttpRequests().find((r: any) => r.method === "POST")!; + return JSON.parse(post.body!).request_id as string; +} + +const REFS = [{ website: "a.com" }, { website: "b.com" }]; + +describe("qualify_leads — derived idempotency key", () => { + it("sends a derived request_id when a paid call omits one", async () => { + const id = await submittedRequestId({ lead_refs: REFS }); + expect(id).toMatch(/^qualify-auto-[0-9a-f]{8}$/); + }); + + it("is stable across identical retries", async () => { + const a = await submittedRequestId({ lead_refs: REFS }); + const b = await submittedRequestId({ lead_refs: REFS }); + expect(a).toBe(b); + }); + + it("does not depend on the clock (a retry after midnight still dedupes)", async () => { + const a = await submittedRequestId({ lead_refs: REFS }); + // Shift the wall clock past a UTC midnight WITHOUT faking timers — fake + // timers would stall the awaited HTTP mock. Stubbing Date.now + the Date + // constructor is enough to catch any date component in the hash. + const RealDate = Date; + const shifted = new RealDate("2031-03-04T00:00:01Z").getTime(); + // @ts-expect-error — deliberate narrow stub for this assertion + globalThis.Date = class extends RealDate { + constructor(...args: any[]) { + // @ts-expect-error — passthrough + super(...(args.length ? args : [shifted])); + } + static now() { + return shifted; + } + }; + try { + const b = await submittedRequestId({ lead_refs: REFS }); + expect(b).toBe(a); + } finally { + globalThis.Date = RealDate; + } + }); + + it("ref order does not change the key", async () => { + const a = await submittedRequestId({ lead_refs: REFS }); + const b = await submittedRequestId({ lead_refs: [...REFS].reverse() }); + expect(a).toBe(b); + }); + + it("a raised max_cost is a NEW approved run", async () => { + const capped = await submittedRequestId({ lead_refs: REFS, max_cost: 500 }); + const raised = await submittedRequestId({ lead_refs: REFS, max_cost: 5000 }); + expect(raised).not.toBe(capped); + }); + + it("different channels key differently", async () => { + const email = await submittedRequestId({ lead_refs: REFS, channels: ["email"] }); + const phone = await submittedRequestId({ lead_refs: REFS, channels: ["phone"] }); + expect(email).not.toBe(phone); + }); + + it("different prior_deliveries slices key differently", async () => { + const first = await submittedRequestId({ + prior_deliveries: { job_id: "j1", limit: 50 }, + }); + const next = await submittedRequestId({ + prior_deliveries: { job_id: "j1", limit: 50, since: "cursor-50" }, + }); + expect(next).not.toBe(first); + }); + + it("an explicit request_id always wins", async () => { + const id = await submittedRequestId({ lead_refs: REFS, request_id: "mine-1" }); + expect(id).toBe("mine-1"); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads.test.ts b/packages/core/test/unit/composite/qualify-leads.test.ts index 66e69a04..1be7e753 100644 --- a/packages/core/test/unit/composite/qualify-leads.test.ts +++ b/packages/core/test/unit/composite/qualify-leads.test.ts @@ -112,8 +112,6 @@ describe("leadbay_qualify_leads", () => { contact_titles: ["Owner"], channels: ["email"], request_id: "vet-austin-1", - // Paid call (channels purchase email) — the spend gate withholds the - // submit without explicit consent. See qualify-leads-spend-gate.test.ts. confirm: true, wait_seconds: 0, }); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index ad110655..52bfb865 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -918,7 +918,10 @@ back from Leadbay. Call \`leadbay_find_new_leads\` with the seed, \`filters\`, \`count\`, \`qualify: false\`, no channels — this is FREE — and a \`request_id\` derived -from the ask + the ARCHETYPE + today's date. The archetype component is not +from the ask + the ARCHETYPE + today's date. \`count\` is the TOTAL I asked +for, not a per-search number: with two archetypes and a request for 10, +split it (5 + 5, or whatever weighting fits my ask) rather than sending 10 +to each — otherwise I get 20 leads and, on the paid pass, pay for 20. The archetype component is not optional: \`request_id\` is the idempotency key, so two archetype searches sharing one id dedupe to the same job and the second archetype is never searched. Render the delivery table and judge fit honestly: are these the diff --git a/packages/mcp/src/prompts.ts b/packages/mcp/src/prompts.ts index 2ad6636e..63149d51 100644 --- a/packages/mcp/src/prompts.ts +++ b/packages/mcp/src/prompts.ts @@ -409,6 +409,15 @@ export function getPrompt( if (!entry) { throw new Error(`Unknown prompt: ${name}`); } + // Filtering prompts/list is not enough: a cached slash command or a direct + // prompts/get by name would still hand back a workflow whose every tool call + // is missing from tools/list. A gated prompt is unavailable, not just unlisted. + const gate = GATED_PROMPTS[name]; + if (gate && !gate()) { + throw new Error( + `Prompt ${name} is not enabled in this deployment (requires LEADBAY_MCP_LEAD_DELIVERY=1).` + ); + } // Validate required arguments. Per spec, missing required args should // surface as a JSON-RPC error so the client can re-prompt the user. const missing = entry.arguments diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index dca585c0..0b0751c3 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -1,6 +1,12 @@ --- name: leadbay_new_leads kind: prompt +# Every phase of this workflow calls leadbay_find_new_leads / +# leadbay_qualify_leads / leadbay_lead_job_status, which are gated off until +# the /1.6/mcp/* backend routes ship. Suppresses the Claude SKILL.md (a static +# file with no runtime gate); the MCP prompt is filtered by listPrompts(). +# DELETE this line in the release that un-gates the tools. +release_gated: true short_description: | Guided net-new lead delivery — turn a natural-language need ("gyms around Dallas that would buy our flooring") into ICP-perfect NEW companies with @@ -86,7 +92,10 @@ back from Leadbay. Call `leadbay_find_new_leads` with the seed, `filters`, `count`, `qualify: false`, no channels — this is FREE — and a `request_id` derived -from the ask + the ARCHETYPE + today's date. The archetype component is not +from the ask + the ARCHETYPE + today's date. `count` is the TOTAL I asked +for, not a per-search number: with two archetypes and a request for 10, +split it (5 + 5, or whatever weighting fits my ask) rather than sending 10 +to each — otherwise I get 20 leads and, on the paid pass, pay for 20. The archetype component is not optional: `request_id` is the idempotency key, so two archetype searches sharing one id dedupe to the same job and the second archetype is never searched. Render the delivery table and judge fit honestly: are these the diff --git a/packages/promptforge/src/frontmatter.ts b/packages/promptforge/src/frontmatter.ts index f30d8790..04265e68 100644 --- a/packages/promptforge/src/frontmatter.ts +++ b/packages/promptforge/src/frontmatter.ts @@ -73,6 +73,12 @@ export const FrontmatterSchema = z.object({ // with routing, "disabled" otherwise; explicit disabled is for tools whose // first-600-char window cannot carry the shared pointer. memory_protocol: z.enum(["enabled", "disabled"]).optional(), + // Set while a prompt's whole workflow depends on tools that are themselves + // gated off (e.g. a backend route that hasn't shipped). The MCP prompt is + // filtered at runtime, but a Claude SKILL.md is a static file with no + // runtime gate — so promptforge simply does not emit one. Delete this line + // in the release that un-gates the tools. + release_gated: z.boolean().optional(), // Compact rendering recipe (1–3 sentences). Promptforge auto-emits // a `## RENDER (quick)` block. The detailed RENDERING block stays // in the body via {{include:rendering/...}}. diff --git a/packages/promptforge/src/skills.ts b/packages/promptforge/src/skills.ts index acb7777b..8832e8ae 100644 --- a/packages/promptforge/src/skills.ts +++ b/packages/promptforge/src/skills.ts @@ -138,9 +138,15 @@ export interface SkillFile { * comparing against disk (cmdCheck) or writing (cmdBuild). */ export function buildSkillFiles(artifacts: AssembledArtifact[]): SkillFile[] { - return artifacts.map((a) => ({ - name: a.frontmatter.name, - relativePath: `${a.frontmatter.name}/SKILL.md`, - content: buildSkillMarkdown(a), - })); + return artifacts + // A SKILL.md is a static file that auto-triggers with no runtime gate, so + // a release_gated prompt must not ship one at all — otherwise Claude Code + // enters a guided workflow whose first tool call is missing from + // tools/list. The MCP prompt is filtered at runtime instead. + .filter((a) => a.frontmatter.release_gated !== true) + .map((a) => ({ + name: a.frontmatter.name, + relativePath: `${a.frontmatter.name}/SKILL.md`, + content: buildSkillMarkdown(a), + })); } diff --git a/packages/promptforge/test/skills.test.ts b/packages/promptforge/test/skills.test.ts index cabbbe8e..542685bb 100644 --- a/packages/promptforge/test/skills.test.ts +++ b/packages/promptforge/test/skills.test.ts @@ -52,16 +52,33 @@ describe("audit: SKILL.md files", () => { const result = assemble({ root: PKG_ROOT, registeredToolNames: registered }); const skillFiles = buildSkillFiles(result.prompts); + // release_gated prompts deliberately emit NO SKILL.md: a skill is a static + // auto-triggering file with no runtime gate, so shipping one while its tools + // are hidden starts a workflow that fails on the first call. + const gatedPrompts = new Set( + result.prompts + .filter((p) => p.frontmatter.release_gated === true) + .map((p) => p.frontmatter.name), + ); + it("every prompt .md.tmpl has a matching SKILL.md", () => { const templates = readdirSync(PROMPTS_DIR) .filter((f) => f.endsWith(".md.tmpl")) - .map((f) => f.replace(/\.md\.tmpl$/, "")); + .map((f) => f.replace(/\.md\.tmpl$/, "")) + .filter((name) => !gatedPrompts.has(name)); for (const name of templates) { const path = join(SKILLS_DIR, name, "SKILL.md"); expect(existsSync(path), `expected ${path}`).toBe(true); } }); + it("release-gated prompts emit no SKILL.md", () => { + for (const name of gatedPrompts) { + const path = join(SKILLS_DIR, name, "SKILL.md"); + expect(existsSync(path), `${path} must not ship while gated`).toBe(false); + } + }); + it("every emitted SKILL.md is in sync with .md.tmpl source", () => { for (const skill of skillFiles) { const path = join(SKILLS_DIR, skill.relativePath); diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index db4cfe8a..21a93fe5 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -56,13 +56,13 @@ annotations: idempotentHint: true openWorldHint: true --- -Submit a net-new lead search: the backend matches an ICP seed against the -full company universe, applies hard filters, skips everything the org already -knows (`novelty: org`), optionally qualifies candidates against the org's own +Submit a net-new lead search: the backend matches an ICP seed against the full +company universe, applies hard filters, skips what the org already knows +(`novelty: org`), optionally qualifies candidates against the org's own intelligence (questions, tags, ideal buyer profile — frozen at submit), and optionally purchases contact channels. The tool polls up to `wait_seconds` -(default 45); a longer job returns `still_running` + `next_poll` — hand off -to `leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. +(default 45); a longer job returns `still_running` + `next_poll` — hand off to +`leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (`qualify: false`, `channels: []`) is FREE: company profile + fit score + cached research + @@ -95,9 +95,8 @@ In a FREE preview, drop violating rows and say the tier doesn't enforce. - `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). -**Read the result honestly.** `funnel` + `explain.scope_notes` tell the -story; zero delivered gets a cause + a concrete next move, never a bare "no -results" (rules in RENDERING). +**Read the result honestly** — `funnel` + `explain.scope_notes` tell the +story; zero delivered gets a cause and a next move (rules in RENDERING). --- From 46d97c761eb4c8b05828e6e89d8a0519829e30e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:25:15 -0700 Subject: [PATCH 15/46] fix(mcp): gate the prompt on write too; escape job ids; untouch prompts.test Sixth Codex pass. - The new-leads prompt is gated on the write surface as well as the rollout flag. Its workflow calls write-tier tools, so a read-only server (LEADBAY_MCP_WRITE=0) offered a slash command whose tools were absent from tools/list even with the flag on. buildServer now threads includeWrite into both listPrompts() and getPrompt(). - job_id is encodeURIComponent'd before building the poll URL. It comes straight from user/agent input and the server does not validate schemas before dispatch, so a value like `../../users/me` normalized out of /mcp/jobs and fired an AUTHENTICATED GET at an unintended endpoint. - prompts.test.ts is byte-identical to main again. The gated-prompt coverage moved wholly into prompt-release-gate.test.ts, which now also covers the read-only case and prompts/get. Both gate suites get an explicit 30s timeout: each case resets the module registry and re-imports a large graph, which overran vitest's 5s default. pnpm -r build, -r test (1364 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 7 ++- .../test/unit/mcp-first-delivery-gate.test.ts | 5 +- packages/mcp/src/prompts.ts | 25 +++++++--- packages/mcp/src/server.ts | 12 ++++- packages/mcp/test/prompt-release-gate.test.ts | 48 +++++++++++++++++-- packages/mcp/test/prompts.test.ts | 15 +----- 6 files changed, 84 insertions(+), 28 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index ad63d7bf..afb4f63c 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -139,8 +139,13 @@ export async function collectJobSnapshot( limit?: number ): Promise { const pageLimit = Math.min(Math.max(limit ?? PAGE_LIMIT, 1), PAGE_LIMIT); + // Escape the handle: job_id comes straight from user/agent input and the + // server does not validate schemas before dispatch, so an unescaped value + // containing path separators (`../../users/me`) would normalize out of + // /mcp/jobs and fire an AUTHENTICATED GET at an unintended endpoint. + const safeJobId = encodeURIComponent(jobId); const qs = (cursor?: string) => - `/mcp/jobs/${jobId}?limit=${pageLimit}` + + `/mcp/jobs/${safeJobId}?limit=${pageLimit}` + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); const maxPages = maxPagesFor(pageLimit); let page = await client.request("GET", qs(since)); diff --git a/packages/core/test/unit/mcp-first-delivery-gate.test.ts b/packages/core/test/unit/mcp-first-delivery-gate.test.ts index 35a9a10f..b320930e 100644 --- a/packages/core/test/unit/mcp-first-delivery-gate.test.ts +++ b/packages/core/test/unit/mcp-first-delivery-gate.test.ts @@ -39,7 +39,10 @@ async function loadCatalogues(flag: string | undefined) { beforeEach(() => vi.resetModules()); -describe("MCP-first delivery release gate", () => { +// Each case resets the module registry and re-imports the full core index, a +// large graph that can take >1s to re-evaluate — well past vitest's 5s default +// once a case does it twice. +describe("MCP-first delivery release gate", { timeout: 30_000 }, () => { it("hides all three tools by default", async () => { const names = await loadCatalogues(undefined); for (const tool of DELIVERY_TOOLS) { diff --git a/packages/mcp/src/prompts.ts b/packages/mcp/src/prompts.ts index 63149d51..d7b9b205 100644 --- a/packages/mcp/src/prompts.ts +++ b/packages/mcp/src/prompts.ts @@ -379,10 +379,20 @@ const CATALOG: CatalogEntry[] = [ /** Prompts whose whole workflow drives tools that are themselves gated off * until the backend routes ship. Exposing the prompt without the tools would * let a user start a guided flow whose every call is missing from tools/list. */ -const GATED_PROMPTS: Record boolean> = { - leadbay_new_leads: () => process.env.LEADBAY_MCP_LEAD_DELIVERY === "1", +const GATED_PROMPTS: Record boolean> = { + // Needs the rollout flag AND the write surface: every phase calls + // leadbay_find_new_leads / leadbay_qualify_leads, which are write-tier, so a + // read-only server (LEADBAY_MCP_WRITE=0) would offer a workflow whose tools + // are absent from tools/list even with the flag on. + leadbay_new_leads: (opts) => + process.env.LEADBAY_MCP_LEAD_DELIVERY === "1" && opts.includeWrite !== false, }; +export interface PromptGateOptions { + /** Mirrors buildServer's includeWrite. Defaults to true (write enabled). */ + includeWrite?: boolean; +} + /** The full catalogue, gates ignored — contract audits assert every prompt * named in WORKFLOWS.md resolves, and a rollout flag must not read as * "this prompt does not exist". */ @@ -394,16 +404,17 @@ export function listAllPrompts(): Prompt[] { })); } -export function listPrompts(): Prompt[] { +export function listPrompts(opts: PromptGateOptions = {}): Prompt[] { return listAllPrompts().filter((p) => { const gate = GATED_PROMPTS[p.name]; - return gate ? gate() : true; + return gate ? gate(opts) : true; }); } export function getPrompt( name: string, - args: Record = {} + args: Record = {}, + opts: PromptGateOptions = {} ): GetPromptResult { const entry = CATALOG.find((c) => c.name === name); if (!entry) { @@ -413,9 +424,9 @@ export function getPrompt( // prompts/get by name would still hand back a workflow whose every tool call // is missing from tools/list. A gated prompt is unavailable, not just unlisted. const gate = GATED_PROMPTS[name]; - if (gate && !gate()) { + if (gate && !gate(opts)) { throw new Error( - `Prompt ${name} is not enabled in this deployment (requires LEADBAY_MCP_LEAD_DELIVERY=1).` + `Prompt ${name} is not enabled in this deployment (requires LEADBAY_MCP_LEAD_DELIVERY=1 and the write surface).` ); } // Validate required arguments. Per spec, missing required args should diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index a16cbfa5..99184689 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -629,11 +629,19 @@ export function buildServer( // Prompts: pull-based slash commands the user can invoke directly. // See packages/mcp/src/prompts.ts for the catalog. + // Pass includeWrite through: a prompt whose workflow needs write-tier tools + // must not be offered on a read-only server, or the user gets a slash + // command whose every call is missing from tools/list. + const promptGate = { includeWrite: opts.includeWrite }; server.setRequestHandler(ListPromptsRequestSchema, async () => ({ - prompts: listPrompts(), + prompts: listPrompts(promptGate), })); server.setRequestHandler(GetPromptRequestSchema, async (req) => { - return getPrompt(req.params.name, (req.params.arguments ?? {}) as Record); + return getPrompt( + req.params.name, + (req.params.arguments ?? {}) as Record, + promptGate + ); }); // Resources: URI-addressable read-only payloads (lead://, lens://, org://). diff --git a/packages/mcp/test/prompt-release-gate.test.ts b/packages/mcp/test/prompt-release-gate.test.ts index 79b71760..3cf9d9c7 100644 --- a/packages/mcp/test/prompt-release-gate.test.ts +++ b/packages/mcp/test/prompt-release-gate.test.ts @@ -12,7 +12,10 @@ import { describe, it, expect, vi, afterEach } from "vitest"; const FLAG = "LEADBAY_MCP_LEAD_DELIVERY"; const GATED = "leadbay_new_leads"; -async function promptNames(flag: string | undefined) { +async function promptNames( + flag: string | undefined, + opts: { includeWrite?: boolean } = {} +) { vi.resetModules(); const previous = process.env[FLAG]; if (flag === undefined) delete process.env[FLAG]; @@ -20,8 +23,9 @@ async function promptNames(flag: string | undefined) { try { const mod = await import("../src/prompts.js"); return { - exposed: mod.listPrompts().map((p) => p.name), + exposed: mod.listPrompts(opts).map((p) => p.name), all: mod.listAllPrompts().map((p) => p.name), + get: (name: string) => mod.getPrompt(name, {}, opts), }; } finally { if (previous === undefined) delete process.env[FLAG]; @@ -31,7 +35,7 @@ async function promptNames(flag: string | undefined) { afterEach(() => vi.resetModules()); -describe("leadbay_new_leads prompt release gate", () => { +describe("leadbay_new_leads prompt release gate", { timeout: 30_000 }, () => { it("is hidden by default", async () => { const { exposed } = await promptNames(undefined); expect(exposed).not.toContain(GATED); @@ -55,4 +59,42 @@ describe("leadbay_new_leads prompt release gate", () => { const difference = on.exposed.filter((n) => !off.exposed.includes(n)); expect(difference).toEqual([GATED]); }); + + it("stays hidden on a read-only server even with the flag on", async () => { + // Every phase calls write-tier delivery tools, so LEADBAY_MCP_WRITE=0 + // leaves the workflow unrunnable regardless of the rollout flag. + const { exposed } = await promptNames("1", { includeWrite: false }); + expect(exposed).not.toContain(GATED); + }); + + it("prompts/get refuses a gated prompt, not just prompts/list", async () => { + // A cached slash command bypasses the list entirely. The gate reads the + // env at CALL time, so each case must assert while its flag is still set — + // hence the calls live inside the helper rather than on a returned closure. + const call = async ( + flag: string | undefined, + opts: { includeWrite?: boolean } = {} + ) => { + vi.resetModules(); + const previous = process.env[FLAG]; + if (flag === undefined) delete process.env[FLAG]; + else process.env[FLAG] = flag; + try { + const mod = await import("../src/prompts.js"); + try { + mod.getPrompt(GATED, {}, opts); + return null; + } catch (err) { + return String(err); + } + } finally { + if (previous === undefined) delete process.env[FLAG]; + else process.env[FLAG] = previous; + } + }; + + expect(await call(undefined)).toMatch(/not enabled/i); + expect(await call("1", { includeWrite: false })).toMatch(/not enabled/i); + expect(await call("1")).toBeNull(); + }); }); diff --git a/packages/mcp/test/prompts.test.ts b/packages/mcp/test/prompts.test.ts index ff71706d..f740f896 100644 --- a/packages/mcp/test/prompts.test.ts +++ b/packages/mcp/test/prompts.test.ts @@ -3,23 +3,11 @@ * commands. */ -import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { httpsMockFactory } from "./harness.js"; vi.mock("node:https", () => httpsMockFactory()); -// leadbay_new_leads is release-gated off by default until the /1.6/mcp/* -// backend routes ship. This suite asserts the FULL catalogue, so enable it. -const PREVIOUS_DELIVERY_FLAG = process.env.LEADBAY_MCP_LEAD_DELIVERY; -beforeAll(() => { - process.env.LEADBAY_MCP_LEAD_DELIVERY = "1"; -}); -afterAll(() => { - if (PREVIOUS_DELIVERY_FLAG === undefined) - delete process.env.LEADBAY_MCP_LEAD_DELIVERY; - else process.env.LEADBAY_MCP_LEAD_DELIVERY = PREVIOUS_DELIVERY_FLAG; -}); - import { LeadbayClient } from "@leadbay/core"; import { buildServer } from "../src/server.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; @@ -47,7 +35,6 @@ describe("prompts/* capability (P2 prompts)", () => { expect(names).toEqual([ "leadbay_daily_check_in", "leadbay_prospecting_overview", - "leadbay_new_leads", "leadbay_research_a_domain", "leadbay_import_file", "leadbay_refine_audience", From e138b863d4e5f6ed0d7c65cce04e0b88635e6799 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:50:19 -0700 Subject: [PATCH 16/46] fix(mcp): prompt-gate truthiness, collision-safe idempotency, retry + top-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seventh Codex pass. Three of the five are follow-ons from earlier fixes. - buildServer normalizes includeWrite with Boolean() for the prompt gate. The tool list uses `if (opts.includeWrite)`, so an omitted flag means NO write tools — but the gate read `undefined` as write-enabled and could expose the slash prompt while its tools were absent. - The derived idempotency key keeps ref FIELD NAMES, so {website:"acme.com"} and {name:"acme.com"} no longer hash alike; the backend resolves them differently and the second approval could be deduped onto the first job. - `lang` joins the key too: re-running an approved batch in another language must not return the earlier job with evidence in the previous one. - Prompt: retries reuse the request_id verbatim. Rederiving it from "today's date" after midnight produced a new key, so a retry of a live or timed-out search launched a second paid, novelty-claiming job. - Prompt: top up after PAID attrition, not just a short preview. A full-count preview can still end short once qualification disqualifies rows or a strict title/channel match misses — the ask is n qualified and contactable, not n examined. Report the gap and offer a shortfall-sized top-up, quoted first. pnpm -r build, -r test (1366 passed), -r typecheck green. Co-Authored-By: Claude --- packages/core/src/composite/qualify-leads.ts | 11 +++++++++-- .../composite/qualify-leads-idempotency.test.ts | 16 ++++++++++++++++ packages/mcp/src/prompts.generated.ts | 16 +++++++++++++++- packages/mcp/src/server.ts | 5 ++++- .../prompts/leadbay_new_leads.md.tmpl | 16 +++++++++++++++- 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 737506a3..7e95e1db 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -58,10 +58,14 @@ const DEFAULT_WAIT_SECONDS = 45; * A caller who wants a deliberate re-run of an identical batch passes an * explicit request_id. */ function derivedRequestId(params: QualifyLeadsParams): string { + // Keep the FIELD NAMES in the shape. Concatenating truthy values alone let + // distinct selectors collapse onto one key — {website:"acme.com"} and + // {name:"acme.com"} resolve differently backend-side but hashed identically, + // so the second approval could be deduped onto the first job. const refs = (params.lead_refs ?? []) .map((r) => - [r.lead_id, r.contact_id, r.website, r.name, r.location] - .filter(Boolean) + (["lead_id", "contact_id", "website", "name", "location"] as const) + .map((k) => `${k}=${r[k] ?? ""}`) .join("~") ) .sort() @@ -86,6 +90,9 @@ function derivedRequestId(params: QualifyLeadsParams): string { // The cap is part of the approval: raising it after a stop_reason:max_cost // is a NEW approved run, and must not dedupe onto the capped job. params.max_cost ?? "", + // Same for the output language — re-running the batch in another language + // must not return the earlier job with evidence in the previous one. + params.lang ?? "", ].join("#"); // FNV-1a — short, dependency-free, and only needs to be collision-resistant // across one org's batches, not cryptographically strong. diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index f0d29b91..cd4a603a 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -131,6 +131,22 @@ describe("qualify_leads — derived idempotency key", () => { expect(next).not.toBe(first); }); + it("a different output language keys differently", async () => { + const en = await submittedRequestId({ lead_refs: REFS, lang: "en" }); + const fr = await submittedRequestId({ lead_refs: REFS, lang: "fr" }); + expect(fr).not.toBe(en); + }); + + it("the same value under a different ref field keys differently", async () => { + // {website:"acme.com"} and {name:"acme.com"} resolve differently backend + // side, so they must not collapse onto one key. + const byWebsite = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + }); + const byName = await submittedRequestId({ lead_refs: [{ name: "acme.com" }] }); + expect(byName).not.toBe(byWebsite); + }); + it("an explicit request_id always wins", async () => { const id = await submittedRequestId({ lead_refs: REFS, request_id: "mine-1" }); expect(id).toBe("mine-1"); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 52bfb865..7a46fe48 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -921,7 +921,13 @@ Call \`leadbay_find_new_leads\` with the seed, \`filters\`, \`count\`, from the ask + the ARCHETYPE + today's date. \`count\` is the TOTAL I asked for, not a per-search number: with two archetypes and a request for 10, split it (5 + 5, or whatever weighting fits my ask) rather than sending 10 -to each — otherwise I get 20 leads and, on the paid pass, pay for 20. The archetype component is not +to each — otherwise I get 20 leads and, on the paid pass, pay for 20. + +When you RETRY a search — it timed out, or the job is still live — reuse the +\`request_id\` you already sent, verbatim. Do not recompute it: rederiving from +"today's date" after midnight yields a new key, the backend cannot dedupe, and +a second paid, novelty-claiming search launches. Roll the date only when I am +genuinely asking for a new batch. The archetype component is not optional: \`request_id\` is the idempotency key, so two archetype searches sharing one id dedupe to the same job and the second archetype is never searched. Render the delivery table and judge fit honestly: are these the @@ -962,6 +968,14 @@ When I want qualification evidence and/or reachable contacts: the paid submit back into the free job; keeping the original count buys a whole second batch, because \`novelty: org\` already excludes everything the preview delivered. + + The same arithmetic applies AFTER the paid pass. A full-count preview can + still end short once qualification disqualifies rows or a strict title / + channel match misses: what I asked for is n QUALIFIED, CONTACTABLE leads, + not n examined. Count the delivered-and-callable rows; if they fall short, + tell me the gap in one line and offer to top it up — another shortfall-sized + search under a NEW \`request_id\`, quoted first like any paid run. Never + silently hand back fewer than I asked for and paid toward. 3. While the job runs, poll with \`leadbay_lead_job_status\` (\`wait_seconds: 60\`); report progress, not silence. diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 99184689..f9f195b5 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -632,7 +632,10 @@ export function buildServer( // Pass includeWrite through: a prompt whose workflow needs write-tier tools // must not be offered on a read-only server, or the user gets a slash // command whose every call is missing from tools/list. - const promptGate = { includeWrite: opts.includeWrite }; + // Normalize to the SAME truthiness the tool list uses above (`if + // (opts.includeWrite)`), so an omitted flag means "no write tools" for the + // prompt gate too rather than being read as write-enabled. + const promptGate = { includeWrite: Boolean(opts.includeWrite) }; server.setRequestHandler(ListPromptsRequestSchema, async () => ({ prompts: listPrompts(promptGate), })); diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index 0b0751c3..ca2f972d 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -95,7 +95,13 @@ Call `leadbay_find_new_leads` with the seed, `filters`, `count`, from the ask + the ARCHETYPE + today's date. `count` is the TOTAL I asked for, not a per-search number: with two archetypes and a request for 10, split it (5 + 5, or whatever weighting fits my ask) rather than sending 10 -to each — otherwise I get 20 leads and, on the paid pass, pay for 20. The archetype component is not +to each — otherwise I get 20 leads and, on the paid pass, pay for 20. + +When you RETRY a search — it timed out, or the job is still live — reuse the +`request_id` you already sent, verbatim. Do not recompute it: rederiving from +"today's date" after midnight yields a new key, the backend cannot dedupe, and +a second paid, novelty-claiming search launches. Roll the date only when I am +genuinely asking for a new batch. The archetype component is not optional: `request_id` is the idempotency key, so two archetype searches sharing one id dedupe to the same job and the second archetype is never searched. Render the delivery table and judge fit honestly: are these the @@ -136,6 +142,14 @@ When I want qualification evidence and/or reachable contacts: the paid submit back into the free job; keeping the original count buys a whole second batch, because `novelty: org` already excludes everything the preview delivered. + + The same arithmetic applies AFTER the paid pass. A full-count preview can + still end short once qualification disqualifies rows or a strict title / + channel match misses: what I asked for is n QUALIFIED, CONTACTABLE leads, + not n examined. Count the delivered-and-callable rows; if they fall short, + tell me the gap in one line and offer to top it up — another shortfall-sized + search under a NEW `request_id`, quoted first like any paid run. Never + silently hand back fewer than I asked for and paid toward. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. From c169b37bc123ed072e7bde9ba08334fed16b5e12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:36:11 -0700 Subject: [PATCH 17/46] fix(mcp): 128-bit idempotency keys, search key synthesis, cursor fallback Eighth Codex pass. The headline is a real hash collision, verified locally. - The derived idempotency key was a 32-bit FNV-1a digest, and two distinct single-ref batches genuinely collide: websites "aeqexh0jh0.com" and "99rcha4ssn.com" both produced qualify-auto-76d7841e. A collision here redirects one paid approval onto another job. Now SHA-256 truncated to 128 bits, via a shared derivedKey() helper. The colliding pair is pinned as a regression test. - find_new_leads synthesizes a key too. `request_id` is schema-`required` but the server does not validate before dispatch, so a confirmed paid search could post with no idempotency handle and a retry would launch a second paid, novelty-claiming job. - collectJobSnapshot falls back to the caller's `since` when a page carries no replacement cursor. An incremental poll of a running job legitimately returns items:[] with no next_since, and nulling the cursor there made the next poll a full re-read of rows already rendered. - skills.test.ts keeps only the minimal filter the old invariant needs; the gated-skill assertions moved to the new skills-release-gate.test.ts. pnpm -r build, -r test (1370 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 15 ++++- packages/core/src/composite/find-new-leads.ts | 32 ++++++++++- packages/core/src/composite/qualify-leads.ts | 10 +--- .../find-new-leads-spend-gate.test.ts | 25 ++++++++ .../qualify-leads-idempotency.test.ts | 15 ++++- .../test/skills-release-gate.test.ts | 57 +++++++++++++++++++ packages/promptforge/test/skills.test.ts | 6 -- 7 files changed, 142 insertions(+), 18 deletions(-) create mode 100644 packages/promptforge/test/skills-release-gate.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index afb4f63c..79f546dc 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -5,6 +5,7 @@ // cumulatively from /mcp/jobs/{id} with an opaque `since` cursor. The // backend caps poll pages at 100 items, while a qualify job can carry up // to 500 refs — so a snapshot collects pages until the cursor drains. +import { createHash } from "node:crypto"; import type { LeadbayClient } from "../client.js"; import type { ToolContext } from "../types.js"; @@ -154,7 +155,10 @@ export async function collectJobSnapshot( // next_since into a page with no items used to overwrite the cursor with that // empty page's (often null) next_since, so a caller that had just received a // full page lost its place and had to re-read everything it had already seen. - let cursor = page.next_since ?? null; + // Fall back to the cursor the CALLER passed in: an incremental poll of a + // running job legitimately returns items:[] with no next_since, and dropping + // to null there would make the next poll a full re-read. + let cursor = page.next_since ?? since ?? null; let pages = 1; while (page.items.length >= pageLimit && page.next_since && pages < maxPages) { const next = await client.request( @@ -218,6 +222,15 @@ export async function waitForJob( return snap; } +/** Stable idempotency key derived from an approved batch's own shape. + * SHA-256 truncated to 128 bits — a 32-bit digest collided in practice, and a + * collision here redirects one paid approval onto a different job. Nothing + * time-based goes into `shape`: a retry of the same approval must dedupe even + * if it lands the next day. */ +export function derivedKey(prefix: string, shape: string): string { + return `${prefix}-${createHash("sha256").update(shape).digest("hex").slice(0, 32)}`; +} + /** LEADBAY_MOCK=1 journals writes and answers the generic * `{mocked, would_call}` envelope instead of a real `{job_id}`. Without a * guard the submit falls through to polling `/mcp/jobs/undefined`, which has diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 5e146cae..59d520ea 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + derivedKey, mockedSubmitPreview, compactBody, normalizeSearchFilters, @@ -204,6 +205,33 @@ export const findNewLeads: Tool = { const vetoed = params.confirm === false; const consented = !vetoed && params.confirm === true; + // `request_id` is schema-`required`, but the server does not validate + // schemas before dispatch, so a caller can omit it and compactBody would + // drop the key entirely — leaving a confirmed PAID search with no + // idempotency handle, so a timeout + retry launches a second paid, + // novelty-claiming job. Synthesize a stable key from the approved search + // itself, exactly as the qualify path does. + const requestId = + params.request_id ?? + derivedKey( + "search-auto", + [ + params.query ?? "", + JSON.stringify(params.example_lead ?? {}), + JSON.stringify(normalizeSearchFilters(params.filters) ?? {}), + params.count ?? "", + params.qualify === true ? "qualify" : "free", + params.min_ai_score ?? "", + (params.contact_titles ?? []).slice().sort().join(","), + params.title_gate ?? "", + (params.channels ?? []).slice().sort().join(","), + params.novelty ?? "", + params.max_cost ?? "", + params.exploration_cap ?? "", + params.lang ?? "", + ].join("#") + ); + const body = compactBody({ query: params.query, example_lead: params.example_lead, @@ -218,7 +246,7 @@ export const findNewLeads: Tool = { novelty: params.novelty, max_cost: params.max_cost, exploration_cap: params.exploration_cap, - request_id: params.request_id, + request_id: requestId, lang: params.lang, dry_run: params.dry_run, }); @@ -287,7 +315,7 @@ export const findNewLeads: Tool = { const { leads, skipped } = splitItems(snapshot); return { job_id: submit.job_id, - request_id: params.request_id, + request_id: requestId, duplicate_submit: submit.duplicate ?? false, state: snapshot.job.state, done, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 7e95e1db..8fed2770 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + derivedKey, mockedSubmitPreview, compactBody, splitItems, @@ -94,14 +95,7 @@ function derivedRequestId(params: QualifyLeadsParams): string { // must not return the earlier job with evidence in the previous one. params.lang ?? "", ].join("#"); - // FNV-1a — short, dependency-free, and only needs to be collision-resistant - // across one org's batches, not cryptographically strong. - let h = 0x811c9dc5; - for (let i = 0; i < shape.length; i++) { - h ^= shape.charCodeAt(i); - h = Math.imul(h, 0x01000193) >>> 0; - } - return `qualify-auto-${h.toString(16).padStart(8, "0")}`; + return derivedKey("qualify-auto", shape); } export const qualifyLeads: Tool = { diff --git a/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts b/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts index 103079c3..6bc61bf0 100644 --- a/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts +++ b/packages/core/test/unit/composite/find-new-leads-spend-gate.test.ts @@ -138,6 +138,31 @@ describe("leadbay_find_new_leads — spend gate", () => { expect(postBodies()[0].dry_run).toBeUndefined(); }); + it("synthesizes a request_id when the caller omits the required field", async () => { + // The server does not validate schemas before dispatch, so `required` is + // not enforced — a confirmed paid search could otherwise post with no + // idempotency handle and a retry would launch a second paid job. + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT, + }, + ]); + + await findNewLeads.execute(newClient(), { + count: 10, + qualify: true, + confirm: true, + wait_seconds: 0, + } as any); + + const body = postBodies()[0]; + expect(body.request_id).toMatch(/^search-auto-[0-9a-f]{32}$/); + }); + it("confirm:true submits the paid search", async () => { mockHttp([ { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index cd4a603a..a911deff 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -69,7 +69,8 @@ const REFS = [{ website: "a.com" }, { website: "b.com" }]; describe("qualify_leads — derived idempotency key", () => { it("sends a derived request_id when a paid call omits one", async () => { const id = await submittedRequestId({ lead_refs: REFS }); - expect(id).toMatch(/^qualify-auto-[0-9a-f]{8}$/); + // 128-bit digest: a 32-bit one collided in practice across distinct batches. + expect(id).toMatch(/^qualify-auto-[0-9a-f]{32}$/); }); it("is stable across identical retries", async () => { @@ -147,6 +148,18 @@ describe("qualify_leads — derived idempotency key", () => { expect(byName).not.toBe(byWebsite); }); + it("does not collide on the pair that broke the 32-bit digest", async () => { + // Both of these hashed to qualify-auto-76d7841e under FNV-1a, which would + // have deduped one paid approval onto the other's job. + const a = await submittedRequestId({ + lead_refs: [{ website: "aeqexh0jh0.com" }], + }); + const b = await submittedRequestId({ + lead_refs: [{ website: "99rcha4ssn.com" }], + }); + expect(a).not.toBe(b); + }); + it("an explicit request_id always wins", async () => { const id = await submittedRequestId({ lead_refs: REFS, request_id: "mine-1" }); expect(id).toBe("mine-1"); diff --git a/packages/promptforge/test/skills-release-gate.test.ts b/packages/promptforge/test/skills-release-gate.test.ts new file mode 100644 index 00000000..43688111 --- /dev/null +++ b/packages/promptforge/test/skills-release-gate.test.ts @@ -0,0 +1,57 @@ +/** + * Release-gated prompts emit no SKILL.md. + * + * A Claude skill is a static file that auto-triggers on matching requests with + * no runtime switch, so shipping one while its tools are gated off starts a + * workflow that fails on the first tool call. The MCP prompt is filtered at + * runtime instead; the skill simply must not exist on disk. + */ + +import { describe, it, expect } from "vitest"; +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join, resolve } from "node:path"; +import { assemble } from "../src/assembler.js"; +import { buildSkillFiles } from "../src/skills.js"; +import { discoverRegisteredTools } from "../src/registry.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PKG_ROOT = resolve(__dirname, ".."); +const REPO_ROOT = resolve(PKG_ROOT, "..", ".."); +const CORE_SRC = resolve(REPO_ROOT, "packages", "core", "src"); +const SKILLS_DIR = resolve( + REPO_ROOT, + ".claude-plugin", + "plugins", + "leadbay", + "skills", +); + +const registered = discoverRegisteredTools(CORE_SRC); +const result = assemble({ root: PKG_ROOT, registeredToolNames: registered }); +const gated = result.prompts.filter( + (p) => p.frontmatter.release_gated === true, +); + +describe("audit: release-gated prompts ship no skill", () => { + it("leadbay_new_leads is currently gated", () => { + // Guards the fixture: if this prompt is un-gated at release, this test + // should be deleted along with the frontmatter flag — not left passing + // vacuously over an empty set. + expect(gated.map((p) => p.frontmatter.name)).toContain("leadbay_new_leads"); + }); + + it("buildSkillFiles emits nothing for a gated prompt", () => { + const emitted = buildSkillFiles(result.prompts).map((s) => s.name); + for (const p of gated) { + expect(emitted).not.toContain(p.frontmatter.name); + } + }); + + it("no gated SKILL.md exists on disk", () => { + for (const p of gated) { + const path = join(SKILLS_DIR, p.frontmatter.name, "SKILL.md"); + expect(existsSync(path), `${path} must not ship while gated`).toBe(false); + } + }); +}); diff --git a/packages/promptforge/test/skills.test.ts b/packages/promptforge/test/skills.test.ts index 542685bb..b83617f0 100644 --- a/packages/promptforge/test/skills.test.ts +++ b/packages/promptforge/test/skills.test.ts @@ -72,12 +72,6 @@ describe("audit: SKILL.md files", () => { } }); - it("release-gated prompts emit no SKILL.md", () => { - for (const name of gatedPrompts) { - const path = join(SKILLS_DIR, name, "SKILL.md"); - expect(existsSync(path), `${path} must not ship while gated`).toBe(false); - } - }); it("every emitted SKILL.md is in sync with .md.tmpl source", () => { for (const skill of skillFiles) { From c5eb0df3498da3387af91b02c843184ae44cee17 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:00:55 -0700 Subject: [PATCH 18/46] fix(mcp): hash exclude_lead_ids into the derived search key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ninth Codex pass — one finding, correct. `exclude_lead_ids` was sent in the POST body but left out of the derived `search-auto-*` shape, so a top-up that differs ONLY by its exclusion list hashed identically to the original search: the backend returns the first job as a duplicate and the exclusions never take effect. Now canonicalized (sorted, so ordering alone never forks the key) and included. Audited both tools for the same class of gap while here. That was the only omission: qualify_leads already hashes every shaping field, and request_id / dry_run are legitimately excluded (the key itself, and a flag that never submits). Added find-new-leads-idempotency.test.ts with a structural guard — one case per body field asserting that changing it changes the key, so a field added later cannot silently drift out of the hash. Verified the guard actually bites by removing the fix and watching exactly the two expected cases fail. pnpm -r build, -r test (1388 passed), -r typecheck green. Co-Authored-By: Claude --- packages/core/src/composite/find-new-leads.ts | 5 + .../find-new-leads-idempotency.test.ts | 133 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 packages/core/test/unit/composite/find-new-leads-idempotency.test.ts diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 59d520ea..ff5026b1 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -225,6 +225,11 @@ export const findNewLeads: Tool = { (params.contact_titles ?? []).slice().sort().join(","), params.title_gate ?? "", (params.channels ?? []).slice().sort().join(","), + // Canonicalized (sorted) so ordering alone never forks the key, but + // PRESENT — a top-up that differs only by exclude_lead_ids is a + // different approved search, and hashing it the same would return + // the first job as a duplicate with the exclusions never applied. + (params.exclude_lead_ids ?? []).slice().sort().join(","), params.novelty ?? "", params.max_cost ?? "", params.exploration_cap ?? "", diff --git a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts new file mode 100644 index 00000000..ca3ae17a --- /dev/null +++ b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts @@ -0,0 +1,133 @@ +/** + * Derived idempotency key for a paid search without an explicit request_id. + * + * The invariant: EVERY field that goes into the POST body and changes what the + * approved search does must also change the key. A field sent to the backend + * but omitted from the hash means two genuinely different approved searches + * collapse onto one job — the backend returns the first as a duplicate and the + * difference (a narrowed filter, a new exclusion list) never takes effect. + * + * The last case below enforces that structurally, so a field added to the body + * later cannot silently drift out of the key. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); +const JOB_ID = "c4e8a1b2-77d3-4f60-9a11-3b5c7e2d9f08"; + +const SUBMIT_202 = { + job_id: JOB_ID, + status_url: `/1.6/mcp/jobs/${JOB_ID}`, + estimated_cost: { max: 1880, unit: "cost_cents" }, + items_requested: 10, + duplicate: false, +}; + +const SNAPSHOT = { + job: { id: JOB_ID, state: "completed" }, + funnel: { delivered: 1 }, + items: [], + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + next_since: null, + explain: { region: "us", model: "m" }, +}; + +beforeEach(() => resetHttpMock()); + +/** Submit a PAID search with no request_id and return the key that went out. */ +async function keyFor(params: Record) { + resetHttpMock(); + mockHttp([ + { method: "POST", path: "/1.6/mcp/search", status: 202, body: SUBMIT_202 }, + { + method: "GET", + path: `/1.6/mcp/jobs/${JOB_ID}?limit=100`, + status: 200, + body: SNAPSHOT, + }, + ]); + await findNewLeads.execute(newClient(), { + count: 10, + qualify: true, + confirm: true, + wait_seconds: 0, + ...params, + } as any); + const post = getHttpRequests().find((r: any) => r.method === "POST")!; + return JSON.parse(post.body!).request_id as string; +} + +describe("find_new_leads — derived search key", () => { + it("is a 128-bit digest", async () => { + expect(await keyFor({})).toMatch(/^search-auto-[0-9a-f]{32}$/); + }); + + it("is stable across identical retries", async () => { + expect(await keyFor({})).toBe(await keyFor({})); + }); + + it("changes when exclude_lead_ids changes", async () => { + // The top-up case: same ask, but avoiding what the first pass returned. + const first = await keyFor({}); + const topUp = await keyFor({ exclude_lead_ids: ["lead-1", "lead-2"] }); + expect(topUp).not.toBe(first); + + const more = await keyFor({ exclude_lead_ids: ["lead-1", "lead-2", "lead-3"] }); + expect(more).not.toBe(topUp); + }); + + it("ignores exclude_lead_ids ORDER", async () => { + const a = await keyFor({ exclude_lead_ids: ["lead-1", "lead-2"] }); + const b = await keyFor({ exclude_lead_ids: ["lead-2", "lead-1"] }); + expect(b).toBe(a); + }); + + it("an explicit request_id always wins", async () => { + expect(await keyFor({ request_id: "gyms-dallas-2026-08-05" })).toBe( + "gyms-dallas-2026-08-05" + ); + }); + + // Structural guard: every body field that shapes the search must move the + // key. `request_id` is the key itself and `dry_run` never submits, so both + // are legitimately excluded. + const SHAPING_FIELDS: Array<[string, unknown]> = [ + ["query", "gyms that buy flooring"], + ["example_lead", { description: "an independent single-site gym" }], + ["filters", { sectors: ["fitness"] }], + ["count", 25], + ["min_ai_score", -10], + ["contact_titles", ["Owner"]], + ["title_gate", "strict"], + ["channels", ["email"]], + ["exclude_lead_ids", ["lead-9"]], + ["novelty", "none"], + ["max_cost", 4200], + ["exploration_cap", 90], + ["lang", "fr"], + ]; + + for (const [field, value] of SHAPING_FIELDS) { + it(`changes when ${field} changes`, async () => { + const base = await keyFor({}); + const changed = await keyFor({ [field]: value }); + expect( + changed, + `${field} is sent to the backend but missing from the hashed shape` + ).not.toBe(base); + }); + } +}); From cf962c18bde434adf800d7f93562d80d4bc2ea77 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:47:41 -0700 Subject: [PATCH 19/46] fix(mcp): canonical JSON idempotency shapes, scalar geo guard, honest mock path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tenth Codex pass — all five findings were real; two verified locally first. - Delimiter forging in the qualify key (verified): the ref shape joined raw values with `~`, so a value CONTAINING a delimiter impersonated another field. {website:"acme~name=Paris"} and {website:"acme", name:"Paris~name="} serialized identically — two distinct paid approvals sharing one key. Both shapes are now canonical JSON, in qualify_leads and find_new_leads, so no free-text value can forge a field boundary. - Defaults are canonicalized before hashing. An approval that omits `novelty` and a retry passing the documented default `"org"` derived different keys, so the retry could not dedupe and launched a second paid, novelty-claiming job. Same for min_ai_score, title_gate, channels, exclude_lead_ids. - Scalar filters.locations bypassed the country guard entirely (verified): `"United States"` as a bare string returned early and reached the backend, reintroducing the silent same-named-town fencing. Non-arrays are normalized to a one-item list instead of skipped. - mockedSubmitPreview only claims "mocked" when LEADBAY_MOCK=1. A real 2xx submit missing job_id now throws MALFORMED_SUBMIT_RESPONSE rather than reporting a successful no-submit envelope, which hid a production job the client could not poll. - Prompt: qualify EVERY preview job after an archetype split (or merge their refs), instead of the singular prior_deliveries handoff that stopped after the first archetype. Both colliding pairs are pinned as regression tests, plus a case asserting omitted fields and their explicit defaults derive the SAME key. pnpm -r build, -r test (1396 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 23 +++++- packages/core/src/composite/find-new-leads.ts | 47 +++++++----- packages/core/src/composite/qualify-leads.ts | 53 ++++++++------ .../find-new-leads-idempotency.test.ts | 22 ++++++ .../composite/mcp-job-helpers-review.test.ts | 73 +++++++++++++++++-- .../qualify-leads-idempotency.test.ts | 12 +++ packages/mcp/src/prompts.generated.ts | 6 +- .../prompts/leadbay_new_leads.md.tmpl | 6 +- 8 files changed, 187 insertions(+), 55 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 79f546dc..6ad8daa5 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -243,6 +243,19 @@ export function mockedSubmitPreview( ): Record | null { const s = (submit ?? {}) as Record; if (typeof s.job_id === "string" && s.job_id) return null; + // Only claim "mocked" when mock mode is actually on. A REAL 2xx submit that + // came back without a job_id (backend contract drift, a proxy eating the + // body) must fail loudly: reporting a successful no-submit envelope would + // hide a production job the client cannot poll, and would tell the user + // LEADBAY_MOCK=1 when it is not set. + if (process.env.LEADBAY_MOCK !== "1") { + throw { + error: true, + code: "MALFORMED_SUBMIT_RESPONSE", + message: `${tool}: the submit succeeded but the response carried no job_id, so the job cannot be polled.`, + hint: "The job may still be running server-side. Do not re-submit blindly — reuse the same request_id so a retry dedupes instead of double-spending.", + }; + } return { mocked: true, tool, @@ -314,8 +327,14 @@ function countryKey(raw: string): string { } export function rejectCountryLocations(locations: unknown): void { - if (!Array.isArray(locations)) return; - for (const loc of locations) { + if (locations === undefined || locations === null) return; + // The server does not validate the schema before dispatch, so an agent can + // send `filters.locations` as a bare string. Treating a non-array as "no + // locations" let a scalar "United States" sail past the guard and reach the + // backend, reintroducing exactly the silent same-named-town fencing this + // exists to stop. Normalize to a one-item list instead of returning. + const list = Array.isArray(locations) ? locations : [locations]; + for (const loc of list) { if (typeof loc === "string" && COUNTRY_LOCATION_VALUES.has(countryKey(loc))) { throw { error: true, diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index ff5026b1..f92b146d 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -215,26 +215,33 @@ export const findNewLeads: Tool = { params.request_id ?? derivedKey( "search-auto", - [ - params.query ?? "", - JSON.stringify(params.example_lead ?? {}), - JSON.stringify(normalizeSearchFilters(params.filters) ?? {}), - params.count ?? "", - params.qualify === true ? "qualify" : "free", - params.min_ai_score ?? "", - (params.contact_titles ?? []).slice().sort().join(","), - params.title_gate ?? "", - (params.channels ?? []).slice().sort().join(","), - // Canonicalized (sorted) so ordering alone never forks the key, but - // PRESENT — a top-up that differs only by exclude_lead_ids is a - // different approved search, and hashing it the same would return - // the first job as a duplicate with the exclusions never applied. - (params.exclude_lead_ids ?? []).slice().sort().join(","), - params.novelty ?? "", - params.max_cost ?? "", - params.exploration_cap ?? "", - params.lang ?? "", - ].join("#") + // JSON, not delimiter-joined: free-text values (query, titles) must not + // be able to forge a field boundary. Fields with a documented backend + // default are canonicalized TO that default, so an approval that omits + // one and a retry that passes it explicitly derive the same key rather + // than launching a second paid, novelty-claiming job. + JSON.stringify({ + query: params.query ?? null, + example_lead: params.example_lead ?? null, + filters: normalizeSearchFilters(params.filters) ?? null, + count: params.count ?? null, + qualify: params.qualify === true, + min_ai_score: params.min_ai_score ?? 0, + contact_titles: (params.contact_titles ?? []).slice().sort(), + title_gate: + params.title_gate ?? + ((params.contact_titles?.length ?? 0) > 0 ? "prefer" : null), + channels: (params.channels ?? []).slice().sort(), + // Sorted so ordering alone never forks the key, but PRESENT — a + // top-up differing only by exclude_lead_ids is a different approved + // search, and hashing it the same would return the first job as a + // duplicate with the exclusions never applied. + exclude_lead_ids: (params.exclude_lead_ids ?? []).slice().sort(), + novelty: params.novelty ?? "org", + max_cost: params.max_cost ?? null, + exploration_cap: params.exploration_cap ?? null, + lang: params.lang ?? null, + }) ); const body = compactBody({ diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 8fed2770..5561ff2d 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -59,42 +59,49 @@ const DEFAULT_WAIT_SECONDS = 45; * A caller who wants a deliberate re-run of an identical batch passes an * explicit request_id. */ function derivedRequestId(params: QualifyLeadsParams): string { - // Keep the FIELD NAMES in the shape. Concatenating truthy values alone let - // distinct selectors collapse onto one key — {website:"acme.com"} and - // {name:"acme.com"} resolve differently backend-side but hashed identically, - // so the second approval could be deduped onto the first job. + // JSON-serialize each ref rather than joining raw values with delimiters. + // Field names alone were not enough: a value CONTAINING the delimiters + // forged a different ref's serialization, so {website:"acme~name=Paris"} and + // {website:"acme", name:"Paris~name="} hashed identically. JSON escapes the + // separators, so no value can impersonate a field boundary. const refs = (params.lead_refs ?? []) .map((r) => - (["lead_id", "contact_id", "website", "name", "location"] as const) - .map((k) => `${k}=${r[k] ?? ""}`) - .join("~") + JSON.stringify( + (["lead_id", "contact_id", "website", "name", "location"] as const).map( + (k) => r[k] ?? null + ) + ) ) .sort() - .join("|"); - const shape = [ + .join(","); + // JSON the WHOLE shape for the same reason as the refs above: free-text + // values (contact_titles, lang) must not be able to forge a field boundary + // by containing a delimiter. + const shape = JSON.stringify({ refs, // The WHOLE selector, not just the job id: qualifying the first 50 of a // delivery job and then the next 50 are different batches, and collapsing // them to one key would make the second submit look like a duplicate and // leave those refs unqualified. - [ - params.prior_deliveries?.job_id, - params.prior_deliveries?.since, - params.prior_deliveries?.limit, - ] - .map((v) => v ?? "") - .join("~"), - params.qualify === false ? "free" : "qualify", - (params.channels ?? []).slice().sort().join(","), - (params.contact_titles ?? []).slice().sort().join(","), - params.title_gate ?? "", + prior: [ + params.prior_deliveries?.job_id ?? null, + params.prior_deliveries?.since ?? null, + params.prior_deliveries?.limit ?? null, + ], + // Canonicalize to the value the BACKEND will apply, so an approval that + // omits a field and a retry that passes that field's documented default + // derive the same key instead of launching a second paid job. + qualify: params.qualify !== false, + channels: (params.channels ?? []).slice().sort(), + contact_titles: (params.contact_titles ?? []).slice().sort(), + title_gate: params.title_gate ?? null, // The cap is part of the approval: raising it after a stop_reason:max_cost // is a NEW approved run, and must not dedupe onto the capped job. - params.max_cost ?? "", + max_cost: params.max_cost ?? null, // Same for the output language — re-running the batch in another language // must not return the earlier job with evidence in the previous one. - params.lang ?? "", - ].join("#"); + lang: params.lang ?? null, + }); return derivedKey("qualify-auto", shape); } diff --git a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts index ca3ae17a..bb537f58 100644 --- a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts @@ -95,6 +95,28 @@ describe("find_new_leads — derived search key", () => { expect(b).toBe(a); }); + // Passing a documented backend default explicitly must NOT fork the key — + // otherwise a retry that materializes defaults launches a second paid job. + it("treats omitted fields and their explicit defaults as the same search", async () => { + const omitted = await keyFor({}); + expect(await keyFor({ novelty: "org" })).toBe(omitted); + expect(await keyFor({ min_ai_score: 0 })).toBe(omitted); + expect(await keyFor({ channels: [] })).toBe(omitted); + expect(await keyFor({ exclude_lead_ids: [] })).toBe(omitted); + + // title_gate defaults to "prefer" only when contact_titles is set. + const withTitles = await keyFor({ contact_titles: ["Owner"] }); + expect(await keyFor({ contact_titles: ["Owner"], title_gate: "prefer" })).toBe( + withTitles + ); + }); + + it("a free-text value cannot forge a field boundary", async () => { + const a = await keyFor({ query: 'x", "count": 99' }); + const b = await keyFor({ query: "x" }); + expect(a).not.toBe(b); + }); + it("an explicit request_id always wins", async () => { expect(await keyFor({ request_id: "gyms-dallas-2026-08-05" })).toBe( "gyms-dallas-2026-08-05" diff --git a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts index c8a49e0e..bf33de31 100644 --- a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts +++ b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts @@ -149,14 +149,21 @@ describe("rejectCountryLocations — alias normalization", () => { describe("mockedSubmitPreview", () => { it("returns a preview when the submit carried no job_id (LEADBAY_MOCK)", () => { - const out = mockedSubmitPreview( - { mocked: true, would_call: { method: "POST", path: "/1.6/mcp/search" } }, - "leadbay_find_new_leads", - "us" - ); - expect(out).not.toBeNull(); - expect(out!.submitted).toBe(false); - expect(out!.tool).toBe("leadbay_find_new_leads"); + const previous = process.env.LEADBAY_MOCK; + process.env.LEADBAY_MOCK = "1"; + try { + const out = mockedSubmitPreview( + { mocked: true, would_call: { method: "POST", path: "/1.6/mcp/search" } }, + "leadbay_find_new_leads", + "us" + ); + expect(out).not.toBeNull(); + expect(out!.submitted).toBe(false); + expect(out!.tool).toBe("leadbay_find_new_leads"); + } finally { + if (previous === undefined) delete process.env.LEADBAY_MOCK; + else process.env.LEADBAY_MOCK = previous; + } }); it("returns null for a real submit so the normal poll proceeds", () => { @@ -165,3 +172,53 @@ describe("mockedSubmitPreview", () => { ).toBeNull(); }); }); + +describe("rejectCountryLocations — scalar input", () => { + // The server does not validate the schema before dispatch, so a bare string + // reaches the tool. Treating a non-array as "nothing to check" let a scalar + // country label through to the silent same-named-town fencing. + it("rejects a bare string country label", () => { + expect(() => rejectCountryLocations("United States")).toThrow( + expect.objectContaining({ code: "COUNTRY_LEVEL_LOCATION" }) + ); + expect(() => rejectCountryLocations("la France")).toThrow( + expect.objectContaining({ code: "COUNTRY_LEVEL_LOCATION" }) + ); + }); + + it("still allows a bare string city", () => { + expect(() => rejectCountryLocations("Austin")).not.toThrow(); + expect(() => rejectCountryLocations("Île-de-France")).not.toThrow(); + }); + + it("ignores null/undefined", () => { + expect(() => rejectCountryLocations(null)).not.toThrow(); + expect(() => rejectCountryLocations(undefined)).not.toThrow(); + }); +}); + +describe("mockedSubmitPreview — only in mock mode", () => { + it("throws on a real submit that carried no job_id", () => { + const previous = process.env.LEADBAY_MOCK; + delete process.env.LEADBAY_MOCK; + try { + expect(() => mockedSubmitPreview({}, "leadbay_find_new_leads", "us")).toThrow( + expect.objectContaining({ code: "MALFORMED_SUBMIT_RESPONSE" }) + ); + } finally { + if (previous !== undefined) process.env.LEADBAY_MOCK = previous; + } + }); + + it("returns the preview when mock mode is on", () => { + const previous = process.env.LEADBAY_MOCK; + process.env.LEADBAY_MOCK = "1"; + try { + const out = mockedSubmitPreview({ mocked: true }, "leadbay_find_new_leads", "us"); + expect(out?.submitted).toBe(false); + } finally { + if (previous === undefined) delete process.env.LEADBAY_MOCK; + else process.env.LEADBAY_MOCK = previous; + } + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index a911deff..6b597cf0 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -160,6 +160,18 @@ describe("qualify_leads — derived idempotency key", () => { expect(a).not.toBe(b); }); + it("a value containing the old delimiters cannot forge another ref", async () => { + // Under the delimiter-joined shape these serialized identically: + // {website:"acme~name=Paris"} == {website:"acme", name:"Paris~name="} + const forged = await submittedRequestId({ + lead_refs: [{ website: "acme~name=Paris" }], + }); + const genuine = await submittedRequestId({ + lead_refs: [{ website: "acme", name: "Paris~name=" }], + }); + expect(forged).not.toBe(genuine); + }); + it("an explicit request_id always wins", async () => { const id = await submittedRequestId({ lead_refs: REFS, request_id: "mine-1" }); expect(id).toBe("mine-1"); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 7a46fe48..9285764f 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -955,7 +955,11 @@ When I want qualification evidence and/or reachable contacts: \`title_gate\`, \`channels\`, \`max_cost\`. **No \`min_ai_score\`.** - \`leadbay_find_new_leads\`: the same, PLUS \`min_ai_score\` and \`count\`. 2. On my go-ahead, prefer feeding the free preview's deliveries to - \`leadbay_qualify_leads\` (\`prior_deliveries: {job_id}\`) — it only spends on + \`leadbay_qualify_leads\` (\`prior_deliveries: {job_id}\`) — one paid pass PER + preview job when Phase 3 ran several archetypes, or merge their delivered + refs into a single \`lead_refs\` call. Never qualify just the first job and + call it done: the other archetypes are part of what I asked for. It only + spends on companies already known to match. Paid calls need \`confirm: true\`; without it the tool withholds the submit and hands back a quote instead of spending. That applies to \`leadbay_find_new_leads\` too whenever you set diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index ca2f972d..71a601a0 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -129,7 +129,11 @@ When I want qualification evidence and/or reachable contacts: `title_gate`, `channels`, `max_cost`. **No `min_ai_score`.** - `leadbay_find_new_leads`: the same, PLUS `min_ai_score` and `count`. 2. On my go-ahead, prefer feeding the free preview's deliveries to - `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — it only spends on + `leadbay_qualify_leads` (`prior_deliveries: {job_id}`) — one paid pass PER + preview job when Phase 3 ran several archetypes, or merge their delivered + refs into a single `lead_refs` call. Never qualify just the first job and + call it done: the other archetypes are part of what I asked for. It only + spends on companies already known to match. Paid calls need `confirm: true`; without it the tool withholds the submit and hands back a quote instead of spending. That applies to `leadbay_find_new_leads` too whenever you set From 7783fcd965749db15f7185e0a737b1456d45cf02 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:07:58 -0700 Subject: [PATCH 20/46] fix(mcp): canonicalize idempotency shapes recursively; dedupe qualify refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleventh Codex pass — both findings real, both verified locally. - Nested property order forked the derived search key: `example_lead` and `filters` were hashed with plain JSON.stringify, so an agent rebuilding {description, employees} as {employees, description} derived a different key and a retry of a live paid search launched a second job. Added a recursive canonicalize() that sorts object keys at every depth, applied INSIDE derivedKey so neither tool can regress by hashing a hand-built string. Sector/location lists are sorted too — they are unordered sets to the backend. - Duplicate refs forked the qualify key even though the contract collapses them into one backend item. A paid submit with repeated websites, retried after the agent deduped, presented a new key and re-ran the batch. The canonical ref list is now uniqued before hashing. pnpm -r build, -r test (1400 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 26 ++++++++++++-- packages/core/src/composite/find-new-leads.ts | 36 ++++++++++++++----- packages/core/src/composite/qualify-leads.ts | 25 +++++++------ .../find-new-leads-idempotency.test.ts | 28 +++++++++++++++ .../qualify-leads-idempotency.test.ts | 11 ++++++ 5 files changed, 106 insertions(+), 20 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 6ad8daa5..87a73796 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -222,13 +222,35 @@ export async function waitForJob( return snap; } +/** Recursively canonicalize a value for hashing: object keys sorted at every + * depth so property ORDER never forks a key, arrays left in place (order can + * be meaningful — callers sort the ones where it is not). Plain + * JSON.stringify is not enough: an agent that rebuilds `example_lead` or + * `filters` with the properties in a different order would otherwise derive a + * different key for the same approved search and re-launch a paid job. */ +export function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === "object") { + const out: Record = {}; + for (const k of Object.keys(value as Record).sort()) { + out[k] = canonicalize((value as Record)[k]); + } + return out; + } + return value; +} + /** Stable idempotency key derived from an approved batch's own shape. * SHA-256 truncated to 128 bits — a 32-bit digest collided in practice, and a * collision here redirects one paid approval onto a different job. Nothing * time-based goes into `shape`: a retry of the same approval must dedupe even * if it lands the next day. */ -export function derivedKey(prefix: string, shape: string): string { - return `${prefix}-${createHash("sha256").update(shape).digest("hex").slice(0, 32)}`; +export function derivedKey(prefix: string, shape: unknown): string { + // Canonicalize HERE rather than at each call site, so neither tool can + // regress by hashing a hand-built string again. + const serialized = + typeof shape === "string" ? shape : JSON.stringify(canonicalize(shape)); + return `${prefix}-${createHash("sha256").update(serialized).digest("hex").slice(0, 32)}`; } /** LEADBAY_MOCK=1 journals writes and answers the generic diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index f92b146d..e45200e9 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -58,6 +58,23 @@ interface FindNewLeadsParams { const DEFAULT_WAIT_SECONDS = 45; +/** Sort the filter arrays that are unordered SETS to the backend, so a retry + * listing the same sectors/locations in another order still derives the same + * idempotency key. Object key order is handled by canonicalize() inside + * derivedKey; only array order needs a decision, and only here. */ +function sortFilterLists( + filters: Record | undefined +): Record | null { + if (!filters) return null; + const out: Record = { ...filters }; + for (const key of ["sectors", "locations"]) { + if (Array.isArray(out[key])) { + out[key] = [...(out[key] as unknown[])].sort(); + } + } + return out; +} + export const findNewLeads: Tool = { name: "leadbay_find_new_leads", annotations: { @@ -215,15 +232,18 @@ export const findNewLeads: Tool = { params.request_id ?? derivedKey( "search-auto", - // JSON, not delimiter-joined: free-text values (query, titles) must not - // be able to forge a field boundary. Fields with a documented backend - // default are canonicalized TO that default, so an approval that omits - // one and a retry that passes it explicitly derive the same key rather - // than launching a second paid, novelty-claiming job. - JSON.stringify({ + // Passed as an OBJECT: derivedKey canonicalizes recursively, so nested + // property order (example_lead, filters) can never fork the key. Fields + // with a documented backend default are canonicalized TO that default, + // so an approval that omits one and a retry that passes it explicitly + // derive the same key rather than launching a second paid, + // novelty-claiming job. + { query: params.query ?? null, example_lead: params.example_lead ?? null, - filters: normalizeSearchFilters(params.filters) ?? null, + // Sector/location lists are unordered sets to the backend — sort + // them so a reordered retry still dedupes. + filters: sortFilterLists(normalizeSearchFilters(params.filters)), count: params.count ?? null, qualify: params.qualify === true, min_ai_score: params.min_ai_score ?? 0, @@ -241,7 +261,7 @@ export const findNewLeads: Tool = { max_cost: params.max_cost ?? null, exploration_cap: params.exploration_cap ?? null, lang: params.lang ?? null, - }) + } ); const body = compactBody({ diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 5561ff2d..bf22baea 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -64,20 +64,25 @@ function derivedRequestId(params: QualifyLeadsParams): string { // forged a different ref's serialization, so {website:"acme~name=Paris"} and // {website:"acme", name:"Paris~name="} hashed identically. JSON escapes the // separators, so no value can impersonate a field boundary. - const refs = (params.lead_refs ?? []) - .map((r) => - JSON.stringify( - (["lead_id", "contact_id", "website", "name", "location"] as const).map( - (k) => r[k] ?? null + // Sorted AND de-duplicated: the backend collapses duplicate refs into one + // item, so a batch repeating a website and the same batch with it listed + // once are the same approved work. Leaving duplicates in forked the key, and + // a retry that happened to dedupe would then re-run the whole paid job. + const refs = [ + ...new Set( + (params.lead_refs ?? []).map((r) => + JSON.stringify( + (["lead_id", "contact_id", "website", "name", "location"] as const).map( + (k) => r[k] ?? null + ) ) ) - ) - .sort() - .join(","); + ), + ].sort(); // JSON the WHOLE shape for the same reason as the refs above: free-text // values (contact_titles, lang) must not be able to forge a field boundary // by containing a delimiter. - const shape = JSON.stringify({ + const shape = { refs, // The WHOLE selector, not just the job id: qualifying the first 50 of a // delivery job and then the next 50 are different batches, and collapsing @@ -101,7 +106,7 @@ function derivedRequestId(params: QualifyLeadsParams): string { // Same for the output language — re-running the batch in another language // must not return the earlier job with evidence in the previous one. lang: params.lang ?? null, - }); + }; return derivedKey("qualify-auto", shape); } diff --git a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts index bb537f58..3f701004 100644 --- a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts @@ -153,3 +153,31 @@ describe("find_new_leads — derived search key", () => { }); } }); + +describe("find_new_leads — canonical shape", () => { + it("nested property ORDER does not fork the key", async () => { + const a = await keyFor({ + example_lead: { description: "an independent gym", employees: 10 }, + }); + const b = await keyFor({ + example_lead: { employees: 10, description: "an independent gym" }, + }); + expect(b).toBe(a); + }); + + it("filter list ORDER does not fork the key", async () => { + const a = await keyFor({ filters: { locations: ["Austin", "Dallas"] } }); + const b = await keyFor({ filters: { locations: ["Dallas", "Austin"] } }); + expect(b).toBe(a); + + const c = await keyFor({ filters: { sectors: ["fitness", "retail"] } }); + const d = await keyFor({ filters: { sectors: ["retail", "fitness"] } }); + expect(d).toBe(c); + }); + + it("filter CONTENT still forks the key", async () => { + const a = await keyFor({ filters: { locations: ["Austin"] } }); + const b = await keyFor({ filters: { locations: ["Dallas"] } }); + expect(b).not.toBe(a); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index 6b597cf0..4d48b743 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -172,6 +172,17 @@ describe("qualify_leads — derived idempotency key", () => { expect(forged).not.toBe(genuine); }); + it("duplicate refs do not fork the key", async () => { + // The backend collapses duplicate refs into one item, so these are the + // same approved work. A retry that happened to dedupe would otherwise + // present a new key and re-run the whole paid batch. + const once = await submittedRequestId({ lead_refs: [{ website: "acme.com" }] }); + const twice = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }, { website: "acme.com" }], + }); + expect(twice).toBe(once); + }); + it("an explicit request_id always wins", async () => { const id = await submittedRequestId({ lead_refs: REFS, request_id: "mine-1" }); expect(id).toBe("mine-1"); From d11f8bb51d1dedf9fe590823de7c76cf5ac15854 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:40:54 -0700 Subject: [PATCH 21/46] fix(mcp): dedupe every set-valued key field; normalize refs like the resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twelfth Codex pass. Two of three counted. - Set-valued lists were sorted but not uniqued, so locations ["Dallas", "Dallas"] and ["Dallas"] request identical work yet derived different keys — a retry that deduped the list re-launched a paid, novelty-claiming job. This was the same defect fixed for lead_refs last round and not generalized; added canonicalSet() (sort + unique) in the shared helper and routed EVERY set field through it: sectors, locations, channels, contact_titles, exclude_lead_ids, refs. - Ref values are now normalized the way the resolver normalizes them, reusing the existing exported normalizeDomain(): a pasted "https://Acme.com/" and a retry's "acme.com" resolve to one company and now to one key. Names and locations are trimmed + lowercased; a non-domain-shaped website falls back to trimmed/lowercased rather than being dropped. pnpm -r build, -r test (1406 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 14 ++++++- packages/core/src/composite/find-new-leads.ts | 9 +++-- packages/core/src/composite/qualify-leads.ts | 32 +++++++++------- .../find-new-leads-idempotency.test.ts | 18 +++++++++ .../qualify-leads-idempotency.test.ts | 38 +++++++++++++++++++ 5 files changed, 93 insertions(+), 18 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 87a73796..3550601c 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -222,9 +222,21 @@ export async function waitForJob( return snap; } +/** Canonicalize a SET-valued list for hashing: sorted AND de-duplicated. + * Every list the backend treats as a set (sectors, locations, channels, + * contact_titles, exclude_lead_ids, refs) must go through this — sorting + * alone leaves `["Dallas","Dallas"]` and `["Dallas"]` hashing differently + * even though they request identical work, so a retry that happens to dedupe + * presents a new key and re-launches a paid job. */ +export function canonicalSet(values: readonly unknown[] | undefined): unknown[] { + return [...new Set((values ?? []).map((v) => JSON.stringify(v)))] + .sort() + .map((v) => JSON.parse(v)); +} + /** Recursively canonicalize a value for hashing: object keys sorted at every * depth so property ORDER never forks a key, arrays left in place (order can - * be meaningful — callers sort the ones where it is not). Plain + * be meaningful — callers pass set-shaped lists through canonicalSet). Plain * JSON.stringify is not enough: an agent that rebuilds `example_lead` or * `filters` with the properties in a different order would otherwise derive a * different key for the same approved search and re-launch a paid job. */ diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index e45200e9..1ed72ca0 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + canonicalSet, derivedKey, mockedSubmitPreview, compactBody, @@ -69,7 +70,7 @@ function sortFilterLists( const out: Record = { ...filters }; for (const key of ["sectors", "locations"]) { if (Array.isArray(out[key])) { - out[key] = [...(out[key] as unknown[])].sort(); + out[key] = canonicalSet(out[key] as unknown[]); } } return out; @@ -247,16 +248,16 @@ export const findNewLeads: Tool = { count: params.count ?? null, qualify: params.qualify === true, min_ai_score: params.min_ai_score ?? 0, - contact_titles: (params.contact_titles ?? []).slice().sort(), + contact_titles: canonicalSet(params.contact_titles), title_gate: params.title_gate ?? ((params.contact_titles?.length ?? 0) > 0 ? "prefer" : null), - channels: (params.channels ?? []).slice().sort(), + channels: canonicalSet(params.channels), // Sorted so ordering alone never forks the key, but PRESENT — a // top-up differing only by exclude_lead_ids is a different approved // search, and hashing it the same would return the first job as a // duplicate with the exclusions never applied. - exclude_lead_ids: (params.exclude_lead_ids ?? []).slice().sort(), + exclude_lead_ids: canonicalSet(params.exclude_lead_ids), novelty: params.novelty ?? "org", max_cost: params.max_cost ?? null, exploration_cap: params.exploration_cap ?? null, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index bf22baea..a320376c 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -12,6 +12,7 @@ import type { Tool, ToolContext } from "../types.js"; import { clampWaitSeconds, collectJobSnapshot, + canonicalSet, derivedKey, mockedSubmitPreview, compactBody, @@ -21,6 +22,7 @@ import { type McpDryRunResponse, type McpSubmitResponse, } from "./_mcp-job-helpers.js"; +import { normalizeDomain } from "./import-leads.js"; import { leadbay_qualify_leads as QUALIFY_LEADS_DESCRIPTION } from "../tool-descriptions.generated.js"; interface QualifyLeadsParams { @@ -68,17 +70,21 @@ function derivedRequestId(params: QualifyLeadsParams): string { // item, so a batch repeating a website and the same batch with it listed // once are the same approved work. Leaving duplicates in forked the key, and // a retry that happened to dedupe would then re-run the whole paid job. - const refs = [ - ...new Set( - (params.lead_refs ?? []).map((r) => - JSON.stringify( - (["lead_id", "contact_id", "website", "name", "location"] as const).map( - (k) => r[k] ?? null - ) - ) - ) - ), - ].sort(); + const refs = canonicalSet( + (params.lead_refs ?? []).map((r) => [ + r.lead_id?.trim() ?? null, + r.contact_id?.trim() ?? null, + // Normalize the website the SAME way the resolver does, so a pasted + // "https://Acme.com/" and a retry's "acme.com" resolve to one company + // AND to one key. Fall back to the trimmed/lowercased raw value when it + // is not domain-shaped, rather than dropping the field. + r.website + ? normalizeDomain(r.website) ?? r.website.trim().toLowerCase() + : null, + r.name?.trim().toLowerCase() ?? null, + r.location?.trim().toLowerCase() ?? null, + ]) + ); // JSON the WHOLE shape for the same reason as the refs above: free-text // values (contact_titles, lang) must not be able to forge a field boundary // by containing a delimiter. @@ -97,8 +103,8 @@ function derivedRequestId(params: QualifyLeadsParams): string { // omits a field and a retry that passes that field's documented default // derive the same key instead of launching a second paid job. qualify: params.qualify !== false, - channels: (params.channels ?? []).slice().sort(), - contact_titles: (params.contact_titles ?? []).slice().sort(), + channels: canonicalSet(params.channels), + contact_titles: canonicalSet(params.contact_titles), title_gate: params.title_gate ?? null, // The cap is part of the approval: raising it after a stop_reason:max_cost // is a NEW approved run, and must not dedupe onto the capped job. diff --git a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts index 3f701004..91c1da62 100644 --- a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts @@ -181,3 +181,21 @@ describe("find_new_leads — canonical shape", () => { expect(b).not.toBe(a); }); }); + +describe("find_new_leads — set-valued fields are deduped", () => { + it("duplicate filter values do not fork the key", async () => { + const once = await keyFor({ filters: { locations: ["Dallas"] } }); + const twice = await keyFor({ filters: { locations: ["Dallas", "Dallas"] } }); + expect(twice).toBe(once); + }); + + it("duplicate exclusions and channels do not fork the key", async () => { + const a = await keyFor({ exclude_lead_ids: ["lead-1"] }); + const b = await keyFor({ exclude_lead_ids: ["lead-1", "lead-1"] }); + expect(b).toBe(a); + + const c = await keyFor({ channels: ["email"] }); + const d = await keyFor({ channels: ["email", "email"] }); + expect(d).toBe(c); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index 4d48b743..4fe3a795 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -188,3 +188,41 @@ describe("qualify_leads — derived idempotency key", () => { expect(id).toBe("mine-1"); }); }); + +describe("qualify_leads — ref + set normalization", () => { + it("a pasted URL and its normalized domain share a key", async () => { + const pasted = await submittedRequestId({ + lead_refs: [{ website: "https://Acme.com/" }], + }); + const clean = await submittedRequestId({ lead_refs: [{ website: "acme.com" }] }); + expect(clean).toBe(pasted); + }); + + it("stray whitespace and casing do not fork the key", async () => { + const messy = await submittedRequestId({ + lead_refs: [{ name: " Franklin Barbecue ", location: " Austin " }], + }); + const tidy = await submittedRequestId({ + lead_refs: [{ name: "franklin barbecue", location: "austin" }], + }); + expect(tidy).toBe(messy); + }); + + it("a genuinely different domain still forks the key", async () => { + const a = await submittedRequestId({ lead_refs: [{ website: "acme.com" }] }); + const b = await submittedRequestId({ lead_refs: [{ website: "other.com" }] }); + expect(b).not.toBe(a); + }); + + it("duplicate channels do not fork the key", async () => { + const once = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + channels: ["email"], + }); + const twice = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + channels: ["email", "email"], + }); + expect(twice).toBe(once); + }); +}); From d298e8742d0e32beed07afb317dce56c5423d3a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:54:06 -0700 Subject: [PATCH 22/46] docs(mcp): real headroom for find_new_leads (78 -> 296 chars) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twelfth-pass P1 on the description budget. The threshold in the finding is still wrong — 17000 is the only enforced cap, there is no ~16k constant, and at 16922 this description was the FOURTH tightest, behind three pre-existing tools this PR never touched. But the fragility point is fair: the cap has broken this build twice, and shaving a few characters each time was not fixing it. Trimmed the shared example-lead-craft snippet instead of the prose that was squeezed last time: rules 7 and 8 duplicated guidance now carried by the prompt and the inputSchema, and rules 1-2 restated their evidence at length. All 8 rules and their measured proof points survive. find_new_leads: 16922 -> 16704 (296 headroom), now clear of prepare_outreach (7), research_lead_by_id (27) and pull_followups (52). The snippet is shared, so qualify_leads and lead_job_status gain the same slack. pnpm -r build, -r test (1406 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/tool-descriptions.generated.ts | 33 +++++++++---------- .../snippets/heuristics/example-lead-craft.md | 33 +++++++++---------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 0de62302..5783f621 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1337,15 +1337,13 @@ and matched against millions of real registry/website company descriptions, which state what a company **IS** — never what is happening. Write the seed the same way or the matcher drifts. Every rule below is measured: -1. **Describe the BUYER, never the seller.** Before writing, answer: "would - this company write a check to my user?" A seed that describes what the user - sells surfaces the user's *competitors and vendors*, not their customers. - Classic trap: if the user's product helps companies of type X serve their - customers of type Y, the seed describes X — never Y. -2. **Put everything in \`description\`; leave \`name\` unset.** A distinctive - invented brand name pulls matching toward name-lookalikes: a seed named - "Meridian Analytics" returned five unrelated companies all named - "Meridian". No name beats any name. +1. **Describe the BUYER, never the seller.** Ask: "would this company write a + check to my user?" A seed describing what the user SELLS surfaces their + *competitors and vendors*. Classic trap: if the product helps companies of + type X serve customers of type Y, the seed describes X — never Y. +2. **Put everything in \`description\`; leave \`name\` unset.** An invented brand + name pulls matching toward name-lookalikes — a seed named "Meridian + Analytics" returned five unrelated "Meridian" companies. 3. **Registry style, one sentence to ~250 chars.** State the business profile: industry niche, business model, what they sell or operate, who they serve, observable scale (sites, membership, fleet). Write it like the first @@ -1370,15 +1368,14 @@ the same way or the matcher drifts. Every rule below is measured: omit \`locations\` (a country name silently matches a same-named town: measured, "France" → the village of Francs). \`example_lead.employees\` does not filter; only \`filters.employees_min/max\` do. -7. **Prefer \`example_lead\` over \`query\`.** Query text matches topic - *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS - as strongly as gym BUYERS (measured: 0 delivered from the query, on-profile - from the example). \`query\` only when the user's wording carries signal an - example can't express. -8. **One seed per buyer archetype.** If the ask spans two distinct segments - (e.g. "gyms and logistics warehouses"), run one search per segment with its - own description — a blended seed lands between the two clusters and matches - neither. Distinct asks need distinct \`request_id\`s. +7. **Prefer \`example_lead\` over \`query\`.** Query matches topic *vocabulary* — + "gyms that need durable flooring" surfaces flooring VENDORS as strongly as + gym BUYERS (measured: 0 delivered vs on-profile from the example). Use + \`query\` only for signal an example can't express. +8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and + warehouses") needs one search each with its own description and + \`request_id\` — a blended seed lands between the clusters and matches + neither. **Parameter notes** diff --git a/packages/promptforge/snippets/heuristics/example-lead-craft.md b/packages/promptforge/snippets/heuristics/example-lead-craft.md index 1d38ba7c..0de6a66d 100644 --- a/packages/promptforge/snippets/heuristics/example-lead-craft.md +++ b/packages/promptforge/snippets/heuristics/example-lead-craft.md @@ -5,15 +5,13 @@ and matched against millions of real registry/website company descriptions, which state what a company **IS** — never what is happening. Write the seed the same way or the matcher drifts. Every rule below is measured: -1. **Describe the BUYER, never the seller.** Before writing, answer: "would - this company write a check to my user?" A seed that describes what the user - sells surfaces the user's *competitors and vendors*, not their customers. - Classic trap: if the user's product helps companies of type X serve their - customers of type Y, the seed describes X — never Y. -2. **Put everything in `description`; leave `name` unset.** A distinctive - invented brand name pulls matching toward name-lookalikes: a seed named - "Meridian Analytics" returned five unrelated companies all named - "Meridian". No name beats any name. +1. **Describe the BUYER, never the seller.** Ask: "would this company write a + check to my user?" A seed describing what the user SELLS surfaces their + *competitors and vendors*. Classic trap: if the product helps companies of + type X serve customers of type Y, the seed describes X — never Y. +2. **Put everything in `description`; leave `name` unset.** An invented brand + name pulls matching toward name-lookalikes — a seed named "Meridian + Analytics" returned five unrelated "Meridian" companies. 3. **Registry style, one sentence to ~250 chars.** State the business profile: industry niche, business model, what they sell or operate, who they serve, observable scale (sites, membership, fleet). Write it like the first @@ -38,12 +36,11 @@ the same way or the matcher drifts. Every rule below is measured: omit `locations` (a country name silently matches a same-named town: measured, "France" → the village of Francs). `example_lead.employees` does not filter; only `filters.employees_min/max` do. -7. **Prefer `example_lead` over `query`.** Query text matches topic - *vocabulary* — "gyms that need durable flooring" surfaces flooring VENDORS - as strongly as gym BUYERS (measured: 0 delivered from the query, on-profile - from the example). `query` only when the user's wording carries signal an - example can't express. -8. **One seed per buyer archetype.** If the ask spans two distinct segments - (e.g. "gyms and logistics warehouses"), run one search per segment with its - own description — a blended seed lands between the two clusters and matches - neither. Distinct asks need distinct `request_id`s. +7. **Prefer `example_lead` over `query`.** Query matches topic *vocabulary* — + "gyms that need durable flooring" surfaces flooring VENDORS as strongly as + gym BUYERS (measured: 0 delivered vs on-profile from the example). Use + `query` only for signal an example can't express. +8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and + warehouses") needs one search each with its own description and + `request_id` — a blended seed lands between the clusters and matches + neither. From 2accf1a5ec1cb4de3212a8066eef9d7b60a8f624 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:44:31 -0700 Subject: [PATCH 23/46] fix(mcp): normalize labels + empty shapes in keys; delete stale gated skills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thirteenth Codex pass. All three real. - contact_titles were hashed raw, so a retry rebuilding "Owner" as "owner " derived a new key and could re-run a paid batch even though the backend matches titles semantically. Added canonicalLabelSet() (trim + lowercase before dedupe) and routed contact_titles and filters.sectors/locations through it. - Three shapes that all mean "no filters" hashed three different ways: omitted, {}, and {locations: []}. canonicalOptionalObject() collapses empty objects and empty set fields to null, so an unfiltered retry dedupes. - promptforge now DELETES a stale gated SKILL.md instead of leaving it on disk. writeIfDifferent only writes, so a prompt marked release_gated after its skill had already been emitted left a static, auto-triggering file to be committed and shipped — I hit exactly this and removed it by hand last time. `prompts:check` now fails on an orphaned gated skill so CI catches it. Verified the full loop: check fails loudly on a planted stale file, build removes it, check then passes. pnpm -r build, -r test (1411 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 29 ++++++++++++ packages/core/src/composite/find-new-leads.ts | 17 ++++--- packages/core/src/composite/qualify-leads.ts | 3 +- .../find-new-leads-idempotency.test.ts | 20 +++++++++ .../qualify-leads-idempotency.test.ts | 26 +++++++++++ packages/promptforge/src/cli.ts | 45 ++++++++++++++++--- 6 files changed, 127 insertions(+), 13 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 3550601c..206bb809 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -234,6 +234,35 @@ export function canonicalSet(values: readonly unknown[] | undefined): unknown[] .map((v) => JSON.parse(v)); } +/** Canonicalize a set of free-text labels (contact titles, sectors): trimmed + * and lower-cased before dedupe, because the backend matches them + * semantically. `["Owner"]` and `["owner "]` request identical work, so a + * retry that re-cased them must not derive a new key and re-spend. */ +export function canonicalLabelSet( + values: readonly string[] | undefined +): string[] { + return canonicalSet( + (values ?? []).map((v) => v.trim().toLowerCase()).filter(Boolean) + ) as string[]; +} + +/** Collapse the several shapes that all mean "nothing specified" to one. + * An omitted `filters`, `{}`, and `{locations: []}` request the same search, + * so they must hash identically or a retry that materializes an empty object + * launches a second paid job. */ +export function canonicalOptionalObject( + value: Record | null | undefined +): Record | null { + if (!value) return null; + const out: Record = {}; + for (const [k, v] of Object.entries(value)) { + if (v === undefined || v === null) continue; + if (Array.isArray(v) && v.length === 0) continue; + out[k] = v; + } + return Object.keys(out).length === 0 ? null : out; +} + /** Recursively canonicalize a value for hashing: object keys sorted at every * depth so property ORDER never forks a key, arrays left in place (order can * be meaningful — callers pass set-shaped lists through canonicalSet). Plain diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 1ed72ca0..c0ac9911 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -13,6 +13,8 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + canonicalLabelSet, + canonicalOptionalObject, derivedKey, mockedSubmitPreview, compactBody, @@ -59,10 +61,11 @@ interface FindNewLeadsParams { const DEFAULT_WAIT_SECONDS = 45; -/** Sort the filter arrays that are unordered SETS to the backend, so a retry - * listing the same sectors/locations in another order still derives the same - * idempotency key. Object key order is handled by canonicalize() inside - * derivedKey; only array order needs a decision, and only here. */ +/** Canonicalize the filter lists that are unordered SETS of free-text labels + * to the backend, so a retry listing the same sectors/locations in another + * order — or with different casing — still derives the same idempotency key. + * Object key order is handled by canonicalize() inside derivedKey; empty + * shapes are collapsed by canonicalOptionalObject at the call site. */ function sortFilterLists( filters: Record | undefined ): Record | null { @@ -70,7 +73,7 @@ function sortFilterLists( const out: Record = { ...filters }; for (const key of ["sectors", "locations"]) { if (Array.isArray(out[key])) { - out[key] = canonicalSet(out[key] as unknown[]); + out[key] = canonicalLabelSet(out[key] as string[]); } } return out; @@ -244,11 +247,11 @@ export const findNewLeads: Tool = { example_lead: params.example_lead ?? null, // Sector/location lists are unordered sets to the backend — sort // them so a reordered retry still dedupes. - filters: sortFilterLists(normalizeSearchFilters(params.filters)), + filters: canonicalOptionalObject(sortFilterLists(normalizeSearchFilters(params.filters))), count: params.count ?? null, qualify: params.qualify === true, min_ai_score: params.min_ai_score ?? 0, - contact_titles: canonicalSet(params.contact_titles), + contact_titles: canonicalLabelSet(params.contact_titles), title_gate: params.title_gate ?? ((params.contact_titles?.length ?? 0) > 0 ? "prefer" : null), diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index a320376c..e827c1fb 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + canonicalLabelSet, derivedKey, mockedSubmitPreview, compactBody, @@ -104,7 +105,7 @@ function derivedRequestId(params: QualifyLeadsParams): string { // derive the same key instead of launching a second paid job. qualify: params.qualify !== false, channels: canonicalSet(params.channels), - contact_titles: canonicalSet(params.contact_titles), + contact_titles: canonicalLabelSet(params.contact_titles), title_gate: params.title_gate ?? null, // The cap is part of the approval: raising it after a stop_reason:max_cost // is a NEW approved run, and must not dedupe onto the capped job. diff --git a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts index 91c1da62..3e369622 100644 --- a/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/find-new-leads-idempotency.test.ts @@ -199,3 +199,23 @@ describe("find_new_leads — set-valued fields are deduped", () => { expect(d).toBe(c); }); }); + +describe("find_new_leads — empty shapes collapse", () => { + it("omitted filters, {} and empty arrays all share a key", async () => { + const omitted = await keyFor({}); + expect(await keyFor({ filters: {} })).toBe(omitted); + expect(await keyFor({ filters: { locations: [] } })).toBe(omitted); + expect(await keyFor({ filters: { sectors: [], locations: [] } })).toBe(omitted); + }); + + it("sector/location casing does not fork the key", async () => { + const a = await keyFor({ filters: { locations: ["Dallas"] } }); + const b = await keyFor({ filters: { locations: ["dallas "] } }); + expect(b).toBe(a); + }); + + it("a non-empty filter still forks the key", async () => { + const omitted = await keyFor({}); + expect(await keyFor({ filters: { locations: ["Dallas"] } })).not.toBe(omitted); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index 4fe3a795..9295094c 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -226,3 +226,29 @@ describe("qualify_leads — ref + set normalization", () => { expect(twice).toBe(once); }); }); + +describe("qualify_leads — label normalization", () => { + it("contact_titles casing and whitespace do not fork the key", async () => { + const a = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + contact_titles: ["Owner"], + }); + const b = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + contact_titles: ["owner "], + }); + expect(b).toBe(a); + }); + + it("a genuinely different title still forks the key", async () => { + const owner = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + contact_titles: ["Owner"], + }); + const cto = await submittedRequestId({ + lead_refs: [{ website: "acme.com" }], + contact_titles: ["CTO"], + }); + expect(cto).not.toBe(owner); + }); +}); diff --git a/packages/promptforge/src/cli.ts b/packages/promptforge/src/cli.ts index 7bbddebb..ec450b8c 100644 --- a/packages/promptforge/src/cli.ts +++ b/packages/promptforge/src/cli.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { assemble, type AssembleResult } from "./assembler.js"; import { emit, emitServerInstructions, diff, writeIfDifferent } from "./emit.js"; import { buildSkillFiles, type SkillFile } from "./skills.js"; @@ -51,6 +51,11 @@ interface AssembleAndEmitOutput { promptsModule: string; toolDescriptionsModule: string; skillFiles: SkillFile[]; + // Prompts marked release_gated emit NO SKILL.md. Their paths are returned so + // the build can DELETE a previously-emitted skill: writeIfDifferent only + // writes, so a prompt gated after the fact would otherwise leave a static, + // auto-triggering skill on disk to be committed and shipped. + gatedSkillPaths: string[]; result: AssembleResult; } @@ -59,11 +64,15 @@ function runAssemble(): AssembleAndEmitOutput { const result = assemble({ root: PKG_ROOT, registeredToolNames: registered }); const { promptsModule, toolDescriptionsModule } = emit(result); const skillFiles = buildSkillFiles(result.prompts); - return { promptsModule, toolDescriptionsModule, skillFiles, result }; + const gatedSkillPaths = result.prompts + .filter((p) => p.frontmatter.release_gated === true) + .map((p) => join(SKILLS_OUT_DIR, p.frontmatter.name, "SKILL.md")); + return { promptsModule, toolDescriptionsModule, skillFiles, gatedSkillPaths, result }; } function cmdBuild(): void { - const { promptsModule, toolDescriptionsModule, skillFiles } = runAssemble(); + const { promptsModule, toolDescriptionsModule, skillFiles, gatedSkillPaths } = + runAssemble(); const serverInstructionsModule = emitServerInstructions(SERVER_INSTRUCTIONS_SNIPPETS); const r1 = writeIfDifferent(PROMPTS_OUT, promptsModule); const r2 = writeIfDifferent(TOOL_DESC_OUT, toolDescriptionsModule); @@ -78,10 +87,22 @@ function cmdBuild(): void { `[forge] ${fullPath.replace(REPO_ROOT + "/", "")}: ${r.changed ? "wrote" : "unchanged"}`, ); } + // A skill is a static file that auto-triggers with no runtime gate, so a + // prompt marked release_gated must not leave one behind. writeIfDifferent + // only ever writes, so deletion has to be explicit. + for (const path of gatedSkillPaths) { + if (existsSync(path)) { + rmSync(dirname(path), { recursive: true, force: true }); + console.log( + `[forge] ${path.replace(REPO_ROOT + "/", "")}: removed (release_gated)`, + ); + } + } } function cmdCheck(): void { - const { promptsModule, toolDescriptionsModule, skillFiles } = runAssemble(); + const { promptsModule, toolDescriptionsModule, skillFiles, gatedSkillPaths } = + runAssemble(); const serverInstructionsModule = emitServerInstructions(SERVER_INSTRUCTIONS_SNIPPETS); const d1 = diff(PROMPTS_OUT, promptsModule); const d2 = diff(TOOL_DESC_OUT, toolDescriptionsModule); @@ -91,13 +112,27 @@ function cmdCheck(): void { const fullPath = join(SKILLS_OUT_DIR, skill.relativePath); if (!diff(fullPath, skill.content).matches) staleSkills.push(fullPath); } - if (!d1.matches || !d2.matches || !d3.matches || staleSkills.length > 0) { + // A gated prompt must ship NO skill — a leftover file would auto-trigger a + // workflow whose tools are hidden, so fail rather than let it be committed. + const orphanedGated = gatedSkillPaths.filter((p) => existsSync(p)); + if ( + !d1.matches || + !d2.matches || + !d3.matches || + staleSkills.length > 0 || + orphanedGated.length > 0 + ) { if (!d1.matches) console.error(`[forge] ${PROMPTS_OUT} is stale. Run pnpm prompts:build.`); if (!d2.matches) console.error(`[forge] ${TOOL_DESC_OUT} is stale. Run pnpm prompts:build.`); if (!d3.matches) console.error(`[forge] ${SERVER_INSTRUCTIONS_OUT} is stale. Run pnpm prompts:build.`); for (const path of staleSkills) { console.error(`[forge] ${path} is stale. Run pnpm prompts:build.`); } + for (const path of orphanedGated) { + console.error( + `[forge] ${path} belongs to a release_gated prompt and must not ship. Run pnpm prompts:build.`, + ); + } process.exit(1); } console.log("[forge] generated files are up-to-date."); From e98b791dd44c1991f837ed78381a14356ec1f613 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:03:47 -0700 Subject: [PATCH 24/46] fix(mcp): scalar filters, UUID casing, top-up exclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourteenth Codex pass. Three of four real. - normalizeSearchFilters wraps a scalar sectors/locations in a one-item array. I had made rejectCountryLocations tolerant of a bare string last round but not the SUBMISSION, so a scalar survived to the wire and 400d on backend deserialization. Verified: {locations:"Dallas"} now POSTs ["Dallas"]. - lead_id / contact_id UUIDs are lower-cased before hashing. UUIDs are case-insensitive to the backend, so an uppercase id and its lowercase form are the same lead and must share an idempotency key. Non-UUID ids keep their casing, since we cannot assume the backend folds those. - Prompt: a paid top-up now carries exclude_lead_ids for every lead already SEEN — delivered, disqualified and skipped, across both passes. novelty:org excludes prior DELIVERIES only, so an examined-and-rejected candidate stays eligible and the top-up would re-pick the same misses and charge again to close no gap. Declined the fourth (restrict the auto request_id to paid calls): the free path would then submit with NO key at all, which is worse — request_id is schema-required, and a timed-out free search retried would launch a fresh novelty-claiming job rather than dedupe. Reasoning posted on the thread. pnpm -r build, -r test (1416 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 10 ++++++++++ packages/core/src/composite/qualify-leads.ts | 19 ++++++++++++++++-- .../composite/mcp-job-helpers-review.test.ts | 20 +++++++++++++++++++ .../qualify-leads-idempotency.test.ts | 18 +++++++++++++++++ packages/mcp/src/prompts.generated.ts | 6 ++++++ .../prompts/leadbay_new_leads.md.tmpl | 6 ++++++ 6 files changed, 77 insertions(+), 2 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 206bb809..e8bac15f 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -429,6 +429,16 @@ export function normalizeSearchFilters( } if (out.employees_min == null) delete out.employees_min; if (out.employees_max == null) delete out.employees_max; + // The server does not validate inputSchema before dispatch, so an agent can + // send `locations: "Dallas"` where the backend expects an array. Wrap a bare + // string in a one-item list — the same tolerance rejectCountryLocations + // already applies — instead of forwarding a scalar that 400s on + // deserialization. + for (const key of ["sectors", "locations"]) { + const v = out[key]; + if (typeof v === "string") out[key] = v.trim() ? [v] : undefined; + if (out[key] === undefined) delete out[key]; + } return out; } diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index e827c1fb..eba027ad 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -53,6 +53,19 @@ interface QualifyLeadsParams { const DEFAULT_WAIT_SECONDS = 45; +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** Lower-case a UUID-shaped id so casing alone never forks an idempotency + * key — the backend resolves `A1B2…` and `a1b2…` to the same record. A value + * that is not UUID-shaped is only trimmed, since we cannot assume the + * backend treats it case-insensitively. */ +function normalizeUuid(value: string | undefined): string | null { + const v = value?.trim(); + if (!v) return null; + return UUID_RE.test(v) ? v.toLowerCase() : v; +} + /** Stable idempotency key for a paid batch the caller didn't key itself. * Deterministic over the APPROVED BATCH ITSELF — refs, selector, paid flags, * spend cap — and nothing time-based: a retry of the same approval must @@ -73,8 +86,10 @@ function derivedRequestId(params: QualifyLeadsParams): string { // a retry that happened to dedupe would then re-run the whole paid job. const refs = canonicalSet( (params.lead_refs ?? []).map((r) => [ - r.lead_id?.trim() ?? null, - r.contact_id?.trim() ?? null, + // UUIDs are case-insensitive to the backend, so an uppercase id and its + // lowercase form are the same lead and must share a key. + normalizeUuid(r.lead_id), + normalizeUuid(r.contact_id), // Normalize the website the SAME way the resolver does, so a pasted // "https://Acme.com/" and a retry's "acme.com" resolve to one company // AND to one key. Fall back to the trimmed/lowercased raw value when it diff --git a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts index bf33de31..0e165fd1 100644 --- a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts +++ b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts @@ -23,6 +23,7 @@ import { collectJobSnapshot, rejectCountryLocations, mockedSubmitPreview, + normalizeSearchFilters, } from "../../../src/composite/_mcp-job-helpers.js"; const BASE = "https://api-us.leadbay.app"; @@ -222,3 +223,22 @@ describe("mockedSubmitPreview — only in mock mode", () => { } }); }); + +describe("normalizeSearchFilters — scalar tolerance", () => { + // The server does not validate inputSchema before dispatch, so a bare string + // reaches the tool and would otherwise be POSTed where an array is expected. + it("wraps a scalar sector/location in a one-item array", () => { + const out = normalizeSearchFilters({ locations: "Dallas", sectors: "fitness" })!; + expect(out.locations).toEqual(["Dallas"]); + expect(out.sectors).toEqual(["fitness"]); + }); + + it("leaves arrays untouched", () => { + const out = normalizeSearchFilters({ locations: ["Dallas", "Austin"] })!; + expect(out.locations).toEqual(["Dallas", "Austin"]); + }); + + it("drops a blank scalar rather than sending an empty string", () => { + expect(normalizeSearchFilters({ locations: " " })!.locations).toBeUndefined(); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index 9295094c..d15d8545 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -252,3 +252,21 @@ describe("qualify_leads — label normalization", () => { expect(cto).not.toBe(owner); }); }); + +describe("qualify_leads — UUID casing", () => { + it("an uppercase lead_id shares a key with its lowercase form", async () => { + const upper = await submittedRequestId({ + lead_refs: [{ lead_id: "AAAA1111-2222-3333-4444-555566667777" }], + }); + const lower = await submittedRequestId({ + lead_refs: [{ lead_id: "aaaa1111-2222-3333-4444-555566667777" }], + }); + expect(lower).toBe(upper); + }); + + it("a non-UUID id keeps its casing (backend may be case-sensitive)", async () => { + const a = await submittedRequestId({ lead_refs: [{ lead_id: "Ref-ABC" }] }); + const b = await submittedRequestId({ lead_refs: [{ lead_id: "ref-abc" }] }); + expect(b).not.toBe(a); + }); +}); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 9285764f..7722b853 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -980,6 +980,12 @@ When I want qualification evidence and/or reachable contacts: tell me the gap in one line and offer to top it up — another shortfall-sized search under a NEW \`request_id\`, quoted first like any paid run. Never silently hand back fewer than I asked for and paid toward. + + Pass every lead already SEEN into that top-up's \`exclude_lead_ids\` — + delivered, disqualified and skipped alike, from both the preview and the + paid pass. \`novelty: org\` only excludes prior DELIVERIES, so a candidate + that was examined and rejected is still eligible; without the exclusion + list the top-up re-picks the same misses and charges again to close no gap. 3. While the job runs, poll with \`leadbay_lead_job_status\` (\`wait_seconds: 60\`); report progress, not silence. diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl index 71a601a0..b9c26aba 100644 --- a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/leadbay_new_leads.md.tmpl @@ -154,6 +154,12 @@ When I want qualification evidence and/or reachable contacts: tell me the gap in one line and offer to top it up — another shortfall-sized search under a NEW `request_id`, quoted first like any paid run. Never silently hand back fewer than I asked for and paid toward. + + Pass every lead already SEEN into that top-up's `exclude_lead_ids` — + delivered, disqualified and skipped alike, from both the preview and the + paid pass. `novelty: org` only excludes prior DELIVERIES, so a candidate + that was examined and rejected is still eligible; without the exclusion + list the top-up re-picks the same misses and charges again to close no gap. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. From 447c40b718bcb92b47aaadec5cb0e451b11d5099 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:22:35 -0700 Subject: [PATCH 25/46] fix(mcp): scalar-safe set canonicalizers; fold UUIDs in exclude_lead_ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifteenth Codex pass. Two of three real. - canonicalSet / canonicalLabelSet threw a TypeError on a scalar. Since the server does not validate inputSchema before dispatch, `channels: "email"` reached the key derivation and CRASHED the call before the user ever saw a quote. All three set canonicalizers now wrap a bare scalar in a one-item list, matching what normalizeSearchFilters does for sectors/locations. - exclude_lead_ids are UUID-folded via a new canonicalIdSet(), so a top-up that lowercases ids copied from a prior result dedupes instead of launching another paid search. normalizeUuid moved into the shared helper and is now used by both tools rather than duplicated. Declined the paging change (drain whenever next_since is set). Checked it against the fixtures and the tool contract: the backend returns next_since on EVERY snapshot as a resumption handle, including a completed job with a short page — that is the normal terminal shape, not evidence of unfetched items. Following the cursor unconditionally adds a wasted round-trip to every terminal poll. I tried it, it broke 10 existing tests for exactly that reason, and I reverted it. A full page remains the drain signal. pnpm -r build, -r test (1418 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 61 +++++++++++++++++-- packages/core/src/composite/find-new-leads.ts | 3 +- packages/core/src/composite/qualify-leads.ts | 14 +---- .../composite/mcp-job-helpers-review.test.ts | 23 +++++++ 4 files changed, 81 insertions(+), 20 deletions(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index e8bac15f..b738dcfc 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -160,6 +160,11 @@ export async function collectJobSnapshot( // to null there would make the next poll a full re-read. let cursor = page.next_since ?? since ?? null; let pages = 1; + // A FULL page is the drain signal, not the cursor alone. `next_since` is a + // resumption handle the backend returns on every snapshot — including a + // completed job with a short page — so following it whenever it is set adds + // a wasted round-trip to every terminal poll. A short page means the cursor + // is caught up; the caller keeps next_since for the next incremental poll. while (page.items.length >= pageLimit && page.next_since && pages < maxPages) { const next = await client.request( "GET", @@ -228,21 +233,65 @@ export async function waitForJob( * alone leaves `["Dallas","Dallas"]` and `["Dallas"]` hashing differently * even though they request identical work, so a retry that happens to dedupe * presents a new key and re-launches a paid job. */ -export function canonicalSet(values: readonly unknown[] | undefined): unknown[] { - return [...new Set((values ?? []).map((v) => JSON.stringify(v)))] +export function canonicalSet(values: unknown): unknown[] { + // The server does not validate inputSchema before dispatch, so a set-shaped + // field can arrive as a bare scalar (`channels: "email"`). Wrap it instead + // of calling .map on a string — that threw a TypeError while DERIVING the + // key, i.e. before the caller even saw a quote. + const list = + values === undefined || values === null + ? [] + : Array.isArray(values) + ? values + : [values]; + return [...new Set(list.map((v) => JSON.stringify(v)))] .sort() .map((v) => JSON.parse(v)); } +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** Lower-case a UUID-shaped id so casing alone never forks an idempotency + * key — the backend resolves `A1B2…` and `a1b2…` to the same record. A value + * that is not UUID-shaped is only trimmed, since we cannot assume the backend + * folds case for arbitrary identifiers. */ +export function normalizeUuid(value: unknown): string | null { + if (typeof value !== "string") return null; + const v = value.trim(); + if (!v) return null; + return UUID_RE.test(v) ? v.toLowerCase() : v; +} + +/** Canonicalize a set of ids: UUID-folded, then sorted + deduped. */ +export function canonicalIdSet(values: unknown): string[] { + const list = + values === undefined || values === null + ? [] + : Array.isArray(values) + ? values + : [values]; + return canonicalSet( + list.map(normalizeUuid).filter((v): v is string => !!v) + ) as string[]; +} + /** Canonicalize a set of free-text labels (contact titles, sectors): trimmed * and lower-cased before dedupe, because the backend matches them * semantically. `["Owner"]` and `["owner "]` request identical work, so a * retry that re-cased them must not derive a new key and re-spend. */ -export function canonicalLabelSet( - values: readonly string[] | undefined -): string[] { +export function canonicalLabelSet(values: unknown): string[] { + const list = + values === undefined || values === null + ? [] + : Array.isArray(values) + ? values + : [values]; return canonicalSet( - (values ?? []).map((v) => v.trim().toLowerCase()).filter(Boolean) + list + .filter((v): v is string => typeof v === "string") + .map((v) => v.trim().toLowerCase()) + .filter(Boolean) ) as string[]; } diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index c0ac9911..6b2c1d92 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + canonicalIdSet, canonicalLabelSet, canonicalOptionalObject, derivedKey, @@ -260,7 +261,7 @@ export const findNewLeads: Tool = { // top-up differing only by exclude_lead_ids is a different approved // search, and hashing it the same would return the first job as a // duplicate with the exclusions never applied. - exclude_lead_ids: canonicalSet(params.exclude_lead_ids), + exclude_lead_ids: canonicalIdSet(params.exclude_lead_ids), novelty: params.novelty ?? "org", max_cost: params.max_cost ?? null, exploration_cap: params.exploration_cap ?? null, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index eba027ad..f1cf5e8e 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + normalizeUuid, canonicalLabelSet, derivedKey, mockedSubmitPreview, @@ -53,19 +54,6 @@ interface QualifyLeadsParams { const DEFAULT_WAIT_SECONDS = 45; -const UUID_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -/** Lower-case a UUID-shaped id so casing alone never forks an idempotency - * key — the backend resolves `A1B2…` and `a1b2…` to the same record. A value - * that is not UUID-shaped is only trimmed, since we cannot assume the - * backend treats it case-insensitively. */ -function normalizeUuid(value: string | undefined): string | null { - const v = value?.trim(); - if (!v) return null; - return UUID_RE.test(v) ? v.toLowerCase() : v; -} - /** Stable idempotency key for a paid batch the caller didn't key itself. * Deterministic over the APPROVED BATCH ITSELF — refs, selector, paid flags, * spend cap — and nothing time-based: a retry of the same approval must diff --git a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts index 0e165fd1..c818dc7f 100644 --- a/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts +++ b/packages/core/test/unit/composite/mcp-job-helpers-review.test.ts @@ -24,6 +24,10 @@ import { rejectCountryLocations, mockedSubmitPreview, normalizeSearchFilters, + canonicalSet, + canonicalLabelSet, + canonicalIdSet, + normalizeUuid, } from "../../../src/composite/_mcp-job-helpers.js"; const BASE = "https://api-us.leadbay.app"; @@ -242,3 +246,22 @@ describe("normalizeSearchFilters — scalar tolerance", () => { expect(normalizeSearchFilters({ locations: " " })!.locations).toBeUndefined(); }); }); + +describe("set canonicalizers — scalar tolerance", () => { + it("does not throw on a scalar set field", () => { + // A bare `channels: "email"` used to TypeError while DERIVING the key, + // i.e. before the caller ever saw a quote. + expect(canonicalSet("email")).toEqual(["email"]); + expect(canonicalLabelSet("Owner ")).toEqual(["owner"]); + expect(canonicalIdSet("AAAA1111-2222-3333-4444-555566667777")).toEqual([ + "aaaa1111-2222-3333-4444-555566667777", + ]); + }); + + it("folds UUID casing but leaves other ids alone", () => { + expect(canonicalIdSet(["AAAA1111-2222-3333-4444-555566667777"])).toEqual( + canonicalIdSet(["aaaa1111-2222-3333-4444-555566667777"]) + ); + expect(normalizeUuid("Ref-ABC")).toBe("Ref-ABC"); + }); +}); From 439a9c31e446659a960e826e5d55a5c0825c3e09 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:46:42 -0700 Subject: [PATCH 26/46] fix(mcp): coerce unvalidated arg shapes once at the entry point Sixteenth Codex pass. All three real, and all three the SAME root cause: the MCP server does not validate inputSchema before dispatch, so any array field can arrive singular. I had been hardening one consumer at a time (key derivation, then filters, then the set canonicalizers) and each round found the next unpatched site. Fixed the class instead. - coerceArrayParams() normalizes the array-typed params ONCE, at the top of execute(), before the spend gate or anything else touches them. A scalar `channels: "email"`, `contact_titles: "Owner"` or a single `lead_refs` object now flows through normally and reaches the wire as an array. Verified: all three previously threw a TypeError BEFORE the gate, so the caller got a crash instead of the promised quote; all three now return needs_confirmation with the correct wire shape. - prior_deliveries.job_id is UUID-folded like the refs, so retrying the same selector with different casing dedupes rather than re-running a paid batch. pnpm -r build, -r test (1422 passed), -r typecheck green. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 27 +++++++++++ packages/core/src/composite/find-new-leads.ts | 8 ++++ packages/core/src/composite/qualify-leads.ts | 14 +++++- .../qualify-leads-idempotency.test.ts | 12 +++++ .../qualify-leads-spend-gate.test.ts | 45 +++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index b738dcfc..f9e29e7e 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -249,6 +249,33 @@ export function canonicalSet(values: unknown): unknown[] { .map((v) => JSON.parse(v)); } +/** Coerce the array-typed params of a job tool into arrays, ONCE, before any + * other code touches them. + * + * The MCP server does not validate `inputSchema` before dispatch, so an agent + * can send any array field in its natural singular form — `channels: "email"`, + * `contact_titles: "Owner"`, `lead_refs: {website: "acme.com"}`. Every site + * that later does `.map` / `.join` / `.length` on those then throws a + * TypeError, and on the paid path that happens BEFORE the spend gate, so the + * caller gets a crash instead of the promised quote. + * + * Normalizing at the entry point fixes the whole class at once, rather than + * hardening each consumer separately. Non-array, non-null values are wrapped; + * null/undefined are left alone so `?? []` defaults still apply. */ +export function coerceArrayParams>( + params: T, + keys: readonly (keyof T)[] +): T { + const out = { ...params }; + for (const key of keys) { + const v = out[key]; + if (v !== undefined && v !== null && !Array.isArray(v)) { + out[key] = [v] as T[keyof T]; + } + } + return out; +} + const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 6b2c1d92..f1b489ca 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + coerceArrayParams, canonicalIdSet, canonicalLabelSet, canonicalOptionalObject, @@ -214,6 +215,13 @@ export const findNewLeads: Tool = { params: FindNewLeadsParams, ctx?: ToolContext ) => { + // Unvalidated MCP args can arrive singular; coerce BEFORE the spend gate + // so a shape slip is never a TypeError in place of a quote. + params = coerceArrayParams(params, [ + "contact_titles", + "channels", + "exclude_lead_ids", + ]); rejectCountryLocations(params.filters?.locations); // Same spend gate as leadbay_qualify_leads. The trigger differs: `qualify` diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index f1cf5e8e..3f0e72e5 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -13,6 +13,7 @@ import { clampWaitSeconds, collectJobSnapshot, canonicalSet, + coerceArrayParams, normalizeUuid, canonicalLabelSet, derivedKey, @@ -99,7 +100,10 @@ function derivedRequestId(params: QualifyLeadsParams): string { // them to one key would make the second submit look like a duplicate and // leave those refs unqualified. prior: [ - params.prior_deliveries?.job_id ?? null, + // UUID-folded like the refs above: the backend resolves the same + // delivery job regardless of casing, so casing alone must not fork + // the key and re-run a paid batch. + normalizeUuid(params.prior_deliveries?.job_id), params.prior_deliveries?.since ?? null, params.prior_deliveries?.limit ?? null, ], @@ -227,6 +231,14 @@ export const qualifyLeads: Tool = { params: QualifyLeadsParams, ctx?: ToolContext ) => { + // Unvalidated MCP args can arrive singular (`channels: "email"`, + // `lead_refs: {website}`); coerce BEFORE the spend gate so a shape slip is + // never a TypeError in place of a quote. + params = coerceArrayParams(params, [ + "lead_refs", + "contact_titles", + "channels", + ]); // Spend gate. `qualify` defaults to TRUE on the backend (~94 cost_cents per // lead needing fresh research), so a bare call carrying only lead_refs is a // PAID submit — up to 500 refs — that the user never approved. Prose in the diff --git a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts index d15d8545..b644e5e0 100644 --- a/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-idempotency.test.ts @@ -270,3 +270,15 @@ describe("qualify_leads — UUID casing", () => { expect(b).not.toBe(a); }); }); + +describe("qualify_leads — prior_deliveries UUID casing", () => { + it("job_id casing does not fork the key", async () => { + const upper = await submittedRequestId({ + prior_deliveries: { job_id: "0A2FCBF5-18E1-4967-B5DE-0C67CD823BCC" }, + }); + const lower = await submittedRequestId({ + prior_deliveries: { job_id: "0a2fcbf5-18e1-4967-b5de-0c67cd823bcc" }, + }); + expect(lower).toBe(upper); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts b/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts index c5836cf2..bdd19a38 100644 --- a/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts +++ b/packages/core/test/unit/composite/qualify-leads-spend-gate.test.ts @@ -246,3 +246,48 @@ describe("leadbay_qualify_leads — render envelope", () => { expect(result.items).toHaveLength(2); }); }); + +describe("leadbay_qualify_leads — unvalidated arg shapes", () => { + // The MCP server does not validate inputSchema before dispatch, so an agent + // can send the natural singular form. These used to TypeError BEFORE the + // spend gate, so the caller got a crash instead of the promised quote. + it("a single lead_refs object is treated as a one-item list", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: { website: "acme.com" }, + } as any); + + expect(result.mode).toBe("needs_confirmation"); + expect(postBodies()[0].lead_refs).toEqual([{ website: "acme.com" }]); + }); + + it("a scalar channels value reaches the wire as an array", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + const result: any = await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + channels: "email", + } as any); + + expect(result.mode).toBe("needs_confirmation"); + expect(postBodies()[0].channels).toEqual(["email"]); + }); + + it("a scalar contact_titles value reaches the wire as an array", async () => { + mockHttp([ + { method: "POST", path: "/1.6/mcp/qualify", status: 200, body: DRY_RUN_200 }, + ]); + + await qualifyLeads.execute(newClient(), { + lead_refs: REFS, + contact_titles: "Owner", + } as any); + + expect(postBodies()[0].contact_titles).toEqual(["Owner"]); + }); +}); From e1efe2414272dda902005f1dee0db2afb43c7211 Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 11 Aug 2026 13:11:48 +0200 Subject: [PATCH 27/46] fix(workflows): close the walkthrough scenario fence so the net-new contract parses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression from my conflict resolution in 70590f1. Concatenating main's contract block and ours left our first `workflow_name` inside main's still-open ```yaml scenario fence, and workflow-contract-schema.test.ts only extracts from fences labelled `yaml expected`. So the net-new lead-delivery workflow — the headline user story of this PR — had a table row and no parsed contract: no required_calls, no success criteria, no eval/audit guard. Measured: 48 contracts parsed at 70590f1, 49 now, the difference being "Net-new lead delivery". The other two blocks kept their own fences and were never affected. Worth noting the gates did NOT catch this — `pnpm -r test` was green with the contract missing, because an absent contract is silence rather than a failure. The audit asserts the shape of contracts it finds, not that a workflow has one. Co-Authored-By: Claude --- WORKFLOWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/WORKFLOWS.md b/WORKFLOWS.md index 9459f9c9..e8652cae 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -1213,6 +1213,9 @@ render_checks: ```yaml scenario prompt: "Walk me through Leadbay." +``` + +```yaml expected workflow_name: Net-new lead delivery (one ask → qualified, contactable leads) prompt_name: leadbay_new_leads required_calls: From 84efd2cb2615bd33e3f15d8bd0776193ff467b0d Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 11 Aug 2026 13:17:08 +0200 Subject: [PATCH 28/46] fix(mcp): reshape string lead_refs before deriving the idempotency key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 on #168. The schema declares `lead_refs` items as objects, but `coerceArrayParams` wraps a scalar, so the common shorthand `lead_refs: "acme.com"` reaches the tool as `["acme.com"]` — a string every reader then treats as an object. Two consequences, both on the paid path. Reading `.lead_id`/`.website` off a string yields an all-null ref, so EVERY string ref canonicalized identically: two unrelated companies derived the same `qualify-auto-*` key and the second paid batch deduped onto the first job. And the raw string was still posted, so the backend 400s a call this tool had already promised a quote for. The shorthand is now mapped onto the shape the schema documents — a UUID is a lead_id, a domain is a website, anything else is a name — before the spend gate, so the quote, the derived key and the submitted body describe the same companies. `isUuidShaped` is exported from the job helpers for the test. New file qualify-leads-string-refs.test.ts: the three shapes, a bare scalar, object refs passing through untouched, and two companies no longer sharing a derived request_id. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 6 + packages/core/src/composite/qualify-leads.ts | 30 ++++ .../qualify-leads-string-refs.test.ts | 144 ++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 packages/core/test/unit/composite/qualify-leads-string-refs.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index f9e29e7e..336b1cd3 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -283,6 +283,12 @@ const UUID_RE = * key — the backend resolves `A1B2…` and `a1b2…` to the same record. A value * that is not UUID-shaped is only trimmed, since we cannot assume the backend * folds case for arbitrary identifiers. */ +/** True when a value is shaped like a Leadbay UUID. Lets a caller tell a + * lead id apart from a website or a company name in an untyped ref. */ +export function isUuidShaped(value: unknown): boolean { + return typeof value === "string" && UUID_RE.test(value.trim()); +} + export function normalizeUuid(value: unknown): string | null { if (typeof value !== "string") return null; const v = value.trim(); diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 3f0e72e5..c99c7a83 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -14,6 +14,7 @@ import { collectJobSnapshot, canonicalSet, coerceArrayParams, + isUuidShaped, normalizeUuid, canonicalLabelSet, derivedKey, @@ -63,6 +64,30 @@ const DEFAULT_WAIT_SECONDS = 45; * after a stop_reason: max_cost) hashes differently and runs as a new job. * A caller who wants a deliberate re-run of an identical batch passes an * explicit request_id. */ +/** MCP args arrive unvalidated, and `coerceArrayParams` turns a scalar + * `lead_refs: "acme.com"` into `["acme.com"]` — a STRING where the schema + * promises an object. Read as an object that yields an all-null ref, so EVERY + * string ref canonicalizes identically: two different companies derive the + * same `qualify-auto-*` key and the second batch dedupes onto the first + * PAID job. The raw string is also posted as-is, which the backend 400s + * after this tool already promised a quote. + * + * Map the shorthand onto the shape the schema documents instead — a UUID is + * a lead_id, a domain is a website, anything else is a name — so the derived + * key and the submitted body agree and describe the company the caller meant. */ +function normalizeLeadRefs( + refs: QualifyLeadsParams["lead_refs"] +): QualifyLeadsParams["lead_refs"] { + if (!Array.isArray(refs)) return refs; + return refs.map((ref) => { + if (typeof ref !== "string") return ref; + const value = (ref as string).trim(); + if (!value) return ref; + if (isUuidShaped(value)) return { lead_id: value }; + return normalizeDomain(value) ? { website: value } : { name: value }; + }); +} + function derivedRequestId(params: QualifyLeadsParams): string { // JSON-serialize each ref rather than joining raw values with delimiters. // Field names alone were not enough: a value CONTAINING the delimiters @@ -239,6 +264,11 @@ export const qualifyLeads: Tool = { "contact_titles", "channels", ]); + // …and a coerced scalar is a STRING inside that array, which every + // downstream reader treats as an object. Reshape before the spend gate so + // the quote, the idempotency key and the posted body all describe the same + // companies. + params = { ...params, lead_refs: normalizeLeadRefs(params.lead_refs) }; // Spend gate. `qualify` defaults to TRUE on the backend (~94 cost_cents per // lead needing fresh research), so a bare call carrying only lead_refs is a // PAID submit — up to 500 refs — that the user never approved. Prose in the diff --git a/packages/core/test/unit/composite/qualify-leads-string-refs.test.ts b/packages/core/test/unit/composite/qualify-leads-string-refs.test.ts new file mode 100644 index 00000000..576ebcfd --- /dev/null +++ b/packages/core/test/unit/composite/qualify-leads-string-refs.test.ts @@ -0,0 +1,144 @@ +/** + * String shorthand in `lead_refs` for leadbay_qualify_leads. + * + * MCP args arrive unvalidated and `coerceArrayParams` wraps a scalar, so + * `lead_refs: "acme.com"` reaches the tool as `["acme.com"]` — a STRING where + * the schema promises an object. Read as an object it yields an all-null ref, + * so every string ref canonicalized identically: two different companies + * derived the SAME `qualify-auto-*` request_id and the second batch would + * dedupe onto the first PAID job. The raw string was also posted as-is, which + * the backend rejects after this tool already promised a quote. + * + * These tests pin the reshape: the submitted body carries the documented + * object shape, and two different companies no longer share a key. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const LEAD_UUID = "7b3c1de2-5f40-4a9c-9d21-0c8ea4f61b55"; + +const DRY_RUN_200 = { + valid: true, + items_requested: 1, + estimated_cost: { max: 94, unit: "cost_cents" }, +}; + +beforeEach(() => resetHttpMock()); + +/** Drive the FREE path so no consent gate is involved: qualify:false, no + * channels. The submit body is what we assert on. */ +async function submittedBody(lead_refs: unknown): Promise { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + // The poller appends cursor params, so match the route, not an exact path. + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await qualifyLeads.execute(newClient(), { + lead_refs, + qualify: false, + wait_seconds: 0, + } as any); + const post = getHttpRequests().find( + (r) => r.method === "POST" && r.path.endsWith("/mcp/qualify") + ); + return JSON.parse(post!.body ?? "{}"); +} + +/** The PAID path, consented, so a `request_id` is actually derived. */ +async function paidSubmittedBody(lead_refs: unknown): Promise { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await qualifyLeads.execute(newClient(), { + lead_refs, + qualify: true, + confirm: true, + wait_seconds: 0, + } as any); + const post = getHttpRequests().find( + (r) => r.method === "POST" && r.path.endsWith("/mcp/qualify") + ); + return JSON.parse(post!.body ?? "{}"); +} + +describe("leadbay_qualify_leads — string lead_refs", () => { + it("a bare domain becomes {website}, not a string the backend 400s on", async () => { + const body = await submittedBody(["acme.com"]); + expect(body.lead_refs).toEqual([{ website: "acme.com" }]); + }); + + it("a UUID becomes {lead_id}", async () => { + const body = await submittedBody([LEAD_UUID]); + expect(body.lead_refs).toEqual([{ lead_id: LEAD_UUID }]); + }); + + it("a non-domain, non-UUID string becomes {name}", async () => { + const body = await submittedBody(["Franklin Barbecue"]); + expect(body.lead_refs).toEqual([{ name: "Franklin Barbecue" }]); + }); + + it("a scalar (not a list) is still reshaped after coercion", async () => { + const body = await submittedBody("acme.com"); + expect(body.lead_refs).toEqual([{ website: "acme.com" }]); + }); + + it("object refs pass through untouched", async () => { + const body = await submittedBody([{ website: "acme.com", name: "Acme" }]); + expect(body.lead_refs).toEqual([{ website: "acme.com", name: "Acme" }]); + }); + + it("two different companies no longer derive the same request_id", async () => { + // The collapse that mattered, and it only bites on the PAID path — that is + // where a key is derived. Read as objects, every string ref produced an + // all-null canonical form, so two unrelated companies hashed to the same + // `qualify-auto-*` key and the second paid batch deduped onto the first job. + const a = await paidSubmittedBody(["acme.com"]); + const b = await paidSubmittedBody(["globex.com"]); + expect(a.request_id).toBeTruthy(); + expect(b.request_id).toBeTruthy(); + expect(a.request_id).not.toEqual(b.request_id); + }); +}); From 35f21e8e1344a30f5ede4ad0e34a862bf256f860 Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 11 Aug 2026 14:16:09 +0200 Subject: [PATCH 29/46] fix(mcp): canonicalize the exploration cap default and reject blank request ids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex P2s on #168, both ways a retry of the SAME approved work derives a DIFFERENT key and escapes backend dedupe into a second paid, novelty-claiming job. exploration_cap was hashed as null when omitted, while every other defaulted field in that shape (min_ai_score, novelty, title_gate) is canonicalized to the value the backend will apply — a principle the comment three lines above states outright. So for a paid `count: 10, qualify: true`, an approval that omits the cap and a retry that materializes the documented default `exploration_cap: 30` asked for identical work under different keys. It is now canonicalized to min(3n,150); an explicit non-default cap still hashes distinctly. `??` also treated a blank request_id as explicit. The field is schema-required on the search, but args are not validated before dispatch, so `""` shipped AS the idempotency key on confirmed paid submits: blank-as-absent means retries double-spend, blank-as-key means unrelated approvals dedupe onto each other. New presentRequestId trims and treats blank as missing, wired into BOTH find_new_leads and qualify_leads — the review noted the same pattern there. New file mcp-key-defaults.test.ts pins the behaviour rather than the implementation: omitted == explicit-default, non-default differs, the default tracks count (40 -> 120) instead of a hard-coded number, it ceilings at 150, and a blank id falls back to the derived key while a real one is honoured trimmed. Co-Authored-By: Claude --- .../core/src/composite/_mcp-job-helpers.ts | 11 ++ packages/core/src/composite/find-new-leads.ts | 15 ++- packages/core/src/composite/qualify-leads.ts | 5 +- .../unit/composite/mcp-key-defaults.test.ts | 113 ++++++++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 packages/core/test/unit/composite/mcp-key-defaults.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 336b1cd3..f31ac73a 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -283,6 +283,17 @@ const UUID_RE = * key — the backend resolves `A1B2…` and `a1b2…` to the same record. A value * that is not UUID-shaped is only trimmed, since we cannot assume the backend * folds case for arbitrary identifiers. */ +/** A caller can fill a schema-`required` string with `""` — the server does not + * validate schemas before dispatch. `??` treats that as an explicit value, so a + * blank id would ship AS the idempotency key: if the backend reads blank as + * absent, a timeout retry launches a second paid job; if it reads blank as a + * key, unrelated blank-key approvals dedupe onto each other. Blank is missing. */ +export function presentRequestId(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed ? trimmed : undefined; +} + /** True when a value is shaped like a Leadbay UUID. Lets a caller tell a * lead id apart from a website or a company name in an untyped ref. */ export function isUuidShaped(value: unknown): boolean { diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index f1b489ca..40d55061 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -18,6 +18,7 @@ import { canonicalLabelSet, canonicalOptionalObject, derivedKey, + presentRequestId, mockedSubmitPreview, compactBody, normalizeSearchFilters, @@ -242,7 +243,7 @@ export const findNewLeads: Tool = { // novelty-claiming job. Synthesize a stable key from the approved search // itself, exactly as the qualify path does. const requestId = - params.request_id ?? + presentRequestId(params.request_id) ?? derivedKey( "search-auto", // Passed as an OBJECT: derivedKey canonicalizes recursively, so nested @@ -272,7 +273,17 @@ export const findNewLeads: Tool = { exclude_lead_ids: canonicalIdSet(params.exclude_lead_ids), novelty: params.novelty ?? "org", max_cost: params.max_cost ?? null, - exploration_cap: params.exploration_cap ?? null, + // Documented backend default is min(3n,150), so an omitted cap is + // canonicalized TO it — same principle as min_ai_score/novelty above. + // Otherwise an approval that omits the cap and a retry that passes the + // materialized default ask for identical work under different keys, + // and the retry escapes dedupe into a second paid, novelty-claiming + // job. An explicit non-default cap still hashes distinctly. + exploration_cap: + params.exploration_cap ?? + (typeof params.count === "number" && params.count > 0 + ? Math.min(3 * params.count, 150) + : null), lang: params.lang ?? null, } ); diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index c99c7a83..7665da09 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -16,6 +16,7 @@ import { coerceArrayParams, isUuidShaped, normalizeUuid, + presentRequestId, canonicalLabelSet, derivedKey, mockedSubmitPreview, @@ -291,8 +292,10 @@ export const qualifyLeads: Tool = { // the same refs. `request_id` is optional on this tool (unlike the search), // so derive a stable one from the batch when the caller omits it: same refs // + same paid flags on the same day = same key = backend dedupe. + // Blank is missing: a caller can fill this optional field with "" and `??` + // would ship it as the key — see presentRequestId. const requestId = - params.request_id ?? + presentRequestId(params.request_id) ?? (isPaid ? derivedRequestId(params) : undefined); const body = compactBody({ diff --git a/packages/core/test/unit/composite/mcp-key-defaults.test.ts b/packages/core/test/unit/composite/mcp-key-defaults.test.ts new file mode 100644 index 00000000..4652f586 --- /dev/null +++ b/packages/core/test/unit/composite/mcp-key-defaults.test.ts @@ -0,0 +1,113 @@ +/** + * Idempotency-key canonicalization edge cases on the MCP-first delivery tools. + * + * Both are ways a retry of the SAME approved work derives a DIFFERENT key and + * so escapes backend dedupe into a second paid, novelty-claiming job: + * + * 1. `exploration_cap` omitted vs. passed as its documented default min(3n,150). + * Every other defaulted field (min_ai_score, novelty, title_gate) is already + * canonicalized to the value the backend will apply; this one was not. + * 2. A blank `request_id`. It is schema-`required` on the search, but args are + * not validated before dispatch, so `""` reached `??` as an "explicit" value + * and shipped as the key. If the backend reads blank as absent a retry + * double-spends; if it reads blank as a key, unrelated approvals dedupe onto + * each other. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +/** Confirmed PAID search — the only path that derives a key. */ +async function searchBody(extra: Record): Promise { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await findNewLeads.execute(newClient(), { + example_lead: { description: "independent gym, 2 sites" }, + count: 10, + qualify: true, + confirm: true, + wait_seconds: 0, + ...extra, + } as any); + const post = getHttpRequests().find( + (r) => r.method === "POST" && r.path.endsWith("/mcp/search") + ); + return JSON.parse(post!.body ?? "{}"); +} + +describe("leadbay_find_new_leads — key canonicalization", () => { + it("an omitted exploration_cap keys the same as its documented default", async () => { + // count: 10 → min(3*10, 150) = 30 + const omitted = await searchBody({}); + const explicit = await searchBody({ exploration_cap: 30 }); + expect(omitted.request_id).toBeTruthy(); + expect(omitted.request_id).toEqual(explicit.request_id); + }); + + it("an explicit NON-default exploration_cap still keys distinctly", async () => { + const omitted = await searchBody({}); + const raised = await searchBody({ exploration_cap: 120 }); + expect(omitted.request_id).not.toEqual(raised.request_id); + }); + + it("the default tracks count rather than being a fixed number", async () => { + // count: 40 → min(120, 150) = 120, so 120 must match the omitted key here + // while 30 (the count:10 default) must not. + const omitted = await searchBody({ count: 40 }); + const matching = await searchBody({ count: 40, exploration_cap: 120 }); + const other = await searchBody({ count: 40, exploration_cap: 30 }); + expect(omitted.request_id).toEqual(matching.request_id); + expect(omitted.request_id).not.toEqual(other.request_id); + }); + + it("the cap is capped at 150 for large counts", async () => { + // count: 50 → min(150, 150) = 150 + const omitted = await searchBody({ count: 50 }); + const explicit = await searchBody({ count: 50, exploration_cap: 150 }); + expect(omitted.request_id).toEqual(explicit.request_id); + }); + + it("a blank request_id falls back to the derived key", async () => { + const blank = await searchBody({ request_id: " " }); + const missing = await searchBody({}); + expect(blank.request_id).toBeTruthy(); + expect(blank.request_id.trim()).not.toEqual(""); + expect(blank.request_id).toEqual(missing.request_id); + }); + + it("a real request_id is still honoured, trimmed", async () => { + const keyed = await searchBody({ request_id: " gyms-texas-2026-07-28 " }); + expect(keyed.request_id).toEqual("gyms-texas-2026-07-28"); + }); +}); From ee537abcf99b019d2652717a180ee81bcf29c756 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 11:14:40 +0200 Subject: [PATCH 30/46] =?UTF-8?q?fix(mcp):=20the=20three=20Codex=20finding?= =?UTF-8?q?s=20on=20#168=20=E2=80=94=20test=20audit,=20country=20fence,=20?= =?UTF-8?q?currency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] Restore packages/promptforge/test/skills.test.ts byte-for-byte. The PR had narrowed its "every prompt template has a SKILL.md" assertion so a release_gated prompt could pass — weakening an existing audit to fit a new feature is exactly what the new-tests-in-new-files rule exists to stop. The fix is layout, not exemption: leadbay_new_leads.md.tmpl moves to prompts/release-gated/. That audit readdirs prompts/ non-recursively while assemble()'s findTemplates() recurses, so the template still builds and the assertion stays at full strength. skills-release-gate.test.ts pins the new contract, and re-asserts the B23 defer-to-tool-rendering rule that assembler.test.ts's equally-shallow readdir no longer reaches — the relocation should not quietly drop that coverage. [P2] rejectCountryLocations recognized only US and France, so Canada, United Kingdom, Germany and every other country reached /mcp/search and hit the exact silent same-named-town fencing the guard exists to prevent. Now built from Intl.DisplayNames over every ISO 3166-1 code, in English and French. Comprehensive detection alone would have been a regression, though: Georgia is a US state before it is a country, and every French overseas region carries its own ISO entry, so "leads in Martinique" would start failing. Those are exempted. Municipality collisions (Lebanon, Peru, Mexico) are deliberately NOT — a bare town name identical to a country is genuinely ambiguous, the rejection is loud, and "Lebanon, Kentucky" folds to a key no country matches. The hint now says so. Two-letter codes are not added wholesale: CA/DE/IN are state abbreviations. Also fixes countryKey, found while testing it: French elides its article with no space, so "l'Allemagne" folded to "lallemagne" and the space-anchored article strip never fired. [P2] The mandatory funnel recipe hard-coded "spent $C.CC" while cost carries no currency and the design doc's own example bills a French job in euros — a French user was told a EUR6.09 job cost $6.09. Now region-derived, with the bare amount as the fallback rather than an invented symbol. Gates: build, 1527 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 76 +++++++++++- .../core/src/tool-descriptions.generated.ts | 18 ++- .../composite/country-location-guard.test.ts | 111 ++++++++++++++++++ .../leadbay_new_leads.md.tmpl | 0 .../snippets/rendering/lead-delivery-table.md | 6 +- .../test/skills-release-gate.test.ts | 41 ++++++- packages/promptforge/test/skills.test.ts | 13 +- 7 files changed, 245 insertions(+), 20 deletions(-) create mode 100644 packages/core/test/unit/composite/country-location-guard.test.ts rename packages/promptforge/prompts/{ => release-gated}/leadbay_new_leads.md.tmpl (100%) diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index f31ac73a..19948dc1 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -455,11 +455,76 @@ export function compactBody( * description saying not to — prose does not prevent this, so the tool * rejects it with a named, actionable error (tracked backend-side in * product#3939). */ -const COUNTRY_LOCATION_VALUES = new Set([ +const COUNTRY_ALIASES = [ "united states", "united states of america", "usa", "us", "america", "etats unis", "etats unis d amerique", "france", "fr", "french republic", "republique francaise", -]); +]; + +/** Country names that are ALSO a legitimate administrative fence inside a + * Leadbay universe, so rejecting them would break a correct search: + * + * - `Georgia` is a US state before it is a country, and one of the most + * common state fences a US account will ask for. + * - The French overseas regions and collectivities each carry their own + * ISO 3166-1 entry, so a comprehensive country list swallows every one of + * them — while "leads in Martinique" is exactly the kind of regional fence + * this parameter exists for. + * + * Municipality collisions (`Lebanon`, `Peru`, `Mexico`, … are all US town + * names) are deliberately NOT exempted: a bare town name identical to a + * country is genuinely ambiguous, and the rejection is loud and recoverable — + * `Lebanon, Kentucky` folds to a two-word key that never matches. A silent + * fence to one village is the failure this guard exists to prevent. */ +const SUBNATIONAL_EXEMPTIONS = new Set( + [ + "georgia", "georgie", + "guadeloupe", "martinique", "reunion", "mayotte", + "french guiana", "guyane francaise", + "new caledonia", "nouvelle caledonie", + "french polynesia", "polynesie francaise", + "saint martin", "saint barthelemy", "saint pierre and miquelon", + "saint pierre et miquelon", "wallis and futuna", "wallis et futuna", + ].map(countryKey), +); + +/** Every ISO 3166-1 country name, in English and French, folded to the same + * comparison key as the input. Built from `Intl.DisplayNames` rather than a + * hand-kept list: the two-country allowlist this replaces let `Canada`, + * `United Kingdom`, `Germany` and every other country through to the same + * silent same-named-town fencing it was written to stop. + * + * Two-letter codes are NOT added wholesale — they collide head-on with US + * state abbreviations (`CA` California, `DE` Delaware, `IN` Indiana). The + * only codes here are the two deliberate aliases in COUNTRY_ALIASES. */ +function buildCountryLocationValues(): Set { + const values = new Set(COUNTRY_ALIASES.map(countryKey)); + try { + const A = "A".charCodeAt(0); + const displays = ["en", "fr"].map( + (locale) => + new Intl.DisplayNames([locale], { type: "region", fallback: "none" }), + ); + for (let i = 0; i < 26; i++) { + for (let j = 0; j < 26; j++) { + const code = String.fromCharCode(A + i) + String.fromCharCode(A + j); + for (const display of displays) { + const name = display.of(code); + if (!name || name === code) continue; + values.add(countryKey(name)); + } + } + } + } catch { + // A Node built without full ICU yields no region names. Falling back to + // the explicit aliases keeps the originally-observed failure covered + // rather than throwing at import time. + } + for (const exempt of SUBNATIONAL_EXEMPTIONS) values.delete(exempt); + return values; +} + +const COUNTRY_LOCATION_VALUES = buildCountryLocationValues(); /** Fold a location label to a comparison key so spelling variants collapse: * strips accents, punctuation (so `U.S.` and `U.S` both become `us`), a @@ -474,6 +539,11 @@ function countryKey(raw: string): string { // Hyphens/underscores separate words; dots and apostrophes do not (so // "U.S" folds to "us", while "etats-unis" stays two words). .replace(/[-_,]/g, " ") + // French elides its article onto the noun with no space — "l'Allemagne", + // "l'Espagne" — so the space-anchored article strip below never fires and + // the label folded to "lallemagne". Drop the elided article first, before + // the apostrophe itself is deleted. Anchored, so "U.S" still folds to "us". + .replace(/^\s*(l|d)['’]\s*/, "") .replace(/['’.]/g, "") .replace(/\s+/g, " ") .trim() @@ -496,7 +566,7 @@ export function rejectCountryLocations(locations: unknown): void { error: true, code: "COUNTRY_LEVEL_LOCATION", message: `filters.locations value "${loc}" is country-level — it would silently fence the search to a same-named town, not the whole country.`, - hint: "Whole-country intent = OMIT filters.locations entirely (each universe is single-country). Use city/state/region names for narrower fences.", + hint: "Whole-country intent = OMIT filters.locations entirely (each universe is single-country). Use city/state/region names for narrower fences. If you meant a town that shares the name, qualify it with its state or region (e.g. \"Lebanon, Kentucky\").", }; } } diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index b293d293..1a82374a 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1437,7 +1437,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent $C.CC. +> the Y asked** · stopped: · spent C.CC. + +**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in +euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid +operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write @@ -2424,7 +2428,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent $C.CC. +> the Y asked** · stopped: · spent C.CC. + +**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in +euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid +operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write @@ -3688,7 +3696,11 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent $C.CC. +> the Y asked** · stopped: · spent C.CC. + +**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in +euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid +operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write diff --git a/packages/core/test/unit/composite/country-location-guard.test.ts b/packages/core/test/unit/composite/country-location-guard.test.ts new file mode 100644 index 00000000..c4841796 --- /dev/null +++ b/packages/core/test/unit/composite/country-location-guard.test.ts @@ -0,0 +1,111 @@ +/** + * `rejectCountryLocations` — the country-level fence guard, both directions. + * + * The guard originally recognized only US and France aliases, so `Canada`, + * `United Kingdom`, `Germany` and every other country sailed through to + * `/mcp/search` and hit the very failure the guard exists to stop: the backend + * excludes countries from admin-area search, so the trigram resolver falls + * through to an arbitrary same-named town and fences the whole job to one + * village — silently. + * + * The fix builds the set from `Intl.DisplayNames` over every ISO 3166-1 code + * in English and French. That is comprehensive, and comprehensiveness is + * exactly what makes the SECOND half of this file load-bearing: a naive + * country list also swallows `Georgia` (a US state, and a common one to fence + * on) and every French overseas region, each of which carries its own ISO + * entry. Rejecting those would trade a silent wrong fence for a loud wrong + * rejection on a legitimate search. + */ + +import { describe, it, expect } from "vitest"; +import { rejectCountryLocations } from "../../../src/composite/_mcp-job-helpers.js"; + +/** The guard throws a plain error envelope, not an Error instance. */ +function rejects(value: unknown): boolean { + try { + rejectCountryLocations(value); + return false; + } catch (e) { + expect((e as { code?: string }).code).toBe("COUNTRY_LEVEL_LOCATION"); + return true; + } +} + +describe("rejectCountryLocations — countries are rejected", () => { + it("rejects the two originally-covered countries", () => { + for (const v of ["United States", "USA", "France", "la France"]) { + expect(rejects([v]), v).toBe(true); + } + }); + + it("rejects the countries the two-country allowlist let through", () => { + // The regression Codex caught on #168: these reached /mcp/search untouched. + for (const v of ["Canada", "United Kingdom", "Germany", "Spain", "Japan"]) { + expect(rejects([v]), v).toBe(true); + } + }); + + it("rejects French-language country names too", () => { + for (const v of ["Allemagne", "Royaume-Uni", "Espagne", "Belgique"]) { + expect(rejects([v]), v).toBe(true); + } + }); + + it("still folds spelling variants — accents, articles, punctuation", () => { + for (const v of [" the United Kingdom ", "l'Allemagne", "ESPAGNE"]) { + expect(rejects([v]), v).toBe(true); + } + }); + + it("still catches a bare string passed where a list was declared", () => { + expect(rejects("Germany")).toBe(true); + }); +}); + +describe("rejectCountryLocations — legitimate sub-national fences survive", () => { + it("does not reject Georgia, which is a US state before it is a country", () => { + for (const v of ["Georgia", "georgia", "Géorgie"]) { + expect(rejects([v]), v).toBe(false); + } + }); + + it("does not reject the French overseas regions", () => { + // Each carries its own ISO 3166-1 entry, so a comprehensive country list + // swallows all of them — while "leads in Martinique" is a normal ask. + for (const v of [ + "Guadeloupe", + "Martinique", + "La Réunion", + "Mayotte", + "Guyane française", + "Nouvelle-Calédonie", + ]) { + expect(rejects([v]), v).toBe(false); + } + }); + + it("does not reject ordinary cities, states and regions", () => { + for (const v of [ + "Austin", + "Texas", + "Lyon", + "Nouvelle-Aquitaine", + "Brooklyn", + ]) { + expect(rejects([v]), v).toBe(false); + } + }); + + it("lets a town be disambiguated by its state, as the hint tells the user", () => { + // "Lebanon" alone IS ambiguous and is rejected on purpose; qualifying it + // folds to a two-word key that no country name matches. + expect(rejects(["Lebanon"])).toBe(true); + expect(rejects(["Lebanon, Kentucky"])).toBe(false); + }); + + it("no-ops on absent locations", () => { + expect(rejects(undefined)).toBe(false); + expect(rejects(null)).toBe(false); + expect(rejects([])).toBe(false); + }); +}); diff --git a/packages/promptforge/prompts/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl similarity index 100% rename from packages/promptforge/prompts/leadbay_new_leads.md.tmpl rename to packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index efab23ef..ab9f25b6 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -43,7 +43,11 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + `explain.scope_notes`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of -> the Y asked** · stopped: · spent $C.CC. +> the Y asked** · stopped: · spent C.CC. + +**Never hard-code `$`.** `cost` carries no currency, and the same job bills in +euros on a France account — `$6.09` for a €6.09 charge misstates a paid +operation. US → `$`, France → `€`; region unknown → write the bare amount. "of the Y asked" needs `summary.items_requested`, which submit results carry but a later `leadbay_lead_job_status` snapshot does not. Without it write diff --git a/packages/promptforge/test/skills-release-gate.test.ts b/packages/promptforge/test/skills-release-gate.test.ts index 43688111..7b3878be 100644 --- a/packages/promptforge/test/skills-release-gate.test.ts +++ b/packages/promptforge/test/skills-release-gate.test.ts @@ -8,7 +8,7 @@ */ import { describe, it, expect } from "vitest"; -import { existsSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join, resolve } from "node:path"; import { assemble } from "../src/assembler.js"; @@ -19,6 +19,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const PKG_ROOT = resolve(__dirname, ".."); const REPO_ROOT = resolve(PKG_ROOT, "..", ".."); const CORE_SRC = resolve(REPO_ROOT, "packages", "core", "src"); +const PROMPTS_DIR = resolve(PKG_ROOT, "prompts"); const SKILLS_DIR = resolve( REPO_ROOT, ".claude-plugin", @@ -54,4 +55,42 @@ describe("audit: release-gated prompts ship no skill", () => { expect(existsSync(path), `${path} must not ship while gated`).toBe(false); } }); + + // The layout is what lets skills.test.ts stay untouched. That audit asserts + // every .md.tmpl sitting DIRECTLY in prompts/ has a matching SKILL.md, and + // reads the directory non-recursively. A gated prompt has no SKILL.md by + // design, so the only honest way to satisfy that audit is to keep the + // template out of the flat directory — not to weaken the audit's assertion. + // assemble()'s findTemplates() recurses, so the subdirectory still builds. + it("a gated prompt template lives in a subdirectory, not flat in prompts/", () => { + const flat = new Set( + readdirSync(PROMPTS_DIR) + .filter((f) => f.endsWith(".md.tmpl")) + .map((f) => f.replace(/\.md\.tmpl$/, "")), + ); + for (const p of gated) { + const rel = p.sourcePath.slice(p.sourcePath.indexOf("prompts/")); + expect( + flat.has(p.frontmatter.name), + `${p.frontmatter.name} is flat in prompts/ but ships no SKILL.md — ` + + `skills.test.ts would fail. Keep it under prompts/release-gated/.`, + ).toBe(false); + expect(rel.startsWith("prompts/release-gated/")).toBe(true); + } + }); + + // Moving the template out of the flat directory also moves it out of reach of + // assembler.test.ts's B23 audit, which readdirs prompts/ the same shallow way. + // That rule still applies to this prompt — it orchestrates composites that + // ship their own RENDERING block — so re-assert it here rather than let the + // relocation quietly drop the coverage. + it("a gated prompt still carries the defer-to-tool-rendering gate (B23)", () => { + for (const p of gated) { + const source = readFileSync(p.sourcePath, "utf8"); + expect( + source.includes("{{include:gates/defer-to-tool-rendering}}"), + `${p.frontmatter.name} orchestrates rendering composites and must defer to their RENDERING blocks`, + ).toBe(true); + } + }); }); diff --git a/packages/promptforge/test/skills.test.ts b/packages/promptforge/test/skills.test.ts index b83617f0..cabbbe8e 100644 --- a/packages/promptforge/test/skills.test.ts +++ b/packages/promptforge/test/skills.test.ts @@ -52,27 +52,16 @@ describe("audit: SKILL.md files", () => { const result = assemble({ root: PKG_ROOT, registeredToolNames: registered }); const skillFiles = buildSkillFiles(result.prompts); - // release_gated prompts deliberately emit NO SKILL.md: a skill is a static - // auto-triggering file with no runtime gate, so shipping one while its tools - // are hidden starts a workflow that fails on the first call. - const gatedPrompts = new Set( - result.prompts - .filter((p) => p.frontmatter.release_gated === true) - .map((p) => p.frontmatter.name), - ); - it("every prompt .md.tmpl has a matching SKILL.md", () => { const templates = readdirSync(PROMPTS_DIR) .filter((f) => f.endsWith(".md.tmpl")) - .map((f) => f.replace(/\.md\.tmpl$/, "")) - .filter((name) => !gatedPrompts.has(name)); + .map((f) => f.replace(/\.md\.tmpl$/, "")); for (const name of templates) { const path = join(SKILLS_DIR, name, "SKILL.md"); expect(existsSync(path), `expected ${path}`).toBe(true); } }); - it("every emitted SKILL.md is in sync with .md.tmpl source", () => { for (const skill of skillFiles) { const path = join(SKILLS_DIR, skill.relativePath); From ebd47779c9adf7741bd9d72cd4d850c7eb4b4e58 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 15:00:57 +0200 Subject: [PATCH 31/46] fix(mcp): move the delivery routing fixtures out of the established file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] The PR appended 8 cases to routing-fixtures.ts. Additive or not, that is an existing test file, and it is the same rule the skills.test.ts fix already answered. Restored byte-for-byte; the cases now live in lead-delivery-routing-fixtures.ts, importing the RoutingFixture type from the untouched module. Worth knowing before trusting these: ROUTING_FIXTURES has NO consumer. Not in this package, not anywhere in the repo — bbf5108 removed the classifier eval in #71 and left the fixture file orphaned. So the 8 cases were not weakly covered, they ran nowhere at all, and relocating them alone would have kept it that way while looking compliant. So they get teeth: lead-delivery-routing-fixtures.test.ts asserts every expected_tool and every forbidden_tools entry resolves to a registered tool, that no fixture forbids the tool it expects, that all three delivery tools stay covered, and that no fixture ships without the false-positive signal. That is what a deterministic run can prove — whether the intents actually route still needs the live classifier and an API key. It stops a rename or a typo from rotting them silently in the meantime. The orphaned ROUTING_FIXTURES itself is left alone: restoring a runner for 160 fixtures is its own change, not a review fix on this PR. Gates: build, 1532 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../lead-delivery-routing-fixtures.test.ts | 88 +++++++++++++++++++ .../lead-delivery-routing-fixtures.ts | 85 ++++++++++++++++++ .../tool-descriptions/routing-fixtures.ts | 42 --------- 3 files changed, 173 insertions(+), 42 deletions(-) create mode 100644 packages/mcp/test/audit/lead-delivery-routing-fixtures.test.ts create mode 100644 packages/mcp/test/eval/tool-descriptions/lead-delivery-routing-fixtures.ts diff --git a/packages/mcp/test/audit/lead-delivery-routing-fixtures.test.ts b/packages/mcp/test/audit/lead-delivery-routing-fixtures.test.ts new file mode 100644 index 00000000..9e4a6f59 --- /dev/null +++ b/packages/mcp/test/audit/lead-delivery-routing-fixtures.test.ts @@ -0,0 +1,88 @@ +/** + * Audit: the MCP-first delivery routing fixtures name real tools. + * + * The live classifier eval that consumes routing fixtures needs an Anthropic + * key and is not part of `pnpm -r test`, so a fixture referencing a tool that + * was renamed — or never existed — sits green forever. `ROUTING_FIXTURES` has + * had no runner at all since #71 (`bbf5108` removed the classifier eval and + * left the fixture file orphaned), which is exactly how that rot happens. + * + * This audit is the deterministic half: it cannot tell you whether an intent + * routes correctly, but it can prove every tool the fixtures name is real and + * that no fixture contradicts itself. That keeps the fixtures honest between + * eval runs instead of decaying unnoticed. + */ + +import { describe, it, expect } from "vitest"; +import { + compositeReadTools, + compositeWriteTools, + granularReadTools, + granularWriteTools, + mcpFirstDeliveryAllTools, + type Tool, +} from "@leadbay/core"; +import { LEAD_DELIVERY_ROUTING_FIXTURES } from "../eval/tool-descriptions/lead-delivery-routing-fixtures.js"; + +const ALL_TOOLS: Tool[] = [ + ...compositeReadTools, + ...compositeWriteTools, + ...granularReadTools, + ...granularWriteTools, + ...mcpFirstDeliveryAllTools, +]; +const REGISTERED = new Set(ALL_TOOLS.map((t) => t.name)); + +describe("audit: lead-delivery routing fixtures", () => { + it("covers all three delivery tools", () => { + // Guards the fixture set itself: silently losing a tool's cases would + // leave this audit passing over a narrower set than it claims to cover. + const covered = new Set( + LEAD_DELIVERY_ROUTING_FIXTURES.map((f) => f.expected_tool), + ); + expect([...covered].sort()).toEqual([ + "leadbay_find_new_leads", + "leadbay_lead_job_status", + "leadbay_qualify_leads", + ]); + }); + + it("every expected_tool is a registered tool", () => { + const unknown = LEAD_DELIVERY_ROUTING_FIXTURES.filter( + (f) => !REGISTERED.has(f.expected_tool), + ).map((f) => `${f.expected_tool} (intent: "${f.intent}")`); + expect(unknown, unknown.join("\n")).toEqual([]); + }); + + it("every forbidden_tools entry is a registered tool", () => { + // A typo here is worse than useless: the fixture would "pass" its + // false-positive check against a tool that cannot be routed to anyway. + const unknown: string[] = []; + for (const f of LEAD_DELIVERY_ROUTING_FIXTURES) { + for (const name of f.forbidden_tools ?? []) { + if (!REGISTERED.has(name)) { + unknown.push(`${name} (intent: "${f.intent}")`); + } + } + } + expect(unknown, unknown.join("\n")).toEqual([]); + }); + + it("no fixture forbids the tool it expects", () => { + const contradictory = LEAD_DELIVERY_ROUTING_FIXTURES.filter((f) => + (f.forbidden_tools ?? []).includes(f.expected_tool), + ).map((f) => f.intent); + expect(contradictory, contradictory.join("\n")).toEqual([]); + }); + + it("every fixture carries forbidden_tools", () => { + // On the delivery tools the neighbour route is the PAID mistake + // (leadbay_pull_leads instead of a net-new search, bulk_qualify instead of + // the supplied-list path), so the false-positive signal is the point of + // these fixtures rather than an optional extra. + const bare = LEAD_DELIVERY_ROUTING_FIXTURES.filter( + (f) => !f.forbidden_tools?.length, + ).map((f) => f.intent); + expect(bare, bare.join("\n")).toEqual([]); + }); +}); diff --git a/packages/mcp/test/eval/tool-descriptions/lead-delivery-routing-fixtures.ts b/packages/mcp/test/eval/tool-descriptions/lead-delivery-routing-fixtures.ts new file mode 100644 index 00000000..65fd213b --- /dev/null +++ b/packages/mcp/test/eval/tool-descriptions/lead-delivery-routing-fixtures.ts @@ -0,0 +1,85 @@ +/** + * Tool-routing classifier fixtures for the MCP-first delivery tools. + * + * Same (intent, expected-tool, forbidden-tools) shape as `ROUTING_FIXTURES`, + * kept in its own module so the established fixture file stays untouched. + * + * These three tools are the ones most at risk of misrouting, because a + * plausible-looking older tool exists for every one of them: + * + * leadbay_find_new_leads vs leadbay_pull_leads (today's batch from an + * existing lens — NOT a net-new search) + * leadbay_qualify_leads vs leadbay_bulk_qualify_leads / import_leads + * (the in-account batch path, not a supplied list) + * leadbay_lead_job_status vs the three other *_status pollers + * + * The `forbidden_tools` entries are therefore the load-bearing half: routing + * to the neighbour is the failure mode, and on the delivery tools it is a + * PAID one. + * + * Note the deliberate split of duties: the live classifier eval (Sonnet with + * the tool catalog bound) is what measures whether the intents actually + * route, and it needs an API key. `lead-delivery-routing-fixtures.test.ts` + * covers what a deterministic run can prove — that every tool named here is + * real and that no fixture forbids the tool it expects — so a rename or typo + * cannot rot these silently while no eval is running. + */ + +import type { RoutingFixture } from "./routing-fixtures.js"; + +export const LEAD_DELIVERY_ROUTING_FIXTURES: RoutingFixture[] = [ + { + intent: + "Find me 10 gyms around Dallas that would buy our flooring, with someone I can call.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_pull_leads", "leadbay_extend_lens"], + }, + { + intent: + "Get me 20 brand-new prospects that look like our best customer, with the VP People's email.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_pull_leads", "leadbay_enrich_titles"], + }, + { + intent: "We're entering the Lyon market — find 15 hotels that fit our ICP.", + expected_tool: "leadbay_find_new_leads", + forbidden_tools: ["leadbay_new_lens"], + }, + { + intent: + "Here are 60 restaurant websites from my Austin sweep — which fit our profile, and who's the owner at each?", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: [ + "leadbay_find_new_leads", + "leadbay_bulk_qualify_leads", + "leadbay_import_leads", + ], + }, + { + intent: + "Re-qualify everything you delivered last week and get phone numbers for the good ones.", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: ["leadbay_bulk_qualify_leads"], + }, + { + intent: + "Vet these 12 accounts from my spreadsheet against our qualification criteria.", + expected_tool: "leadbay_qualify_leads", + forbidden_tools: ["leadbay_import_leads", "leadbay_bulk_qualify_leads"], + }, + { + intent: "Any results yet from that lead search job you started?", + expected_tool: "leadbay_lead_job_status", + forbidden_tools: [ + "leadbay_bulk_enrich_status", + "leadbay_import_status", + "leadbay_qualify_status", + ], + }, + { + intent: + "Wait for the lead delivery job to finish and show me everything it found.", + expected_tool: "leadbay_lead_job_status", + forbidden_tools: ["leadbay_bulk_enrich_status"], + }, +]; diff --git a/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts b/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts index 60f5d0f8..d9dc38db 100644 --- a/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts +++ b/packages/mcp/test/eval/tool-descriptions/routing-fixtures.ts @@ -155,46 +155,4 @@ export const ROUTING_FIXTURES: RoutingFixture[] = [ { intent: "Remove the epilogue entry for this lead.", expected_tool: "leadbay_remove_epilogue" }, { intent: "Preview the cost of bulk enrichment before launching.", expected_tool: "leadbay_preview_bulk_enrichment" }, { intent: "Launch the bulk enrichment job.", expected_tool: "leadbay_launch_bulk_enrichment" }, - - // MCP-first lead delivery (net-new search / batch qualify / job poll) - { - intent: "Find me 10 gyms around Dallas that would buy our flooring, with someone I can call.", - expected_tool: "leadbay_find_new_leads", - forbidden_tools: ["leadbay_pull_leads", "leadbay_extend_lens"], - }, - { - intent: "Get me 20 brand-new prospects that look like our best customer, with the VP People's email.", - expected_tool: "leadbay_find_new_leads", - forbidden_tools: ["leadbay_pull_leads", "leadbay_enrich_titles"], - }, - { - intent: "We're entering the Lyon market — find 15 hotels that fit our ICP.", - expected_tool: "leadbay_find_new_leads", - forbidden_tools: ["leadbay_new_lens"], - }, - { - intent: "Here are 60 restaurant websites from my Austin sweep — which fit our profile, and who's the owner at each?", - expected_tool: "leadbay_qualify_leads", - forbidden_tools: ["leadbay_find_new_leads", "leadbay_bulk_qualify_leads", "leadbay_import_leads"], - }, - { - intent: "Re-qualify everything you delivered last week and get phone numbers for the good ones.", - expected_tool: "leadbay_qualify_leads", - forbidden_tools: ["leadbay_bulk_qualify_leads"], - }, - { - intent: "Vet these 12 accounts from my spreadsheet against our qualification criteria.", - expected_tool: "leadbay_qualify_leads", - forbidden_tools: ["leadbay_import_leads", "leadbay_bulk_qualify_leads"], - }, - { - intent: "Any results yet from that lead search job you started?", - expected_tool: "leadbay_lead_job_status", - forbidden_tools: ["leadbay_bulk_enrich_status", "leadbay_import_status", "leadbay_qualify_status"], - }, - { - intent: "Wait for the lead delivery job to finish and show me everything it found.", - expected_tool: "leadbay_lead_job_status", - forbidden_tools: ["leadbay_bulk_enrich_status"], - }, ]; From bb9fdd97fe3baa7ac02c14406e6aacdf39a20cf6 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 15:11:48 +0200 Subject: [PATCH 32/46] fix(mcp): restore workflows.test.ts, enable the full registry in config instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] Third instance of the same rule on this PR. The audit had been widened to import mcpFirstDeliveryAllTools and listAllPrompts so the new WORKFLOWS.md rows would resolve. Restored byte-for-byte. The fix is configuration, not a wider import. workflows.test.ts asks "does every backtick'd leadbay_* name resolve to something that exists?" — a question about the REGISTRY, not about one deployment's exposure. So the mcp test run now sets LEADBAY_MCP_LEAD_DELIVERY=1 in vitest.config.ts and the established audit reads the full registry unchanged. A rollout flag should not be able to make WORKFLOWS.md look like it names tools that were never built. The seams this needs already existed — mcpFirstDeliveryAllTools is even documented as "the ungated registry the contract audits read". Only the audit's own imports needed to stay put. Checked rather than assumed: - The two tests that exercise the gate itself (prompt-release-gate.test.ts, mcp-first-delivery-gate.test.ts) set and DELETE the variable themselves and restore the previous value, so the new default cannot mask them. Their "hidden by default" cases still pass. - The audit still bites: appending a row naming `leadbay_does_not_exist` to WORKFLOWS.md fails it with that exact identifier. It passes because the names are real, not because the flag silenced it. Gates: 1532 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- packages/mcp/test/audit/workflows.test.ts | 9 ++------- packages/mcp/vitest.config.ts | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/mcp/test/audit/workflows.test.ts b/packages/mcp/test/audit/workflows.test.ts index 050bffb7..4cc72d38 100644 --- a/packages/mcp/test/audit/workflows.test.ts +++ b/packages/mcp/test/audit/workflows.test.ts @@ -20,9 +20,8 @@ import { compositeWriteTools, granularReadTools, granularWriteTools, - mcpFirstDeliveryAllTools, } from "@leadbay/core"; -import { listAllPrompts } from "../../src/prompts.js"; +import { listPrompts } from "../../src/prompts.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -35,13 +34,9 @@ const KNOWN_TOOLS = new Set( ...compositeWriteTools, ...granularReadTools, ...granularWriteTools, - // Gated off by default until the backend routes ship; still registered. - ...mcpFirstDeliveryAllTools, ].map((t) => t.name), ); -// listAllPrompts: release-gated prompts are still registered, and a rollout -// flag must not make WORKFLOWS.md look like it names a nonexistent prompt. -const KNOWN_PROMPTS = new Set(listAllPrompts().map((p) => p.name)); +const KNOWN_PROMPTS = new Set(listPrompts().map((p) => p.name)); // Skills shipped via the Claude Code plugin marketplace. Each is a // directory under .claude-plugin/plugins/leadbay/skills//SKILL.md; diff --git a/packages/mcp/vitest.config.ts b/packages/mcp/vitest.config.ts index 9c2e1292..84712f76 100644 --- a/packages/mcp/vitest.config.ts +++ b/packages/mcp/vitest.config.ts @@ -11,5 +11,20 @@ export default defineConfig({ environment: "node", include: ["test/**/*.test.ts"], exclude: ["test/smoke/**", "node_modules", "dist"], + env: { + // Run the contract audits against the FULL registry. WORKFLOWS.md is a + // map of what Leadbay can do, so `workflows.test.ts` asks "does every + // leadbay_* name resolve to something that exists?" — a question about + // the registry, not about one deployment's exposure. With the flag off, + // the delivery tools are absent from the exported arrays and that audit + // would read documented, registered tools as typos. + // + // Set here rather than by widening the audit's imports: the audit is an + // established test file, and a rollout flag should not be able to make + // WORKFLOWS.md look wrong. Tests that exercise the gate itself + // (prompt-release-gate.test.ts, mcp-first-delivery-gate.test.ts) set and + // delete the variable themselves, so they are unaffected by this default. + LEADBAY_MCP_LEAD_DELIVERY: "1", + }, }, }); From 2a43db14ef404f8ffeb9f1c408922cfc48cc5264 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 15:15:08 +0200 Subject: [PATCH 33/46] fix(mcp): the last modified test file, and the doc that told us to modify it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth and final instance on this PR. routing-block.test.ts had the three delivery tools appended to TOOLS_WITH_ROUTING plus a widened ALL_TOOLS. Restored byte-for-byte; lead-delivery-routing-block.test.ts now asserts the same contract for those three — WHEN TO USE inside the 600-char window, trigger phrases, memory pointer, >=2 positive AND >=2 negative examples. The ALL_TOOLS hunk turned out to be unnecessary anyway: with the delivery flag now set in vitest.config.ts, the exported arrays already carry those tools. And route_to resolution needed nothing — that check iterates every registered tool rather than only the listed ones, so it covered them all along. This one was not the author's mistake. CLAUDE.md said, twice, "add the tool name to TOOLS_WITH_ROUTING in the routing audit", and AGENTS.md repeated it three bullets below the rule forbidding exactly that. Following the documented process produced the violation. Both docs now say to assert the contract in a new audit file and point at the worked example, so the next tool does not re-earn this review. With this, every test-path change on the PR is a new file — no established test is touched. Gates: 1532 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 11 ++- CLAUDE.md | 17 +++- .../audit/lead-delivery-routing-block.test.ts | 95 +++++++++++++++++++ packages/mcp/test/audit/routing-block.test.ts | 6 -- 4 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 packages/mcp/test/audit/lead-delivery-routing-block.test.ts diff --git a/AGENTS.md b/AGENTS.md index 04a3a8fb..21986a4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,10 +34,13 @@ high-priority (P0/P1) issues: - **New user-facing tools must be wired completely.** They must declare `routing` and `rendering_hint` frontmatter (plus `next_steps` when the - tool has a NEXT STEPS table), be added to `TOOLS_WITH_ROUTING` in - `packages/mcp/test/audit/routing-block.test.ts`, and carry ≥2 positive - AND ≥2 negative routing examples. Every `route_to` anti-trigger must - resolve to a registered tool name. + tool has a NEXT STEPS table), have the routing contract asserted for + them in a NEW audit file (never by appending to `TOOLS_WITH_ROUTING` in + the existing `packages/mcp/test/audit/routing-block.test.ts` — the + new-coverage-in-new-files rule above wins; see + `packages/mcp/test/audit/lead-delivery-routing-block.test.ts`), and + carry ≥2 positive AND ≥2 negative routing examples. Every `route_to` + anti-trigger must resolve to a registered tool name. - **`WORKFLOWS.md` is normative.** A new user story needs a row, and every backtick-wrapped `leadbay_*` identifier must resolve to a diff --git a/CLAUDE.md b/CLAUDE.md index 08cc9749..057772a3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,8 +218,16 @@ next_steps: pull-leads # snippets/next-steps/pull-leads.md description over, trim the BODY — never disable the audit. When you add a new user-facing tool: declare `routing`, -`rendering_hint`, `next_steps`; add the tool name to -`TOOLS_WITH_ROUTING` in the routing audit; run `pnpm -r test`. +`rendering_hint`, `next_steps`; assert the routing contract for it; +run `pnpm -r test`. + +**Do not append to `TOOLS_WITH_ROUTING` in +`packages/mcp/test/audit/routing-block.test.ts`** — that is an existing +test file, and the "new coverage lives in new files" rule wins. Add a new +audit file asserting the same contract for your tool instead; +`test/audit/lead-delivery-routing-block.test.ts` is the worked example. +The `route_to` resolution check needs nothing: it already iterates every +registered tool. ## Tool descriptions are generated, not hand-edited @@ -426,7 +434,10 @@ pnpm -r typecheck # must stay green If `prompts:build` fails: check `name:` matches filename, `kind: tool-description` is set, every `{{include:...}}` resolves, and `route_to:` values match registered tool names. -Also: add the tool name to `TOOLS_WITH_ROUTING` in `packages/mcp/test/audit/routing-block.test.ts`. +Also: assert the routing contract for the new tool in a **new** audit file — do +not append its name to `TOOLS_WITH_ROUTING` in the existing +`packages/mcp/test/audit/routing-block.test.ts`. See +`packages/mcp/test/audit/lead-delivery-routing-block.test.ts`. ## Composite vs granular diff --git a/packages/mcp/test/audit/lead-delivery-routing-block.test.ts b/packages/mcp/test/audit/lead-delivery-routing-block.test.ts new file mode 100644 index 00000000..dd4b3356 --- /dev/null +++ b/packages/mcp/test/audit/lead-delivery-routing-block.test.ts @@ -0,0 +1,95 @@ +/** + * Audit: the MCP-first delivery tools carry a usable routing block. + * + * Same contract `routing-block.test.ts` enforces for the tools listed in its + * TOOLS_WITH_ROUTING set — `## WHEN TO USE` inside the 600-char window every + * chat host loads even when truncating, trigger phrases, the shared memory + * pointer, and ≥2 positive AND ≥2 negative examples. Asserted here instead of + * by appending three names to that established set, so the audit file itself + * stays unchanged. + * + * This matters more for these three than for most tools. Each has a + * plausible-looking older neighbour (`leadbay_pull_leads`, + * `leadbay_bulk_qualify_leads`, the other `*_status` pollers), and two of the + * three SPEND. A truncating host that never sees the routing block routes to + * the neighbour, which is a wrong answer on the read path and a wrong charge + * on the write path. + * + * Not re-asserted here: `anti_triggers[].route_to` resolution. That check in + * routing-block.test.ts iterates every registered tool rather than only the + * listed ones, so it already covers these three. + */ + +import { describe, it, expect } from "vitest"; +import { mcpFirstDeliveryAllTools, type Tool } from "@leadbay/core"; + +const ROUTING_HEAD_WINDOW = 600; +const EXAMPLE_WINDOW = 1500; +const MEMORY_POINTER = + "**Memory:** recall + capture via `leadbay_agent_memory_*` tools."; + +const POS_BLOCK_RE = + /Examples that SHOULD invoke this tool:\n([\s\S]+?)(?:\n\n|$)/; +const NEG_BLOCK_RE = + /Examples that should NOT invoke this tool[^:]*:\n([\s\S]+?)(?:\n\n|$)/; + +const DELIVERY_TOOLS: Tool[] = mcpFirstDeliveryAllTools; + +function countBullets(block: RegExpMatchArray | null): number { + if (!block) return 0; + return block[1].split("\n").filter((l) => l.trim().startsWith("- ")).length; +} + +describe("audit: routing block on the MCP-first delivery tools", () => { + it("covers all three delivery tools", () => { + // Guards the fixture: mcpFirstDeliveryAllTools is the ungated registry, so + // a tool dropped from it would silently shrink everything below. + expect(DELIVERY_TOOLS.map((t) => t.name).sort()).toEqual([ + "leadbay_find_new_leads", + "leadbay_lead_job_status", + "leadbay_qualify_leads", + ]); + }); + + it("each has WHEN TO USE in the first 600 chars", () => { + const violations = DELIVERY_TOOLS.filter( + (t) => !t.description.slice(0, ROUTING_HEAD_WINDOW).includes("## WHEN TO USE"), + ).map( + (t) => + `${t.name}: '## WHEN TO USE' missing from first ${ROUTING_HEAD_WINDOW} chars (description length ${t.description.length})`, + ); + expect(violations, violations.join("\n")).toEqual([]); + }); + + it("each lists at least one trigger phrase", () => { + const violations = DELIVERY_TOOLS.filter( + (t) => !t.description.slice(0, ROUTING_HEAD_WINDOW).match(/Trigger phrases: "/), + ).map((t) => `${t.name}: WHEN TO USE block has no trigger phrases`); + expect(violations, violations.join("\n")).toEqual([]); + }); + + it("each carries the shared agent-memory pointer in the first 600 chars", () => { + const violations = DELIVERY_TOOLS.filter( + (t) => !t.description.slice(0, ROUTING_HEAD_WINDOW).includes(MEMORY_POINTER), + ).map((t) => `${t.name}: missing memory pointer in first ${ROUTING_HEAD_WINDOW} chars`); + expect(violations, violations.join("\n")).toEqual([]); + }); + + it("each carries ≥2 positive AND ≥2 negative example messages", () => { + // The negatives are the load-bearing half on these three: every one of + // them sounds like the neighbour it must not route to. + const violations: string[] = []; + for (const t of DELIVERY_TOOLS) { + const head = t.description.slice(0, EXAMPLE_WINDOW); + const posCount = countBullets(head.match(POS_BLOCK_RE)); + const negCount = countBullets(head.match(NEG_BLOCK_RE)); + if (posCount < 2) { + violations.push(`${t.name}: only ${posCount} positive example(s) (need ≥2)`); + } + if (negCount < 2) { + violations.push(`${t.name}: only ${negCount} negative example(s) (need ≥2)`); + } + } + expect(violations, violations.join("\n")).toEqual([]); + }); +}); diff --git a/packages/mcp/test/audit/routing-block.test.ts b/packages/mcp/test/audit/routing-block.test.ts index d4f81c7b..cd2fe3f2 100644 --- a/packages/mcp/test/audit/routing-block.test.ts +++ b/packages/mcp/test/audit/routing-block.test.ts @@ -18,7 +18,6 @@ import { compositeWriteTools, granularReadTools, granularWriteTools, - mcpFirstDeliveryAllTools, type Tool, } from "@leadbay/core"; @@ -29,9 +28,6 @@ const MEMORY_POINTER = "**Memory:** recall + capture via `leadbay_agent_memory_* // The audit ensures none of them regresses. Other tools MAY add routing // later — when they do, append here. const TOOLS_WITH_ROUTING = new Set([ - "leadbay_find_new_leads", - "leadbay_qualify_leads", - "leadbay_lead_job_status", "leadbay_prepare_outreach", "leadbay_account_status", "leadbay_add_leads_to_campaign", @@ -79,8 +75,6 @@ const ALL_TOOLS: Tool[] = [ ...compositeWriteTools, ...granularReadTools, ...granularWriteTools, - // Gated off by default until the backend routes ship; still registered. - ...mcpFirstDeliveryAllTools, ]; const ALL_TOOL_NAMES = new Set(ALL_TOOLS.map((t) => t.name)); From fb38867ce4459216acbd7abccafb32795fecb082 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 15:42:49 +0200 Subject: [PATCH 34/46] fix(mcp): bound the top-up exclusion list, and stop claiming qualify enforces ad-hoc exclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] The workflow prompt said to pass every lead already SEEN into the shortfall top-up's exclude_lead_ids. That parameter caps at 500, and a paid run may examine up to exploration_cap's ceiling (min(20n, 1000)) — so on a wide run the instruction builds a list the backend refuses outright, killing the one call that exists to close a gap the user already paid toward. The prompt now sends only the examined-and-REJECTED ids, which is also the correct set: novelty:'org' already excludes prior deliveries, so delivered ids were the redundant half of the list all along. Dropping them is normally enough to fit under 500; past that, most recent 500. Belt as well as braces: rejectOversizedExclusions refuses an over-long list in code with the bounding rule attached, so the agent gets an actionable error instead of an opaque 400 arriving after the spend. It counts what would actually be SENT — canonicalIdSet dedupes, so a merely-repetitive list is not an overflow. [P2] The description claimed ad-hoc exclusions ("no chains") are "enforced by QUALIFICATION". They are not. qualify scores against the org's FROZEN questions and IBP, which need not mention chains, and the positive inverse in the seed only shifts ranking — so a violator can survive min_ai_score, be paid for, and be delivered. Now described honestly: no tier enforces it, post-filter before rendering, and leadbay_refine_prompt is what makes it durable. Trimmed to stay inside the 17000-char budget (16964). One correction to my own test while writing it: normalizeUuid passes any non-empty string through and only lowercases real uuids, so a malformed id still counts toward the cap. Asserting it was free would have let a list the backend refuses slip past the guard. Gates: 1544 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 25 +++++ packages/core/src/composite/find-new-leads.ts | 5 +- .../core/src/tool-descriptions.generated.ts | 11 ++- .../composite/exclude-lead-ids-cap.test.ts | 93 +++++++++++++++++++ packages/mcp/src/prompts.generated.ts | 13 ++- .../release-gated/leadbay_new_leads.md.tmpl | 13 ++- .../composite/find-new-leads.md.tmpl | 11 ++- 7 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 packages/core/test/unit/composite/exclude-lead-ids-cap.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 19948dc1..f6074b38 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -552,6 +552,31 @@ function countryKey(raw: string): string { .trim(); } +/** `exclude_lead_ids` accepts at most 500 ids. The shortfall top-up is where + * this bites: a paid run may examine up to `exploration_cap`'s ceiling + * (min(20n, 1000)) candidates, so "exclude everything already seen" overflows + * the cap and the backend refuses the whole call — the one call that exists + * to close a gap the user already paid toward. Reject it here, carrying the + * bounding rule, rather than letting an opaque 400 surface after the spend. + * + * `novelty: org` already excludes prior DELIVERIES, so delivered ids are the + * redundant half of the list and dropping them is usually enough to fit. */ +export const MAX_EXCLUDE_LEAD_IDS = 500; + +export function rejectOversizedExclusions(ids: unknown): void { + if (ids === undefined || ids === null) return; + // Count what would actually be SENT — canonicalIdSet drops non-uuids and + // dedupes, so a list that merely repeats itself is not a real overflow. + const unique = canonicalIdSet(ids); + if (unique.length <= MAX_EXCLUDE_LEAD_IDS) return; + throw { + error: true, + code: "TOO_MANY_EXCLUSIONS", + message: `exclude_lead_ids carries ${unique.length} ids — the maximum is ${MAX_EXCLUDE_LEAD_IDS}.`, + hint: "Drop the DELIVERED ids first: novelty:'org' already excludes those. Send the examined-but-rejected ones (disqualified + skipped), most recent first, capped at 500.", + }; +} + export function rejectCountryLocations(locations: unknown): void { if (locations === undefined || locations === null) return; // The server does not validate the schema before dispatch, so an agent can diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 40d55061..24772b4c 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -23,6 +23,7 @@ import { compactBody, normalizeSearchFilters, rejectCountryLocations, + rejectOversizedExclusions, splitItems, TERMINAL_JOB_STATES, waitForJob, @@ -168,7 +169,8 @@ export const findNewLeads: Tool = { exclude_lead_ids: { type: "array", items: { type: "string" }, - description: "Caller-side novelty belt on top of the server-side one (max 500 ids).", + description: + "Caller-side novelty belt on top of the server-side one (max 500 ids — over that the call is refused, so drop the DELIVERED ids first: novelty:'org' already covers those, and the examined-but-rejected ones are what it misses).", }, novelty: { type: "string", @@ -224,6 +226,7 @@ export const findNewLeads: Tool = { "exclude_lead_ids", ]); rejectCountryLocations(params.filters?.locations); + rejectOversizedExclusions(params.exclude_lead_ids); // Same spend gate as leadbay_qualify_leads. The trigger differs: `qualify` // defaults to FALSE here, so the default ask really is free and only an diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 1a82374a..5cae23ef 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1324,11 +1324,12 @@ phone 250c, success-only). The gate is enforced in code: a paid call **Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an off-profile seed is free, exploring it with \`qualify: true\` is not. -**Exclusions ("no chains") are enforced by QUALIFICATION, not the free -match** — \`filters\` has no exclusion key. Put the positive inverse in the -seed, then \`qualify: true\`: IBP anti-patterns score violators negative and -\`min_ai_score: 0\` drops them. Durable exclusion → \`leadbay_refine_prompt\`. -In a FREE preview, drop violating rows and say the tier doesn't enforce. +**Ad-hoc exclusions ("no chains") are enforced by NO tier** — \`filters\` has +no exclusion key, and \`qualify\` scores against the org's FROZEN questions and +IBP, which need not mention chains; the seed's positive inverse only shifts +ranking. Violators can survive, be paid for and be delivered — post-filter +them yourself and say the tier didn't enforce it. Durable enforcement → +\`leadbay_refine_prompt\`. ### Crafting the \`example_lead\` seed — the input that decides result quality diff --git a/packages/core/test/unit/composite/exclude-lead-ids-cap.test.ts b/packages/core/test/unit/composite/exclude-lead-ids-cap.test.ts new file mode 100644 index 00000000..245ede84 --- /dev/null +++ b/packages/core/test/unit/composite/exclude-lead-ids-cap.test.ts @@ -0,0 +1,93 @@ +/** + * `exclude_lead_ids` is capped at 500, and the shortfall top-up is exactly + * where that bites. + * + * The workflow prompt used to say "pass every lead already SEEN" into the + * top-up's exclusion list. A paid run may examine up to `exploration_cap`'s + * ceiling — min(20n, 1000) — so on a wide run that instruction builds a list + * the backend refuses outright, killing the one call that exists to close a + * gap the user has already paid toward. + * + * Two halves to the fix, both pinned here: the tool refuses an over-long list + * itself, carrying the bounding rule, instead of letting an opaque 400 land + * after the spend; and the rule it carries is actionable, because + * `novelty: "org"` already excludes prior DELIVERIES — dropping those is + * normally enough to fit under the cap. + */ + +import { describe, it, expect } from "vitest"; +import { + rejectOversizedExclusions, + MAX_EXCLUDE_LEAD_IDS, +} from "../../../src/composite/_mcp-job-helpers.js"; + +/** Distinct, well-formed uuids — canonicalIdSet drops anything else. */ +function ids(n: number, seed = 0): string[] { + return Array.from({ length: n }, (_, i) => { + const h = (seed * 100000 + i).toString(16).padStart(12, "0"); + return `7b3c1de2-5f40-4a9c-9d21-${h}`; + }); +} + +function rejects(value: unknown): boolean { + try { + rejectOversizedExclusions(value); + return false; + } catch (e) { + expect((e as { code?: string }).code).toBe("TOO_MANY_EXCLUSIONS"); + return true; + } +} + +describe("rejectOversizedExclusions", () => { + it("accepts a list exactly at the cap", () => { + expect(rejects(ids(MAX_EXCLUDE_LEAD_IDS))).toBe(false); + }); + + it("refuses one id over the cap", () => { + expect(rejects(ids(MAX_EXCLUDE_LEAD_IDS + 1))).toBe(true); + }); + + it("refuses the list a wide exploration_cap would build", () => { + // count:50 → exploration_cap ceiling min(20n, 1000) = 1000 candidates, + // so "exclude everything seen" is twice the cap. + expect(rejects(ids(1000))).toBe(true); + }); + + it("counts what would be SENT, not what was passed", () => { + // canonicalIdSet dedupes, so a list that merely repeats itself is not a + // real overflow and must not be refused. + const repeated = [...ids(MAX_EXCLUDE_LEAD_IDS), ...ids(MAX_EXCLUDE_LEAD_IDS)]; + expect(repeated.length).toBeGreaterThan(MAX_EXCLUDE_LEAD_IDS); + expect(rejects(repeated)).toBe(false); + }); + + it("drops blanks but still counts non-uuid entries", () => { + // normalizeUuid only lowercases actual uuids — it passes any other + // non-empty string through, so a malformed id is still SENT and still + // consumes cap. Only blanks disappear. Counting it as free would let a + // list the backend refuses slip past this guard. + expect(rejects([...ids(MAX_EXCLUDE_LEAD_IDS), "", " "])).toBe(false); + expect(rejects([...ids(MAX_EXCLUDE_LEAD_IDS), "not-a-uuid"])).toBe(true); + }); + + it("no-ops on absent exclusions", () => { + expect(rejects(undefined)).toBe(false); + expect(rejects(null)).toBe(false); + expect(rejects([])).toBe(false); + }); + + it("the refusal explains how to get under the cap", () => { + // An error the agent cannot act on just converts a backend 400 into a + // local one. It has to name the half of the list that is redundant. + try { + rejectOversizedExclusions(ids(1000)); + throw new Error("expected a refusal"); + } catch (e) { + const hint = (e as { hint?: string }).hint ?? ""; + expect(hint).toMatch(/novelty/i); + expect(hint).toMatch(/deliver/i); + expect((e as { message?: string }).message).toContain("1000"); + } + }); +}); diff --git a/packages/mcp/src/prompts.generated.ts b/packages/mcp/src/prompts.generated.ts index 3755ab6f..02653872 100644 --- a/packages/mcp/src/prompts.generated.ts +++ b/packages/mcp/src/prompts.generated.ts @@ -1584,11 +1584,14 @@ When I want qualification evidence and/or reachable contacts: search under a NEW \`request_id\`, quoted first like any paid run. Never silently hand back fewer than I asked for and paid toward. - Pass every lead already SEEN into that top-up's \`exclude_lead_ids\` — - delivered, disqualified and skipped alike, from both the preview and the - paid pass. \`novelty: org\` only excludes prior DELIVERIES, so a candidate - that was examined and rejected is still eligible; without the exclusion - list the top-up re-picks the same misses and charges again to close no gap. + Pass the leads already EXAMINED-AND-REJECTED into that top-up's + \`exclude_lead_ids\` — disqualified and skipped, from both the preview and + the paid pass. \`novelty: org\` already excludes prior DELIVERIES, so + delivered ids are redundant there; the rejected ones are exactly what it + misses, and without them the top-up re-picks the same misses and charges + again to close no gap. **\`exclude_lead_ids\` caps at 500** — a wide + \`exploration_cap\` can examine more than that, so send the most recent 500 + rejects rather than an over-long list the tool refuses outright. 3. While the job runs, poll with \`leadbay_lead_job_status\` (\`wait_seconds: 60\`); report progress, not silence. diff --git a/packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl b/packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl index b9c26aba..13316260 100644 --- a/packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl +++ b/packages/promptforge/prompts/release-gated/leadbay_new_leads.md.tmpl @@ -155,11 +155,14 @@ When I want qualification evidence and/or reachable contacts: search under a NEW `request_id`, quoted first like any paid run. Never silently hand back fewer than I asked for and paid toward. - Pass every lead already SEEN into that top-up's `exclude_lead_ids` — - delivered, disqualified and skipped alike, from both the preview and the - paid pass. `novelty: org` only excludes prior DELIVERIES, so a candidate - that was examined and rejected is still eligible; without the exclusion - list the top-up re-picks the same misses and charges again to close no gap. + Pass the leads already EXAMINED-AND-REJECTED into that top-up's + `exclude_lead_ids` — disqualified and skipped, from both the preview and + the paid pass. `novelty: org` already excludes prior DELIVERIES, so + delivered ids are redundant there; the rejected ones are exactly what it + misses, and without them the top-up re-picks the same misses and charges + again to close no gap. **`exclude_lead_ids` caps at 500** — a wide + `exploration_cap` can examine more than that, so send the most recent 500 + rejects rather than an over-long list the tool refuses outright. 3. While the job runs, poll with `leadbay_lead_job_status` (`wait_seconds: 60`); report progress, not silence. diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 21a93fe5..33e60b1d 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -78,11 +78,12 @@ phone 250c, success-only). The gate is enforced in code: a paid call **Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an off-profile seed is free, exploring it with `qualify: true` is not. -**Exclusions ("no chains") are enforced by QUALIFICATION, not the free -match** — `filters` has no exclusion key. Put the positive inverse in the -seed, then `qualify: true`: IBP anti-patterns score violators negative and -`min_ai_score: 0` drops them. Durable exclusion → `leadbay_refine_prompt`. -In a FREE preview, drop violating rows and say the tier doesn't enforce. +**Ad-hoc exclusions ("no chains") are enforced by NO tier** — `filters` has +no exclusion key, and `qualify` scores against the org's FROZEN questions and +IBP, which need not mention chains; the seed's positive inverse only shifts +ranking. Violators can survive, be paid for and be delivered — post-filter +them yourself and say the tier didn't enforce it. Durable enforcement → +`leadbay_refine_prompt`. {{include:heuristics/example-lead-craft}} From bc4d41db0a21d3ac37e58e9b3510b848a020e187 Mon Sep 17 00:00:00 2001 From: Arty Date: Mon, 17 Aug 2026 16:02:13 +0200 Subject: [PATCH 35/46] =?UTF-8?q?fix(mcp):=20four=20Codex=20findings=20?= =?UTF-8?q?=E2=80=94=20wire/guard=20mismatch,=20region=20scope,=20abortabl?= =?UTF-8?q?e=20poll,=20long-runners?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] The cap guard counted canonicalIdSet(exclude_lead_ids) while the submit body posted the raw array. A 600-entry list that dedupes to 400 therefore cleared the local guard and was refused by the backend anyway — on the top-up call the user had already paid toward. My own regression from last round. The body now posts the canonical list, which is also what the idempotency key was derived from, so all three finally agree. Kept undefined when absent so compactBody drops the key instead of sending []. [P2] The country-guard exemptions were process-wide. Georgia is a US state and must survive there, but on a FRANCE account it can only mean the country; the French overseas regions are the exact mirror. A global set gave each a free pass on the wrong side and let it reach the backend. Exemptions are now keyed by client.region. Unknown region falls back to the union, deliberately: without a region we cannot tell a state fence from a foreign country, and wrongly REJECTING a correct search is the louder failure. [P2] The poll slept on a bare setTimeout(4000) and only re-checked ctx.signal.aborted after it fired, so a cancel mid-sleep waited out the full interval — twice the "exits within <=2 seconds" bound the server advertises in its own instructions. sleepUnlessAborted races the timer against the signal and removes its listener either way, so a 180s wait loop cannot accumulate one listener per poll and trip Node's warning. [P2] The longRunners list still named only the legacy bulk tools, so nothing told clients to send a progressToken for calls that block-poll 45-180s: the per-poll ctx.progress was normally absent and the call looked frozen. All three added, behind the same .filter(has) that keeps the iter-12 invariant — a deployment without the delivery flag never sees them named. Gates: 1560 tests, typecheck, prompts:check all green. Every test-path change on the PR is still a new file. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 98 +++++++++++++++---- packages/core/src/composite/find-new-leads.ts | 11 ++- .../country-guard-region-scope.test.ts | 71 ++++++++++++++ .../composite/exclude-lead-ids-wire.test.ts | 89 +++++++++++++++++ .../composite/job-poll-cancellation.test.ts | 74 ++++++++++++++ packages/mcp/src/server.ts | 8 ++ .../test/audit/delivery-long-runners.test.ts | 65 ++++++++++++ 7 files changed, 393 insertions(+), 23 deletions(-) create mode 100644 packages/core/test/unit/composite/country-guard-region-scope.test.ts create mode 100644 packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts create mode 100644 packages/core/test/unit/composite/job-poll-cancellation.test.ts create mode 100644 packages/mcp/test/audit/delivery-long-runners.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index f6074b38..740e3ed7 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -184,6 +184,30 @@ export async function collectJobSnapshot( return { ...page, items, next_since: cursor }; } +/** Sleep, but wake immediately if the request is cancelled. + * + * A bare `setTimeout` cannot observe the signal until it fires, so a cancel + * landing just after a poll waited out the FULL 4s interval — while the + * server's own instructions promise the polling loop exits "within ≤2 + * seconds". Racing the timer against `abort` keeps that promise, and the + * listener is removed either way so a long wait loop cannot accumulate one + * listener per poll. */ +export function sleepUnlessAborted( + ms: number, + signal?: AbortSignal +): Promise { + if (signal?.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", done); + resolve(); + }; + const timer = setTimeout(done, ms); + signal?.addEventListener("abort", done, { once: true }); + }); +} + /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). * Fires ctx.progress per poll and respects ctx.signal cancellation. * `since`/`limit` are forwarded to every snapshot so a caller that block-waits @@ -210,8 +234,9 @@ export async function waitForJob( // bound by most of an interval (MCP clients time calls out). const remainingMs = waitSeconds * 1000 - (Date.now() - startedAt); if (remainingMs <= 0) break; - await new Promise((resolve) => - setTimeout(resolve, Math.min(MCP_JOB_POLL.intervalMs, remainingMs)) + await sleepUnlessAborted( + Math.min(MCP_JOB_POLL.intervalMs, remainingMs), + ctx?.signal ); if (ctx?.signal?.aborted) break; snap = await collectJobSnapshot(client, jobId, since, limit); @@ -461,33 +486,51 @@ const COUNTRY_ALIASES = [ "republique francaise", ]; -/** Country names that are ALSO a legitimate administrative fence inside a - * Leadbay universe, so rejecting them would break a correct search: +/** Country names that are ALSO a legitimate administrative fence — but only + * inside ONE universe, which is why these are keyed by region rather than + * subtracted globally. Each universe is single-country, so a name that is a + * state in the US universe is nothing but a country in the French one: * * - `Georgia` is a US state before it is a country, and one of the most - * common state fences a US account will ask for. + * common state fences a US account will ask for. On a FRANCE account it + * can only mean the country, and must still be rejected. * - The French overseas regions and collectivities each carry their own * ISO 3166-1 entry, so a comprehensive country list swallows every one of * them — while "leads in Martinique" is exactly the kind of regional fence - * this parameter exists for. + * this parameter exists for. On a US account they are foreign countries. * * Municipality collisions (`Lebanon`, `Peru`, `Mexico`, … are all US town * names) are deliberately NOT exempted: a bare town name identical to a * country is genuinely ambiguous, and the rejection is loud and recoverable — * `Lebanon, Kentucky` folds to a two-word key that never matches. A silent * fence to one village is the failure this guard exists to prevent. */ -const SUBNATIONAL_EXEMPTIONS = new Set( - [ - "georgia", "georgie", - "guadeloupe", "martinique", "reunion", "mayotte", - "french guiana", "guyane francaise", - "new caledonia", "nouvelle caledonie", - "french polynesia", "polynesie francaise", - "saint martin", "saint barthelemy", "saint pierre and miquelon", - "saint pierre et miquelon", "wallis and futuna", "wallis et futuna", - ].map(countryKey), +const SUBNATIONAL_EXEMPTIONS: Record> = { + us: new Set(["georgia", "georgie"].map(countryKey)), + fr: new Set( + [ + "guadeloupe", "martinique", "reunion", "mayotte", + "french guiana", "guyane francaise", + "new caledonia", "nouvelle caledonie", + "french polynesia", "polynesie francaise", + "saint martin", "saint barthelemy", "saint pierre and miquelon", + "saint pierre et miquelon", "wallis and futuna", "wallis et futuna", + ].map(countryKey), + ), +}; + +/** Union of every region's exemptions — the fallback when the caller's region + * is unknown. Deliberately permissive: without a region we cannot tell a + * legitimate state fence from a foreign country, and wrongly REJECTING a + * correct search is the louder failure. */ +const ALL_EXEMPTIONS: ReadonlySet = new Set( + Object.values(SUBNATIONAL_EXEMPTIONS).flatMap((s) => [...s]), ); +function exemptionsFor(region?: string): ReadonlySet { + const key = typeof region === "string" ? region.trim().toLowerCase() : ""; + return SUBNATIONAL_EXEMPTIONS[key] ?? ALL_EXEMPTIONS; +} + /** Every ISO 3166-1 country name, in English and French, folded to the same * comparison key as the input. Built from `Intl.DisplayNames` rather than a * hand-kept list: the two-country allowlist this replaces let `Canada`, @@ -520,7 +563,8 @@ function buildCountryLocationValues(): Set { // the explicit aliases keeps the originally-observed failure covered // rather than throwing at import time. } - for (const exempt of SUBNATIONAL_EXEMPTIONS) values.delete(exempt); + // Exemptions are NOT subtracted here — they are region-scoped and applied + // per call. Baking them in would exempt a French region on a US account. return values; } @@ -565,8 +609,10 @@ export const MAX_EXCLUDE_LEAD_IDS = 500; export function rejectOversizedExclusions(ids: unknown): void { if (ids === undefined || ids === null) return; - // Count what would actually be SENT — canonicalIdSet drops non-uuids and - // dedupes, so a list that merely repeats itself is not a real overflow. + // Count what would actually be SENT. That is only true because the submit + // body posts `canonicalIdSet(exclude_lead_ids)` too — counting the canonical + // list while wiring the raw one would clear a 600-entry array that dedupes + // to 400 and then let the backend refuse it anyway. const unique = canonicalIdSet(ids); if (unique.length <= MAX_EXCLUDE_LEAD_IDS) return; throw { @@ -577,8 +623,16 @@ export function rejectOversizedExclusions(ids: unknown): void { }; } -export function rejectCountryLocations(locations: unknown): void { +export function rejectCountryLocations( + locations: unknown, + region?: string +): void { if (locations === undefined || locations === null) return; + // Exemptions depend on WHICH universe is asking: `Georgia` is a state on a + // US account and nothing but a country on a French one, and the French + // overseas regions are the mirror image. A process-wide exemption set let + // each one bypass the guard on the wrong side and reach the backend. + const exempt = exemptionsFor(region); // The server does not validate the schema before dispatch, so an agent can // send `filters.locations` as a bare string. Treating a non-array as "no // locations" let a scalar "United States" sail past the guard and reach the @@ -586,7 +640,9 @@ export function rejectCountryLocations(locations: unknown): void { // exists to stop. Normalize to a one-item list instead of returning. const list = Array.isArray(locations) ? locations : [locations]; for (const loc of list) { - if (typeof loc === "string" && COUNTRY_LOCATION_VALUES.has(countryKey(loc))) { + if (typeof loc !== "string") continue; + const key = countryKey(loc); + if (!exempt.has(key) && COUNTRY_LOCATION_VALUES.has(key)) { throw { error: true, code: "COUNTRY_LEVEL_LOCATION", diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 24772b4c..70eeec20 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -225,7 +225,7 @@ export const findNewLeads: Tool = { "channels", "exclude_lead_ids", ]); - rejectCountryLocations(params.filters?.locations); + rejectCountryLocations(params.filters?.locations, client.region); rejectOversizedExclusions(params.exclude_lead_ids); // Same spend gate as leadbay_qualify_leads. The trigger differs: `qualify` @@ -301,7 +301,14 @@ export const findNewLeads: Tool = { contact_titles: params.contact_titles, title_gate: params.title_gate, channels: params.channels, - exclude_lead_ids: params.exclude_lead_ids, + // Wire the SAME list the cap guard counted and the idempotency key was + // derived from. Posting the raw array instead let a 600-entry list that + // dedupes to 400 clear the guard and still be refused by the backend. + // Kept undefined when absent so compactBody drops it rather than + // sending an empty array. + exclude_lead_ids: params.exclude_lead_ids + ? canonicalIdSet(params.exclude_lead_ids) + : undefined, novelty: params.novelty, max_cost: params.max_cost, exploration_cap: params.exploration_cap, diff --git a/packages/core/test/unit/composite/country-guard-region-scope.test.ts b/packages/core/test/unit/composite/country-guard-region-scope.test.ts new file mode 100644 index 00000000..d7d762d7 --- /dev/null +++ b/packages/core/test/unit/composite/country-guard-region-scope.test.ts @@ -0,0 +1,71 @@ +/** + * The country-guard exemptions are per-universe, not process-wide. + * + * `Georgia` has to survive the guard on a US account — it is a state there, + * and a common one to fence on. It must NOT survive on a France account, + * where it can only mean the country and would hit the same arbitrary-locality + * fencing the guard exists to prevent. The French overseas regions are the + * exact mirror: legitimate fences on a France account, foreign countries on a + * US one. + * + * A single global exemption set gave each of them a free pass on the wrong + * side of that line. + */ + +import { describe, it, expect } from "vitest"; +import { rejectCountryLocations } from "../../../src/composite/_mcp-job-helpers.js"; + +function rejects(value: unknown, region?: string): boolean { + try { + rejectCountryLocations(value, region); + return false; + } catch (e) { + expect((e as { code?: string }).code).toBe("COUNTRY_LEVEL_LOCATION"); + return true; + } +} + +describe("rejectCountryLocations — region-scoped exemptions", () => { + it("Georgia is a state on US and a country on FR", () => { + expect(rejects(["Georgia"], "us")).toBe(false); + expect(rejects(["Georgia"], "fr")).toBe(true); + expect(rejects(["Géorgie"], "fr")).toBe(true); + }); + + it("the French overseas regions are fences on FR and countries on US", () => { + for (const v of ["Martinique", "Guadeloupe", "La Réunion", "Mayotte"]) { + expect(rejects([v], "fr"), `${v} on fr`).toBe(false); + expect(rejects([v], "us"), `${v} on us`).toBe(true); + } + }); + + it("an unknown region falls back to permissive — never a false rejection", () => { + // Without a region we cannot tell a state fence from a foreign country. + // Wrongly rejecting a correct search is the louder failure, so the union + // of exemptions applies. + expect(rejects(["Georgia"], undefined)).toBe(false); + expect(rejects(["Martinique"], "")).toBe(false); + expect(rejects(["Georgia"], "zz")).toBe(false); + }); + + it("region casing and padding do not change the verdict", () => { + expect(rejects(["Georgia"], " US ")).toBe(false); + expect(rejects(["Georgia"], "FR")).toBe(true); + }); + + it("real countries are still rejected in every region", () => { + for (const region of ["us", "fr", undefined]) { + for (const v of ["Canada", "Germany", "United Kingdom", "France"]) { + expect(rejects([v], region), `${v} on ${region}`).toBe(true); + } + } + }); + + it("ordinary cities and states survive in every region", () => { + for (const region of ["us", "fr", undefined]) { + for (const v of ["Austin", "Texas", "Lyon", "Nouvelle-Aquitaine"]) { + expect(rejects([v], region), `${v} on ${region}`).toBe(false); + } + } + }); +}); diff --git a/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts b/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts new file mode 100644 index 00000000..b9009f84 --- /dev/null +++ b/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts @@ -0,0 +1,89 @@ +/** + * The exclusion list that is COUNTED is the exclusion list that is SENT. + * + * `rejectOversizedExclusions` counts `canonicalIdSet(ids)` — deduped and + * UUID-folded — while the submit body used to post `params.exclude_lead_ids` + * raw. A 600-entry array that collapses to 400 therefore cleared the local + * guard and was still refused by the backend, on the top-up call the user had + * already paid toward. + * + * The body now posts the canonical list, which also matches what the + * idempotency key was derived from. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +const UUID_A = "7b3c1de2-5f40-4a9c-9d21-0c8ea4f61b55"; +const UUID_B = "9f2e8a10-3c77-4b6d-8e12-5a4b7c9d0e33"; + +beforeEach(() => resetHttpMock()); + +async function submittedBody(extra: Record): Promise { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await findNewLeads.execute(newClient(), { + example_lead: { description: "independent gym, 2 sites" }, + count: 5, + request_id: "wire-test", + wait_seconds: 0, + ...extra, + } as any); + const post = getHttpRequests().find( + (r) => r.method === "POST" && r.path.endsWith("/mcp/search") + ); + return JSON.parse(post!.body ?? "{}"); +} + +describe("leadbay_find_new_leads — exclude_lead_ids on the wire", () => { + it("posts the canonical list, not the raw array", async () => { + const body = await submittedBody({ + exclude_lead_ids: [UUID_A, UUID_A.toUpperCase(), UUID_B, " "], + }); + // Deduped case-insensitively, blanks dropped — exactly what the cap guard + // counted. + expect(body.exclude_lead_ids).toEqual([UUID_A, UUID_B].sort()); + }); + + it("a list that only repeats itself is sent deduped, not refused", async () => { + const many = Array.from({ length: 600 }, () => UUID_A); + const body = await submittedBody({ exclude_lead_ids: many }); + expect(body.exclude_lead_ids).toEqual([UUID_A]); + }); + + it("omits the key entirely when no exclusions were given", async () => { + // compactBody drops undefined; sending [] instead would be a different + // request body for the same ask. + const body = await submittedBody({}); + expect("exclude_lead_ids" in body).toBe(false); + }); +}); diff --git a/packages/core/test/unit/composite/job-poll-cancellation.test.ts b/packages/core/test/unit/composite/job-poll-cancellation.test.ts new file mode 100644 index 00000000..9eacb5c2 --- /dev/null +++ b/packages/core/test/unit/composite/job-poll-cancellation.test.ts @@ -0,0 +1,74 @@ +/** + * The poll delay must observe cancellation, not outlive it. + * + * `buildProtocolPrimitivesParagraph` tells every client that on Cancel "the + * polling loop exits within ≤2 seconds". The job poller slept on a bare + * `setTimeout(MCP_JOB_POLL.intervalMs)` — 4000ms — and only re-checked + * `ctx.signal.aborted` after the timer fired. A cancel landing just after a + * poll therefore waited out the full interval: twice the advertised bound, + * on a promise the server makes in its own instructions. + */ + +import { describe, it, expect, vi } from "vitest"; +import { sleepUnlessAborted } from "../../../src/composite/_mcp-job-helpers.js"; + +describe("sleepUnlessAborted", () => { + it("resolves early when the signal aborts mid-sleep", async () => { + const ac = new AbortController(); + const started = Date.now(); + const sleeping = sleepUnlessAborted(4000, ac.signal); + setTimeout(() => ac.abort(), 20); + await sleeping; + // Generous bound — the point is "nowhere near 4000", not a tight timing + // assertion that would flake on a loaded CI box. + expect(Date.now() - started).toBeLessThan(1000); + }); + + it("returns immediately when the signal is already aborted", async () => { + const ac = new AbortController(); + ac.abort(); + const started = Date.now(); + await sleepUnlessAborted(4000, ac.signal); + expect(Date.now() - started).toBeLessThan(200); + }); + + it("still sleeps the full duration with no signal", async () => { + vi.useFakeTimers(); + try { + let done = false; + const sleeping = sleepUnlessAborted(4000).then(() => { + done = true; + }); + await vi.advanceTimersByTimeAsync(3999); + expect(done).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await sleeping; + expect(done).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it("removes its abort listener so a long poll loop cannot leak them", async () => { + // waitForJob calls this once per poll against ONE signal. Leaving the + // listener attached would accumulate one per iteration for the life of the + // request, and Node warns at 11. + const ac = new AbortController(); + const added: string[] = []; + const removed: string[] = []; + const realAdd = ac.signal.addEventListener.bind(ac.signal); + const realRemove = ac.signal.removeEventListener.bind(ac.signal); + ac.signal.addEventListener = ((t: string, ...rest: unknown[]) => { + added.push(t); + return (realAdd as never as (...a: unknown[]) => void)(t, ...rest); + }) as typeof ac.signal.addEventListener; + ac.signal.removeEventListener = ((t: string, ...rest: unknown[]) => { + removed.push(t); + return (realRemove as never as (...a: unknown[]) => void)(t, ...rest); + }) as typeof ac.signal.removeEventListener; + + for (let i = 0; i < 5; i++) await sleepUnlessAborted(1, ac.signal); + expect(added.filter((t) => t === "abort")).toHaveLength(5); + expect(removed.filter((t) => t === "abort")).toHaveLength(5); + }); +}); diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 564dfef8..8ddef8ba 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -281,6 +281,14 @@ function buildProtocolPrimitivesParagraph(has: (name: string) => boolean): strin "enrich_titles", "bulk_enrich_status", "qualify_status", + // The MCP-first delivery jobs block-poll for 45s by default and up to + // 180s. Without a progressToken ctx.progress is absent, so the call looks + // frozen for minutes — the exact case this paragraph exists to prevent. + // `.filter(has)` keeps the iter-12 invariant: a deployment without the + // delivery flag never sees them named. + "find_new_leads", + "qualify_leads", + "lead_job_status", ].filter((n) => has(`leadbay_${n}`)); const elicitTools = [ "refine_prompt clarifications", diff --git a/packages/mcp/test/audit/delivery-long-runners.test.ts b/packages/mcp/test/audit/delivery-long-runners.test.ts new file mode 100644 index 00000000..d05ffc24 --- /dev/null +++ b/packages/mcp/test/audit/delivery-long-runners.test.ts @@ -0,0 +1,65 @@ +/** + * Audit: the delivery jobs are advertised as progress-capable long runners. + * + * `ctx.progress` only exists when the MCP request carried a progressToken, and + * the only thing that tells a client to send one is the "Protocol primitives" + * paragraph in the server instructions. The delivery tools block-poll for 45s + * by default and up to 180s, so if they are missing from that list the call + * looks frozen for minutes and the per-poll progress callback is simply never + * invoked. + * + * Also pinned: the iter-12 invariant that buildServerInstructions never names + * a tool the deployment does not expose. These three are release-gated, so a + * default deployment must not see them advertised. + */ + +import { describe, it, expect } from "vitest"; +import { buildServerInstructions } from "../../src/server.js"; + +const DELIVERY = [ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", +]; + +/** The paragraph names tools only in the progressToken sentence. */ +function instructionsFor(exposed: string[]): string { + return buildServerInstructions(new Set(exposed)); +} + +describe("audit: delivery tools advertised as long runners", () => { + it("names all three when they are exposed", () => { + const text = instructionsFor([...DELIVERY, "leadbay_pull_leads"]); + const progressLine = text + .split("\n") + .find((l) => l.includes("Pass a progressToken on")); + expect(progressLine, "no progressToken sentence in instructions").toBeTruthy(); + for (const name of DELIVERY) { + expect(progressLine, `${name} missing from the long-runner list`).toContain( + name + ); + } + }); + + it("names none of them on a deployment without the delivery tools", () => { + // iter-12 invariant: never advertise a tool the agent cannot call. + const text = instructionsFor([ + "leadbay_pull_leads", + "leadbay_bulk_qualify_leads", + ]); + for (const name of DELIVERY) { + expect(text, `${name} advertised while not exposed`).not.toContain(name); + } + }); + + it("keeps the legacy long runners listed", () => { + // The delivery tools are an addition, not a replacement. + const text = instructionsFor([ + ...DELIVERY, + "leadbay_bulk_qualify_leads", + "leadbay_enrich_titles", + ]); + expect(text).toContain("leadbay_bulk_qualify_leads"); + expect(text).toContain("leadbay_enrich_titles"); + }); +}); From de0abe69d815877085b5e65fa1912eb7ffdeefe9 Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 09:49:27 +0200 Subject: [PATCH 36/46] fix(mcp): restore real headroom on the find_new_leads description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] 36 chars under the 17000 cap is not a margin — any shared snippet edit would have broken tool-description-source.test.ts for whoever made it, not for us. Now 639. Trimmed wording, not rules. Every rule the description carried it still carries: - heuristics/example-lead-craft (exclusive to this tool): compressed the intro and rules 1/3/4/5/7/8, and cut rule 6's country digression down to a clause — that rule is now enforced in code by rejectCountryLocations, which returns a better-targeted message than the prose could. - The body: tightened the opening paragraph and folded "free preview first" into the free-vs-paid paragraph it was restating. - rendering/lead-delivery-table (shared with qualify_leads and lead_job_status): compressed the funnel/zero-delivered/weak-batch/ sanity-check prose. Both other tools keep 3000+ chars of headroom. One correction to the finding's premise, worth recording because it points at real work: this was never the tightest description in the repo, and it is now 4th. On main today, untouched by this PR: leadbay_prepare_outreach 16993 headroom 7 leadbay_research_lead_by_id 16973 headroom 27 leadbay_pull_followups 16948 headroom 52 So the fragility Codex describes is real but pre-existing, and 90x worse on prepare_outreach than on ours. A shared-snippet edit breaks those three first. The audit also has no early warning — it passes at 16999 and fails at 17001 — so nobody sees it coming. Both belong in their own PR. Gates: 1560 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../core/src/tool-descriptions.generated.ts | 190 +++++++++--------- .../snippets/heuristics/example-lead-craft.md | 46 ++--- .../snippets/rendering/lead-delivery-table.md | 43 ++-- .../composite/find-new-leads.md.tmpl | 15 +- 4 files changed, 138 insertions(+), 156 deletions(-) diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 5cae23ef..cdb3378d 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1304,10 +1304,10 @@ delivered. Full algorithm below. Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips what the org already knows -(\`novelty: org\`), optionally qualifies candidates against the org's own -intelligence (questions, tags, ideal buyer profile — frozen at submit), and -optionally purchases contact channels. The tool polls up to \`wait_seconds\` -(default 45); a longer job returns \`still_running\` + \`next_poll\` — hand off to +(\`novelty: org\`), optionally qualifies against the org's own intelligence +(questions, tags, ideal buyer profile — frozen at submit), and optionally +purchases contact channels. Polls up to \`wait_seconds\` (default 45); a longer +job returns \`still_running\` + \`next_poll\` — hand off to \`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (\`qualify: false\`, @@ -1319,10 +1319,9 @@ phone 250c, success-only). The gate is enforced in code: a paid call \`confirm: true\` — otherwise nothing is submitted and you get \`mode: "needs_confirmation"\` with a real quote to show the user. Re-call with \`confirm: true\` on their go-ahead ("spend / get their emails" counts). -\`confirm: false\` vetoes. Free needs no consent. - -**Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an -off-profile seed is free, exploring it with \`qualify: true\` is not. +\`confirm: false\` vetoes. Free needs no consent. **Preview free first** — +reshaping an off-profile seed is free, exploring it with \`qualify: true\` is +not. **Ad-hoc exclusions ("no chains") are enforced by NO tier** — \`filters\` has no exclusion key, and \`qualify\` scores against the org's FROZEN questions and @@ -1333,22 +1332,21 @@ them yourself and say the tier didn't enforce it. Durable enforcement → ### Crafting the \`example_lead\` seed — the input that decides result quality -The \`example_lead\` is a FICTIONAL typical ideal customer. Its text is embedded -and matched against millions of real registry/website company descriptions, -which state what a company **IS** — never what is happening. Write the seed -the same way or the matcher drifts. Every rule below is measured: +The \`example_lead\` is a FICTIONAL typical ideal customer, embedded and matched +against real registry/website descriptions — which state what a company **IS**, +never what is happening. Write it the same way or the matcher drifts. Every +rule below is measured: 1. **Describe the BUYER, never the seller.** Ask: "would this company write a check to my user?" A seed describing what the user SELLS surfaces their - *competitors and vendors*. Classic trap: if the product helps companies of - type X serve customers of type Y, the seed describes X — never Y. + *competitors and vendors*. If the product helps companies of type X serve + customers of type Y, the seed describes X — never Y. 2. **Put everything in \`description\`; leave \`name\` unset.** An invented brand name pulls matching toward name-lookalikes — a seed named "Meridian Analytics" returned five unrelated "Meridian" companies. -3. **Registry style, one sentence to ~250 chars.** State the business profile: - industry niche, business model, what they sell or operate, who they serve, - observable scale (sites, membership, fleet). Write it like the first - paragraph of the company's About-Us page. +3. **Registry style, one sentence to ~250 chars.** Industry niche, business + model, what they sell or operate, who they serve, observable scale. Write + it like the first paragraph of their About-Us page. - STRONG: "Operator of full-service fitness centers offering strength training areas, group classes and personal training to individual members across multiple club locations." @@ -1356,25 +1354,20 @@ the same way or the matcher drifts. Every rule below is measured: - WRONG (seller-side): "Supplier of durable modular flooring for gyms." 4. **No event language.** "hiring", "expanding", "just raised" are not filters — registry descriptions never contain them, so they dilute the - profile. Purchase-trigger criteria belong in the org's qualification - questions, where the paid stage scores them from fresh research. -5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — - real descriptions don't carry them. + profile. Purchase triggers belong in the org's qualification questions, + which the paid stage scores from fresh research. +5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)". 6. **Hard constraints go in \`filters\`, not prose — exact keys:** \`sectors: string[]\`, \`locations: string[]\`, \`employees_min: number\`, - \`employees_max: number\`. FLAT numbers — a nested \`employees: {min, max}\` - object exists only in RESULT payloads, never on input. \`locations\` take - city/state/region names ("Dallas, TX", "Texas", "Île-de-France"); NEVER - a country — each universe is single-country, so whole-country intent = - omit \`locations\` (a country name silently matches a same-named town: - measured, "France" → the village of Francs). \`example_lead.employees\` - does not filter; only \`filters.employees_min/max\` do. -7. **Prefer \`example_lead\` over \`query\`.** Query matches topic *vocabulary* — - "gyms that need durable flooring" surfaces flooring VENDORS as strongly as - gym BUYERS (measured: 0 delivered vs on-profile from the example). Use - \`query\` only for signal an example can't express. + \`employees_max: number\`. FLAT numbers — nested \`employees: {min, max}\` + exists only in RESULT payloads. \`example_lead.employees\` does not filter. + \`locations\` take city/state/region names ("Dallas, TX", "Île-de-France"); + a country name is refused in code — whole-country intent = omit it. +7. **Prefer \`example_lead\` over \`query\`.** Query matches topic *vocabulary*: + "gyms that need durable flooring" surfaced flooring VENDORS, 0 delivered. + Use \`query\` only for signal an example can't express. 8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and - warehouses") needs one search each with its own description and + warehouses") needs one search each, with its own description and \`request_id\` — a blended seed lands between the clusters and matches neither. @@ -1440,37 +1433,36 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in -euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid -operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. +**Never hard-code \`$\`** — \`cost\` carries no currency and a France account +bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. +US → \`$\`, France → \`€\`; region unknown → bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those -count candidates, not the target) and never guess it. +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they +count candidates, not the target), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give -the funnel line plus the relevant \`explain.scope_notes\` (they carry the -backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen -rejections), then propose the concrete fix (reshape the seed per the -example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a -filter) as NEXT STEPS options. - -**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not -present the table as an answer — open with "weak matches only", show at most -the top 3, and propose reshaping the seed/filters first. The count was -filled with barely-better-than-random candidates, not good ones. - -**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must -sit inside any requested fence; drop and call out leaks (a same-named city -in another state slips through). (b) When \`explain.seed_strategy\` is -\`text_match_exemplars\` (the standard FR path), fit is calibrated for -lead-to-lead distances, not exemplar centroids — treat high scores -skeptically and verify each row's \`description\` actually matches the ask. +the funnel line plus the relevant \`explain.scope_notes\` (the backend's own +diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose +the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, +raise \`max_cost\`, drop a filter) as NEXT STEPS options. + +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present +the table as an answer — open with "weak matches only", show at most the top +3, propose reshaping the seed/filters first. The count was filled with +barely-better-than-random candidates. + +**Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any +requested fence; drop and call out leaks (a same-named city in another state +slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` +(the standard FR path), fit is calibrated for lead-to-lead distances, not +exemplar centroids — treat high scores skeptically and verify each row's +\`description\` matches the ask. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: @@ -2431,37 +2423,36 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in -euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid -operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. +**Never hard-code \`$\`** — \`cost\` carries no currency and a France account +bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. +US → \`$\`, France → \`€\`; region unknown → bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those -count candidates, not the target) and never guess it. +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they +count candidates, not the target), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give -the funnel line plus the relevant \`explain.scope_notes\` (they carry the -backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen -rejections), then propose the concrete fix (reshape the seed per the -example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a -filter) as NEXT STEPS options. - -**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not -present the table as an answer — open with "weak matches only", show at most -the top 3, and propose reshaping the seed/filters first. The count was -filled with barely-better-than-random candidates, not good ones. - -**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must -sit inside any requested fence; drop and call out leaks (a same-named city -in another state slips through). (b) When \`explain.seed_strategy\` is -\`text_match_exemplars\` (the standard FR path), fit is calibrated for -lead-to-lead distances, not exemplar centroids — treat high scores -skeptically and verify each row's \`description\` actually matches the ask. +the funnel line plus the relevant \`explain.scope_notes\` (the backend's own +diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose +the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, +raise \`max_cost\`, drop a filter) as NEXT STEPS options. + +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present +the table as an answer — open with "weak matches only", show at most the top +3, propose reshaping the seed/filters first. The count was filled with +barely-better-than-random candidates. + +**Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any +requested fence; drop and call out leaks (a same-named city in another state +slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` +(the standard FR path), fit is calibrated for lead-to-lead distances, not +exemplar centroids — treat high scores skeptically and verify each row's +\`description\` matches the ask. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: @@ -3699,37 +3690,36 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`.** \`cost\` carries no currency, and the same job bills in -euros on a France account — \`$6.09\` for a €6.09 charge misstates a paid -operation. US → \`$\`, France → \`€\`; region unknown → write the bare amount. +**Never hard-code \`$\`** — \`cost\` carries no currency and a France account +bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. +US → \`$\`, France → \`€\`; region unknown → bare amount. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (those -count candidates, not the target) and never guess it. +**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they +count candidates, not the target), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → "hit an org quota", \`time_budget\` → "hit the 30-min time budget". **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give -the funnel line plus the relevant \`explain.scope_notes\` (they carry the -backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen -rejections), then propose the concrete fix (reshape the seed per the -example_lead craft rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a -filter) as NEXT STEPS options. - -**Weak batch**: when the BEST delivered \`fit.score\` is under 30, do not -present the table as an answer — open with "weak matches only", show at most -the top 3, and propose reshaping the seed/filters first. The count was -filled with barely-better-than-random candidates, not good ones. - -**Sanity-check every row before rendering**: (a) geo — \`city\`/\`region\` must -sit inside any requested fence; drop and call out leaks (a same-named city -in another state slips through). (b) When \`explain.seed_strategy\` is -\`text_match_exemplars\` (the standard FR path), fit is calibrated for -lead-to-lead distances, not exemplar centroids — treat high scores -skeptically and verify each row's \`description\` actually matches the ask. +the funnel line plus the relevant \`explain.scope_notes\` (the backend's own +diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose +the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, +raise \`max_cost\`, drop a filter) as NEXT STEPS options. + +**Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present +the table as an answer — open with "weak matches only", show at most the top +3, propose reshaping the seed/filters first. The count was filled with +barely-better-than-random candidates. + +**Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any +requested fence; drop and call out leaks (a same-named city in another state +slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` +(the standard FR path), fit is calibrated for lead-to-lead distances, not +exemplar centroids — treat high scores skeptically and verify each row's +\`description\` matches the ask. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: diff --git a/packages/promptforge/snippets/heuristics/example-lead-craft.md b/packages/promptforge/snippets/heuristics/example-lead-craft.md index 0de6a66d..03e08505 100644 --- a/packages/promptforge/snippets/heuristics/example-lead-craft.md +++ b/packages/promptforge/snippets/heuristics/example-lead-craft.md @@ -1,21 +1,20 @@ ### Crafting the `example_lead` seed — the input that decides result quality -The `example_lead` is a FICTIONAL typical ideal customer. Its text is embedded -and matched against millions of real registry/website company descriptions, -which state what a company **IS** — never what is happening. Write the seed -the same way or the matcher drifts. Every rule below is measured: +The `example_lead` is a FICTIONAL typical ideal customer, embedded and matched +against real registry/website descriptions — which state what a company **IS**, +never what is happening. Write it the same way or the matcher drifts. Every +rule below is measured: 1. **Describe the BUYER, never the seller.** Ask: "would this company write a check to my user?" A seed describing what the user SELLS surfaces their - *competitors and vendors*. Classic trap: if the product helps companies of - type X serve customers of type Y, the seed describes X — never Y. + *competitors and vendors*. If the product helps companies of type X serve + customers of type Y, the seed describes X — never Y. 2. **Put everything in `description`; leave `name` unset.** An invented brand name pulls matching toward name-lookalikes — a seed named "Meridian Analytics" returned five unrelated "Meridian" companies. -3. **Registry style, one sentence to ~250 chars.** State the business profile: - industry niche, business model, what they sell or operate, who they serve, - observable scale (sites, membership, fleet). Write it like the first - paragraph of the company's About-Us page. +3. **Registry style, one sentence to ~250 chars.** Industry niche, business + model, what they sell or operate, who they serve, observable scale. Write + it like the first paragraph of their About-Us page. - STRONG: "Operator of full-service fitness centers offering strength training areas, group classes and personal training to individual members across multiple club locations." @@ -23,24 +22,19 @@ the same way or the matcher drifts. Every rule below is measured: - WRONG (seller-side): "Supplier of durable modular flooring for gyms." 4. **No event language.** "hiring", "expanding", "just raised" are not filters — registry descriptions never contain them, so they dilute the - profile. Purchase-trigger criteria belong in the org's qualification - questions, where the paid stage scores them from fresh research. -5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)" — - real descriptions don't carry them. + profile. Purchase triggers belong in the org's qualification questions, + which the paid stage scores from fresh research. +5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)". 6. **Hard constraints go in `filters`, not prose — exact keys:** `sectors: string[]`, `locations: string[]`, `employees_min: number`, - `employees_max: number`. FLAT numbers — a nested `employees: {min, max}` - object exists only in RESULT payloads, never on input. `locations` take - city/state/region names ("Dallas, TX", "Texas", "Île-de-France"); NEVER - a country — each universe is single-country, so whole-country intent = - omit `locations` (a country name silently matches a same-named town: - measured, "France" → the village of Francs). `example_lead.employees` - does not filter; only `filters.employees_min/max` do. -7. **Prefer `example_lead` over `query`.** Query matches topic *vocabulary* — - "gyms that need durable flooring" surfaces flooring VENDORS as strongly as - gym BUYERS (measured: 0 delivered vs on-profile from the example). Use - `query` only for signal an example can't express. + `employees_max: number`. FLAT numbers — nested `employees: {min, max}` + exists only in RESULT payloads. `example_lead.employees` does not filter. + `locations` take city/state/region names ("Dallas, TX", "Île-de-France"); + a country name is refused in code — whole-country intent = omit it. +7. **Prefer `example_lead` over `query`.** Query matches topic *vocabulary*: + "gyms that need durable flooring" surfaced flooring VENDORS, 0 delivered. + Use `query` only for signal an example can't express. 8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and - warehouses") needs one search each with its own description and + warehouses") needs one search each, with its own description and `request_id` — a blended seed lands between the clusters and matches neither. diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index ab9f25b6..62eb3e1a 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -45,37 +45,36 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code `$`.** `cost` carries no currency, and the same job bills in -euros on a France account — `$6.09` for a €6.09 charge misstates a paid -operation. US → `$`, France → `€`; region unknown → write the bare amount. +**Never hard-code `$`** — `cost` carries no currency and a France account +bills in euros, so `$6.09` for a €6.09 charge misstates a paid operation. +US → `$`, France → `€`; region unknown → bare amount. "of the Y asked" needs `summary.items_requested`, which submit results carry but a later `leadbay_lead_job_status` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from `matched`/`examined` (those -count candidates, not the target) and never guess it. +**delivered X** and stop — never back-fill Y from `matched`/`examined` (they +count candidates, not the target), never guess it. Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → "ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → "hit an org quota", `time_budget` → "hit the 30-min time budget". **When `delivered` is 0**: NEVER say just "no results". Render no table; give -the funnel line plus the relevant `explain.scope_notes` (they carry the -backend's own diagnosis, e.g. vendor-vocabulary queries or pre-screen -rejections), then propose the concrete fix (reshape the seed per the -example_lead craft rules, lower `min_ai_score`, raise `max_cost`, drop a -filter) as NEXT STEPS options. - -**Weak batch**: when the BEST delivered `fit.score` is under 30, do not -present the table as an answer — open with "weak matches only", show at most -the top 3, and propose reshaping the seed/filters first. The count was -filled with barely-better-than-random candidates, not good ones. - -**Sanity-check every row before rendering**: (a) geo — `city`/`region` must -sit inside any requested fence; drop and call out leaks (a same-named city -in another state slips through). (b) When `explain.seed_strategy` is -`text_match_exemplars` (the standard FR path), fit is calibrated for -lead-to-lead distances, not exemplar centroids — treat high scores -skeptically and verify each row's `description` actually matches the ask. +the funnel line plus the relevant `explain.scope_notes` (the backend's own +diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose +the concrete fix (reshape the seed per the craft rules, lower `min_ai_score`, +raise `max_cost`, drop a filter) as NEXT STEPS options. + +**Weak batch**: when the BEST delivered `fit.score` is under 30, don't present +the table as an answer — open with "weak matches only", show at most the top +3, propose reshaping the seed/filters first. The count was filled with +barely-better-than-random candidates. + +**Sanity-check every row**: (a) geo — `city`/`region` must sit inside any +requested fence; drop and call out leaks (a same-named city in another state +slips through). (b) When `explain.seed_strategy` is `text_match_exemplars` +(the standard FR path), fit is calibrated for lead-to-lead distances, not +exemplar centroids — treat high scores skeptically and verify each row's +`description` matches the ask. **Skipped items** (`skipped[]`, qualify jobs mostly): render a compact second table `Ref → Outcome` translating `status_reason` to plain words: diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 33e60b1d..c2c0f13a 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -58,10 +58,10 @@ annotations: --- Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips what the org already knows -(`novelty: org`), optionally qualifies candidates against the org's own -intelligence (questions, tags, ideal buyer profile — frozen at submit), and -optionally purchases contact channels. The tool polls up to `wait_seconds` -(default 45); a longer job returns `still_running` + `next_poll` — hand off to +(`novelty: org`), optionally qualifies against the org's own intelligence +(questions, tags, ideal buyer profile — frozen at submit), and optionally +purchases contact channels. Polls up to `wait_seconds` (default 45); a longer +job returns `still_running` + `next_poll` — hand off to `leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. **Free vs paid — never spend silently.** The default ask (`qualify: false`, @@ -73,10 +73,9 @@ phone 250c, success-only). The gate is enforced in code: a paid call `confirm: true` — otherwise nothing is submitted and you get `mode: "needs_confirmation"` with a real quote to show the user. Re-call with `confirm: true` on their go-ahead ("spend / get their emails" counts). -`confirm: false` vetoes. Free needs no consent. - -**Free preview first**: run FREE, eyeball fit, THEN pay — reshaping an -off-profile seed is free, exploring it with `qualify: true` is not. +`confirm: false` vetoes. Free needs no consent. **Preview free first** — +reshaping an off-profile seed is free, exploring it with `qualify: true` is +not. **Ad-hoc exclusions ("no chains") are enforced by NO tier** — `filters` has no exclusion key, and `qualify` scores against the org's FROZEN questions and From db921e5b11910970e1687c79fee4ee435cef15db Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 10:07:13 +0200 Subject: [PATCH 37/46] =?UTF-8?q?fix(mcp):=20four=20Codex=20findings=20?= =?UTF-8?q?=E2=80=94=20dead=20cross-routes,=20drain=20bound,=20qualify=20k?= =?UTF-8?q?ey,=20cancel=20guidance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] Exposed tools routed at gated ones. On a default deployment (LEADBAY_MCP_LEAD_DELIVERY unset) leadbay_pull_leads stays exposed while leadbay_find_new_leads is absent from tools/list — but tool descriptions are static strings, not filtered against the exposed set, so pull_leads, extend_lens and bulk_qualify_leads still pointed net-new asks at tools the agent cannot see. Ordinary "find me 10 gyms in Dallas" requests routed into a void until the flag is set. Fixed by making the description self-aware rather than deleting the routes: snippets/gates/delivery-release-gate tells the agent to check its own tool list first and say net-new search isn't enabled when the target is missing. That keeps the routing useful once the flag IS on, which deleting would have thrown away. Kept to 391 chars — pull_leads is the tight one and still holds 467 of headroom. [P2] The page drain was sized for qualify only. MAX_JOB_ITEMS was 500 (a qualify job's ref ceiling), but a SEARCH may examine exploration_cap's ceiling of min(20n, 1000) and emit an outcome for each. At limit=1 the drain stopped at 501 pages and still returned a terminal snapshot with no cursor, so paid deliveries — and the rejected ids the top-up needs for exclude_lead_ids — silently never reached the render. Now 1000. [P2] qualify-leads hashed an omitted title_gate as null while the backend applies `prefer` when contact_titles are present. An approval that omitted the field and a retry that passed the materialized default derived different qualify-auto-* keys, so the retry escaped dedupe into a second paid qualification and channel purchase. Mirrors find-new-leads.ts, which already canonicalized it — the two tools now agree. [P2] My own regression from last round: adding the delivery tools to longRunners put them in the CANCELLATION sentence too, which promises a bulk-store transition to 'cancelled' and BULK_CANCELLED on later polls. Those jobs own no bulk-store record — cancelling stops our wait while the backend job keeps running and keeps spending. Telling the agent to stop polling was the worst possible advice there. The two families now get separate clauses; delivery keeps its place in the progress list. One correction to my own test: it banned the substring BULK_CANCELLED, but the honest text mentions it in a negation ("there is no BULK_CANCELLED"). It now bans the PROMISE and requires the negation. Its clause slicing also anchored on bullet (3), which is itself conditional — it leaked the rest of the instructions in when elicitation was not exposed. Gates: 1572 tests, typecheck, prompts:check all green. Every test-path change on this PR is still a new file. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 22 ++-- packages/core/src/composite/qualify-leads.ts | 9 +- .../core/src/tool-descriptions.generated.ts | 24 +++++ .../composite/job-drain-search-sized.test.ts | 73 +++++++++++++ .../composite/qualify-title-gate-key.test.ts | 100 ++++++++++++++++++ packages/mcp/src/server.ts | 38 ++++++- .../delivery-cancellation-guidance.test.ts | 85 +++++++++++++++ .../snippets/gates/delivery-release-gate.md | 6 ++ .../composite/bulk-qualify-leads.md.tmpl | 2 + .../composite/extend-lens.md.tmpl | 2 + .../composite/pull-leads.md.tmpl | 2 + 11 files changed, 351 insertions(+), 12 deletions(-) create mode 100644 packages/core/test/unit/composite/job-drain-search-sized.test.ts create mode 100644 packages/core/test/unit/composite/qualify-title-gate-key.test.ts create mode 100644 packages/mcp/test/audit/delivery-cancellation-guidance.test.ts create mode 100644 packages/promptforge/snippets/gates/delivery-release-gate.md diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 740e3ed7..a435d53d 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -117,15 +117,23 @@ export const TERMINAL_JOB_STATES: ReadonlySet = new Set([ export const MCP_JOB_POLL = { intervalMs: 4000 }; const PAGE_LIMIT = 100; -// A qualify job carries at most 500 refs, so the drain is bounded by the page -// SIZE, not by a flat page count: at limit=5 the worst case is 100 pages, and a -// flat 20 would silently return the first 100 items while reporting done:true. -// Derive the bound instead, with a floor so a large page size still gets a few -// follow-ups and a ceiling that stays a runaway backstop. -const MAX_JOB_ITEMS = 500; +// The drain is bounded by the page SIZE, not by a flat page count: at limit=5 +// the worst case is 100 pages, and a flat 20 would silently return the first +// 100 items while reporting done:true. Derive the bound instead, with a floor +// so a large page size still gets a few follow-ups and a ceiling that stays a +// runaway backstop. +// +// Size it for the LARGEST job either endpoint can produce, not just qualify: +// a qualify job carries at most 500 refs, but a SEARCH may examine up to +// `exploration_cap`'s ceiling of min(20n, 1000) candidates and emit an +// outcome (delivered or skipped) for each. Bounding at 500 truncated a wide +// paid search mid-drain and still reported done:true with no next_poll — so +// deliveries the user paid for, and the rejected ids the top-up needs for +// `exclude_lead_ids`, silently never reached the render. +const MAX_JOB_ITEMS = 1000; const MIN_PAGES = 20; // The bound must let EVERY allowed page size reach MAX_JOB_ITEMS — at limit=1 -// that is 500 pages, and a lower flat cap would return a partial batch while +// that is 1000 pages, and a lower flat cap would return a partial batch while // still reporting done:true, with no cursor on a terminal submit response to // fetch the rest. Capping below the drain would hide items, not just slow them. const maxPagesFor = (pageLimit: number) => diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 7665da09..b7a915ea 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -139,7 +139,14 @@ function derivedRequestId(params: QualifyLeadsParams): string { qualify: params.qualify !== false, channels: canonicalSet(params.channels), contact_titles: canonicalLabelSet(params.contact_titles), - title_gate: params.title_gate ?? null, + // Same canonicalization as the search path: with contact_titles present + // the backend applies `prefer` when the field is omitted, so an approval + // that omits it and a retry that passes the materialized default describe + // identical work. Hashing the omission as null forked the key and let the + // retry escape dedupe into a second paid qualification / channel purchase. + title_gate: + params.title_gate ?? + ((params.contact_titles?.length ?? 0) > 0 ? "prefer" : null), // The cap is part of the approval: raising it after a stop_reason:max_cost // is a NEW approved run, and must not dedupe onto the capped job. max_cost: params.max_cost ?? null, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index cdb3378d..fe0ef86f 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -657,6 +657,14 @@ Exactly two offers — keep it terse, this is a status tool: |--------------------------------------|-----------------------------------------------|--------------------------------| | Qualification kicked off (async) | "Check progress in ~30s" | leadbay_qualify_status | | Job is done / blocking call returned | "Refresh leads view — the new qualifications should be on the top" | leadbay_pull_leads(lensId = pinned) | + + +**Before routing to net-new delivery, check your tool list.** +\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` +are release-gated: on a default deployment they are ABSENT from \`tools/list\` +even though this description names them. Route there only if the tool is in +your list — otherwise say net-new search isn't enabled here. Never call a name +you cannot see. `; // endregion: leadbay_bulk_qualify_leads @@ -1268,6 +1276,14 @@ Pick the row matching the response \`status\`. Seed-picking is internal; do NOT | \`no_valid_seeds\` | (silent retry — re-call \`leadbay_seed_candidates\` then \`leadbay_extend_lens\`) | internal — only surface if the second attempt also fails | If nothing matches cleanly, default to "pull leads now to see what's queued" — never invent a tool that doesn't exist. + + +**Before routing to net-new delivery, check your tool list.** +\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` +are release-gated: on a default deployment they are ABSENT from \`tools/list\` +even though this description names them. Route there only if the tool is in +your list — otherwise say net-new search isn't enabled here. Never call a name +you cannot see. `; // endregion: leadbay_extend_lens @@ -3555,6 +3571,14 @@ Pick 2–3 items below based on what was actually observed in the response. The | User wants a narrower / wider audience | "Adjust the lens filters (sector / size)" | leadbay_adjust_audience(...) | | Phase 4 research was run (\`research_lead_by_id\` called) AND top contacts lack direct email/phone | "Enrich contacts on [Lead1], [Lead2] to get direct emails and phone numbers" | leadbay_enrich_contacts(leadId, contactId) — ONE call per contact (the tool takes a single leadId + contactId, never a list) | If nothing in the menu applies cleanly, suggest only "pull next page" and "research a specific lead in depth" — never invent a tool that doesn't exist. + + +**Before routing to net-new delivery, check your tool list.** +\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` +are release-gated: on a default deployment they are ABSENT from \`tools/list\` +even though this description names them. Route there only if the tool is in +your list — otherwise say net-new search isn't enabled here. Never call a name +you cannot see. `; // endregion: leadbay_pull_leads diff --git a/packages/core/test/unit/composite/job-drain-search-sized.test.ts b/packages/core/test/unit/composite/job-drain-search-sized.test.ts new file mode 100644 index 00000000..e8646f40 --- /dev/null +++ b/packages/core/test/unit/composite/job-drain-search-sized.test.ts @@ -0,0 +1,73 @@ +/** + * The page drain must be sized for the LARGEST job, not just qualify. + * + * `maxPagesFor` derives its bound from MAX_JOB_ITEMS. That constant was 500 — + * a qualify job's ref ceiling — but a SEARCH may examine up to + * `exploration_cap`'s ceiling of min(20n, 1000) candidates and emit an outcome + * for each. With a small `limit` on `leadbay_lead_job_status`, the drain then + * stopped mid-job and still returned a terminal snapshot with no cursor, so + * paid deliveries — and the rejected ids the top-up needs for + * `exclude_lead_ids` — silently never reached the render. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { collectJobSnapshot } from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +/** A job that emits `total` outcomes, one item per page (limit=1) — the worst + * case for the page bound, and the one Codex named. */ +function mockPagedJob(total: number) { + const pages = Array.from({ length: total }, (_, i) => ({ + method: "GET" as const, + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "completed" }, + funnel: { delivered: total, examined: total }, + items: [{ status: "delivered", seq: i + 1 }], + next_since: i + 1 < total ? `cur-${i + 1}` : null, + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + explain: { region: "us", model: "m" }, + }, + })); + mockHttp(pages); +} + +describe("collectJobSnapshot — drain sized for a wide search", () => { + it("drains a 1000-outcome search at limit=1 without truncating", async () => { + // count:50 -> exploration_cap ceiling min(20n, 1000). The old bound of 500 + // stopped here at 501 pages and reported the job done. + mockPagedJob(1000); + const snap = await collectJobSnapshot(newClient(), "job-1", undefined, 1); + expect(snap.items).toHaveLength(1000); + }); + + it("still drains the qualify ceiling of 500", async () => { + mockPagedJob(500); + const snap = await collectJobSnapshot(newClient(), "job-1", undefined, 1); + expect(snap.items).toHaveLength(500); + }); + + it("stops as soon as a short page says the cursor is caught up", async () => { + // The bound is a backstop, not the drain signal — a short page must still + // end it immediately rather than burning the full page budget. + mockPagedJob(3); + const snap = await collectJobSnapshot(newClient(), "job-1", undefined, 1); + expect(snap.items).toHaveLength(3); + expect(getHttpRequests().filter((r) => r.method === "GET")).toHaveLength(3); + }); +}); diff --git a/packages/core/test/unit/composite/qualify-title-gate-key.test.ts b/packages/core/test/unit/composite/qualify-title-gate-key.test.ts new file mode 100644 index 00000000..d77b80ec --- /dev/null +++ b/packages/core/test/unit/composite/qualify-title-gate-key.test.ts @@ -0,0 +1,100 @@ +/** + * `title_gate` canonicalization on the qualify idempotency key. + * + * With `contact_titles` present the backend applies `prefer` when the field is + * omitted. The key hashed the omission as `null`, so an approval that left the + * field out and an unkeyed retry that materialized the same documented default + * described identical work under two different `qualify-auto-*` keys — and the + * retry escaped dedupe into a second PAID qualification and channel purchase. + * + * The search path already canonicalized this; qualify did not. Same class as + * the `exploration_cap` fix, same consequence: a double charge on a retry. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +/** PAID path — the only one that derives a key. */ +async function paidKey(extra: Record): Promise { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await qualifyLeads.execute(newClient(), { + lead_refs: [{ website: "acme.com" }], + qualify: true, + confirm: true, + wait_seconds: 0, + ...extra, + } as any); + const post = getHttpRequests().find( + (r) => r.method === "POST" && r.path.endsWith("/mcp/qualify") + ); + return JSON.parse(post!.body ?? "{}").request_id; +} + +describe("leadbay_qualify_leads — title_gate key canonicalization", () => { + it("an omitted title_gate keys the same as the applied default", async () => { + const omitted = await paidKey({ contact_titles: ["Head of Ops"] }); + const explicit = await paidKey({ + contact_titles: ["Head of Ops"], + title_gate: "prefer", + }); + expect(omitted).toBeTruthy(); + expect(omitted).toEqual(explicit); + }); + + it("an explicit non-default title_gate still keys distinctly", async () => { + const omitted = await paidKey({ contact_titles: ["Head of Ops"] }); + const strict = await paidKey({ + contact_titles: ["Head of Ops"], + title_gate: "strict", + }); + expect(omitted).not.toEqual(strict); + }); + + it("with no contact_titles the default is absent, not 'prefer'", async () => { + // Nothing to gate on, so the backend applies no gate — canonicalizing to + // "prefer" here would collapse two genuinely different asks. + const bare = await paidKey({}); + const prefer = await paidKey({ title_gate: "prefer" }); + expect(bare).not.toEqual(prefer); + }); + + it("matches the search path's rule", async () => { + // Both tools must materialize the same default, or the same approval + // hashes differently depending on which tool the agent reached for. + const a = await paidKey({ contact_titles: ["VP People"] }); + const b = await paidKey({ contact_titles: ["VP People"], title_gate: "prefer" }); + expect(a).toEqual(b); + }); +}); diff --git a/packages/mcp/src/server.ts b/packages/mcp/src/server.ts index 8ddef8ba..f28014d3 100644 --- a/packages/mcp/src/server.ts +++ b/packages/mcp/src/server.ts @@ -290,6 +290,19 @@ function buildProtocolPrimitivesParagraph(has: (name: string) => boolean): strin "qualify_leads", "lead_job_status", ].filter((n) => has(`leadbay_${n}`)); + // Cancellation is NOT the same story for both families, so they get separate + // lists. The legacy bulk tools own a bulk-store entry that flips to + // 'cancelled' and makes later status polls return BULK_CANCELLED. The + // delivery jobs own no such record: cancelling stops OUR wait, while the + // backend job keeps running. Naming them in the bulk sentence promised a + // transition that never happens and told the agent to stop polling a job + // that was still live. + const bulkStoreRunners = longRunners.filter( + (n) => !["find_new_leads", "qualify_leads", "lead_job_status"].includes(n) + ); + const deliveryRunners = longRunners.filter((n) => + ["find_new_leads", "qualify_leads", "lead_job_status"].includes(n) + ); const elicitTools = [ "refine_prompt clarifications", "report_outreach.user_confirmed", @@ -317,11 +330,28 @@ function buildProtocolPrimitivesParagraph(has: (name: string) => boolean): strin ); } - if (longRunners.length > 0) { + if (bulkStoreRunners.length > 0 || deliveryRunners.length > 0) { + const clauses: string[] = []; + if (bulkStoreRunners.length > 0) { + clauses.push( + "On " + + bulkStoreRunners.map((n) => `leadbay_${n}`).join(", ") + + " the polling loop exits within \u22642 seconds AND the bulk-store entry transitions to " + + "'cancelled'; subsequent status polls return `BULK_CANCELLED` so the agent stops polling." + ); + } + if (deliveryRunners.length > 0) { + clauses.push( + "On " + + deliveryRunners.map((n) => `leadbay_${n}`).join(", ") + + " the wait exits within \u22642 seconds but the job is BACKEND-owned and keeps running \u2014 " + + "there is no bulk-store entry and no `BULK_CANCELLED`. Any work already paid for still completes; " + + "poll `leadbay_lead_job_status` later to collect it." + ); + } parts.push( - "(2) `notifications/cancelled` — when the user clicks Cancel in the host UI, the polling loop exits " + - "within ≤2 seconds AND the bulk-store entry transitions to 'cancelled'; subsequent status polls " + - "return `BULK_CANCELLED` so the agent stops polling." + "(2) `notifications/cancelled` — when the user clicks Cancel in the host UI. " + + clauses.join(" ") ); } else { parts.push( diff --git a/packages/mcp/test/audit/delivery-cancellation-guidance.test.ts b/packages/mcp/test/audit/delivery-cancellation-guidance.test.ts new file mode 100644 index 00000000..7496dae9 --- /dev/null +++ b/packages/mcp/test/audit/delivery-cancellation-guidance.test.ts @@ -0,0 +1,85 @@ +/** + * Audit: cancellation guidance tells the truth for BOTH job families. + * + * The legacy bulk tools own a bulk-store entry that flips to 'cancelled', so a + * later status poll returns BULK_CANCELLED and the agent knows to stop. The + * MCP-first delivery jobs own no such record: cancelling stops OUR wait while + * the BACKEND job keeps running. + * + * Adding the delivery tools to the shared long-runner list made the + * cancellation sentence promise them a transition that never happens — and + * told the agent to stop polling a job that was still live and still spending. + * The two families now get separate clauses. + */ + +import { describe, it, expect } from "vitest"; +import { buildServerInstructions } from "../../src/server.js"; + +const DELIVERY = [ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", +]; +const BULK = ["leadbay_bulk_qualify_leads", "leadbay_enrich_titles"]; + +/** The cancellation bullet, isolated from the rest of the instructions. + * Bounded by the paragraph break — bullet (3) is itself conditional on the + * elicitation tools being exposed, so anchoring on it leaked the whole tail + * of the instructions into the clause. */ +function cancellationClause(exposed: string[]): string { + const text = buildServerInstructions(new Set(exposed)); + const start = text.indexOf("(2) `notifications/cancelled`"); + expect(start, "no cancellation bullet in instructions").toBeGreaterThan(-1); + const break_ = text.indexOf("\n\n", start); + const next = text.indexOf("(3) `elicitation/create`", start); + const ends = [break_, next].filter((i) => i > -1); + return text.slice(start, ends.length ? Math.min(...ends) : undefined); +} + +describe("audit: cancellation guidance per job family", () => { + it("never promises BULK_CANCELLED for a delivery job", () => { + // The word may appear — the honest text says there is NO BULK_CANCELLED. + // What must never appear is the PROMISE: a bulk-store transition, or a + // status poll returning that code. + const clause = cancellationClause(DELIVERY); + expect(clause).not.toMatch(/return `BULK_CANCELLED`/); + expect(clause).not.toMatch(/transitions to 'cancelled'/); + expect(clause).toMatch(/no `BULK_CANCELLED`/); + }); + + it("says a cancelled delivery job keeps running backend-side", () => { + const clause = cancellationClause(DELIVERY); + expect(clause).toMatch(/BACKEND-owned|backend-owned/); + expect(clause).toContain("leadbay_lead_job_status"); + }); + + it("still promises BULK_CANCELLED for the bulk tools", () => { + const clause = cancellationClause(BULK); + expect(clause).toContain("BULK_CANCELLED"); + }); + + it("keeps the two families in separate clauses when both are exposed", () => { + const clause = cancellationClause([...BULK, ...DELIVERY]); + // BULK_CANCELLED must be claimed, but never about a delivery tool: the + // delivery names must not appear before the bulk-store promise. + expect(clause).toContain("BULK_CANCELLED"); + const bulkSentenceEnd = clause.indexOf("BULK_CANCELLED"); + const bulkHalf = clause.slice(0, bulkSentenceEnd); + for (const name of DELIVERY) { + expect(bulkHalf, `${name} named inside the bulk-store promise`).not.toContain( + name + ); + } + }); + + it("names all three delivery tools in the progress list still", () => { + // The split must not have dropped them from bullet (1) — being absent + // there is what made the calls look frozen in the first place. + const text = buildServerInstructions(new Set([...BULK, ...DELIVERY])); + const progress = text.slice( + text.indexOf("(1) `notifications/progress`"), + text.indexOf("(2) `notifications/cancelled`") + ); + for (const name of DELIVERY) expect(progress).toContain(name); + }); +}); diff --git a/packages/promptforge/snippets/gates/delivery-release-gate.md b/packages/promptforge/snippets/gates/delivery-release-gate.md new file mode 100644 index 00000000..b04dfe14 --- /dev/null +++ b/packages/promptforge/snippets/gates/delivery-release-gate.md @@ -0,0 +1,6 @@ +**Before routing to net-new delivery, check your tool list.** +`leadbay_find_new_leads` / `leadbay_qualify_leads` / `leadbay_lead_job_status` +are release-gated: on a default deployment they are ABSENT from `tools/list` +even though this description names them. Route there only if the tool is in +your list — otherwise say net-new search isn't enabled here. Never call a name +you cannot see. diff --git a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl index ed643ef9..fc47c5b7 100644 --- a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl @@ -41,3 +41,5 @@ Do not enumerate the affected leads — that's the job of `leadbay_pull_leads`. --- {{include:next-steps/bulk-qualify-leads}} + +{{include:gates/delivery-release-gate}} diff --git a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl index 0334a55a..d106df4c 100644 --- a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl @@ -72,3 +72,5 @@ Queue an additive extra-refill on a lens — more leads on the same criteria, wi --- {{include:next-steps/extend-lens}} + +{{include:gates/delivery-release-gate}} diff --git a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl index 27d7d90c..92e03275 100644 --- a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl @@ -96,3 +96,5 @@ The active lens can change between calls (5-min cache + backend `last_requested_ --- {{include:next-steps/pull-leads}} + +{{include:gates/delivery-release-gate}} From 73d999166a466ba11126b45ba7868a1ca0e6b5ba Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 11:08:00 +0200 Subject: [PATCH 38/46] =?UTF-8?q?fix(mcp):=20four=20Codex=20findings=20?= =?UTF-8?q?=E2=80=94=20malformed=20refs,=20quota=20recovery,=20currency,?= =?UTF-8?q?=20expired=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] `lead_refs: [null]` threw a raw TypeError before the spend gate. normalizeLeadRefs deliberately passes non-strings through untouched so object refs survive, so a null reached derivedRequestId and died on the first property access — a stack trace where the contract promises a named error, and past the point where a quote could be returned. rejectMalformedLeadRefs now refuses null / numbers / arrays with INVALID_LEAD_REF naming the offending indexes, after the string reshape (so the bare-string shorthand still works) and before both the spend gate and key derivation. Refusing rather than dropping is deliberate: silently filtering bad entries would qualify and BILL a subset of the batch the user listed without saying so. [P2] The NEXT STEPS table treated `stop_reason: quota` and `max_cost` as one row and offered "raise the cap and continue" for both. Raising max_cost cannot clear an org quota — the re-run burns a submit and a rate-limit slot to stop in exactly the same place. Split: max_cost gets the higher cap under a new request_id; quota gets account_status then the top-up flow, and an explicit "never a re-run". [P2] The same row hard-coded "$X" while the delivery renderer had just been taught that a France job bills in euros. Now deferred to the funnel-line currency rule. [P2] lead_job_status had no `expired` branch. It is a terminal state, and the tool description already documents the recovery (items are no longer listed; re-read the billed leads via qualify_leads.prior_deliveries) — but the table told the agent to render the delivery, so an expired job surfaced as an empty result. Added the row, and the header now says pick the ONE row matching the state instead of claiming exactly two. delivery-next-steps-contract.test.ts pins all of it against the GENERATED descriptions rather than the snippets, so a template that stops being included is caught too. Headroom: the new quota row cost find_new_leads 435 chars, which walked back last round's fix. Trimmed the currency enumeration (it duplicates the shared funnel-line rule) and two verbose cells to land at 356. Gates: 1583 tests, typecheck, prompts:check all green. Every test-path change on this PR is still a new file. Co-Authored-By: Claude Opus 5 --- packages/core/src/composite/qualify-leads.ts | 26 ++++ .../core/src/tool-descriptions.generated.ts | 13 +- .../composite/qualify-malformed-refs.test.ts | 115 ++++++++++++++++++ .../delivery-next-steps-contract.test.ts | 65 ++++++++++ .../snippets/next-steps/find-new-leads.md | 9 +- .../snippets/next-steps/lead-job-status.md | 4 +- 6 files changed, 222 insertions(+), 10 deletions(-) create mode 100644 packages/core/test/unit/composite/qualify-malformed-refs.test.ts create mode 100644 packages/mcp/test/audit/delivery-next-steps-contract.test.ts diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index b7a915ea..c2c8b0b3 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -89,6 +89,29 @@ function normalizeLeadRefs( }); } +/** MCP args are not schema-validated before dispatch, so `lead_refs` can carry + * `null`, a number, or an array where an object belongs. `normalizeLeadRefs` + * deliberately passes non-strings through untouched, so those entries reached + * `derivedRequestId`, where the first property access threw a raw TypeError — + * BEFORE the spend gate could return a quote. A malformed ref is a caller + * mistake, and the tool's contract is to answer with a named, actionable + * error rather than a stack trace or a silent drop: dropping would qualify + * and BILL a subset of the batch the user listed, without saying so. */ +function rejectMalformedLeadRefs(refs: QualifyLeadsParams["lead_refs"]): void { + if (!Array.isArray(refs)) return; + const bad: number[] = []; + refs.forEach((ref, i) => { + if (ref === null || typeof ref !== "object" || Array.isArray(ref)) bad.push(i); + }); + if (bad.length === 0) return; + throw { + error: true, + code: "INVALID_LEAD_REF", + message: `lead_refs contains ${bad.length} entr${bad.length === 1 ? "y" : "ies"} that are not a reference object (index ${bad.join(", ")}).`, + hint: "Each ref is an object — {lead_id} | {website} | {name, location?} | {contact_id}. A bare string is accepted and reshaped; null, numbers and arrays are not. Drop the bad entries and re-call.", + }; +} + function derivedRequestId(params: QualifyLeadsParams): string { // JSON-serialize each ref rather than joining raw values with delimiters. // Field names alone were not enough: a value CONTAINING the delimiters @@ -277,6 +300,9 @@ export const qualifyLeads: Tool = { // the quote, the idempotency key and the posted body all describe the same // companies. params = { ...params, lead_refs: normalizeLeadRefs(params.lead_refs) }; + // AFTER the string reshape (so a bare string is not called malformed) and + // BEFORE the spend gate and key derivation, both of which read ref fields. + rejectMalformedLeadRefs(params.lead_refs); // Spend gate. `qualify` defaults to TRUE on the backend (~94 cost_cents per // lead needing fresh research), so a bare call carrying only lead_refs is a // PAID submit — up to 500 refs — that the user never approved. Prose in the diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index fe0ef86f..965792da 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1527,15 +1527,16 @@ User picks → call the matching \`Calls\` tool. Constraints: 2–4 mutually-exc -Pick the 2-3 options that match what actually happened — never all six: +Pick the 2-3 options that match what actually happened — never all seven: | Observation | Suggest | Calls | |---|---|---| | Job still running (\`still_running: true\`) | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | -| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote \`dry_run\` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | +| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — \`dry_run\` first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | -| Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap / quota | "Raise the cap to $X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| Delivered 0 or off-profile | "Reshape the example and retry" (name the fix from funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | +| Stopped at cost cap (\`stop_reason: max_cost\`) | "Raise the cap to X and continue" — X in the account's currency per the funnel-line rule, never a hard-coded \`$\` | leadbay_find_new_leads, NEW request_id + higher max_cost (a same-id re-submit only dedupes onto a LIVE job) | +| Stopped on org quota (\`stop_reason: quota\`) | "Wait for the reset, or top up" — never a re-run: it cannot clear an org quota and burns a submit slot to stop in the same place | leadbay_account_status, then leadbay_create_topup_link | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | `; // endregion: leadbay_find_new_leads @@ -2517,12 +2518,14 @@ User picks → call the matching \`Calls\` tool. Constraints: 2–4 mutually-exc -Exactly two offers — this is a status tool, keep it terse: +Pick the ONE row matching the job's state and offer at most two options — this +is a status tool, keep it terse: | Observation | Suggest | Calls | |---|---|---| | Still running | "Keep waiting (~1 min) or leave it — results are kept 30 days" | leadbay_lead_job_status(job_id, wait_seconds: 60) | | Terminal (completed / partial / failed) | Render the delivery per the RENDERING block, then offer the matching find_new_leads / qualify_leads NEXT STEPS | — | +| \`expired\` (past the 30-day window) | "Re-read the billed leads from your delivery ledger" — there is nothing left to render: the job terminalized and its items are no longer listed, so do NOT present an empty delivery as a result | leadbay_qualify_leads(prior_deliveries: {job_id}) | `; // endregion: leadbay_lead_job_status diff --git a/packages/core/test/unit/composite/qualify-malformed-refs.test.ts b/packages/core/test/unit/composite/qualify-malformed-refs.test.ts new file mode 100644 index 00000000..a6905219 --- /dev/null +++ b/packages/core/test/unit/composite/qualify-malformed-refs.test.ts @@ -0,0 +1,115 @@ +/** + * Malformed `lead_refs` entries are refused, not crashed on. + * + * MCP args are not schema-validated before dispatch, and `normalizeLeadRefs` + * deliberately passes non-strings through untouched (so object refs survive). + * A `null` therefore reached `derivedRequestId`, whose first property access + * threw a raw TypeError — BEFORE the spend gate could return a quote. The + * caller got a stack trace where the contract promises a named error. + * + * Dropping the bad entries instead would be worse: the batch would qualify and + * BILL a subset of what the user listed without saying so. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +async function run(lead_refs: unknown, extra: Record = {}) { + mockHttp([]); + return qualifyLeads.execute(newClient(), { + lead_refs, + wait_seconds: 0, + ...extra, + } as any); +} + +async function refusal(lead_refs: unknown, extra: Record = {}) { + try { + await run(lead_refs, extra); + return null; + } catch (e) { + return e as { code?: string; message?: string; hint?: string }; + } +} + +describe("leadbay_qualify_leads — malformed lead_refs", () => { + it("refuses a null ref with a named error, not a TypeError", async () => { + const e = await refusal([null], { qualify: true, confirm: true }); + expect(e).toBeTruthy(); + expect(e).not.toBeInstanceOf(TypeError); + expect(e!.code).toBe("INVALID_LEAD_REF"); + }); + + it("refuses before any network call — nothing is submitted", async () => { + // The whole point: the throw used to happen while deriving the key, which + // is past the spend gate's quote path. Nothing may reach the wire. + await refusal([null], { qualify: true, confirm: true }); + expect(getHttpRequests()).toHaveLength(0); + }); + + it("names the offending indexes so the caller can fix the batch", async () => { + const e = await refusal([{ website: "acme.com" }, null, 42], { + qualify: true, + confirm: true, + }); + expect(e!.code).toBe("INVALID_LEAD_REF"); + expect(e!.message).toMatch(/index 1, 2/); + }); + + it("refuses on the FREE path too — a crash is not free-path-specific", async () => { + const e = await refusal([null], { qualify: false }); + expect(e!.code).toBe("INVALID_LEAD_REF"); + }); + + it("rejects arrays and numbers where a ref object belongs", async () => { + expect((await refusal([[]], { qualify: false }))!.code).toBe("INVALID_LEAD_REF"); + expect((await refusal([7], { qualify: false }))!.code).toBe("INVALID_LEAD_REF"); + }); + + it("still accepts the bare-string shorthand", async () => { + // The guard runs AFTER normalizeLeadRefs, so a string is already an object + // by then and must not be called malformed. + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/qualify", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await qualifyLeads.execute(newClient(), { + lead_refs: ["acme.com"], + qualify: false, + wait_seconds: 0, + } as any); + const post = getHttpRequests().find((r) => r.method === "POST"); + expect(JSON.parse(post!.body ?? "{}").lead_refs).toEqual([ + { website: "acme.com" }, + ]); + }); +}); diff --git a/packages/mcp/test/audit/delivery-next-steps-contract.test.ts b/packages/mcp/test/audit/delivery-next-steps-contract.test.ts new file mode 100644 index 00000000..4c606f47 --- /dev/null +++ b/packages/mcp/test/audit/delivery-next-steps-contract.test.ts @@ -0,0 +1,65 @@ +/** + * Audit: the delivery NEXT STEPS tables offer a valid action for every state + * the tools can actually return. + * + * These tables are normative — the agent is told to pick a row and offer it. + * A state with no matching row leaves the agent instructed to offer something + * while holding nothing valid to offer, which is how a terminal `expired` job + * ended up rendered as an empty delivery. + * + * Read from the GENERATED descriptions, not the snippets, so a template that + * stops being included is caught too. + */ + +import { describe, it, expect } from "vitest"; +import { + leadbay_find_new_leads, + leadbay_lead_job_status, +} from "@leadbay/core/dist/tool-descriptions.generated.js"; + +describe("audit: find_new_leads NEXT STEPS", () => { + it("separates the cost-cap stop from the org-quota stop", () => { + // Raising max_cost cannot clear an org quota: re-running burns a submit + // and a rate-limit slot to stop in the same place. + expect(leadbay_find_new_leads).toMatch(/stop_reason: max_cost/); + expect(leadbay_find_new_leads).toMatch(/stop_reason: quota/); + }); + + it("routes a quota stop at quota recovery, not another search", () => { + const row = leadbay_find_new_leads + .split("\n") + .find((l) => l.includes("stop_reason: quota")); + expect(row, "no quota row in the NEXT STEPS table").toBeTruthy(); + expect(row).toMatch(/leadbay_(account_status|create_topup_link|open_billing_portal)/); + expect(row).not.toMatch(/leadbay_find_new_leads/); + }); + + it("does not hard-code a dollar sign on the cap-raise amount", () => { + const row = leadbay_find_new_leads + .split("\n") + .find((l) => l.includes("stop_reason: max_cost")); + expect(row).toBeTruthy(); + expect(row).not.toMatch(/Raise the cap to \$/); + expect(row).toMatch(/currency/i); + }); +}); + +describe("audit: lead_job_status NEXT STEPS", () => { + it("covers every terminal state the tool documents", () => { + // TERMINAL_JOB_STATES: completed, completed_partial, failed, expired. + for (const state of ["completed", "partial", "failed", "expired"]) { + expect( + leadbay_lead_job_status.toLowerCase(), + `${state} has no NEXT STEPS row` + ).toContain(state); + } + }); + + it("gives expired the ledger re-read, not a render", () => { + const row = leadbay_lead_job_status + .split("\n") + .find((l) => l.includes("expired") && l.includes("|")); + expect(row, "no expired row").toBeTruthy(); + expect(row).toMatch(/prior_deliveries/); + }); +}); diff --git a/packages/promptforge/snippets/next-steps/find-new-leads.md b/packages/promptforge/snippets/next-steps/find-new-leads.md index 2f5fab25..8ebb7650 100644 --- a/packages/promptforge/snippets/next-steps/find-new-leads.md +++ b/packages/promptforge/snippets/next-steps/find-new-leads.md @@ -2,13 +2,14 @@ {{include:next-steps/ask-user-input-routing}} -Pick the 2-3 options that match what actually happened — never all six: +Pick the 2-3 options that match what actually happened — never all seven: | Observation | Suggest | Calls | |---|---|---| | Job still running (`still_running: true`) | "Check on it in ~1 min" | leadbay_lead_job_status(job_id, wait_seconds: 60) | -| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — quote `dry_run` estimate first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | +| Free run delivered on-profile leads | "Qualify these N against your criteria (paid — `dry_run` first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | -| Delivered 0 or off-profile | "Reshape the example and retry" (state the specific fix from the funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap / quota | "Raise the cap to $X and continue" | leadbay_find_new_leads (SAME request_id re-submits are dedup-safe only for live jobs — use a new request_id with higher max_cost) | +| Delivered 0 or off-profile | "Reshape the example and retry" (name the fix from funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | +| Stopped at cost cap (`stop_reason: max_cost`) | "Raise the cap to X and continue" — X in the account's currency per the funnel-line rule, never a hard-coded `$` | leadbay_find_new_leads, NEW request_id + higher max_cost (a same-id re-submit only dedupes onto a LIVE job) | +| Stopped on org quota (`stop_reason: quota`) | "Wait for the reset, or top up" — never a re-run: it cannot clear an org quota and burns a submit slot to stop in the same place | leadbay_account_status, then leadbay_create_topup_link | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | diff --git a/packages/promptforge/snippets/next-steps/lead-job-status.md b/packages/promptforge/snippets/next-steps/lead-job-status.md index 86c16c9e..2e189408 100644 --- a/packages/promptforge/snippets/next-steps/lead-job-status.md +++ b/packages/promptforge/snippets/next-steps/lead-job-status.md @@ -2,9 +2,11 @@ {{include:next-steps/ask-user-input-routing}} -Exactly two offers — this is a status tool, keep it terse: +Pick the ONE row matching the job's state and offer at most two options — this +is a status tool, keep it terse: | Observation | Suggest | Calls | |---|---|---| | Still running | "Keep waiting (~1 min) or leave it — results are kept 30 days" | leadbay_lead_job_status(job_id, wait_seconds: 60) | | Terminal (completed / partial / failed) | Render the delivery per the RENDERING block, then offer the matching find_new_leads / qualify_leads NEXT STEPS | — | +| `expired` (past the 30-day window) | "Re-read the billed leads from your delivery ledger" — there is nothing left to render: the job terminalized and its items are no longer listed, so do NOT present an empty delivery as a result | leadbay_qualify_leads(prior_deliveries: {job_id}) | From 0da75c62ec6501edcd88a14a713644cbb75f507a Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 17:03:23 +0200 Subject: [PATCH 39/46] fix(mcp): validate lead-ref FIELD types, not just the ref container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] My previous guard checked that each lead_refs entry was an object and stopped there. `{website: 123}` cleared it and died one level in: normalizeDomain returns null for a non-string, so the `??` fallback ran `123.trim()` — the same raw TypeError, in the same place, still before the spend gate could return its quote. It was not only `website`. Enumerated every ref read in derivedRequestId instead of patching the one named: normalizeUuid(r.lead_id) safe — typeof check inside normalizeUuid(r.contact_id) safe — same r.website ... .trim() UNSAFE via the ?? fallback r.name?.trim() UNSAFE — ?. guards null/undefined, not a number r.location?.trim() UNSAFE — same Checked the sibling too: find-new-leads.ts has no unguarded .trim() at all — its ids and labels go through canonicalIdSet / canonicalLabelSet, both typeof-filtered. The class is confined to this file. Two layers: 1. rejectMalformedLeadRefs now validates every declared field's TYPE and names each offender as `index.field (type)`. The field list is `satisfies keyof` the ref shape, so adding a field to lead_refs without adding it here fails the typecheck rather than silently reopening the hole. 2. Key derivation no longer holds the crash primitive: a `text()` helper folds a non-string to null instead of throwing on .trim(). Layer 2 is NOT covered by a test, and I am not claiming it is: derivedRequestId is module-private with exactly one call site, and the guard runs before it, so there is no reachable path to exercise it. It is there so the derivation cannot become the crash site again if a future caller skips the guard — exporting a private function purely to test that would be worse than the gap. One behaviour change worth recording: `text()` folds an empty-string field to null, where the old code kept "". `{name: ""}` and `{}` now hash alike. That is more correct — an empty name is no name — and safe today because the feature is unreleased, so no live keys exist under the old scheme. Also updated qualify-malformed-refs.test.ts, which this PR added last round, to match the richer message format. Confirmed against origin/main that the file is not pre-existing before editing it. Gates: 1589 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- packages/core/src/composite/qualify-leads.ts | 78 +++++++++--- .../composite/qualify-malformed-refs.test.ts | 6 +- .../composite/qualify-ref-field-types.test.ts | 116 ++++++++++++++++++ 3 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 packages/core/test/unit/composite/qualify-ref-field-types.test.ts diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index c2c8b0b3..7475b972 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -97,21 +97,56 @@ function normalizeLeadRefs( * mistake, and the tool's contract is to answer with a named, actionable * error rather than a stack trace or a silent drop: dropping would qualify * and BILL a subset of the batch the user listed, without saying so. */ +/** Every identifying field the ref shape declares. Validated as a set rather + * than one-by-one so adding a field to `lead_refs` cannot silently reopen the + * crash: a new field left off this list is caught by the typecheck below. */ +const LEAD_REF_FIELDS = [ + "lead_id", + "website", + "name", + "location", + "contact_id", +] as const satisfies ReadonlyArray< + keyof NonNullable[number] +>; + function rejectMalformedLeadRefs(refs: QualifyLeadsParams["lead_refs"]): void { if (!Array.isArray(refs)) return; - const bad: number[] = []; + const bad: string[] = []; refs.forEach((ref, i) => { - if (ref === null || typeof ref !== "object" || Array.isArray(ref)) bad.push(i); + if (ref === null || typeof ref !== "object" || Array.isArray(ref)) { + bad.push(`${i} (not an object)`); + return; + } + // Field TYPES, not just the container. `{website: 123}` cleared the object + // check and then died on `.trim()` while deriving the key — the same crash + // one level in. `undefined` and absent are both fine; anything present and + // non-string is not. + for (const field of LEAD_REF_FIELDS) { + const value = (ref as Record)[field]; + if (value !== undefined && typeof value !== "string") { + bad.push(`${i}.${field} (${value === null ? "null" : typeof value})`); + } + } }); if (bad.length === 0) return; throw { error: true, code: "INVALID_LEAD_REF", - message: `lead_refs contains ${bad.length} entr${bad.length === 1 ? "y" : "ies"} that are not a reference object (index ${bad.join(", ")}).`, - hint: "Each ref is an object — {lead_id} | {website} | {name, location?} | {contact_id}. A bare string is accepted and reshaped; null, numbers and arrays are not. Drop the bad entries and re-call.", + message: `lead_refs has ${bad.length} invalid entr${bad.length === 1 ? "y" : "ies"}: ${bad.join(", ")}.`, + hint: "Each ref is an object whose fields are STRINGS — {lead_id} | {website} | {name, location?} | {contact_id}. A bare string is accepted and reshaped; null, numbers, arrays and non-string field values are not. Fix or drop those entries and re-call.", }; } +/** Trim + lowercase a value that SHOULD be a string, without trusting that it + * is. Non-strings fold to null rather than throwing, so an unvalidated caller + * cannot turn key derivation into a TypeError. */ +function text(value: unknown): string | null { + if (typeof value !== "string") return null; + const v = value.trim().toLowerCase(); + return v ? v : null; +} + function derivedRequestId(params: QualifyLeadsParams): string { // JSON-serialize each ref rather than joining raw values with delimiters. // Field names alone were not enough: a value CONTAINING the delimiters @@ -123,21 +158,26 @@ function derivedRequestId(params: QualifyLeadsParams): string { // once are the same approved work. Leaving duplicates in forked the key, and // a retry that happened to dedupe would then re-run the whole paid job. const refs = canonicalSet( - (params.lead_refs ?? []).map((r) => [ - // UUIDs are case-insensitive to the backend, so an uppercase id and its - // lowercase form are the same lead and must share a key. - normalizeUuid(r.lead_id), - normalizeUuid(r.contact_id), - // Normalize the website the SAME way the resolver does, so a pasted - // "https://Acme.com/" and a retry's "acme.com" resolve to one company - // AND to one key. Fall back to the trimmed/lowercased raw value when it - // is not domain-shaped, rather than dropping the field. - r.website - ? normalizeDomain(r.website) ?? r.website.trim().toLowerCase() - : null, - r.name?.trim().toLowerCase() ?? null, - r.location?.trim().toLowerCase() ?? null, - ]) + (params.lead_refs ?? []).map((r) => { + // Defence in depth: rejectMalformedLeadRefs already guarantees every + // field is a string, but key derivation must not be the thing that + // crashes if a future caller reaches it without that guard. `text()` + // folds a non-string to null instead of throwing on .trim(). + const website = text(r.website); + return [ + // UUIDs are case-insensitive to the backend, so an uppercase id and + // its lowercase form are the same lead and must share a key. + normalizeUuid(r.lead_id), + normalizeUuid(r.contact_id), + // Normalize the website the SAME way the resolver does, so a pasted + // "https://Acme.com/" and a retry's "acme.com" resolve to one company + // AND to one key. Fall back to the trimmed/lowercased raw value when + // it is not domain-shaped, rather than dropping the field. + website ? normalizeDomain(website) ?? website : null, + text(r.name), + text(r.location), + ]; + }) ); // JSON the WHOLE shape for the same reason as the refs above: free-text // values (contact_titles, lang) must not be able to forge a field boundary diff --git a/packages/core/test/unit/composite/qualify-malformed-refs.test.ts b/packages/core/test/unit/composite/qualify-malformed-refs.test.ts index a6905219..4681178a 100644 --- a/packages/core/test/unit/composite/qualify-malformed-refs.test.ts +++ b/packages/core/test/unit/composite/qualify-malformed-refs.test.ts @@ -68,7 +68,11 @@ describe("leadbay_qualify_leads — malformed lead_refs", () => { confirm: true, }); expect(e!.code).toBe("INVALID_LEAD_REF"); - expect(e!.message).toMatch(/index 1, 2/); + // Index-and-reason per bad entry, so the caller can fix the batch rather + // than bisect it. Ref 0 is well-formed and must not be listed. + expect(e!.message).toMatch(/1 \(not an object\)/); + expect(e!.message).toMatch(/2 \(not an object\)/); + expect(e!.message).not.toMatch(/\b0 \(/); }); it("refuses on the FREE path too — a crash is not free-path-specific", async () => { diff --git a/packages/core/test/unit/composite/qualify-ref-field-types.test.ts b/packages/core/test/unit/composite/qualify-ref-field-types.test.ts new file mode 100644 index 00000000..e1d88e8a --- /dev/null +++ b/packages/core/test/unit/composite/qualify-ref-field-types.test.ts @@ -0,0 +1,116 @@ +/** + * Lead-ref FIELD types are validated, not just the ref container. + * + * The first guard checked that each entry was an object. `{website: 123}` + * cleared it and then died one level in: `normalizeDomain` returns null for a + * non-string, so the `??` fallback ran `123.trim()` — a raw TypeError while + * deriving the automatic idempotency key, again BEFORE the spend gate could + * return its quote. `name` and `location` had the same hole via `?.trim()`, + * which guards null/undefined but not a number. + * + * Two layers are pinned here: the guard rejects non-string fields with + * INVALID_LEAD_REF, and key derivation itself no longer throws on one — so a + * future caller that reaches it without the guard degrades instead of crashing. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { qualifyLeads } from "../../../src/composite/qualify-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +/** The PAID, unkeyed path — the one that derives a fallback request_id. */ +async function refusal(lead_refs: unknown) { + mockHttp([]); + try { + await qualifyLeads.execute(newClient(), { + lead_refs, + qualify: true, + confirm: true, + wait_seconds: 0, + } as any); + return null; + } catch (e) { + return e as { code?: string; message?: string }; + } +} + +const OK_JOB = [ + { + method: "POST" as const, + path: "/1.6/mcp/qualify", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET" as const, + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, +]; + +describe("leadbay_qualify_leads — lead-ref field types", () => { + it("refuses a numeric website instead of throwing a TypeError", async () => { + const e = await refusal([{ website: 123 }]); + expect(e).toBeTruthy(); + expect(e).not.toBeInstanceOf(TypeError); + expect(e!.code).toBe("INVALID_LEAD_REF"); + expect(e!.message).toMatch(/0\.website \(number\)/); + }); + + it("refuses non-string name and location too", async () => { + expect((await refusal([{ name: 42 }]))!.message).toMatch(/0\.name \(number\)/); + expect((await refusal([{ location: {} }]))!.message).toMatch( + /0\.location \(object\)/ + ); + }); + + it("refuses an explicit null field", async () => { + // `?? null` and `?.` both tolerate null, so this never crashed — but it is + // still not a usable identifier and must not reach the backend silently. + expect((await refusal([{ website: null }]))!.message).toMatch( + /0\.website \(null\)/ + ); + }); + + it("refuses nothing before the wire — no submit happens", async () => { + await refusal([{ website: 123 }]); + expect(getHttpRequests()).toHaveLength(0); + }); + + it("reports every bad field across every ref", async () => { + const e = await refusal([{ website: "acme.com" }, { name: 1, location: 2 }]); + expect(e!.message).toMatch(/1\.name/); + expect(e!.message).toMatch(/1\.location/); + expect(e!.message).not.toMatch(/0\./); + }); + + it("still accepts absent fields and well-formed refs", async () => { + mockHttp(OK_JOB); + await qualifyLeads.execute(newClient(), { + lead_refs: [{ website: "acme.com" }, { name: "Acme", location: "Paris" }], + qualify: false, + wait_seconds: 0, + } as any); + expect( + getHttpRequests().filter((r) => r.method === "POST") + ).toHaveLength(1); + }); +}); From d4f91343ff63b987f5838a3645e9ed7a3d5489cb Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 17:25:48 +0200 Subject: [PATCH 40/46] fix(mcp): put the gated-route condition where hosts actually read it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P1] My previous fix was placed where it could not be read. Measured on the generated description: pull_leads names leadbay_find_new_leads at char 302 — inside the ~600-char window every host loads — while the availability caveat I appended sat at char 16,146. A truncating host saw the route and never the condition, so the exact deployment the caveat existed for kept routing net-new asks at a tool absent from tools/list. It was decorative. Fixed at the generator instead of in prose: anti_triggers take an optional `gated: true`, and emitRoutingBlock renders "(only if listed)" beside the target — inside the block, at char ~300. Marked the three cross-routes (pull_leads x2, extend_lens x1) and the bulk_qualify_leads prose mention, which had the same defect at char 1,690 and was not in the finding. The route_to audit is unaffected: its regex matches the backticked name and ignores trailing text. [P1] Deleting the 391-char snippet is also the headroom fix. pull_leads goes 16,533 -> 16,176, headroom 467 -> 824, back under the ~16k target the finding cited. The in-header marker costs 18 chars per gated route instead. [P2] Cancellation now reaches the in-flight request. Making the sleep abortable last round was half the job: waitForJob still checked ctx.signal only AFTER awaiting collectJobSnapshot, and that GET took no signal at all, so a cancel during a slow /mcp/jobs response stayed blocked past the advertised <=2s exit. - LeadbayClient.request accepts an AbortSignal and passes it to https.request, which aborts the socket rather than leaking it. - The 401 retry is skipped when already aborted — its 250ms backoff would only lengthen a wait nobody is listening to. - collectJobSnapshot forwards the signal and stops paging on abort. - waitForJob checks BEFORE the first poll and throws REQUEST_CANCELLED rather than opening a request; a cancel mid-loop returns the last good snapshot instead of surfacing an AbortError, because the user cancelled the WAIT and the job keeps running backend-side. Two of my own slips while doing this, both caught before commit: I first patched the httpsRequestWithRetry call inside requestVoid (identical code shape, first match won) — typecheck caught it; and my drain test omitted a terminating page, which was my fixture, not the code. Gates: 1596 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- packages/core/src/client.ts | 23 +++- .../core/src/composite/_mcp-job-helpers.ts | 45 +++++++- .../core/src/tool-descriptions.generated.ts | 30 +---- .../composite/job-poll-abort-inflight.test.ts | 105 ++++++++++++++++++ .../audit/gated-anti-trigger-header.test.ts | 65 +++++++++++ .../snippets/gates/delivery-release-gate.md | 6 - packages/promptforge/src/assembler.ts | 6 +- packages/promptforge/src/frontmatter.ts | 6 + .../composite/bulk-qualify-leads.md.tmpl | 4 +- .../composite/extend-lens.md.tmpl | 3 +- .../composite/pull-leads.md.tmpl | 4 +- 11 files changed, 244 insertions(+), 53 deletions(-) create mode 100644 packages/core/test/unit/composite/job-poll-abort-inflight.test.ts create mode 100644 packages/mcp/test/audit/gated-anti-trigger-header.test.ts delete mode 100644 packages/promptforge/snippets/gates/delivery-release-gate.md diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 718c17d0..be2b2e32 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -42,7 +42,8 @@ function httpsRequest( method: string, url: string, headers: Record, - body?: string | Buffer + body?: string | Buffer, + signal?: AbortSignal ): Promise { return new Promise((resolve, reject) => { const start = Date.now(); @@ -58,6 +59,11 @@ function httpsRequest( path: parsed.pathname + parsed.search, method, headers: reqHeaders, + // Node aborts the socket and emits an AbortError on `error`, which the + // handler below rejects with. Without this a cancelled tool call sat on + // an in-flight GET until the server answered — the polling loop cannot + // honour its advertised <=2s exit while blocked inside one. + signal, }, (res) => { const chunks: Buffer[] = []; @@ -441,9 +447,10 @@ export class LeadbayClient { method: string, url: string, headers: Record, - body?: string | Buffer + body?: string | Buffer, + signal?: AbortSignal ): Promise => { - const res = await httpsRequest(method, url, headers, body); + const res = await httpsRequest(method, url, headers, body, signal); if (res.status === 401 && method.toUpperCase() === "GET") { this.releaseSemaphore(); try { @@ -451,7 +458,10 @@ export class LeadbayClient { } finally { await this.acquireSemaphore(); } - return httpsRequest(method, url, headers, body); + // Don't burn the retry on an already-cancelled call: the caller is gone, + // and the 250ms backoff above just made the wait longer. + if (signal?.aborted) return res; + return httpsRequest(method, url, headers, body, signal); } return res; }; @@ -460,7 +470,7 @@ export class LeadbayClient { method: string, path: string, body?: unknown, - opts?: { retryOn401?: boolean } + opts?: { retryOn401?: boolean; signal?: AbortSignal } ): Promise { // Mock mode short-circuit (no auth required). if (process.env.LEADBAY_MOCK === "1") { @@ -491,7 +501,8 @@ export class LeadbayClient { method, url, headers, - body ? JSON.stringify(body) : undefined + body ? JSON.stringify(body) : undefined, + opts?.signal ); this._lastMeta = { diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index a435d53d..b43a80fb 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -145,7 +145,8 @@ export async function collectJobSnapshot( client: LeadbayClient, jobId: string, since?: string, - limit?: number + limit?: number, + signal?: AbortSignal ): Promise { const pageLimit = Math.min(Math.max(limit ?? PAGE_LIMIT, 1), PAGE_LIMIT); // Escape the handle: job_id comes straight from user/agent input and the @@ -157,7 +158,9 @@ export async function collectJobSnapshot( `/mcp/jobs/${safeJobId}?limit=${pageLimit}` + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); const maxPages = maxPagesFor(pageLimit); - let page = await client.request("GET", qs(since)); + let page = await client.request("GET", qs(since), undefined, { + signal, + }); const items = [...page.items]; // The resumption cursor must survive an empty drain page. Following // next_since into a page with no items used to overwrite the cursor with that @@ -173,10 +176,17 @@ export async function collectJobSnapshot( // completed job with a short page — so following it whenever it is set adds // a wasted round-trip to every terminal poll. A short page means the cursor // is caught up; the caller keeps next_since for the next incremental poll. - while (page.items.length >= pageLimit && page.next_since && pages < maxPages) { + while ( + page.items.length >= pageLimit && + page.next_since && + pages < maxPages && + !signal?.aborted + ) { const next = await client.request( "GET", - qs(page.next_since) + qs(page.next_since), + undefined, + { signal } ); items.push(...next.items); pages += 1; @@ -216,6 +226,17 @@ export function sleepUnlessAborted( }); } +/** A wait that was cancelled before it ever read the job. There is no snapshot + * to return, and inventing one would report a state nobody observed. */ +function cancelledError(jobId: string): unknown { + return { + error: true, + code: "REQUEST_CANCELLED", + message: `The wait for job ${jobId} was cancelled before any status was read.`, + hint: "The job itself is backend-owned and keeps running. Poll leadbay_lead_job_status when you want its result.", + }; +} + /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). * Fires ctx.progress per poll and respects ctx.signal cancellation. * `since`/`limit` are forwarded to every snapshot so a caller that block-waits @@ -231,7 +252,11 @@ export async function waitForJob( limit?: number ): Promise { const startedAt = Date.now(); - let snap = await collectJobSnapshot(client, jobId, since, limit); + // Cancellation can arrive BEFORE the first poll — the wait then has no reason + // to open a request at all. Checked here rather than only in the loop + // condition, which is not reached until after that request returns. + if (ctx?.signal?.aborted) throw cancelledError(jobId); + let snap = await collectJobSnapshot(client, jobId, since, limit, ctx?.signal); while ( !TERMINAL_JOB_STATES.has(snap.job.state) && (Date.now() - startedAt) / 1000 < waitSeconds && @@ -247,7 +272,15 @@ export async function waitForJob( ctx?.signal ); if (ctx?.signal?.aborted) break; - snap = await collectJobSnapshot(client, jobId, since, limit); + // A cancel landing mid-flight rejects this request. We already hold a good + // snapshot, so return it instead of surfacing an AbortError: the user + // cancelled the WAIT, not the job, and the job keeps running backend-side. + try { + snap = await collectJobSnapshot(client, jobId, since, limit, ctx?.signal); + } catch (e) { + if (ctx?.signal?.aborted) break; + throw e; + } const f = snap.funnel; ctx?.progress?.({ progress: f.delivered ?? 0, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 965792da..67a39fe4 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -597,7 +597,7 @@ Context: Leadbay auto-qualifies roughly the top 10 of each daily batch. Leads be WHEN TO USE: when the user wants more qualified leads than what's currently shown, or when a lead looks promising in leadbay_pull_leads but has an empty \`qualification_summary\`. -WHEN NOT TO USE: to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. +WHEN NOT TO USE: to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (only if listed — it is release-gated; server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. This tool MUTATES state. The caller (agent or human-in-the-loop) is responsible for confirming intent before invocation; the MCP server does not soft-prompt for confirmation. See \`annotations.destructiveHint\`. @@ -657,14 +657,6 @@ Exactly two offers — keep it terse, this is a status tool: |--------------------------------------|-----------------------------------------------|--------------------------------| | Qualification kicked off (async) | "Check progress in ~30s" | leadbay_qualify_status | | Job is done / blocking call returned | "Refresh leads view — the new qualifications should be on the top" | leadbay_pull_leads(lensId = pinned) | - - -**Before routing to net-new delivery, check your tool list.** -\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` -are release-gated: on a default deployment they are ABSENT from \`tools/list\` -even though this description names them. Route there only if the tool is in -your list — otherwise say net-new search isn't enabled here. Never call a name -you cannot see. `; // endregion: leadbay_bulk_qualify_leads @@ -1195,7 +1187,7 @@ Trigger phrases: "I want more leads on this lens", "extend the lens", "I need a **Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. -Do NOT use for: "show me today's leads" → \`leadbay_pull_leads\`; "find me companies that " → \`leadbay_find_new_leads\`; "narrow the audience" → \`leadbay_adjust_audience\`; "stop showing me X" → \`leadbay_refine_prompt\`. +Do NOT use for: "show me today's leads" → \`leadbay_pull_leads\`; "find me companies that " → \`leadbay_find_new_leads\` (only if listed); "narrow the audience" → \`leadbay_adjust_audience\`; "stop showing me X" → \`leadbay_refine_prompt\`. Prefer when: user has bigger appetite than the daily lens fill delivers — additive refill on same criteria @@ -1276,14 +1268,6 @@ Pick the row matching the response \`status\`. Seed-picking is internal; do NOT | \`no_valid_seeds\` | (silent retry — re-call \`leadbay_seed_candidates\` then \`leadbay_extend_lens\`) | internal — only surface if the second attempt also fails | If nothing matches cleanly, default to "pull leads now to see what's queued" — never invent a tool that doesn't exist. - - -**Before routing to net-new delivery, check your tool list.** -\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` -are release-gated: on a default deployment they are ABSENT from \`tools/list\` -even though this description names them. Route there only if the tool is in -your list — otherwise say net-new search isn't enabled here. Never call a name -you cannot see. `; // endregion: leadbay_extend_lens @@ -3416,7 +3400,7 @@ Trigger phrases: "show me leads", "show me new leads", "show me today's leads", **Memory:** recall + capture via \`leadbay_agent_memory_*\` tools. -Do NOT use for: "find me N companies that " → \`leadbay_find_new_leads\`; "new prospects like with their emails" → \`leadbay_find_new_leads\`; "leads I should follow up with" → \`leadbay_pull_followups\`; "I'm going to " → \`leadbay_tour_plan\`; "I'm in next week — who's worth meeting" → \`leadbay_tour_plan\`; "who should I meet in " → \`leadbay_tour_plan\`; "visiting — who's worth meeting / seeing" → \`leadbay_tour_plan\`; "leads I should reach out to" → \`leadbay_pull_followups\`; "leads to get back to" → \`leadbay_pull_followups\`; "leads to contact today" → \`leadbay_pull_followups\`; "should I contact" → \`leadbay_pull_followups\`; "reconnect with" → \`leadbay_pull_followups\`; "re-engage" → \`leadbay_pull_followups\`. +Do NOT use for: "find me N companies that " → \`leadbay_find_new_leads\` (only if listed); "new prospects like with their emails" → \`leadbay_find_new_leads\` (only if listed); "leads I should follow up with" → \`leadbay_pull_followups\`; "I'm going to " → \`leadbay_tour_plan\`; "I'm in next week — who's worth meeting" → \`leadbay_tour_plan\`; "who should I meet in " → \`leadbay_tour_plan\`; "visiting — who's worth meeting / seeing" → \`leadbay_tour_plan\`; "leads I should reach out to" → \`leadbay_pull_followups\`; "leads to get back to" → \`leadbay_pull_followups\`; "leads to contact today" → \`leadbay_pull_followups\`; "should I contact" → \`leadbay_pull_followups\`; "reconnect with" → \`leadbay_pull_followups\`; "re-engage" → \`leadbay_pull_followups\`. Prefer when: fresh Discover leads; if a lens is named, pass \`lensId\` and pin it @@ -3574,14 +3558,6 @@ Pick 2–3 items below based on what was actually observed in the response. The | User wants a narrower / wider audience | "Adjust the lens filters (sector / size)" | leadbay_adjust_audience(...) | | Phase 4 research was run (\`research_lead_by_id\` called) AND top contacts lack direct email/phone | "Enrich contacts on [Lead1], [Lead2] to get direct emails and phone numbers" | leadbay_enrich_contacts(leadId, contactId) — ONE call per contact (the tool takes a single leadId + contactId, never a list) | If nothing in the menu applies cleanly, suggest only "pull next page" and "research a specific lead in depth" — never invent a tool that doesn't exist. - - -**Before routing to net-new delivery, check your tool list.** -\`leadbay_find_new_leads\` / \`leadbay_qualify_leads\` / \`leadbay_lead_job_status\` -are release-gated: on a default deployment they are ABSENT from \`tools/list\` -even though this description names them. Route there only if the tool is in -your list — otherwise say net-new search isn't enabled here. Never call a name -you cannot see. `; // endregion: leadbay_pull_leads diff --git a/packages/core/test/unit/composite/job-poll-abort-inflight.test.ts b/packages/core/test/unit/composite/job-poll-abort-inflight.test.ts new file mode 100644 index 00000000..b33be5ee --- /dev/null +++ b/packages/core/test/unit/composite/job-poll-abort-inflight.test.ts @@ -0,0 +1,105 @@ +/** + * Cancellation must reach the in-flight request, not just the sleep between + * polls. + * + * Making the delay abortable was only half of it: `waitForJob` still checked + * `ctx.signal` only AFTER awaiting `collectJobSnapshot`, and the snapshot's GET + * took no signal at all. A cancel arriving before or during a slow `/mcp/jobs` + * response therefore sat blocked until the server answered — well past the + * "polling loop exits within <=2 seconds" the server advertises in its own + * instructions. + * + * Three properties pinned here: an already-cancelled wait opens no request at + * all, the signal reaches the HTTP layer, and a cancel mid-drain stops paging. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { + waitForJob, + collectJobSnapshot, +} from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +const runningPage = (n: number) => ({ + method: "GET" as const, + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "running" }, + funnel: { delivered: n, examined: n }, + items: [{ status: "delivered", seq: n }], + next_since: `cur-${n}`, + cost: { spent: 0, unit: "cost_cents", breakdown: {} }, + explain: { region: "us", model: "m" }, + }, +}); + +describe("waitForJob — cancellation reaches the request", () => { + it("opens NO request when the signal is already aborted", async () => { + mockHttp([runningPage(1)]); + const ac = new AbortController(); + ac.abort(); + await expect( + waitForJob(newClient(), "job-1", 60, { signal: ac.signal } as any) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + // The point: not one byte went out for a wait nobody is listening to. + expect(getHttpRequests()).toHaveLength(0); + }); + + it("says the job keeps running, since cancelling the wait does not stop it", async () => { + mockHttp([runningPage(1)]); + const ac = new AbortController(); + ac.abort(); + const err = await waitForJob( + newClient(), + "job-1", + 60, + { signal: ac.signal } as any + ).catch((e) => e); + expect(err.hint).toMatch(/backend-owned|keeps running/i); + expect(err.hint).toMatch(/leadbay_lead_job_status/); + }); +}); + +describe("collectJobSnapshot — abort stops the drain", () => { + it("stops paging once the signal aborts", async () => { + // Full pages keep the drain going; aborting after the first must end it + // rather than walking every page the bound allows. + mockHttp([runningPage(1), runningPage(2), runningPage(3)]); + const ac = new AbortController(); + const client = newClient(); + const p = collectJobSnapshot(client, "job-1", undefined, 1, ac.signal); + ac.abort(); + const snap = await p.catch(() => null); + // Either it rejected (abort hit the socket) or it stopped early — what it + // must NOT do is drain all three pages as if nothing happened. + const gets = getHttpRequests().filter((r) => r.method === "GET"); + expect(gets.length).toBeLessThan(3); + if (snap) expect(snap.items.length).toBeLessThan(3); + }); + + it("still drains normally with no signal", async () => { + // Last page carries no cursor, which is what ends a normal drain. + const lastPage = { + ...runningPage(2), + body: { ...runningPage(2).body, next_since: null }, + }; + mockHttp([runningPage(1), lastPage]); + const snap = await collectJobSnapshot(newClient(), "job-1", undefined, 1); + expect(snap.items.length).toBe(2); + }); +}); diff --git a/packages/mcp/test/audit/gated-anti-trigger-header.test.ts b/packages/mcp/test/audit/gated-anti-trigger-header.test.ts new file mode 100644 index 00000000..d27477af --- /dev/null +++ b/packages/mcp/test/audit/gated-anti-trigger-header.test.ts @@ -0,0 +1,65 @@ +/** + * Audit: a cross-route to a release-gated tool carries its condition INSIDE + * the routing header. + * + * Hosts load roughly the first 600 characters of a tool description — that is + * the constraint routing-block.test.ts exists to enforce. A caveat placed in + * the body is therefore invisible to exactly the host that would misroute: + * `leadbay_pull_leads` named `leadbay_find_new_leads` at char ~300 while the + * availability caveat sat at char ~16,100, so a truncating host saw the route + * and never the condition, and sent net-new asks at a tool absent from + * tools/list. + * + * The `gated: true` marker on an anti-trigger now emits "(only if listed)" + * beside the target, inside the block every host reads. + */ + +import { describe, it, expect } from "vitest"; +import * as Generated from "@leadbay/core/dist/tool-descriptions.generated.js"; + +const HEAD = 600; +const GATED_TARGETS = [ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", +]; + +/** Tools that are always exposed but name a gated tool in their routing. */ +const CROSS_ROUTERS = ["leadbay_pull_leads", "leadbay_extend_lens"]; + +describe("audit: gated cross-routes are conditioned in the header", () => { + it("every gated target named in the header carries the condition", () => { + const offenders: string[] = []; + for (const tool of CROSS_ROUTERS) { + const head = (Generated as Record)[tool].slice(0, HEAD); + for (const target of GATED_TARGETS) { + let i = head.indexOf(target); + while (i !== -1) { + // "(only if listed)" must follow the backticked name closely. + const after = head.slice(i, i + target.length + 24); + if (!after.includes("only if listed")) { + offenders.push(`${tool}: ${target} at ${i} has no condition`); + } + i = head.indexOf(target, i + 1); + } + } + } + expect(offenders, offenders.join("\n")).toEqual([]); + }); + + it("the condition really is inside the loaded window, not the body", () => { + // Guards the fixture: if the routing block stopped emitting the marker, + // the test above would pass vacuously on a head containing no target. + const head = Generated.leadbay_pull_leads.slice(0, HEAD); + expect(head).toContain("leadbay_find_new_leads"); + expect(head).toContain("only if listed"); + }); + + it("ungated cross-routes are left alone", () => { + // The marker is opt-in; a normal route must not sprout a condition. + const head = Generated.leadbay_pull_leads.slice(0, HEAD); + const i = head.indexOf("leadbay_pull_followups"); + expect(i).toBeGreaterThan(-1); + expect(head.slice(i, i + 46)).not.toContain("only if listed"); + }); +}); diff --git a/packages/promptforge/snippets/gates/delivery-release-gate.md b/packages/promptforge/snippets/gates/delivery-release-gate.md deleted file mode 100644 index b04dfe14..00000000 --- a/packages/promptforge/snippets/gates/delivery-release-gate.md +++ /dev/null @@ -1,6 +0,0 @@ -**Before routing to net-new delivery, check your tool list.** -`leadbay_find_new_leads` / `leadbay_qualify_leads` / `leadbay_lead_job_status` -are release-gated: on a default deployment they are ABSENT from `tools/list` -even though this description names them. Route there only if the tool is in -your list — otherwise say net-new search isn't enabled here. Never call a name -you cannot see. diff --git a/packages/promptforge/src/assembler.ts b/packages/promptforge/src/assembler.ts index f17b5d35..3417889a 100644 --- a/packages/promptforge/src/assembler.ts +++ b/packages/promptforge/src/assembler.ts @@ -24,7 +24,11 @@ function emitRoutingBlock( } if (routing.anti_triggers && routing.anti_triggers.length > 0) { const formatted = routing.anti_triggers - .map((a) => `"${a.phrase}" → \`${a.route_to}\``) + .map( + (a) => + `"${a.phrase}" → \`${a.route_to}\`` + + (a.gated ? " (only if listed)" : "") + ) .join("; "); lines.push(`Do NOT use for: ${formatted}.`); } diff --git a/packages/promptforge/src/frontmatter.ts b/packages/promptforge/src/frontmatter.ts index 04265e68..a7f10a9a 100644 --- a/packages/promptforge/src/frontmatter.ts +++ b/packages/promptforge/src/frontmatter.ts @@ -25,6 +25,12 @@ export const RoutingAntiTriggerSchema = z.object({ route_to: z.string().regex(/^leadbay_[a-z0-9_]+$/, { message: "route_to must be a leadbay_* tool name", }), + // The target is release-gated, so it may be absent from tools/list on a + // default deployment. Marked HERE rather than explained in the body: the + // routing block is the part every host loads even when truncating, and a + // caveat further down the description is never read by the host that most + // needs it. + gated: z.boolean().optional(), }); // Full-sentence examples — community best practice (Anthropic diff --git a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl index fc47c5b7..51f91f7f 100644 --- a/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/bulk-qualify-leads.md.tmpl @@ -19,7 +19,7 @@ Context: Leadbay auto-qualifies roughly the top 10 of each daily batch. Leads be {{include:headers/tool-when-to-use}} when the user wants more qualified leads than what's currently shown, or when a lead looks promising in leadbay_pull_leads but has an empty `qualification_summary`. -{{include:headers/tool-when-not-to-use}} to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. +{{include:headers/tool-when-not-to-use}} to qualify a single specific lead — that's leadbay_qualify_lead (granular, advanced). And NOT for companies the user names or lists themselves (CRM rows, websites, prior deliveries) — that's leadbay_qualify_leads (only if listed — it is release-gated; server-side batch with per-item verdicts and contact matching); this tool only walks the ACTIVE LENS top-down. {{include:headers/verification-required-if-write}} @@ -41,5 +41,3 @@ Do not enumerate the affected leads — that's the job of `leadbay_pull_leads`. --- {{include:next-steps/bulk-qualify-leads}} - -{{include:gates/delivery-release-gate}} diff --git a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl index d106df4c..29c577e9 100644 --- a/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/extend-lens.md.tmpl @@ -20,6 +20,7 @@ routing: route_to: leadbay_pull_leads - phrase: "find me companies that " route_to: leadbay_find_new_leads + gated: true - phrase: "narrow the audience" route_to: leadbay_adjust_audience - phrase: "stop showing me X" @@ -72,5 +73,3 @@ Queue an additive extra-refill on a lens — more leads on the same criteria, wi --- {{include:next-steps/extend-lens}} - -{{include:gates/delivery-release-gate}} diff --git a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl index 92e03275..1ef95794 100644 --- a/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/pull-leads.md.tmpl @@ -23,8 +23,10 @@ routing: anti_triggers: - phrase: "find me N companies that " route_to: leadbay_find_new_leads + gated: true - phrase: "new prospects like with their emails" route_to: leadbay_find_new_leads + gated: true - phrase: "leads I should follow up with" route_to: leadbay_pull_followups - phrase: "I'm going to " @@ -96,5 +98,3 @@ The active lens can change between calls (5-min cache + backend `last_requested_ --- {{include:next-steps/pull-leads}} - -{{include:gates/delivery-release-gate}} From a7037b795239bfe8045de5b5803f2b75337b8c8b Mon Sep 17 00:00:00 2001 From: Arty Date: Tue, 18 Aug 2026 17:43:45 +0200 Subject: [PATCH 41/46] fix(mcp): reject dropped exclusions, convert cost_cents, re-point stale indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] exclude_lead_ids silently shrank — my own regression. Once the submit body started sending canonicalIdSet(...) instead of the raw array, a non-string entry stopped reaching the backend at all: canonicalIdSet maps it to null and filters it, and blanks go the same way. The paid search then ran WITHOUT an exclusion the caller asked for, and could re-deliver and charge for exactly the lead they were trying to skip. rejectMalformedExclusions refuses the list instead, before the cap check (counting a list still full of junk would size the cap against entries that were never going to be sent). [P2] The funnel line never converted cost_cents. The recipe said `spent C.CC` and I had just taught it to pick a currency symbol — so `spent: 165` (which is 1.65) rendered as `$165.00`, a 100x overstatement of a real charge, now with a confident symbol in front of it. The rule states the division first and covers every amount, not just the flagged line: cost.spent, estimated_cost.max and the needs_confirmation quote all carry the same unit and had no formatting rule at all. [P2] Stale ref.input_indexes on a deduped retry. The qualify key is order-insensitive by design, so a reordered retry returns the ORIGINAL job, whose indexes describe the ORIGINAL order — retry [B, A] after [A, B] and A is reported at index 0. remapInputIndexes re-points them at the current caller's lead_refs by ref identity (same website folding as the key, so a pasted URL matches a bare-domain echo), and nulls the indexes for ALL items when any one cannot be matched. A missing index is a gap; a wrong index is a false statement about which company was skipped. Only runs on duplicate submits, so the normal path is untouched, and `input_indexes_remapped` tells the caller which they got. Two things the full suite caught that I had not: - error-hints.test.ts is a PRE-EXISTING audit requiring every hint to name a concrete recovery action. My new INVALID_EXCLUDE_LEAD_ID hint failed it. Rewrote the hint to lead with "Drop or fix those entries and re-call" rather than touch the audit. - exclude-lead-ids-wire.test.ts (added by this PR two rounds ago) asserted a blank entry is silently dropped — the exact behaviour now refused. Updated it; verified against origin/main first that the file is not pre-existing. Headroom: the money rule cost find_new_leads 141 chars; trimmed it plus two verbose passages in the same shared snippet to land at 318. Gates: 1607 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 112 ++++++++++++++++++ packages/core/src/composite/find-new-leads.ts | 4 + packages/core/src/composite/qualify-leads.ts | 22 +++- .../core/src/tool-descriptions.generated.ts | 54 ++++----- .../composite/exclude-ids-malformed.test.ts | 103 ++++++++++++++++ .../composite/exclude-lead-ids-wire.test.ts | 7 +- .../composite/qualify-input-indexes.test.ts | 90 ++++++++++++++ .../snippets/rendering/lead-delivery-table.md | 18 +-- 8 files changed, 369 insertions(+), 41 deletions(-) create mode 100644 packages/core/test/unit/composite/exclude-ids-malformed.test.ts create mode 100644 packages/core/test/unit/composite/qualify-input-indexes.test.ts diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index b43a80fb..8dc930ee 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -6,6 +6,10 @@ // backend caps poll pages at 100 items, while a qualify job can carry up // to 500 refs — so a snapshot collects pages until the cursor drains. import { createHash } from "node:crypto"; +// Same website folding the idempotency key uses, so a caller's +// "https://Acme.com/" and a backend echo of "acme.com" compare equal. +// import-leads does not import this module, so no cycle. +import { normalizeDomain } from "./import-leads.js"; import type { LeadbayClient } from "../client.js"; import type { ToolContext } from "../types.js"; @@ -293,6 +297,89 @@ export async function waitForJob( return snap; } +/** Fold one ref-shaped object to a comparison key. Uses the same fields and + * the same website normalization as the idempotency key, so a caller's + * `https://Acme.com/` and a backend echo of `acme.com` compare equal. */ +function refIdentity(ref: unknown): string | null { + if (!ref || typeof ref !== "object" || Array.isArray(ref)) return null; + const o = ref as Record; + const str = (f: string): string | null => { + const v = o[f]; + if (typeof v !== "string") return null; + const t = v.trim().toLowerCase(); + return t ? t : null; + }; + const website = str("website"); + const parts = [ + normalizeUuid(o.lead_id) ?? null, + normalizeUuid(o.contact_id) ?? null, + website ? normalizeDomain(website) ?? website : null, + str("name"), + str("location"), + ]; + return parts.some((p) => p !== null) ? JSON.stringify(parts) : null; +} + +/** Re-point `ref.input_indexes` at the CURRENT caller's `lead_refs`. + * + * The qualify key is order-insensitive on purpose — the same refs in another + * order are the same approved work — so a reordered retry dedupes onto the + * ORIGINAL job, whose `input_indexes` describe the ORIGINAL order. Relaying + * them then maps every skipped-item verdict onto the wrong company for this + * caller: retrying `[B, A]` after `[A, B]` reports A at index 0. + * + * Remap by ref identity where every item resolves. Where any item cannot be + * matched — the backend echo is missing or shaped differently than assumed — + * null the indexes for ALL items rather than relay a mix of correct and + * stale ones. A missing index is a gap; a wrong index is a false statement + * about which company was skipped. */ +export function remapInputIndexes( + items: McpJobItem[], + refs: unknown +): { items: McpJobItem[]; remapped: boolean } { + const list = Array.isArray(refs) ? refs : []; + if (list.length === 0) return { items, remapped: false }; + const byIdentity = new Map(); + list.forEach((ref, i) => { + const key = refIdentity(ref); + if (!key) return; + const at = byIdentity.get(key); + if (at) at.push(i); + else byIdentity.set(key, [i]); + }); + + const next: McpJobItem[] = []; + let ok = true; + for (const item of items) { + const ref = item.ref; + if (!ref || ref.input_indexes == null) { + next.push(item); + continue; + } + const key = + refIdentity(ref.requested_as) ?? + refIdentity({ lead_id: ref.lead_id ?? undefined }); + const found = key ? byIdentity.get(key) : undefined; + if (!found) { + ok = false; + break; + } + next.push({ ...item, ref: { ...ref, input_indexes: found } }); + } + + if (!ok) { + return { + items: items.map((item) => + item.ref && item.ref.input_indexes != null + ? { ...item, ref: { ...item.ref, input_indexes: null } } + : item + ), + remapped: false, + }; + } + return { items: next, remapped: true }; +} + /** Canonicalize a SET-valued list for hashing: sorted AND de-duplicated. * Every list the backend treats as a set (sectors, locations, channels, * contact_titles, exclude_lead_ids, refs) must go through this — sorting @@ -648,6 +735,31 @@ function countryKey(raw: string): string { * redundant half of the list and dropping them is usually enough to fit. */ export const MAX_EXCLUDE_LEAD_IDS = 500; +/** `canonicalIdSet` maps a non-string to null and drops blanks. That is right + * for hashing, but the canonical list is now what goes ON THE WIRE — so a + * silent drop means the paid search runs WITHOUT an exclusion the caller + * asked for, and can reselect and charge for exactly the lead they were + * trying to skip. Refuse the list instead of quietly shrinking it. */ +export function rejectMalformedExclusions(ids: unknown): void { + if (ids === undefined || ids === null) return; + const list = Array.isArray(ids) ? ids : [ids]; + const bad: string[] = []; + list.forEach((v, i) => { + if (typeof v !== "string") { + bad.push(`${i} (${v === null ? "null" : typeof v})`); + } else if (!v.trim()) { + bad.push(`${i} (blank)`); + } + }); + if (bad.length === 0) return; + throw { + error: true, + code: "INVALID_EXCLUDE_LEAD_ID", + message: `exclude_lead_ids has ${bad.length} entr${bad.length === 1 ? "y" : "ies"} that is not a lead id: ${bad.join(", ")}.`, + hint: "Drop or fix those entries and re-call — every entry must be a non-blank lead id string. Silently skipping them would run the search without an exclusion you asked for, and could re-deliver and charge for that exact lead.", + }; +} + export function rejectOversizedExclusions(ids: unknown): void { if (ids === undefined || ids === null) return; // Count what would actually be SENT. That is only true because the submit diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 70eeec20..a7b72e20 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -23,6 +23,7 @@ import { compactBody, normalizeSearchFilters, rejectCountryLocations, + rejectMalformedExclusions, rejectOversizedExclusions, splitItems, TERMINAL_JOB_STATES, @@ -226,6 +227,9 @@ export const findNewLeads: Tool = { "exclude_lead_ids", ]); rejectCountryLocations(params.filters?.locations, client.region); + // Types first, then the cap: counting a list that still contains junk + // would size the cap against entries that were never going to be sent. + rejectMalformedExclusions(params.exclude_lead_ids); rejectOversizedExclusions(params.exclude_lead_ids); // Same spend gate as leadbay_qualify_leads. The trigger differs: `qualify` diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 7475b972..a5effff1 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -17,6 +17,7 @@ import { isUuidShaped, normalizeUuid, presentRequestId, + remapInputIndexes, canonicalLabelSet, derivedKey, mockedSubmitPreview, @@ -450,6 +451,17 @@ export const qualifyLeads: Tool = { : await collectJobSnapshot(client, submit.job_id); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); + + // A duplicate submit returns the ORIGINAL job. Its ref.input_indexes + // describe the order THAT request used, and this key is deliberately + // order-insensitive, so a reordered retry would map each verdict onto the + // wrong company for the current caller. Re-point them at this caller's + // lead_refs, or null them when the mapping cannot be proven. + const indexed = (submit.duplicate ?? false) + ? remapInputIndexes(snapshot.items, params.lead_refs) + : { items: snapshot.items, remapped: true }; + const view = { ...snapshot, items: indexed.items }; + return { job_id: submit.job_id, // Echo the key actually sent, so a retry can reuse it verbatim. @@ -473,14 +485,20 @@ export const qualifyLeads: Tool = { // QualifiedLead payload when delivered/degraded, and an honest // status_reason (not_in_universe, low_confidence_identity, ...) when // skipped — a skip is an ANSWER about that ref, not an error. - items: snapshot.items, + items: view.items, + // On a duplicate submit whose indexes could not be re-pointed at this + // caller's refs, input_indexes are null rather than stale — match items + // by `ref.requested_as` / `lead_id` instead. + input_indexes_remapped: (submit.duplicate ?? false) + ? indexed.remapped + : null, // ...and the same outcomes pre-split, because the shared // rendering/lead-delivery-table contract this tool's description // mandates reads deliveries from `leads[]` and skips from `skipped[]`. // Returning only `items` left an agent following the RENDER block with // two empty tables; the sibling tools (find_new_leads, lead_job_status) // both split. `items` stays for input-order per-ref mapping. - ...splitItems(snapshot), + ...splitItems(view), cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 67a39fe4..6ea22101 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1433,14 +1433,15 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`** — \`cost\` carries no currency and a France account -bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. -US → \`$\`, France → \`€\`; region unknown → bare amount. +**Money: divide, then symbol.** Every amount (\`cost.spent\`, +\`estimated_cost.max\`, quotes) is \`cost_cents\` — divide by 100, two decimals, +so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US +\`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates, not the target), never guess it. +count candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -1458,11 +1459,10 @@ the table as an answer — open with "weak matches only", show at most the top barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any -requested fence; drop and call out leaks (a same-named city in another state -slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` -(the standard FR path), fit is calibrated for lead-to-lead distances, not -exemplar centroids — treat high scores skeptically and verify each row's -\`description\` matches the ask. +requested fence; drop and call out leaks (same-named cities slip through). +(b) When \`explain.seed_strategy\` is \`text_match_exemplars\` (the standard FR +path), fit is calibrated for lead-to-lead distances, not exemplar centroids — +treat high scores skeptically and verify each row's \`description\`. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: @@ -2424,14 +2424,15 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`** — \`cost\` carries no currency and a France account -bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. -US → \`$\`, France → \`€\`; region unknown → bare amount. +**Money: divide, then symbol.** Every amount (\`cost.spent\`, +\`estimated_cost.max\`, quotes) is \`cost_cents\` — divide by 100, two decimals, +so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US +\`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates, not the target), never guess it. +count candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -2449,11 +2450,10 @@ the table as an answer — open with "weak matches only", show at most the top barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any -requested fence; drop and call out leaks (a same-named city in another state -slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` -(the standard FR path), fit is calibrated for lead-to-lead distances, not -exemplar centroids — treat high scores skeptically and verify each row's -\`description\` matches the ask. +requested fence; drop and call out leaks (same-named cities slip through). +(b) When \`explain.seed_strategy\` is \`text_match_exemplars\` (the standard FR +path), fit is calibrated for lead-to-lead distances, not exemplar centroids — +treat high scores skeptically and verify each row's \`description\`. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: @@ -3693,14 +3693,15 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code \`$\`** — \`cost\` carries no currency and a France account -bills in euros, so \`$6.09\` for a €6.09 charge misstates a paid operation. -US → \`$\`, France → \`€\`; region unknown → bare amount. +**Money: divide, then symbol.** Every amount (\`cost.spent\`, +\`estimated_cost.max\`, quotes) is \`cost_cents\` — divide by 100, two decimals, +so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US +\`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. "of the Y asked" needs \`summary.items_requested\`, which submit results carry but a later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates, not the target), never guess it. +count candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -3718,11 +3719,10 @@ the table as an answer — open with "weak matches only", show at most the top barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any -requested fence; drop and call out leaks (a same-named city in another state -slips through). (b) When \`explain.seed_strategy\` is \`text_match_exemplars\` -(the standard FR path), fit is calibrated for lead-to-lead distances, not -exemplar centroids — treat high scores skeptically and verify each row's -\`description\` matches the ask. +requested fence; drop and call out leaks (same-named cities slip through). +(b) When \`explain.seed_strategy\` is \`text_match_exemplars\` (the standard FR +path), fit is calibrated for lead-to-lead distances, not exemplar centroids — +treat high scores skeptically and verify each row's \`description\`. **Skipped items** (\`skipped[]\`, qualify jobs mostly): render a compact second table \`Ref → Outcome\` translating \`status_reason\` to plain words: diff --git a/packages/core/test/unit/composite/exclude-ids-malformed.test.ts b/packages/core/test/unit/composite/exclude-ids-malformed.test.ts new file mode 100644 index 00000000..c8c80be0 --- /dev/null +++ b/packages/core/test/unit/composite/exclude-ids-malformed.test.ts @@ -0,0 +1,103 @@ +/** + * Malformed exclusion ids are refused, not silently dropped. + * + * Once the submit body started sending `canonicalIdSet(exclude_lead_ids)` + * rather than the raw array, a non-string entry stopped reaching the backend + * at all: canonicalIdSet maps it to null and filters it. So a paid search ran + * WITHOUT an exclusion the caller asked for, and could re-deliver — and + * charge for — exactly the lead they were trying to skip. Blanks drop the + * same way. + * + * That silent narrowing is the regression the wire fix introduced; refusing + * the list keeps the caller's intent intact. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); +const UUID = "7b3c1de2-5f40-4a9c-9d21-0c8ea4f61b55"; + +beforeEach(() => resetHttpMock()); + +async function refusal(exclude_lead_ids: unknown) { + mockHttp([]); + try { + await findNewLeads.execute(newClient(), { + example_lead: { description: "independent gym" }, + count: 5, + request_id: "excl-test", + exclude_lead_ids, + wait_seconds: 0, + } as any); + return null; + } catch (e) { + return e as { code?: string; message?: string; hint?: string }; + } +} + +describe("leadbay_find_new_leads — malformed exclude_lead_ids", () => { + it("refuses a numeric entry rather than dropping it", async () => { + const e = await refusal([123, UUID]); + expect(e!.code).toBe("INVALID_EXCLUDE_LEAD_ID"); + expect(e!.message).toMatch(/0 \(number\)/); + }); + + it("refuses null and blank entries", async () => { + expect((await refusal([null]))!.message).toMatch(/0 \(null\)/); + expect((await refusal([" "]))!.message).toMatch(/0 \(blank\)/); + }); + + it("says why silence would be worse", async () => { + // The hint has to carry the consequence: this is a PAID path and the + // dropped exclusion is the lead the caller is trying not to pay for. + const e = await refusal([123]); + expect(e!.hint).toMatch(/charge|pay/i); + }); + + it("nothing reaches the wire", async () => { + await refusal([123, UUID]); + expect(getHttpRequests()).toHaveLength(0); + }); + + it("still accepts a clean list", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + { + method: "GET", + path: /^\/1\.6\/mcp\/jobs\//, + status: 200, + body: { + job: { id: "job-1", state: "succeeded" }, + funnel: { delivered: 0, examined: 0 }, + items: [], + }, + }, + ]); + await findNewLeads.execute(newClient(), { + example_lead: { description: "independent gym" }, + count: 5, + request_id: "excl-ok", + exclude_lead_ids: [UUID], + wait_seconds: 0, + } as any); + const post = getHttpRequests().find((r) => r.method === "POST"); + expect(JSON.parse(post!.body ?? "{}").exclude_lead_ids).toEqual([UUID]); + }); +}); diff --git a/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts b/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts index b9009f84..009ff36d 100644 --- a/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts +++ b/packages/core/test/unit/composite/exclude-lead-ids-wire.test.ts @@ -67,10 +67,11 @@ async function submittedBody(extra: Record): Promise { describe("leadbay_find_new_leads — exclude_lead_ids on the wire", () => { it("posts the canonical list, not the raw array", async () => { const body = await submittedBody({ - exclude_lead_ids: [UUID_A, UUID_A.toUpperCase(), UUID_B, " "], + exclude_lead_ids: [UUID_A, UUID_A.toUpperCase(), UUID_B], }); - // Deduped case-insensitively, blanks dropped — exactly what the cap guard - // counted. + // Deduped case-insensitively — exactly what the cap guard counted. + // (A blank entry is no longer silently dropped here; it is refused + // outright by rejectMalformedExclusions — see exclude-ids-malformed.) expect(body.exclude_lead_ids).toEqual([UUID_A, UUID_B].sort()); }); diff --git a/packages/core/test/unit/composite/qualify-input-indexes.test.ts b/packages/core/test/unit/composite/qualify-input-indexes.test.ts new file mode 100644 index 00000000..c5db046b --- /dev/null +++ b/packages/core/test/unit/composite/qualify-input-indexes.test.ts @@ -0,0 +1,90 @@ +/** + * `ref.input_indexes` must describe THIS caller's `lead_refs`, or say nothing. + * + * The qualify idempotency key is order-insensitive on purpose — the same refs + * in another order are the same approved work — so a reordered retry dedupes + * onto the ORIGINAL job. That job's `input_indexes` describe the ORIGINAL + * order, and relaying them maps each skipped-item verdict onto the wrong + * company: retry `[B, A]` after `[A, B]` and A is reported at index 0. + * + * Remap where every item resolves by ref identity; null the indexes for ALL + * items where any one cannot be matched. A missing index is a gap; a wrong + * index is a false statement about which company was skipped. + */ + +import { describe, it, expect } from "vitest"; +import { remapInputIndexes } from "../../../src/composite/_mcp-job-helpers.js"; + +const item = (requested_as: unknown, input_indexes: number[] | null) => + ({ + status: "skipped", + seq: 1, + ref: { input_indexes, requested_as }, + }) as any; + +describe("remapInputIndexes", () => { + it("re-points indexes at the current order after a reordered retry", () => { + // Original job ran [A, B]; this caller sent [B, A]. + const items = [ + item({ website: "a.com" }, [0]), + item({ website: "b.com" }, [1]), + ]; + const { items: out, remapped } = remapInputIndexes(items, [ + { website: "b.com" }, + { website: "a.com" }, + ]); + expect(remapped).toBe(true); + expect(out[0].ref.input_indexes).toEqual([1]); // a.com is now index 1 + expect(out[1].ref.input_indexes).toEqual([0]); // b.com is now index 0 + }); + + it("folds website spelling the same way the key does", () => { + // The caller pasted a URL; the backend echoed the bare domain. + const items = [item({ website: "acme.com" }, [0])]; + const { items: out, remapped } = remapInputIndexes(items, [ + { name: "Other" }, + { website: "https://Acme.com/" }, + ]); + expect(remapped).toBe(true); + expect(out[0].ref.input_indexes).toEqual([1]); + }); + + it("matches on lead_id when requested_as is absent", () => { + const id = "7b3c1de2-5f40-4a9c-9d21-0c8ea4f61b55"; + const items = [{ status: "skipped", seq: 1, ref: { input_indexes: [0], lead_id: id } } as any]; + const { items: out, remapped } = remapInputIndexes(items, [ + { name: "Other" }, + { lead_id: id.toUpperCase() }, + ]); + expect(remapped).toBe(true); + expect(out[0].ref.input_indexes).toEqual([1]); + }); + + it("nulls EVERY index when any item cannot be matched", () => { + // Partial remapping would leave a mix of correct and stale indexes with + // no way for the caller to tell them apart. + const items = [ + item({ website: "a.com" }, [0]), + item({ website: "unknown.com" }, [1]), + ]; + const { items: out, remapped } = remapInputIndexes(items, [ + { website: "a.com" }, + ]); + expect(remapped).toBe(false); + expect(out[0].ref.input_indexes).toBeNull(); + expect(out[1].ref.input_indexes).toBeNull(); + }); + + it("leaves items that carry no indexes alone", () => { + const items = [item({ website: "a.com" }, null)]; + const { items: out } = remapInputIndexes(items, [{ website: "a.com" }]); + expect(out[0].ref.input_indexes).toBeNull(); + }); + + it("no refs to map against is a no-op, not a wipe", () => { + const items = [item({ website: "a.com" }, [0])]; + const { items: out, remapped } = remapInputIndexes(items, undefined); + expect(remapped).toBe(false); + expect(out[0].ref.input_indexes).toEqual([0]); + }); +}); diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index 62eb3e1a..65b2f6d6 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -45,14 +45,15 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + > Matched N · examined E · qualified Q · disqualified D → **delivered X of > the Y asked** · stopped: · spent C.CC. -**Never hard-code `$`** — `cost` carries no currency and a France account -bills in euros, so `$6.09` for a €6.09 charge misstates a paid operation. -US → `$`, France → `€`; region unknown → bare amount. +**Money: divide, then symbol.** Every amount (`cost.spent`, +`estimated_cost.max`, quotes) is `cost_cents` — divide by 100, two decimals, +so `165` renders `1.65`, NEVER `165.00`. Symbol from the account region: US +`$`, France `€`, unknown → bare. Never hard-code `$`: it misstates a charge. "of the Y asked" needs `summary.items_requested`, which submit results carry but a later `leadbay_lead_job_status` snapshot does not. Without it write **delivered X** and stop — never back-fill Y from `matched`/`examined` (they -count candidates, not the target), never guess it. +count candidates), never guess it. Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → "ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → @@ -70,11 +71,10 @@ the table as an answer — open with "weak matches only", show at most the top barely-better-than-random candidates. **Sanity-check every row**: (a) geo — `city`/`region` must sit inside any -requested fence; drop and call out leaks (a same-named city in another state -slips through). (b) When `explain.seed_strategy` is `text_match_exemplars` -(the standard FR path), fit is calibrated for lead-to-lead distances, not -exemplar centroids — treat high scores skeptically and verify each row's -`description` matches the ask. +requested fence; drop and call out leaks (same-named cities slip through). +(b) When `explain.seed_strategy` is `text_match_exemplars` (the standard FR +path), fit is calibrated for lead-to-lead distances, not exemplar centroids — +treat high scores skeptically and verify each row's `description`. **Skipped items** (`skipped[]`, qualify jobs mostly): render a compact second table `Ref → Outcome` translating `status_reason` to plain words: From 73f0f039774cf3167b64c4d2e0839f3065dd9e98 Mon Sep 17 00:00:00 2001 From: Arty Date: Wed, 19 Aug 2026 09:52:04 +0200 Subject: [PATCH 42/46] fix(mcp): cancel the zero-wait polls too, and stop rendering rejects as prospects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [P2] The cancellation fix covered only the wait path. Threading ctx.signal through waitForJob left the three DIRECT collectJobSnapshot calls — the default wait_seconds:0 branch of lead_job_status, find_new_leads and qualify_leads — passing no signal at all, so a cancelled status poll still opened a GET and sat on it. Same defect the previous round was meant to close, one call site over. Fixed at the helper rather than at each site: collectJobSnapshot now throws REQUEST_CANCELLED on an already-aborted signal, so every caller inherits the guard instead of having to remember it, and all three sites now forward ctx?.signal. The paid SUBMIT stays deliberately non-abortable — it may already have committed server-side; what must not happen is polling it afterwards for a caller who has gone. [P2] A rejected account could be rendered as a prospect. qualify_leads DELIVERS org-owned companies that failed qualification, carrying their negative evidence — that is the documented contract ("here's why to skip this account" is a deliverable) — so splitItems puts them in leads[], where the shared recipe gives them a fit bar and a "Why it fits" column. The `disqualified` translation only ever existed in the SKIPPED table, which these never reach. On a paid result, with a high firmographic score, that reads as a recommendation to call the account the evaluation just rejected. They now get their own "Evaluated — does not fit" section, branched on status_reason / a negative qualification ai_score. Scoped to the tools that can actually emit one: qualify_leads and lead_job_status. find_new_leads filters disqualified candidates out via min_ai_score and never delivers one, so it pays nothing for a rule it cannot hit — its headroom is unchanged at 318, while the two emitters keep 2373 and 4840. Test-suite note, reported rather than smoothed over: one full run showed bulk-store.test.ts and import-status.test.ts failing on 5s timeouts. Both are files this PR never touches. They pass in isolation and passed two further full core runs plus the final workspace run. I could not reproduce it and cannot prove it is unrelated to the new import edge (_mcp-job-helpers -> import-leads for normalizeDomain); flagging it rather than calling it clean. Gates: 1614 tests, typecheck, prompts:check all green. Co-Authored-By: Claude Opus 5 --- .../core/src/composite/_mcp-job-helpers.ts | 4 + packages/core/src/composite/find-new-leads.ts | 8 +- .../core/src/composite/lead-job-status.ts | 3 +- packages/core/src/composite/qualify-leads.ts | 8 +- .../core/src/tool-descriptions.generated.ts | 32 +++++++ .../composite/zero-wait-cancellation.test.ts | 94 +++++++++++++++++++ .../disqualified-delivered-rendering.test.ts | 50 ++++++++++ .../rendering/disqualified-delivered.md | 14 +++ .../composite/lead-job-status.md.tmpl | 2 + .../composite/qualify-leads.md.tmpl | 2 + 10 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 packages/core/test/unit/composite/zero-wait-cancellation.test.ts create mode 100644 packages/mcp/test/audit/disqualified-delivered-rendering.test.ts create mode 100644 packages/promptforge/snippets/rendering/disqualified-delivered.md diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 8dc930ee..cf6b907b 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -152,6 +152,10 @@ export async function collectJobSnapshot( limit?: number, signal?: AbortSignal ): Promise { + // Centralised so every caller inherits it — the wait path had this check + // and the three zero-wait paths did not, which is precisely how a cancelled + // `wait_seconds: 0` status poll still opened a request. + if (signal?.aborted) throw cancelledError(jobId); const pageLimit = Math.min(Math.max(limit ?? PAGE_LIMIT, 1), PAGE_LIMIT); // Escape the handle: job_id comes straight from user/agent input and the // server does not validate schemas before dispatch, so an unescaped value diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index a7b72e20..181aae37 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -379,7 +379,13 @@ export const findNewLeads: Tool = { const snapshot = waitSeconds > 0 ? await waitForJob(client, submit.job_id, waitSeconds, ctx, params.count) - : await collectJobSnapshot(client, submit.job_id); + : await collectJobSnapshot( + client, + submit.job_id, + undefined, + undefined, + ctx?.signal + ); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); const { leads, skipped } = splitItems(snapshot); diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts index 113917b5..268a3c0a 100644 --- a/packages/core/src/composite/lead-job-status.ts +++ b/packages/core/src/composite/lead-job-status.ts @@ -80,7 +80,8 @@ export const leadJobStatus: Tool = { client, params.job_id, params.since, - params.limit + params.limit, + ctx?.signal ); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index a5effff1..db121713 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -448,7 +448,13 @@ export const qualifyLeads: Tool = { ctx, submit.items_requested ) - : await collectJobSnapshot(client, submit.job_id); + : await collectJobSnapshot( + client, + submit.job_id, + undefined, + undefined, + ctx?.signal + ); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 6ea22101..285e22b4 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -2480,6 +2480,22 @@ Never link a person's name to the company's LinkedIn page (and vice versa) — t +**Delivered ≠ endorsed.** This tool DELIVERS org-owned companies that FAILED +qualification, carrying their negative evidence — so a delivered item is not +automatically a prospect. An item whose \`status_reason\` is \`disqualified\`, or +whose \`fit.components.qualification\` is available with a negative \`ai_score\`, +must NOT go in the fit table: its firmographic score can still be high, and a +full bar beside "why it fits" reads as a recommendation to call an account the +evaluation just rejected. + +Give those their own short section after the fit table, titled +**Evaluated — does not fit**: linked company, then the verdict in plain +words from the +qualification evidence (failed question verdicts, missed tags, IBP reasoning). +That is the deliverable — "here's why to skip this account" — not a defect to +hide. + + --- ## NEXT STEPS — after a job status poll @@ -3749,6 +3765,22 @@ Never link a person's name to the company's LinkedIn page (and vice versa) — t +**Delivered ≠ endorsed.** This tool DELIVERS org-owned companies that FAILED +qualification, carrying their negative evidence — so a delivered item is not +automatically a prospect. An item whose \`status_reason\` is \`disqualified\`, or +whose \`fit.components.qualification\` is available with a negative \`ai_score\`, +must NOT go in the fit table: its firmographic score can still be high, and a +full bar beside "why it fits" reads as a recommendation to call an account the +evaluation just rejected. + +Give those their own short section after the fit table, titled +**Evaluated — does not fit**: linked company, then the verdict in plain +words from the +qualification evidence (failed question verdicts, missed tags, IBP reasoning). +That is the deliverable — "here's why to skip this account" — not a defect to +hide. + + --- ## NEXT STEPS — after a qualify_leads delivery diff --git a/packages/core/test/unit/composite/zero-wait-cancellation.test.ts b/packages/core/test/unit/composite/zero-wait-cancellation.test.ts new file mode 100644 index 00000000..9180aeb3 --- /dev/null +++ b/packages/core/test/unit/composite/zero-wait-cancellation.test.ts @@ -0,0 +1,94 @@ +/** + * Cancellation reaches the ZERO-WAIT snapshot paths too. + * + * Threading the signal through `waitForJob` covered `wait_seconds > 0` and + * left the three direct `collectJobSnapshot` calls — the default + * `wait_seconds: 0` branch of lead_job_status, find_new_leads and + * qualify_leads — passing no signal at all. A cancelled status poll therefore + * still opened a GET and sat on it, which is the same defect the previous fix + * was supposed to close. + * + * The guard now lives INSIDE collectJobSnapshot, so every call site inherits + * it rather than each needing to remember. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { + mockHttp, + resetHttpMock, + httpsMockFactory, + getHttpRequests, +} from "../../harness.js"; + +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { collectJobSnapshot } from "../../../src/composite/_mcp-job-helpers.js"; +import { leadJobStatus } from "../../../src/composite/lead-job-status.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +const aborted = () => { + const ac = new AbortController(); + ac.abort(); + return ac.signal; +}; + +describe("collectJobSnapshot — the guard every caller inherits", () => { + it("throws REQUEST_CANCELLED and opens no request", async () => { + mockHttp([]); + await expect( + collectJobSnapshot(newClient(), "job-1", undefined, undefined, aborted()) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(getHttpRequests()).toHaveLength(0); + }); +}); + +describe("leadbay_lead_job_status — wait_seconds: 0", () => { + it("a cancelled poll never reaches the network", async () => { + mockHttp([]); + await expect( + leadJobStatus.execute( + newClient(), + { job_id: "job-1", wait_seconds: 0 } as any, + { signal: aborted() } as any + ) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(getHttpRequests()).toHaveLength(0); + }); +}); + +describe("leadbay_find_new_leads — wait_seconds: 0", () => { + it("the submit still lands, then the snapshot is cancelled", async () => { + // The paid submit is deliberately NOT abortable — it may already have + // committed server-side. What must not happen is polling it afterwards + // on a call nobody is listening to. + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-1", state: "queued", items: [] }, + }, + ]); + await expect( + findNewLeads.execute( + newClient(), + { + example_lead: { description: "independent gym" }, + count: 5, + request_id: "cancel-test", + wait_seconds: 0, + } as any, + { signal: aborted() } as any + ) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + const reqs = getHttpRequests(); + expect(reqs.filter((r) => r.method === "POST")).toHaveLength(1); + expect(reqs.filter((r) => r.method === "GET")).toHaveLength(0); + }); +}); diff --git a/packages/mcp/test/audit/disqualified-delivered-rendering.test.ts b/packages/mcp/test/audit/disqualified-delivered-rendering.test.ts new file mode 100644 index 00000000..5ed356a4 --- /dev/null +++ b/packages/mcp/test/audit/disqualified-delivered-rendering.test.ts @@ -0,0 +1,50 @@ +/** + * Audit: a delivered-but-disqualified item is not rendered as a prospect. + * + * leadbay_qualify_leads deliberately DELIVERS org-owned companies that failed + * qualification, carrying their negative evidence ("here's why to skip this + * account" is a deliverable). splitItems therefore places them in `leads[]`, + * where the shared delivery recipe renders a fit bar and a "Why it fits" + * column — so a rejected account could be presented as a positive prospect, + * with a high firmographic score, on a PAID result. + * + * The rule belongs only to the tools that can actually emit such an item: + * qualify_leads, and lead_job_status when polling a qualify job. find_new_leads + * filters disqualified candidates out via min_ai_score and never delivers one, + * so it must NOT pay the char budget for a rule it cannot hit. + */ + +import { describe, it, expect } from "vitest"; +import * as Generated from "@leadbay/core/dist/tool-descriptions.generated.js"; + +const G = Generated as unknown as Record; +const EMITTERS = ["leadbay_qualify_leads", "leadbay_lead_job_status"]; + +describe("audit: delivered-but-disqualified rendering", () => { + it("both emitting tools carry the branch", () => { + for (const tool of EMITTERS) { + expect(G[tool], `${tool} missing the rule`).toMatch(/Delivered ≠ endorsed/); + } + }); + + it("the branch names the fields the agent can actually test", () => { + for (const tool of EMITTERS) { + expect(G[tool]).toContain("status_reason"); + expect(G[tool]).toMatch(/ai_score/); + } + }); + + it("it gives them a section of their own, not the fit table", () => { + for (const tool of EMITTERS) { + // Whitespace-tolerant: the snippet is hard-wrapped, so the label can + // legitimately break across a line. + expect(G[tool]).toMatch(/Evaluated\s+—\s+does not fit/); + } + }); + + it("find_new_leads does not carry it — it cannot deliver one", () => { + // Guards the scoping decision: this tool is the budget-critical one, and + // adding a rule it can never hit is how its headroom evaporates. + expect(G.leadbay_find_new_leads).not.toMatch(/Delivered ≠ endorsed/); + }); +}); diff --git a/packages/promptforge/snippets/rendering/disqualified-delivered.md b/packages/promptforge/snippets/rendering/disqualified-delivered.md new file mode 100644 index 00000000..3c17408d --- /dev/null +++ b/packages/promptforge/snippets/rendering/disqualified-delivered.md @@ -0,0 +1,14 @@ +**Delivered ≠ endorsed.** This tool DELIVERS org-owned companies that FAILED +qualification, carrying their negative evidence — so a delivered item is not +automatically a prospect. An item whose `status_reason` is `disqualified`, or +whose `fit.components.qualification` is available with a negative `ai_score`, +must NOT go in the fit table: its firmographic score can still be high, and a +full bar beside "why it fits" reads as a recommendation to call an account the +evaluation just rejected. + +Give those their own short section after the fit table, titled +**Evaluated — does not fit**: linked company, then the verdict in plain +words from the +qualification evidence (failed question verdicts, missed tags, IBP reasoning). +That is the deliverable — "here's why to skip this account" — not a defect to +hide. diff --git a/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl b/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl index f453b0e0..01498112 100644 --- a/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/lead-job-status.md.tmpl @@ -59,6 +59,8 @@ unknown job or another org's job. {{include:rendering/lead-delivery-table}} +{{include:rendering/disqualified-delivered}} + --- {{include:next-steps/lead-job-status}} diff --git a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl index 85b4f475..01658ca2 100644 --- a/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/qualify-leads.md.tmpl @@ -98,6 +98,8 @@ Retry-After beyond — wait, don't hammer), 30-min job wall clock. {{include:rendering/lead-delivery-table}} +{{include:rendering/disqualified-delivered}} + --- {{include:next-steps/qualify-leads}} From f1629f3f4731bb4337e7ea6e410d1ff42661b616 Mon Sep 17 00:00:00 2001 From: Arty Date: Thu, 20 Aug 2026 11:08:41 +0200 Subject: [PATCH 43/46] fix(mcp): make the cancel path reach the queue, the job handle a real handle, and the wait a real bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three P2s from the Codex round, all confirmed before fixing. 1. Cancellation stopped at the socket. `opts.signal` was only forwarded to https.request, i.e. AFTER acquireSemaphore() had returned. A delivery poll cancelled while QUEUED behind five busy slots could not observe its own abort until an unrelated request finished — against a stalled peer, well past the <=2s exit the delivery tools advertise. acquireSemaphore now takes the signal and SPLICES the waiter out of the queue on abort; tombstoning it instead would have leaked one slot per cancellation, since releaseSemaphore shifts blindly and the dead waiter would still take the ++. The 401-retry path now checks for abort BEFORE surrendering its slot, so a cancelled call never re-queues just to hand the slot straight back, and its 250ms backoff is abort-aware. The re-acquire in that `finally` is deliberately NOT abortable — the request must return holding exactly one slot or the caller's unconditional release drifts the counter permanently. 2. The path-traversal guard did not guard. encodeURIComponent does not escape `.`, so a job_id of `..` survived it verbatim and new URL() normalized /1.6/mcp/jobs/..?limit=100 down to /1.6/mcp/?limit=100 — sending the bearer token to an authenticated endpoint nobody asked for. Verified directly before fixing; `../..` and friends were already safe because their slashes DO get escaped, which is why the original comment looked right. Replaced with an allowlist of the RFC 3986 unreserved set plus an explicit dot-segment rejection: it accepts every handle the backend actually issues (UUIDs, search-auto-*/qualify-auto-* keys) while structurally excluding separators, query/fragment injection and encoded traversal. 3. wait_seconds did not bound the first read. The deadline was enforced only by the loop condition, which is not evaluated until the initial snapshot returns, so even `wait_seconds: 1` could block forever on a stalled GET. Every snapshot request now carries the REMAINING budget, capped by a per-request ceiling; zero-wait callers get the ceiling, which is what stops their single poll being unbounded too. This uses the `timeoutMs` that main's Stargate work added to the client — the two changes compose. A timeout on a LATER read returns the last good snapshot rather than discarding a live job over one slow read. A timeout on the FIRST read throws an envelope that CARRIES the job_id, so a caller who just paid for a submit does not lose the only handle to what they bought. Coverage in three new files; each fix was mutation-tested by reverting it and confirming the new tests fail (the semaphore ones fail by hanging, which is the bug's actual signature). Gates: build, typecheck, prompts:check, and two consecutive full runs — 1707 tests (core 888, mcp 751, components 47, promptforge 21). Co-Authored-By: Claude Opus 5 --- packages/core/src/client.ts | 69 +++++++++-- .../core/src/composite/_mcp-job-helpers.ts | 116 +++++++++++++++++- .../test/unit/client-semaphore-abort.test.ts | 107 ++++++++++++++++ .../unit/composite/job-id-traversal.test.ts | 80 ++++++++++++ .../unit/composite/job-wait-deadline.test.ts | 108 ++++++++++++++++ 5 files changed, 467 insertions(+), 13 deletions(-) create mode 100644 packages/core/test/unit/client-semaphore-abort.test.ts create mode 100644 packages/core/test/unit/composite/job-id-traversal.test.ts create mode 100644 packages/core/test/unit/composite/job-wait-deadline.test.ts diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index cc40cc57..80da98f8 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -425,19 +425,50 @@ export class LeadbayClient { return { active: this.activeRequests, queued: this.waitQueue.length }; } - private async acquireSemaphore(): Promise { + // `signal` makes a QUEUED acquisition abortable. Without it a cancelled call + // that arrived when all MAX_CONCURRENT slots were busy could not observe the + // abort until an unrelated request finished — the signal was only forwarded + // to the socket, which this call had not reached yet. Against slow or stalled + // peers that stranded the caller well past the <=2s exit the delivery tools + // advertise. + private async acquireSemaphore(signal?: AbortSignal): Promise { + if (signal?.aborted) throw this.cancelledBeforeSendError(); if (this.activeRequests < MAX_CONCURRENT) { this.activeRequests++; return; } - return new Promise((resolve) => { - this.waitQueue.push(() => { + return new Promise((resolve, reject) => { + const waiter = () => { + cleanup(); this.activeRequests++; resolve(); - }); + }; + const onAbort = () => { + // SPLICE the waiter out rather than flagging it dead: releaseSemaphore() + // shifts the queue blindly, so a tombstoned waiter would still take the + // ++ and resolve nothing — leaking one slot per cancellation until the + // client can serve no requests at all. + const i = this.waitQueue.indexOf(waiter); + if (i !== -1) this.waitQueue.splice(i, 1); + cleanup(); + reject(this.cancelledBeforeSendError()); + }; + const cleanup = () => signal?.removeEventListener("abort", onAbort); + signal?.addEventListener("abort", onAbort, { once: true }); + this.waitQueue.push(waiter); }); } + // Cancelled while queued — nothing was ever put on the wire, which is what + // makes this safe to report as "not sent" even for a write. + private cancelledBeforeSendError(): LeadbayError { + return this.makeError( + "REQUEST_CANCELLED", + "The request was cancelled before it was sent.", + "Re-call the tool if you still want the result — nothing reached the API, so nothing was charged." + ); + } + private releaseSemaphore(): void { this.activeRequests--; const next = this.waitQueue.shift(); @@ -489,14 +520,34 @@ export class LeadbayClient { ): Promise => { const res = await httpsRequest(method, url, headers, body, timeoutMs, signal); if (res.status === 401 && method.toUpperCase() === "GET") { + // Check BEFORE letting go of the slot: an already-cancelled call that + // releases here has to re-queue behind every other waiter just to hand + // the slot straight back, which is the unbounded wait this whole path + // is trying to avoid. + if (signal?.aborted) return res; this.releaseSemaphore(); try { - await new Promise((r) => setTimeout(r, 250)); + // Abort-aware so a cancel landing mid-backoff doesn't sit out the full + // 250ms before anyone notices. + await new Promise((resolve) => { + const t = setTimeout(done, 250); + function done() { + clearTimeout(t); + signal?.removeEventListener("abort", done); + resolve(); + } + signal?.addEventListener("abort", done, { once: true }); + }); } finally { + // Deliberately NOT abort-aware. This request must return holding + // exactly one slot, because the caller's `finally` releases one + // unconditionally; throwing here would decrement a slot we never took + // and drift the counter permanently. The window it can block for is + // bounded by the check above plus the 250ms backoff. await this.acquireSemaphore(); } - // Don't burn the retry on an already-cancelled call: the caller is gone, - // and the 250ms backoff above just made the wait longer. + // Don't burn the retry on a call cancelled during the backoff: the caller + // is gone, and the retry would only make the wait longer. if (signal?.aborted) return res; return httpsRequest(method, url, headers, body, timeoutMs, signal); } @@ -528,7 +579,9 @@ export class LeadbayClient { // Auto-retry a transient 401 on normal calls; the startup auth-probe opts // out (retryOn401:false) so a bad token fails fast instead of double-probing. const retryOn401 = opts?.retryOn401 !== false; - await this.acquireSemaphore(); + // Pass the signal: a cancel that lands while this call is QUEUED must not + // wait on unrelated in-flight requests to drain first. + await this.acquireSemaphore(opts?.signal); try { const url = `${this._baseUrl}${API_PREFIX}${path}`; const headers: Record = { diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index cf6b907b..fb96f0f3 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -120,6 +120,14 @@ export const TERMINAL_JOB_STATES: ReadonlySet = new Set([ // Poll cadence seam — tests shrink this so wait loops don't sleep for real. export const MCP_JOB_POLL = { intervalMs: 4000 }; +// Per-request ceiling for a job snapshot. node:https sets NO socket timeout, so +// without this a peer that completes the handshake and then stalls leaves the +// read pending forever. A block-waiting caller narrows this further to whatever +// is left of its own wait_seconds; a zero-wait caller ("one poll, now") has no +// budget of its own to inherit, and this is what stops its single poll being +// unbounded. +export const SNAPSHOT_TIMEOUT_MS = 30_000; + const PAGE_LIMIT = 100; // The drain is bounded by the page SIZE, not by a flat page count: at limit=5 // the worst case is 100 pages, and a flat 20 would silently return the first @@ -143,6 +151,36 @@ const MIN_PAGES = 20; const maxPagesFor = (pageLimit: number) => Math.max(MIN_PAGES, Math.ceil(MAX_JOB_ITEMS / pageLimit) + 1); +// A job handle is opaque, but it is always ONE path segment. Allowlisting the +// RFC 3986 unreserved set accepts every handle the backend actually issues +// (UUIDs, and the `search-auto-*` / `qualify-auto-*` derived keys) while +// structurally excluding separators, query/fragment injection, and encoded +// traversal. The dot-segment check is separate because `.` is unreserved and +// therefore passes the charset test while still being a traversal token. +const JOB_ID_CHARSET = /^[A-Za-z0-9._~-]+$/; +const MAX_JOB_ID_LENGTH = 200; + +export function assertSafeJobId(jobId: unknown): string { + const reject = (why: string): never => { + throw { + error: true, + code: "INVALID_JOB_ID", + message: `job_id ${why}.`, + hint: "Pass the job_id exactly as leadbay_find_new_leads or leadbay_qualify_leads returned it — it is an opaque handle, not a path.", + }; + }; + if (typeof jobId !== "string" || jobId.length === 0) + return reject("must be a non-empty string"); + if (jobId.length > MAX_JOB_ID_LENGTH) + return reject(`is ${jobId.length} chars — the maximum is ${MAX_JOB_ID_LENGTH}`); + // `.` and `..` are the two values that survive escaping and then get + // normalized AWAY by the URL parser, re-pointing the request at a parent path. + if (/^\.+$/.test(jobId)) return reject("cannot be a dot segment"); + if (!JOB_ID_CHARSET.test(jobId)) + return reject("contains characters that are not valid in a job handle"); + return encodeURIComponent(jobId); +} + /** One cumulative snapshot of the job, paging the item cursor dry. Job/funnel/ * cost/explain come from the LAST page fetched (the freshest projection). */ export async function collectJobSnapshot( @@ -150,7 +188,8 @@ export async function collectJobSnapshot( jobId: string, since?: string, limit?: number, - signal?: AbortSignal + signal?: AbortSignal, + timeoutMs: number = SNAPSHOT_TIMEOUT_MS ): Promise { // Centralised so every caller inherits it — the wait path had this check // and the three zero-wait paths did not, which is precisely how a cancelled @@ -161,13 +200,22 @@ export async function collectJobSnapshot( // server does not validate schemas before dispatch, so an unescaped value // containing path separators (`../../users/me`) would normalize out of // /mcp/jobs and fire an AUTHENTICATED GET at an unintended endpoint. - const safeJobId = encodeURIComponent(jobId); + // + // Escaping ALONE is not enough, which is what this guard originally got + // wrong. encodeURIComponent() does not touch `.`, so a job_id of `..` + // survives it verbatim and `new URL()` then resolves + // `/1.6/mcp/jobs/..?limit=100` to `/1.6/mcp/?limit=100` — the bearer token + // goes to a different authenticated endpoint than the one this function + // claims to read. (`../..` and friends are already safe: their slashes DO + // get escaped.) So validate the shape first. + const safeJobId = assertSafeJobId(jobId); const qs = (cursor?: string) => `/mcp/jobs/${safeJobId}?limit=${pageLimit}` + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); const maxPages = maxPagesFor(pageLimit); let page = await client.request("GET", qs(since), undefined, { signal, + timeoutMs, }); const items = [...page.items]; // The resumption cursor must survive an empty drain page. Following @@ -194,7 +242,7 @@ export async function collectJobSnapshot( "GET", qs(page.next_since), undefined, - { signal } + { signal, timeoutMs } ); items.push(...next.items); pages += 1; @@ -245,6 +293,32 @@ function cancelledError(jobId: string): unknown { }; } +// Each snapshot gets whatever is LEFT of the caller's wait, capped by the +// per-request ceiling. Floored at 1s so a nearly-spent budget still makes a +// real attempt rather than a request born already expired; the loop condition +// is what actually ends the wait, so this floor cannot extend it by more than +// one in-flight read. +function snapshotBudget(remainingMs: number): number { + return Math.min(SNAPSHOT_TIMEOUT_MS, Math.max(remainingMs, 1000)); +} + +function isTimeout(e: unknown): boolean { + return ( + typeof e === "object" && + e !== null && + (e as { code?: unknown }).code === "TIMEOUT" + ); +} + +function jobReadTimedOutError(jobId: string, waitSeconds: number): unknown { + return { + error: true, + code: "JOB_READ_TIMEOUT", + message: `Job ${jobId} was submitted and is running, but its status could not be read within ${waitSeconds}s.`, + hint: `Pass job_id ${jobId} to leadbay_lead_job_status to read it — the job is backend-owned, still running, and its results are kept for 30 days.`, + }; +} + /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). * Fires ctx.progress per poll and respects ctx.signal cancellation. * `since`/`limit` are forwarded to every snapshot so a caller that block-waits @@ -260,11 +334,32 @@ export async function waitForJob( limit?: number ): Promise { const startedAt = Date.now(); + const remainingMsOf = () => waitSeconds * 1000 - (Date.now() - startedAt); // Cancellation can arrive BEFORE the first poll — the wait then has no reason // to open a request at all. Checked here rather than only in the loop // condition, which is not reached until after that request returns. if (ctx?.signal?.aborted) throw cancelledError(jobId); - let snap = await collectJobSnapshot(client, jobId, since, limit, ctx?.signal); + // The FIRST snapshot needs the deadline as much as the later ones. Before + // this it had none: wait_seconds was enforced only by the loop condition, + // which is not evaluated until this call returns, so a stalled first GET made + // even `wait_seconds: 1` block indefinitely. + let snap: McpJobSnapshot; + try { + snap = await collectJobSnapshot( + client, + jobId, + since, + limit, + ctx?.signal, + snapshotBudget(remainingMsOf()) + ); + } catch (e) { + // We hold no snapshot yet, so there is nothing honest to return — but the + // job_id must survive, or a caller that just PAID for a submit loses the + // only handle to the job it bought. + if (isTimeout(e)) throw jobReadTimedOutError(jobId, waitSeconds); + throw e; + } while ( !TERMINAL_JOB_STATES.has(snap.job.state) && (Date.now() - startedAt) / 1000 < waitSeconds && @@ -284,9 +379,20 @@ export async function waitForJob( // snapshot, so return it instead of surfacing an AbortError: the user // cancelled the WAIT, not the job, and the job keeps running backend-side. try { - snap = await collectJobSnapshot(client, jobId, since, limit, ctx?.signal); + snap = await collectJobSnapshot( + client, + jobId, + since, + limit, + ctx?.signal, + snapshotBudget(remainingMsOf()) + ); } catch (e) { if (ctx?.signal?.aborted) break; + // A snapshot that outran the remaining wait is the wait expiring, not a + // failure: we already hold a good snapshot, so return it and let the + // caller poll on. Throwing here would discard a live job over a slow read. + if (isTimeout(e)) break; throw e; } const f = snap.funnel; diff --git a/packages/core/test/unit/client-semaphore-abort.test.ts b/packages/core/test/unit/client-semaphore-abort.test.ts new file mode 100644 index 00000000..8603aaf2 --- /dev/null +++ b/packages/core/test/unit/client-semaphore-abort.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { EventEmitter } from "node:events"; + +// Own node:https double: this suite needs requests that never answer, so it can +// occupy every concurrency slot and inspect what a queued caller does. The +// shared harness always responds on setImmediate, which is precisely the case +// that hides this bug. +const open: Array<() => void> = []; + +vi.mock("node:https", () => ({ + default: { + request: (_options: Record, cb?: (res: unknown) => void) => { + const req = new EventEmitter() as EventEmitter & { + write: () => void; + end: () => void; + destroy: () => void; + }; + req.write = () => {}; + req.destroy = () => {}; + req.end = () => { + // Park it. `open` holds the completion trigger so the test decides when + // (or whether) a slot is ever given back. + open.push(() => { + const res = new EventEmitter() as EventEmitter & { + statusCode: number; + headers: Record; + }; + res.statusCode = 200; + res.headers = {}; + cb?.(res); + res.emit("data", Buffer.from("{}")); + res.emit("end"); + }); + }; + return req; + }, + }, +})); + +import { LeadbayClient } from "../../src/client.js"; + +const MAX_CONCURRENT = 5; +const newClient = () => new LeadbayClient("https://api-us.leadbay.app", "u.test-token", "us"); + +beforeEach(() => { + open.length = 0; +}); + +const settled = () => new Promise((r) => setImmediate(r)); + +describe("client — a cancelled call queued on the semaphore does not wait for unrelated traffic", () => { + it("rejects the queued waiter on abort instead of blocking until a slot frees", async () => { + const client = newClient(); + // Occupy every slot with requests that never answer. + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + + const ac = new AbortController(); + const queued = client.request("GET", "/users/me", undefined, { signal: ac.signal }); + await settled(); + expect(client._semaphoreState.queued).toBe(1); + + ac.abort(); + + // The point of the test: this resolves while all five slots are STILL held. + await expect(queued).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + void stalled; + }); + + it("removes the cancelled waiter from the queue, so no slot is leaked", async () => { + const client = newClient(); + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + + const ac = new AbortController(); + const queued = client.request("GET", "/users/me", undefined, { signal: ac.signal }); + await settled(); + ac.abort(); + await expect(queued).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + + // A tombstoned (rather than spliced) waiter would still be shifted here, + // taking the ++ and resolving nobody — one slot lost per cancellation. + expect(client._semaphoreState.queued).toBe(0); + + open.forEach((finish) => finish()); + await Promise.all(stalled); + expect(client._semaphoreState.active).toBe(0); + expect(client._semaphoreState.queued).toBe(0); + }); + + it("refuses an already-aborted signal without taking a slot at all", async () => { + const client = newClient(); + const ac = new AbortController(); + ac.abort(); + + await expect( + client.request("GET", "/users/me", undefined, { signal: ac.signal }) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(client._semaphoreState.active).toBe(0); + }); +}); diff --git a/packages/core/test/unit/composite/job-id-traversal.test.ts b/packages/core/test/unit/composite/job-id-traversal.test.ts new file mode 100644 index 00000000..b7518121 --- /dev/null +++ b/packages/core/test/unit/composite/job-id-traversal.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { mockHttp, resetHttpMock, httpsMockFactory, getHttpRequests } from "../../harness.js"; +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { collectJobSnapshot, assertSafeJobId } from "../../../src/composite/_mcp-job-helpers.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.test-token", "us"); + +beforeEach(() => resetHttpMock()); + +describe("job_id path-traversal guard", () => { + // The regression this exists for: encodeURIComponent does NOT escape `.`, so + // `..` reached the URL verbatim and `new URL()` normalized + // /1.6/mcp/jobs/..?limit=100 down to /1.6/mcp/?limit=100 — sending the bearer + // token to an endpoint the caller never asked for. + it.each([[".."], ["."], ["..."]])("rejects the dot segment %j", async (jobId) => { + mockHttp([]); + await expect( + collectJobSnapshot(newClient(), jobId) + ).rejects.toMatchObject({ code: "INVALID_JOB_ID" }); + // Nothing may reach the wire: the whole point is that the token never goes + // to the re-pointed path. + expect(getHttpRequests()).toHaveLength(0); + }); + + it.each([["a/b"], ["../users/me"], ["x?y=1"], ["x#f"], ["x y"], [""]])( + "rejects the malformed handle %j without calling the API", + async (jobId) => { + mockHttp([]); + await expect( + collectJobSnapshot(newClient(), jobId) + ).rejects.toMatchObject({ code: "INVALID_JOB_ID" }); + expect(getHttpRequests()).toHaveLength(0); + } + ); + + it("rejects a non-string handle", async () => { + mockHttp([]); + await expect( + collectJobSnapshot(newClient(), undefined as unknown as string) + ).rejects.toMatchObject({ code: "INVALID_JOB_ID" }); + }); + + it("rejects an absurdly long handle", () => { + expect(() => assertSafeJobId("a".repeat(201))).toThrow(); + expect(() => assertSafeJobId("a".repeat(200))).not.toThrow(); + }); + + it("still accepts the handles the backend actually issues", async () => { + const ids = [ + "0a2fcbf5-18e1-4967-b5de-0c67cd823bcc", + "search-auto-gyms-dallas-2026-07-28", + "qualify-auto-abc_123", + "j1", + ]; + for (const id of ids) { + resetHttpMock(); + mockHttp([ + { + method: "GET", + path: `/1.6/mcp/jobs/${id}?limit=100`, + status: 200, + body: { + job: { state: "running" }, + items: [], + funnel: {}, + cost: { spent: 0 }, + next_since: null, + }, + }, + ]); + await collectJobSnapshot(newClient(), id); + // Asserting the PATH, not just that it succeeded: a handle that survives + // validation must also land on /mcp/jobs/ unchanged. + expect(getHttpRequests()[0].path).toBe(`/1.6/mcp/jobs/${id}?limit=100`); + } + }); +}); diff --git a/packages/core/test/unit/composite/job-wait-deadline.test.ts b/packages/core/test/unit/composite/job-wait-deadline.test.ts new file mode 100644 index 00000000..96618375 --- /dev/null +++ b/packages/core/test/unit/composite/job-wait-deadline.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import type { LeadbayClient } from "../../../src/client.js"; +import { + waitForJob, + collectJobSnapshot, + SNAPSHOT_TIMEOUT_MS, + MCP_JOB_POLL, +} from "../../../src/composite/_mcp-job-helpers.js"; + +type Opts = { signal?: AbortSignal; timeoutMs?: number }; + +const snapshot = (state: string) => ({ + job: { state }, + items: [], + funnel: {}, + cost: { spent: 0 }, + next_since: null, +}); + +// Records the opts of every request so the deadline can be asserted where it +// actually matters — on the wire call, not on the helper's signature. +function stubClient( + handler: (call: number) => unknown +): { client: LeadbayClient; opts: Opts[] } { + const opts: Opts[] = []; + let call = 0; + const client = { + request: async (_m: string, _p: string, _b: unknown, o?: Opts) => { + opts.push(o ?? {}); + const out = handler(call++); + if (out instanceof Error) throw out; + return out; + }, + } as unknown as LeadbayClient; + return { client, opts }; +} + +const timeoutErr = () => { + const e = new Error("Request timed out after 1000ms: GET /x") as Error & { code?: string }; + e.code = "TIMEOUT"; + return e; +}; + +// Use the documented cadence seam so the multi-poll cases don't sleep for real. +const REAL_INTERVAL = MCP_JOB_POLL.intervalMs; +beforeEach(() => { + MCP_JOB_POLL.intervalMs = 5; +}); +afterEach(() => { + MCP_JOB_POLL.intervalMs = REAL_INTERVAL; +}); + +describe("job snapshots are bounded by the caller's wait budget", () => { + it("bounds the FIRST snapshot, not just the ones after the loop condition", async () => { + const { client, opts } = stubClient(() => snapshot("completed")); + await waitForJob(client, "job-1", 5); + + expect(opts).toHaveLength(1); + // Before the fix this was undefined: wait_seconds was enforced only by the + // loop condition, which is not evaluated until the first GET has returned. + expect(opts[0].timeoutMs).toBeDefined(); + expect(opts[0].timeoutMs!).toBeLessThanOrEqual(5000); + }); + + it("never lets a snapshot outlive the requested wait", async () => { + const { client, opts } = stubClient(() => snapshot("completed")); + await waitForJob(client, "job-1", 1); + expect(opts[0].timeoutMs!).toBeLessThanOrEqual(1000); + }); + + it("caps a generous wait at the per-request ceiling", async () => { + const { client, opts } = stubClient(() => snapshot("completed")); + await waitForJob(client, "job-1", 600); + expect(opts[0].timeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); + }); + + it("gives a zero-wait poll a bound of its own", async () => { + const { client, opts } = stubClient(() => snapshot("running")); + await collectJobSnapshot(client, "job-1"); + expect(opts[0].timeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); + }); + + it("keeps the job_id when the first read times out, rather than losing a paid job", async () => { + const { client } = stubClient(() => timeoutErr()); + await expect(waitForJob(client, "job-abc", 2)).rejects.toMatchObject({ + code: "JOB_READ_TIMEOUT", + }); + // The handle must be recoverable from the error itself. + await waitForJob(client, "job-abc", 2).catch((e) => { + expect(e.message).toContain("job-abc"); + expect(e.hint).toContain("job-abc"); + expect(e.hint).toContain("leadbay_lead_job_status"); + }); + }); + + it("returns the last good snapshot when a LATER read times out", async () => { + // First read succeeds (running), the follow-up poll times out. + const { client } = stubClient((n) => (n === 0 ? snapshot("running") : timeoutErr())); + const snap = await waitForJob(client, "job-1", 5); + // A live job must not be discarded over one slow read. + expect(snap.job.state).toBe("running"); + }); + + it("propagates a non-timeout failure untouched", async () => { + const { client } = stubClient(() => new Error("boom")); + await expect(waitForJob(client, "job-1", 2)).rejects.toThrow("boom"); + }); +}); From f0fcd4fdb01df017134695a79da5984bdce5f74e Mon Sep 17 00:00:00 2001 From: Arty Date: Thu, 20 Aug 2026 11:57:36 +0200 Subject: [PATCH 44/46] fix(mcp): cancel a submit while it can still prove it spent nothing, and make the wait bound the whole drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings, two of them fair hits on last round's fixes. 1. (P1) A confirmed PAID submit passed no signal at all, so a search or qualify POST queued behind the client's five slots still went out after the user hit Cancel — charging, and claiming novelty on leads nobody asked for any more. The previous round called the submit "deliberately non-abortable" because it may already have committed server-side. That is true only ONCE IT IS ON THE WIRE; while queued, nothing has been sent and cancelling is provably free. So `request` now has two cancellation scopes: `signal` (queue + socket, for reads) and `preSendSignal` (queue only — aborts up to dispatch, then lets an in-flight request finish rather than leaving the spend ambiguous). Both paid submits pass the latter. zero-wait-cancellation.test.ts asserted the old contract, so it is updated with the reasoning recorded rather than quietly flipped. 2. (P2) Last round's deadline was per-REQUEST, which multiplies the budget by the page count: at limit:1 over a 1000-item job, 1000 sequential GETs each entitled to the full wait_seconds, and waitForJob cannot re-check its own deadline until the whole drain returns. Now one ABSOLUTE deadline is computed per snapshot and each page gets what is LEFT; the drain stops rather than starting a page that cannot finish. That makes early exit reachable, so truncation is now REPORTED, not silent: a full last page plus a live cursor means items are a prefix, and the tools surface `items_truncated` with a rendering rule. Without it a partial drain of a completed job renders "delivered 40" over 12 rows. The rule is shared by all three delivery tools rather than scoped to the drain-heavy pair — scoping would leave find_new_leads silently misreporting the day its item count crosses a page, which is the exact defect being fixed. Its budget was paid for by trimming its own body (250 chars headroom). 3. (P2) The 401 retry's re-acquisition was left non-abortable last round, and documented as a deliberate residual because the caller's `finally` releases unconditionally. That was the wrong trade: it can still block indefinitely. Fixed properly by making slot ownership explicit — a box the caller passes, so re-acquisition can reject on abort and the `finally` releases only what is actually held. Callers that do not track ownership keep the unconditional re-acquire, which is what preserves their balance. Every fix was mutation-tested by reverting it. The first attempt at the drain test passed against its own bug (a stubbed client never advances the clock, so a per-page budget and a remaining-budget are indistinguishable) — it now moves Date.now and asserts the budgets strictly decrease. The slot-accounting mutant also survived the first pass, so a dedicated 401-retry test was added; it now fails with the exact one-slot drift (`expected 4 to be 5`). Gates: build, typecheck, prompts:check, two consecutive full runs — 1720 tests (core 898, mcp 754, components 47, promptforge 21). Co-Authored-By: Claude Opus 5 --- packages/core/src/client.ts | 88 +++++++++--- .../core/src/composite/_mcp-job-helpers.ts | 32 ++++- packages/core/src/composite/find-new-leads.ts | 10 +- .../core/src/composite/lead-job-status.ts | 3 + packages/core/src/composite/qualify-leads.ts | 7 +- .../core/src/tool-descriptions.generated.ts | 26 ++-- .../client-401-retry-slot-accounting.test.ts | 96 +++++++++++++ .../test/unit/client-semaphore-abort.test.ts | 47 +++++++ .../unit/composite/job-wait-deadline.test.ts | 73 +++++++++- .../paid-submit-presend-cancel.test.ts | 128 ++++++++++++++++++ .../composite/zero-wait-cancellation.test.ts | 14 +- .../audit/items-truncated-rendering.test.ts | 52 +++++++ .../snippets/rendering/lead-delivery-table.md | 3 + .../composite/find-new-leads.md.tmpl | 17 ++- 14 files changed, 546 insertions(+), 50 deletions(-) create mode 100644 packages/core/test/unit/client-401-retry-slot-accounting.test.ts create mode 100644 packages/core/test/unit/composite/paid-submit-presend-cancel.test.ts create mode 100644 packages/mcp/test/audit/items-truncated-rendering.test.ts diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 80da98f8..358a8f16 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -516,7 +516,15 @@ export class LeadbayClient { headers: Record, body?: string | Buffer, timeoutMs?: number, - signal?: AbortSignal + signal?: AbortSignal, + // Slot-ownership box shared with the caller. The 401 path is the only place + // that hands the semaphore slot back mid-request, so it is the only place + // where "does this call still hold a slot?" can stop being a constant. A + // caller that passes the box gets an ABORTABLE re-acquisition, because the + // box tells its `finally` whether there is anything to release; a caller + // that omits it keeps the unconditional re-acquire, which is what preserves + // the balance for the paths that do not track ownership. + held?: { value: boolean } ): Promise => { const res = await httpsRequest(method, url, headers, body, timeoutMs, signal); if (res.status === 401 && method.toUpperCase() === "GET") { @@ -526,6 +534,7 @@ export class LeadbayClient { // is trying to avoid. if (signal?.aborted) return res; this.releaseSemaphore(); + if (held) held.value = false; try { // Abort-aware so a cancel landing mid-backoff doesn't sit out the full // 250ms before anyone notices. @@ -539,12 +548,12 @@ export class LeadbayClient { signal?.addEventListener("abort", done, { once: true }); }); } finally { - // Deliberately NOT abort-aware. This request must return holding - // exactly one slot, because the caller's `finally` releases one - // unconditionally; throwing here would decrement a slot we never took - // and drift the counter permanently. The window it can block for is - // bounded by the check above plus the 250ms backoff. - await this.acquireSemaphore(); + // Abortable ONLY when the caller tracks ownership. Otherwise a throw + // here would leave that caller's `finally` decrementing a slot it never + // obtained, drifting the counter permanently — so those paths keep the + // unconditional re-acquire instead. + await this.acquireSemaphore(held ? signal : undefined); + if (held) held.value = true; } // Don't burn the retry on a call cancelled during the backoff: the caller // is gone, and the retry would only make the wait longer. @@ -560,9 +569,24 @@ export class LeadbayClient { body?: unknown, // `timeoutMs` bounds a single attempt (each retry gets its own deadline) and // surfaces as a `TIMEOUT`-coded Error — never an auth code, so a caller that - // classifies failures reads it as a transient fault. `signal` is the - // caller's own cancellation, forwarded to every attempt. - opts?: { retryOn401?: boolean; timeoutMs?: number; signal?: AbortSignal } + // classifies failures reads it as a transient fault. + // + // Two cancellation scopes, because a paid POST needs half of one: + // `signal` — full cancellation. Aborts the queue wait AND the + // in-flight socket. Right for reads. + // `preSendSignal` — cancels ONLY up to the moment of dispatch. Aborts the + // queue wait, but once the request is on the wire it is + // left to finish. Right for a paid submit: while queued + // nothing has been sent so cancelling is free and + // honest, but tearing down an in-flight POST leaves the + // caller unable to say whether the backend already + // committed and charged for it. + opts?: { + retryOn401?: boolean; + timeoutMs?: number; + signal?: AbortSignal; + preSendSignal?: AbortSignal; + } ): Promise { // Mock mode short-circuit (no auth required). if (process.env.LEADBAY_MOCK === "1") { @@ -580,9 +604,15 @@ export class LeadbayClient { // out (retryOn401:false) so a bad token fails fast instead of double-probing. const retryOn401 = opts?.retryOn401 !== false; // Pass the signal: a cancel that lands while this call is QUEUED must not - // wait on unrelated in-flight requests to drain first. - await this.acquireSemaphore(opts?.signal); + // wait on unrelated in-flight requests to drain first. A pre-send-only + // signal governs the queue wait too — that phase is exactly what it covers. + const held = { value: true }; + await this.acquireSemaphore(opts?.signal ?? opts?.preSendSignal); try { + // Last point at which "nothing has been sent" is still true. A submit + // cancelled here provably spent nothing; one cancelled a line later + // provably nothing — which is exactly why it is allowed to finish. + if (opts?.preSendSignal?.aborted) throw this.cancelledBeforeSendError(); const url = `${this._baseUrl}${API_PREFIX}${path}`; const headers: Record = { Authorization: `Bearer ${this.token}`, @@ -591,14 +621,27 @@ export class LeadbayClient { headers["Content-Type"] = "application/json"; } - const res = await (retryOn401 ? this.httpsRequestWithRetry : httpsRequest)( - method, - url, - headers, - body ? JSON.stringify(body) : undefined, - opts?.timeoutMs, - opts?.signal - ); + const payload = body ? JSON.stringify(body) : undefined; + // Spelled out rather than a ternary over the two functions: only the + // retrying variant takes the ownership box, so their arities differ. + const res = retryOn401 + ? await this.httpsRequestWithRetry( + method, + url, + headers, + payload, + opts?.timeoutMs, + opts?.signal, + held + ) + : await httpsRequest( + method, + url, + headers, + payload, + opts?.timeoutMs, + opts?.signal + ); this._lastMeta = { region: this._region, @@ -617,7 +660,10 @@ export class LeadbayClient { return JSON.parse(res.body) as T; } finally { - this.releaseSemaphore(); + // Only if we still hold one: the 401 path can hand the slot back and then + // fail to re-acquire on abort, and releasing unconditionally there would + // decrement a slot this call no longer owns. + if (held.value) this.releaseSemaphore(); } } diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index fb96f0f3..20d3d6b2 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -93,6 +93,10 @@ export interface McpJobSnapshot { funnel: McpFunnel; items: McpJobItem[]; next_since?: string | null; + /** Set when the page drain stopped before the cursor ran dry, so `items` is a + * prefix of what the job holds. Resume from `next_since`. Never set by the + * backend — collectJobSnapshot adds it. */ + items_truncated?: boolean; cost: { spent: number; unit: string; @@ -213,9 +217,16 @@ export async function collectJobSnapshot( `/mcp/jobs/${safeJobId}?limit=${pageLimit}` + (cursor ? `&since=${encodeURIComponent(cursor)}` : ""); const maxPages = maxPagesFor(pageLimit); + // ABSOLUTE deadline, computed once. Applying `timeoutMs` per request instead + // multiplies the caller's budget by the page count: at limit:1 over a + // 1000-item job that is 1000 sequential GETs each entitled to the full + // wait_seconds, and waitForJob cannot re-check its own deadline until the + // whole drain returns. The wait is a bound on the drain, not on each page. + const deadlineAt = Date.now() + timeoutMs; + const remaining = () => deadlineAt - Date.now(); let page = await client.request("GET", qs(since), undefined, { signal, - timeoutMs, + timeoutMs: remaining(), }); const items = [...page.items]; // The resumption cursor must survive an empty drain page. Following @@ -238,11 +249,14 @@ export async function collectJobSnapshot( pages < maxPages && !signal?.aborted ) { + // Out of budget mid-drain: stop with what we have rather than start a page + // that cannot finish in time. The cursor below makes it resumable. + if (remaining() <= 0) break; const next = await client.request( "GET", qs(page.next_since), undefined, - { signal, timeoutMs } + { signal, timeoutMs: remaining() } ); items.push(...next.items); pages += 1; @@ -255,7 +269,19 @@ export async function collectJobSnapshot( } cursor = next.next_since ?? cursor; } - return { ...page, items, next_since: cursor }; + // Truncation is REPORTED, never silent. A full last page that still carries a + // cursor means the drain stopped early — deadline spent, abort, or the + // maxPages backstop — and every one of those exits leaves items unread. The + // natural finish is a SHORT page, which fails this test. Without the flag a + // partial drain of a completed job is indistinguishable from a complete one, + // and the tools would render "delivered N" over fewer rows than N. + const itemsTruncated = page.items.length >= pageLimit && !!page.next_since; + return { + ...page, + items, + next_since: cursor, + ...(itemsTruncated ? { items_truncated: true } : {}), + }; } /** Sleep, but wake immediately if the request is cancelled. diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 181aae37..3c161573 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -361,10 +361,17 @@ export const findNewLeads: Tool = { }; } + // preSendSignal, NOT signal. While this POST is queued behind the client's + // concurrency slots nothing has been sent, so a cancel there is free and + // provably spends nothing — that is the window this closes. Once it is on + // the wire it is deliberately left to finish: aborting mid-flight would + // leave us unable to say whether the backend already committed the job, + // charged for it, and claimed novelty on the leads. const submit = await client.request( "POST", "/mcp/search", - body + body, + { preSendSignal: ctx?.signal } ); const mocked = mockedSubmitPreview( submit, @@ -409,6 +416,7 @@ export const findNewLeads: Tool = { funnel: snapshot.funnel, leads, skipped, + items_truncated: snapshot.items_truncated ?? false, cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts index 268a3c0a..2bd9cf1b 100644 --- a/packages/core/src/composite/lead-job-status.ts +++ b/packages/core/src/composite/lead-job-status.ts @@ -93,6 +93,9 @@ export const leadJobStatus: Tool = { funnel: snapshot.funnel, leads, skipped, + // Surfaced so the renderer never presents a partial page set as the whole + // result: `leads` is a prefix, and next_since resumes it. + items_truncated: snapshot.items_truncated ?? false, next_since: snapshot.next_since ?? null, cost: snapshot.cost, explain: snapshot.explain, diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index db121713..1d933187 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -424,10 +424,14 @@ export const qualifyLeads: Tool = { }; } + // preSendSignal, NOT signal — same reasoning as the /mcp/search submit: a + // cancel while QUEUED provably spent nothing, but an in-flight POST may + // already have committed and charged, so it is left to finish. const submit = await client.request( "POST", "/mcp/qualify", - body + body, + { preSendSignal: ctx?.signal } ); const mocked = mockedSubmitPreview( submit, @@ -505,6 +509,7 @@ export const qualifyLeads: Tool = { // two empty tables; the sibling tools (find_new_leads, lead_job_status) // both split. `items` stays for input-order per-ref mapping. ...splitItems(view), + items_truncated: snapshot.items_truncated ?? false, cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index 285e22b4..f19e635e 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1310,24 +1310,23 @@ purchases contact channels. Polls up to \`wait_seconds\` (default 45); a longer job returns \`still_running\` + \`next_poll\` — hand off to \`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. -**Free vs paid — never spend silently.** The default ask (\`qualify: false\`, +**Free vs paid — never spend silently.** Default (\`qualify: false\`, \`channels: []\`) is FREE: company profile + fit score + cached research + contact identity. Paid: \`qualify: true\` (~94 cost_cents per candidate EXAMINED, capped by \`exploration_cap\`/\`max_cost\`) and \`channels\` (email 25c / -phone 250c, success-only). The gate is enforced in code: a paid call -(\`qualify: true\` and/or \`channels\`) is WITHHELD unless it carries -\`confirm: true\` — otherwise nothing is submitted and you get +phone 250c, success-only). Enforced in code: a paid call is WITHHELD unless it +carries \`confirm: true\` — nothing is submitted and you get \`mode: "needs_confirmation"\` with a real quote to show the user. Re-call with \`confirm: true\` on their go-ahead ("spend / get their emails" counts). \`confirm: false\` vetoes. Free needs no consent. **Preview free first** — reshaping an off-profile seed is free, exploring it with \`qualify: true\` is not. -**Ad-hoc exclusions ("no chains") are enforced by NO tier** — \`filters\` has -no exclusion key, and \`qualify\` scores against the org's FROZEN questions and -IBP, which need not mention chains; the seed's positive inverse only shifts -ranking. Violators can survive, be paid for and be delivered — post-filter -them yourself and say the tier didn't enforce it. Durable enforcement → +**Ad-hoc exclusions ("no chains") are enforced by NO tier** — \`filters\` has no +exclusion key, and \`qualify\` scores against the org's FROZEN questions and IBP, +which need not mention chains; the seed's inverse only shifts ranking. +Violators can survive, be paid for and be delivered — post-filter them yourself +and say the tier didn't enforce it. Durable enforcement → \`leadbay_refine_prompt\`. ### Crafting the \`example_lead\` seed — the input that decides result quality @@ -1472,6 +1471,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". +**\`items_truncated\`**: rows are a PREFIX, not the batch. Say so, and offer +\`leadbay_lead_job_status(job_id, since: next_since)\` for the rest. + **Hide from the user:** UUIDs (keep for tool calls, never render), cursors, \`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, \`seq\`/\`from_cache\`, empty arrays. @@ -2463,6 +2465,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". +**\`items_truncated\`**: rows are a PREFIX, not the batch. Say so, and offer +\`leadbay_lead_job_status(job_id, since: next_since)\` for the rest. + **Hide from the user:** UUIDs (keep for tool calls, never render), cursors, \`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, \`seq\`/\`from_cache\`, empty arrays. @@ -3748,6 +3753,9 @@ table \`Ref → Outcome\` translating \`status_reason\` to plain words: \`disqualified\` → "evaluated: does not fit" (evidence is in the item when owned), \`enrichment_failed\` → "channel could not be sourced (not billed)". +**\`items_truncated\`**: rows are a PREFIX, not the batch. Say so, and offer +\`leadbay_lead_job_status(job_id, since: next_since)\` for the rest. + **Hide from the user:** UUIDs (keep for tool calls, never render), cursors, \`explain.model\`/\`intelligence_snapshot\`, raw \`distance\`/\`calibration\`, \`seq\`/\`from_cache\`, empty arrays. diff --git a/packages/core/test/unit/client-401-retry-slot-accounting.test.ts b/packages/core/test/unit/client-401-retry-slot-accounting.test.ts new file mode 100644 index 00000000..31d7ca99 --- /dev/null +++ b/packages/core/test/unit/client-401-retry-slot-accounting.test.ts @@ -0,0 +1,96 @@ +/** + * The 401 retry is the only path that hands its semaphore slot back mid-request + * and then tries to take it again. That makes "does this call still hold a + * slot?" a variable rather than a constant, and it is the one place where an + * abortable re-acquisition can corrupt the counter: if re-acquisition rejects, + * the caller's `finally` must NOT release a slot it no longer owns. + * + * Drift here is permanent and one-directional — every occurrence costs the + * client another slot until it can serve nothing at all. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { EventEmitter } from "node:events"; + +let nextStatus = 200; +const pending: Array<() => void> = []; + +vi.mock("node:https", () => ({ + default: { + request: (_o: Record, cb?: (res: unknown) => void) => { + const status = nextStatus; + const req = new EventEmitter() as EventEmitter & { + write: () => void; + end: () => void; + destroy: () => void; + }; + req.write = () => {}; + req.destroy = () => {}; + req.end = () => { + const finish = () => { + const res = new EventEmitter() as EventEmitter & { + statusCode: number; + headers: Record; + }; + res.statusCode = status; + res.headers = {}; + cb?.(res); + res.emit("data", Buffer.from("{}")); + res.emit("end"); + }; + // A 401 answers at once so the retry path is entered; everything else + // parks, so the test controls when slots free up. + if (status === 401) setImmediate(finish); + else pending.push(finish); + }; + return req; + }, + }, +})); + +import { LeadbayClient } from "../../src/client.js"; + +const MAX_CONCURRENT = 5; +const newClient = () => new LeadbayClient("https://api-us.leadbay.app", "u.tok", "us"); +const settled = () => new Promise((r) => setImmediate(r)); + +beforeEach(() => { + nextStatus = 200; + pending.length = 0; +}); + +describe("401 retry — slot accounting survives an abort during the backoff", () => { + it("does not release a slot it failed to re-acquire", async () => { + const client = newClient(); + + // A GET that 401s: it releases its slot and enters the 250ms backoff. + nextStatus = 401; + const ac = new AbortController(); + const retrying = client + .request("GET", "/users/me", undefined, { signal: ac.signal }) + .catch(() => "rejected"); + await settled(); + + // While it is backing off, unrelated traffic takes every slot, so its + // re-acquisition will have to queue. + nextStatus = 200; + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + + // Cancel while it is queued for its slot back. + ac.abort(); + await retrying; + + // THE ASSERTION: the five stalled requests still hold exactly five slots. + // Releasing unconditionally in the caller's `finally` would decrement one + // of THEIR slots — the counter drifts to 4 and one slot is gone for good. + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + expect(client._semaphoreState.queued).toBe(0); + + pending.splice(0).forEach((f) => f()); + await Promise.all(stalled); + expect(client._semaphoreState.active).toBe(0); + }); +}); diff --git a/packages/core/test/unit/client-semaphore-abort.test.ts b/packages/core/test/unit/client-semaphore-abort.test.ts index 8603aaf2..04fcb555 100644 --- a/packages/core/test/unit/client-semaphore-abort.test.ts +++ b/packages/core/test/unit/client-semaphore-abort.test.ts @@ -105,3 +105,50 @@ describe("client — a cancelled call queued on the semaphore does not wait for expect(client._semaphoreState.active).toBe(0); }); }); + +describe("client — the 401 retry gives its slot back safely", () => { + // The retry path is the only place a request hands its slot back mid-flight. + // That makes "does this call still hold a slot?" a variable, and getting it + // wrong drifts the counter permanently in one direction or the other. + it("does not surrender the slot for a call already cancelled", async () => { + const client = newClient(); + const ac = new AbortController(); + const p = client.request("GET", "/users/me", undefined, { signal: ac.signal }); + await settled(); + + ac.abort(); + open.forEach((finish) => finish()); + await p.catch(() => {}); + + // Whatever happened, the accounting must balance: no slot left held, none + // released twice into a negative count. + expect(client._semaphoreState.active).toBe(0); + expect(client._semaphoreState.queued).toBe(0); + }); + + it("leaves the counter balanced across a burst of cancellations", async () => { + const client = newClient(); + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + + const controllers = Array.from({ length: 4 }, () => new AbortController()); + const queued = controllers.map((ac) => + client.request("GET", "/users/me", undefined, { signal: ac.signal }).catch(() => {}) + ); + await settled(); + expect(client._semaphoreState.queued).toBe(4); + + controllers.forEach((ac) => ac.abort()); + await Promise.all(queued); + // A leaked slot per cancellation would show up here as a queue that never + // empties, and then as a client that can serve nothing. + expect(client._semaphoreState.queued).toBe(0); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + + open.forEach((finish) => finish()); + await Promise.all(stalled); + expect(client._semaphoreState.active).toBe(0); + }); +}); diff --git a/packages/core/test/unit/composite/job-wait-deadline.test.ts b/packages/core/test/unit/composite/job-wait-deadline.test.ts index 96618375..1d139937 100644 --- a/packages/core/test/unit/composite/job-wait-deadline.test.ts +++ b/packages/core/test/unit/composite/job-wait-deadline.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import type { LeadbayClient } from "../../../src/client.js"; import { waitForJob, @@ -106,3 +106,74 @@ describe("job snapshots are bounded by the caller's wait budget", () => { await expect(waitForJob(client, "job-1", 2)).rejects.toThrow("boom"); }); }); + +describe("the wait bounds the whole drain, not each page of it", () => { + const page = (n: number, full: boolean) => ({ + job: { state: "completed" }, + items: full ? Array.from({ length: 2 }, (_, i) => ({ id: `${n}-${i}` })) : [], + funnel: {}, + cost: { spent: 0 }, + next_since: full ? `cur-${n}` : null, + }); + + it("spends one budget across the pages instead of handing each page a fresh one", async () => { + // Every page comes back FULL with a cursor, so the drain would run until + // maxPages. Per-request timeouts let each of those pages claim the caller's + // whole wait_seconds — the budget multiplied by the page count. + // + // The clock MUST advance for this to mean anything: with a frozen clock + // `remaining()` and a per-page `timeoutMs` are indistinguishable, and the + // test passes against the very bug it is meant to catch. + let now = 0; + const spy = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + const { client, opts } = stubClient((n) => { + now += 100; + return page(n, true); + }); + await waitForJob(client, "job-1", 5, undefined, undefined, undefined, 2); + + expect(opts.length).toBeGreaterThan(1); + const budgets = opts.map((o) => o.timeoutMs!); + // STRICTLY decreasing: each page is bounded by what is LEFT, so the drain + // as a whole cannot outlast the wait. A per-page budget would hold flat. + for (let i = 1; i < budgets.length; i++) { + expect(budgets[i]).toBeLessThan(budgets[i - 1]); + } + expect(Math.max(...budgets)).toBeLessThanOrEqual(5000); + // And the total handed out must not exceed the wait, which is precisely + // what "one budget per page" violated. + expect(budgets[budgets.length - 1]).toBeLessThanOrEqual(5000 - 100 * (budgets.length - 1)); + } finally { + spy.mockRestore(); + } + }); + + it("stops paging once the budget is spent rather than starting a doomed page", async () => { + let now = 0; + const spy = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + // Each page consumes 400ms of a 1s budget. + const { client, opts } = stubClient((n) => { + now += 400; + return page(n, true); + }); + const snap = await waitForJob(client, "job-1", 1, undefined, undefined, undefined, 2); + // 1000ms / 400ms — the drain must stop, not run to maxPages. + expect(opts.length).toBeLessThanOrEqual(3); + // And it must SAY it stopped early: a full last page plus a cursor is a + // prefix, not a finished read. + expect(snap.items_truncated).toBe(true); + expect(snap.next_since).toBeTruthy(); + } finally { + spy.mockRestore(); + } + }); + + it("does not flag a drain that finished naturally", async () => { + // Short page = the cursor ran dry. + const { client } = stubClient(() => page(0, false)); + const snap = await waitForJob(client, "job-1", 5, undefined, undefined, undefined, 2); + expect(snap.items_truncated).toBeUndefined(); + }); +}); diff --git a/packages/core/test/unit/composite/paid-submit-presend-cancel.test.ts b/packages/core/test/unit/composite/paid-submit-presend-cancel.test.ts new file mode 100644 index 00000000..65d7a536 --- /dev/null +++ b/packages/core/test/unit/composite/paid-submit-presend-cancel.test.ts @@ -0,0 +1,128 @@ +/** + * A paid submit is cancellable up to the moment of dispatch — and not after. + * + * The two halves are a single rule: cancellation is honoured exactly as long as + * we can still PROVE nothing was spent. Queued behind the client's concurrency + * slots, that proof holds. On the wire it does not: the backend may already + * have committed the job, charged for it and claimed novelty on the leads, and + * a torn-down socket cannot tell us which. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { EventEmitter } from "node:events"; + +const sent: string[] = []; +// Every dispatched request parks here until the test releases it. Nothing +// completes on its own — a double that auto-answers can never produce the +// "all slots busy, this one is queued" state these tests are about. +const pending: Array<() => void> = []; + +vi.mock("node:https", () => ({ + default: { + request: (options: Record, cb?: (res: unknown) => void) => { + const req = new EventEmitter() as EventEmitter & { + write: () => void; + end: () => void; + destroy: () => void; + }; + req.write = () => {}; + req.destroy = () => {}; + req.end = () => { + // Record at DISPATCH: this array is the record of what actually left + // the process, which is the only thing that can cost money. + sent.push(`${options.method} ${options.path}`); + const finish = () => { + const res = new EventEmitter() as EventEmitter & { + statusCode: number; + headers: Record; + }; + res.statusCode = 200; + res.headers = {}; + cb?.(res); + res.emit("data", Buffer.from(JSON.stringify({ job_id: "job-1", state: "queued" }))); + res.emit("end"); + }; + pending.push(finish); + }; + return req; + }, + }, +})); + +import { LeadbayClient } from "../../../src/client.js"; + +const newClient = () => new LeadbayClient("https://api-us.leadbay.app", "u.tok", "us"); +const settled = () => new Promise((r) => setImmediate(r)); + +beforeEach(() => { + sent.length = 0; + pending.length = 0; +}); + +const flush = () => { + const queued = pending.splice(0); + queued.forEach((f) => f()); +}; + +describe("preSendSignal — the pre-dispatch cancellation window", () => { + it("does not send a submit whose signal is already aborted", async () => { + const ac = new AbortController(); + ac.abort(); + await expect( + newClient().request("POST", "/mcp/search", { q: 1 }, { preSendSignal: ac.signal }) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(sent).toHaveLength(0); + }); + + it("cancels a submit still QUEUED behind busy slots, spending nothing", async () => { + const client = newClient(); + // Occupy all five slots with requests that never answer. + const stalled = Array.from({ length: 5 }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + sent.length = 0; + + const ac = new AbortController(); + const submit = client.request("POST", "/mcp/search", { q: 1 }, { + preSendSignal: ac.signal, + }); + await settled(); + ac.abort(); + + await expect(submit).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + // The whole point: the POST never left the process, so it cannot have + // charged, and it did not have to wait for the stalled traffic either. + expect(sent.filter((s) => s.includes("/mcp/search"))).toHaveLength(0); + void stalled; + }); + + it("lets an IN-FLIGHT submit finish rather than leaving the spend ambiguous", async () => { + const client = newClient(); + const ac = new AbortController(); + const submit = client.request<{ job_id: string }>( + "POST", + "/mcp/search", + { q: 1 }, + { preSendSignal: ac.signal } + ); + await settled(); + // Dispatched — past the provable-no-spend boundary. + expect(sent.filter((s) => s.includes("/mcp/search"))).toHaveLength(1); + + ac.abort(); + flush(); + + // It resolves normally: the job_id survives, so a job that may have been + // paid for is never orphaned by a late cancel. + await expect(submit).resolves.toMatchObject({ job_id: "job-1" }); + }); + + it("keeps full `signal` cancellation available for reads", async () => { + const ac = new AbortController(); + ac.abort(); + await expect( + newClient().request("GET", "/mcp/jobs/x", undefined, { signal: ac.signal }) + ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); + expect(sent).toHaveLength(0); + }); +}); diff --git a/packages/core/test/unit/composite/zero-wait-cancellation.test.ts b/packages/core/test/unit/composite/zero-wait-cancellation.test.ts index 9180aeb3..9b5a87f6 100644 --- a/packages/core/test/unit/composite/zero-wait-cancellation.test.ts +++ b/packages/core/test/unit/composite/zero-wait-cancellation.test.ts @@ -63,10 +63,14 @@ describe("leadbay_lead_job_status — wait_seconds: 0", () => { }); describe("leadbay_find_new_leads — wait_seconds: 0", () => { - it("the submit still lands, then the snapshot is cancelled", async () => { - // The paid submit is deliberately NOT abortable — it may already have - // committed server-side. What must not happen is polling it afterwards - // on a call nobody is listening to. + it("a submit cancelled BEFORE dispatch never reaches the network", async () => { + // Superseded contract: this used to assert the POST still landed, on the + // reasoning that a paid submit may already have committed server-side. That + // is true only ONCE IT IS ON THE WIRE. A submit still queued behind the + // client's concurrency slots has provably spent nothing, so cancelling it + // there is free — and letting it through charged the user for a job they + // had already cancelled. The in-flight half of the rule is unchanged and is + // covered in paid-submit-presend-cancel.test.ts. mockHttp([ { method: "POST", @@ -88,7 +92,7 @@ describe("leadbay_find_new_leads — wait_seconds: 0", () => { ) ).rejects.toMatchObject({ code: "REQUEST_CANCELLED" }); const reqs = getHttpRequests(); - expect(reqs.filter((r) => r.method === "POST")).toHaveLength(1); + expect(reqs.filter((r) => r.method === "POST")).toHaveLength(0); expect(reqs.filter((r) => r.method === "GET")).toHaveLength(0); }); }); diff --git a/packages/mcp/test/audit/items-truncated-rendering.test.ts b/packages/mcp/test/audit/items-truncated-rendering.test.ts new file mode 100644 index 00000000..18e7f79e --- /dev/null +++ b/packages/mcp/test/audit/items-truncated-rendering.test.ts @@ -0,0 +1,52 @@ +/** + * Audit: a partial page set is never presented as the whole batch. + * + * collectJobSnapshot stops draining when the caller's wait budget runs out (and + * at the maxPages backstop, and on abort). `items` is then a PREFIX of what the + * job holds, while `funnel.delivered` still reports the backend's full count — + * so an undocumented truncation renders "delivered 40" over 12 rows and reads + * as though 28 leads simply did not exist. + * + * Unlike the disqualified rule, this one is SHARED by all three delivery tools. + * Scoping it to the drain-heavy pair would leave find_new_leads silently + * misreporting on the day its item count crosses a page — the precise failure + * being fixed here — and that divergence risk is worth more than the chars. + */ + +import { describe, it, expect } from "vitest"; +import * as Generated from "@leadbay/core/dist/tool-descriptions.generated.js"; + +const G = Generated as unknown as Record; +const DELIVERY_TOOLS = [ + "leadbay_find_new_leads", + "leadbay_qualify_leads", + "leadbay_lead_job_status", +]; + +describe("audit: items_truncated rendering", () => { + it("every delivery tool carries the rule", () => { + for (const tool of DELIVERY_TOOLS) { + expect(G[tool], `${tool} missing the truncation rule`).toMatch( + /items_truncated/ + ); + } + }); + + it("names the field the agent tests and the recovery it offers", () => { + for (const tool of DELIVERY_TOOLS) { + const d = G[tool]; + // The rows are a prefix... + expect(d, `${tool} does not say the rows are partial`).toMatch(/PREFIX/); + // ...and the cursor is how the rest is fetched. + expect(d, `${tool} does not name the resumption path`).toMatch( + /leadbay_lead_job_status\(job_id, since: next_since\)/ + ); + } + }); + + it("stays within the per-tool description budget", () => { + for (const tool of DELIVERY_TOOLS) { + expect(G[tool].length, `${tool} over budget`).toBeLessThanOrEqual(17000); + } + }); +}); diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index 65b2f6d6..dc503bcd 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -84,6 +84,9 @@ table `Ref → Outcome` translating `status_reason` to plain words: `disqualified` → "evaluated: does not fit" (evidence is in the item when owned), `enrichment_failed` → "channel could not be sourced (not billed)". +**`items_truncated`**: rows are a PREFIX, not the batch. Say so, and offer +`leadbay_lead_job_status(job_id, since: next_since)` for the rest. + **Hide from the user:** UUIDs (keep for tool calls, never render), cursors, `explain.model`/`intelligence_snapshot`, raw `distance`/`calibration`, `seq`/`from_cache`, empty arrays. diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index c2c0f13a..703703c3 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -64,24 +64,23 @@ purchases contact channels. Polls up to `wait_seconds` (default 45); a longer job returns `still_running` + `next_poll` — hand off to `leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. -**Free vs paid — never spend silently.** The default ask (`qualify: false`, +**Free vs paid — never spend silently.** Default (`qualify: false`, `channels: []`) is FREE: company profile + fit score + cached research + contact identity. Paid: `qualify: true` (~94 cost_cents per candidate EXAMINED, capped by `exploration_cap`/`max_cost`) and `channels` (email 25c / -phone 250c, success-only). The gate is enforced in code: a paid call -(`qualify: true` and/or `channels`) is WITHHELD unless it carries -`confirm: true` — otherwise nothing is submitted and you get +phone 250c, success-only). Enforced in code: a paid call is WITHHELD unless it +carries `confirm: true` — nothing is submitted and you get `mode: "needs_confirmation"` with a real quote to show the user. Re-call with `confirm: true` on their go-ahead ("spend / get their emails" counts). `confirm: false` vetoes. Free needs no consent. **Preview free first** — reshaping an off-profile seed is free, exploring it with `qualify: true` is not. -**Ad-hoc exclusions ("no chains") are enforced by NO tier** — `filters` has -no exclusion key, and `qualify` scores against the org's FROZEN questions and -IBP, which need not mention chains; the seed's positive inverse only shifts -ranking. Violators can survive, be paid for and be delivered — post-filter -them yourself and say the tier didn't enforce it. Durable enforcement → +**Ad-hoc exclusions ("no chains") are enforced by NO tier** — `filters` has no +exclusion key, and `qualify` scores against the org's FROZEN questions and IBP, +which need not mention chains; the seed's inverse only shifts ranking. +Violators can survive, be paid for and be delivered — post-filter them yourself +and say the tier didn't enforce it. Durable enforcement → `leadbay_refine_prompt`. {{include:heuristics/example-lead-craft}} From c70a6f059ed6faf24db31b7377298352fb1046ee Mon Sep 17 00:00:00 2001 From: Arty Date: Thu, 20 Aug 2026 12:47:11 +0200 Subject: [PATCH 45/46] fix(mcp): bound the queue wait too, and size the cost-cap rerun to what is missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. (P2) `timeoutMs` only started once httpsRequest ran — i.e. after the semaphore was acquired. Five slow or stalled unrelated requests therefore let a bounded call sit in the queue without limit, so even `wait_seconds: 1` could overrun its contract while every individual request looked well-behaved. The wait a caller asked for is wall-clock, not socket time, so acquireSemaphore now takes an absolute deadline, rejects a queued waiter when it expires, and splices it out; the socket then gets what is LEFT rather than a fresh allowance. The 401 RETRY is the deliberate exception: it gets a fresh window measured after the backoff. `timeoutMs` is documented as bounding a single attempt and the hosted auth probe depends on that — its 250ms 401-backoff alone outlasts a 200ms probe budget, so charging the retry against the first attempt's deadline deletes the retry instead of bounding it. auth-http-mixed-outcome- retry.test.ts (from main, untouched) is what says so: it drops to 3 host attempts instead of 4. I verified that failure is real rather than assuming it, by re-testing the shared-deadline variant against a freshly built core. 2. (P2) A search that stops at `max_cost` has delivered PART of the batch, but the NEXT STEPS row only said "raise the cap and continue" — so a 10-lead ask that delivered 6 re-requested 10 rather than 4, and because `novelty: org` excludes only DELIVERED leads, the rerun also paid to re-examine the same examined-and-rejected candidates. Both overspend while the action is presented to the user as a continuation. The row now prescribes `count` = the shortfall (`items_requested` − delivered) and `exclude_lead_ids` = the examined-but-rejected ids, with the reason named. That row cost more than find_new_leads had left, so the chars were bought back where they were genuinely redundant — verbose phrasing in the shared delivery-table snippet and the seed-craft heuristics (rules kept intact), plus its own body. Headroom 198, roughly where it started. Both fixes mutation-tested: dropping the acquisition deadline hangs the queued call (the bug's real signature), restarting the clock after acquisition lands at 501ms against a 450ms bound, and reverting the rerun row fails the new audit. Gates: build, typecheck, prompts:check, two consecutive full runs — 1729 tests (core 902, mcp 759, components 47, promptforge 21). Co-Authored-By: Claude Opus 5 --- packages/core/src/client.ts | 112 ++++++++++++--- .../core/src/tool-descriptions.generated.ts | 92 ++++++------- .../test/unit/client-queue-deadline.test.ts | 130 ++++++++++++++++++ .../test/audit/cost-cap-rerun-sizing.test.ts | 46 +++++++ .../snippets/heuristics/example-lead-craft.md | 17 ++- .../snippets/next-steps/find-new-leads.md | 2 +- .../snippets/rendering/lead-delivery-table.md | 19 ++- .../composite/find-new-leads.md.tmpl | 16 +-- 8 files changed, 340 insertions(+), 94 deletions(-) create mode 100644 packages/core/test/unit/client-queue-deadline.test.ts create mode 100644 packages/mcp/test/audit/cost-cap-rerun-sizing.test.ts diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 358a8f16..85217d9f 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -36,6 +36,15 @@ interface HttpResult { latency_ms: number; } +// One shape for every deadline expiry — the socket one below and the queue one +// in acquireSemaphore. Callers classify on `code`, so a second shape would make +// a queued timeout look like an unrelated failure. +function timeoutError(what: string): Error & { code?: string } { + const err = new Error(what) as Error & { code?: string }; + err.code = "TIMEOUT"; // not an auth code — callers treat it as a transient fault + return err; +} + // Use node:https directly — the OpenClaw gateway patches globalThis.fetch // which intercepts outgoing requests and causes auth failures. function httpsRequest( @@ -101,11 +110,9 @@ function httpsRequest( // promise. Optional-called because the node:https test double is a bare // EventEmitter with no destroy(). (req as { destroy?: (e?: Error) => void }).destroy?.(); - const err = new Error( - `Request timed out after ${timeoutMs}ms: ${method} ${url}` - ) as Error & { code?: string }; - err.code = "TIMEOUT"; // not an auth code — callers treat it as a transient fault - reject(err); + reject( + timeoutError(`Request timed out after ${timeoutMs}ms: ${method} ${url}`) + ); }, timeoutMs); // Never hold the process open on a probe deadline. (deadline as unknown as { unref?: () => void }).unref?.(); @@ -431,30 +438,59 @@ export class LeadbayClient { // to the socket, which this call had not reached yet. Against slow or stalled // peers that stranded the caller well past the <=2s exit the delivery tools // advertise. - private async acquireSemaphore(signal?: AbortSignal): Promise { + // `deadlineAt` is an ABSOLUTE epoch-ms bound covering the queue wait itself. + // Without it a bounded call could still be stranded here without limit: the + // deadline was only handed to httpsRequest, which does not start until this + // resolves, so five slow peers let even `wait_seconds: 1` run unbounded. The + // wait a caller asked for is wall-clock, not socket time. + private async acquireSemaphore( + signal?: AbortSignal, + deadlineAt?: number + ): Promise { if (signal?.aborted) throw this.cancelledBeforeSendError(); + if (deadlineAt !== undefined && Date.now() >= deadlineAt) { + throw timeoutError("Request deadline expired before a request slot was free"); + } if (this.activeRequests < MAX_CONCURRENT) { this.activeRequests++; return; } return new Promise((resolve, reject) => { + let timer: ReturnType | undefined; const waiter = () => { cleanup(); this.activeRequests++; resolve(); }; - const onAbort = () => { - // SPLICE the waiter out rather than flagging it dead: releaseSemaphore() - // shifts the queue blindly, so a tombstoned waiter would still take the - // ++ and resolve nothing — leaking one slot per cancellation until the - // client can serve no requests at all. + // SPLICE the waiter out rather than flagging it dead: releaseSemaphore() + // shifts the queue blindly, so a tombstoned waiter would still take the + // ++ and resolve nothing — leaking one slot per abandonment until the + // client can serve no requests at all. + const drop = () => { const i = this.waitQueue.indexOf(waiter); if (i !== -1) this.waitQueue.splice(i, 1); cleanup(); + }; + const onAbort = () => { + drop(); reject(this.cancelledBeforeSendError()); }; - const cleanup = () => signal?.removeEventListener("abort", onAbort); + const onDeadline = () => { + drop(); + reject( + timeoutError("Request deadline expired while queued for a request slot") + ); + }; + const cleanup = () => { + signal?.removeEventListener("abort", onAbort); + if (timer !== undefined) clearTimeout(timer); + }; signal?.addEventListener("abort", onAbort, { once: true }); + if (deadlineAt !== undefined) { + timer = setTimeout(onDeadline, Math.max(deadlineAt - Date.now(), 0)); + // Never hold the process open on a queue deadline. + (timer as unknown as { unref?: () => void }).unref?.(); + } this.waitQueue.push(waiter); }); } @@ -524,9 +560,22 @@ export class LeadbayClient { // box tells its `finally` whether there is anything to release; a caller // that omits it keeps the unconditional re-acquire, which is what preserves // the balance for the paths that do not track ownership. - held?: { value: boolean } + held?: { value: boolean }, + // Absolute deadline shared by BOTH attempts and the re-acquisition between + // them. `timeoutMs` alone would give the retry a fresh full budget, so a + // 401 could double the wait the caller asked for. + deadlineAt?: number ): Promise => { - const res = await httpsRequest(method, url, headers, body, timeoutMs, signal); + // Throws rather than returning 0: httpsRequest treats a non-positive + // timeoutMs as "no deadline", so passing the spent budget through would + // silently restore the unbounded behaviour this is here to remove. + const firstBudget = (): number | undefined => { + if (deadlineAt === undefined) return timeoutMs; + const left = deadlineAt - Date.now(); + if (left <= 0) throw timeoutError(`Request deadline expired: ${method} ${url}`); + return left; + }; + const res = await httpsRequest(method, url, headers, body, firstBudget(), signal); if (res.status === 401 && method.toUpperCase() === "GET") { // Check BEFORE letting go of the slot: an already-cancelled call that // releases here has to re-queue behind every other waiter just to hand @@ -552,7 +601,15 @@ export class LeadbayClient { // here would leave that caller's `finally` decrementing a slot it never // obtained, drifting the counter permanently — so those paths keep the // unconditional re-acquire instead. - await this.acquireSemaphore(held ? signal : undefined); + // FRESH window, measured from here — after the backoff, not before it. + // `timeoutMs` is documented as bounding a single attempt, and the + // hosted auth probe depends on that: its 250ms 401-backoff alone + // outlasts a 200ms probe budget, so a deadline that started before the + // sleep would delete the retry rather than bound it. + await this.acquireSemaphore( + held ? signal : undefined, + timeoutMs !== undefined ? Date.now() + timeoutMs : undefined + ); if (held) held.value = true; } // Don't burn the retry on a call cancelled during the backoff: the caller @@ -607,7 +664,23 @@ export class LeadbayClient { // wait on unrelated in-flight requests to drain first. A pre-send-only // signal governs the queue wait too — that phase is exactly what it covers. const held = { value: true }; - await this.acquireSemaphore(opts?.signal ?? opts?.preSendSignal); + // Start the clock BEFORE queueing. `timeoutMs` is the caller's total budget + // for this call, and time spent waiting for a slot is time they waited. + const deadlineAt = + opts?.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : undefined; + const remainingBudget = (): number | undefined => { + if (deadlineAt === undefined) return undefined; + const left = deadlineAt - Date.now(); + // Never hand back 0: httpsRequest reads a non-positive timeout as "no + // deadline at all", which would turn an exhausted budget into an + // unbounded request. + if (left <= 0) throw timeoutError(`Request deadline expired: ${method} ${path}`); + return left; + }; + await this.acquireSemaphore( + opts?.signal ?? opts?.preSendSignal, + deadlineAt + ); try { // Last point at which "nothing has been sent" is still true. A submit // cancelled here provably spent nothing; one cancelled a line later @@ -632,14 +705,17 @@ export class LeadbayClient { payload, opts?.timeoutMs, opts?.signal, - held + held, + deadlineAt ) : await httpsRequest( method, url, headers, payload, - opts?.timeoutMs, + // What is LEFT after queueing, not the original budget — otherwise + // the queue wait and the socket wait each get the full allowance. + remainingBudget(), opts?.signal ); diff --git a/packages/core/src/tool-descriptions.generated.ts b/packages/core/src/tool-descriptions.generated.ts index f19e635e..12491108 100644 --- a/packages/core/src/tool-descriptions.generated.ts +++ b/packages/core/src/tool-descriptions.generated.ts @@ -1305,10 +1305,10 @@ delivered. Full algorithm below. Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips what the org already knows (\`novelty: org\`), optionally qualifies against the org's own intelligence -(questions, tags, ideal buyer profile — frozen at submit), and optionally -purchases contact channels. Polls up to \`wait_seconds\` (default 45); a longer -job returns \`still_running\` + \`next_poll\` — hand off to -\`leadbay_lead_job_status\`. Jobs run ≤30 min, results kept 30 days. +(questions, tags, ideal buyer profile — frozen at submit), and optionally buys +contact channels. Polls up to \`wait_seconds\` (default 45); a longer job returns +\`still_running\` + \`next_poll\` — hand off to \`leadbay_lead_job_status\`. Jobs run +≤30 min, results kept 30 days. **Free vs paid — never spend silently.** Default (\`qualify: false\`, \`channels: []\`) is FREE: company profile + fit score + cached research + @@ -1331,10 +1331,10 @@ and say the tier didn't enforce it. Durable enforcement → ### Crafting the \`example_lead\` seed — the input that decides result quality -The \`example_lead\` is a FICTIONAL typical ideal customer, embedded and matched -against real registry/website descriptions — which state what a company **IS**, -never what is happening. Write it the same way or the matcher drifts. Every -rule below is measured: +The \`example_lead\` is a FICTIONAL typical ideal customer, matched against real +registry/website descriptions — which state what a company **IS**, never what +is happening. Write it the same way or the matcher drifts. Every rule below is +measured: 1. **Describe the BUYER, never the seller.** Ask: "would this company write a check to my user?" A seed describing what the user SELLS surfaces their @@ -1347,14 +1347,13 @@ rule below is measured: model, what they sell or operate, who they serve, observable scale. Write it like the first paragraph of their About-Us page. - STRONG: "Operator of full-service fitness centers offering strength - training areas, group classes and personal training to individual members - across multiple club locations." + areas, group classes and personal training to members across multiple + clubs." - WEAK (generic): "A gym in Texas." - WRONG (seller-side): "Supplier of durable modular flooring for gyms." 4. **No event language.** "hiring", "expanding", "just raised" are not filters — registry descriptions never contain them, so they dilute the - profile. Purchase triggers belong in the org's qualification questions, - which the paid stage scores from fresh research. + profile. Purchase triggers belong in the org's qualification questions. 5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)". 6. **Hard constraints go in \`filters\`, not prose — exact keys:** \`sectors: string[]\`, \`locations: string[]\`, \`employees_min: number\`, @@ -1366,22 +1365,22 @@ rule below is measured: "gyms that need durable flooring" surfaced flooring VENDORS, 0 delivered. Use \`query\` only for signal an example can't express. 8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and - warehouses") needs one search each, with its own description and + warehouses") needs one search each with its own description and \`request_id\` — a blended seed lands between the clusters and matches neither. **Parameter notes** - \`request_id\` (REQUIRED) is the retry contract: SAME value retrying the same - ask (same live job, no double spend); NEW value for a changed ask. Derive - from ask + archetype + date: \`gyms-dallas-2026-07-28\`. + ask (same live job, no double spend); NEW for a changed ask. Derive from ask + + archetype + date: \`gyms-dallas-2026-07-28\`. - Never lower \`min_ai_score\` together with \`channels\` — that buys emails for leads the AI just scored as junk. - \`count\` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). -**Read the result honestly** — \`funnel\` + \`explain.scope_notes\` tell the -story; zero delivered gets a cause and a next move (rules in RENDERING). +**Read the result honestly** — \`funnel\` + \`explain.scope_notes\` tell the story; +zero delivered gets a cause and a next move (rules in RENDERING). --- @@ -1426,7 +1425,7 @@ when nothing was delivered. **The funnel line (mandatory, after the table):** -One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +One short line narrating the delivery honestly, from \`funnel\` + \`cost\` + \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of @@ -1437,10 +1436,10 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US \`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. -"of the Y asked" needs \`summary.items_requested\`, which submit results carry -but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates), never guess it. +"of the Y asked" needs \`summary.items_requested\`, which submits carry but a +later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered +X** and stop — never back-fill Y from \`matched\`/\`examined\` (they count +candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -1448,13 +1447,12 @@ Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give the funnel line plus the relevant \`explain.scope_notes\` (the backend's own -diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose -the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, -raise \`max_cost\`, drop a filter) as NEXT STEPS options. +diagnosis), then propose the concrete fix (reshape the seed per the craft +rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS. **Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present -the table as an answer — open with "weak matches only", show at most the top -3, propose reshaping the seed/filters first. The count was filled with +the table as an answer — open with "weak matches only", show at most the top 3, +propose reshaping the seed/filters first. The count was filled with barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any @@ -1521,7 +1519,7 @@ Pick the 2-3 options that match what actually happened — never all seven: | Free run delivered on-profile leads | "Qualify these N against your criteria (paid — \`dry_run\` first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | | Delivered 0 or off-profile | "Reshape the example and retry" (name the fix from funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap (\`stop_reason: max_cost\`) | "Raise the cap to X and continue" — X in the account's currency per the funnel-line rule, never a hard-coded \`$\` | leadbay_find_new_leads, NEW request_id + higher max_cost (a same-id re-submit only dedupes onto a LIVE job) | +| Stopped at cost cap (\`stop_reason: max_cost\`) | "Raise the cap to X and get the remaining N" — X in the account's currency per the funnel-line rule, never a hard-coded \`$\` | leadbay_find_new_leads, NEW request_id (same-id only dedupes onto a LIVE job) + higher max_cost + \`count\` = the SHORTFALL (\`items_requested\` − delivered), not the original + \`exclude_lead_ids\` = the examined-but-REJECTED ids (novelty covers delivered; these are what it misses — without them the rerun re-buys the same losers) | | Stopped on org quota (\`stop_reason: quota\`) | "Wait for the reset, or top up" — never a re-run: it cannot clear an org quota and burns a submit slot to stop in the same place | leadbay_account_status, then leadbay_create_topup_link | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | `; @@ -2420,7 +2418,7 @@ when nothing was delivered. **The funnel line (mandatory, after the table):** -One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +One short line narrating the delivery honestly, from \`funnel\` + \`cost\` + \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of @@ -2431,10 +2429,10 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US \`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. -"of the Y asked" needs \`summary.items_requested\`, which submit results carry -but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates), never guess it. +"of the Y asked" needs \`summary.items_requested\`, which submits carry but a +later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered +X** and stop — never back-fill Y from \`matched\`/\`examined\` (they count +candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -2442,13 +2440,12 @@ Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give the funnel line plus the relevant \`explain.scope_notes\` (the backend's own -diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose -the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, -raise \`max_cost\`, drop a filter) as NEXT STEPS options. +diagnosis), then propose the concrete fix (reshape the seed per the craft +rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS. **Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present -the table as an answer — open with "weak matches only", show at most the top -3, propose reshaping the seed/filters first. The count was filled with +the table as an answer — open with "weak matches only", show at most the top 3, +propose reshaping the seed/filters first. The count was filled with barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any @@ -3708,7 +3705,7 @@ when nothing was delivered. **The funnel line (mandatory, after the table):** -One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` + +One short line narrating the delivery honestly, from \`funnel\` + \`cost\` + \`explain.scope_notes\`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of @@ -3719,10 +3716,10 @@ One short line narrating the delivery honestly, built from \`funnel\` + \`cost\` so \`165\` renders \`1.65\`, NEVER \`165.00\`. Symbol from the account region: US \`$\`, France \`€\`, unknown → bare. Never hard-code \`$\`: it misstates a charge. -"of the Y asked" needs \`summary.items_requested\`, which submit results carry -but a later \`leadbay_lead_job_status\` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from \`matched\`/\`examined\` (they -count candidates), never guess it. +"of the Y asked" needs \`summary.items_requested\`, which submits carry but a +later \`leadbay_lead_job_status\` snapshot does not. Without it write **delivered +X** and stop — never back-fill Y from \`matched\`/\`examined\` (they count +candidates), never guess it. Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted\` → "ran out of matching candidates", \`max_cost\` → "hit the cost cap", \`quota\` → @@ -3730,13 +3727,12 @@ Plain-word stop reasons: \`target_reached\` → omit (success), \`pool_exhausted **When \`delivered\` is 0**: NEVER say just "no results". Render no table; give the funnel line plus the relevant \`explain.scope_notes\` (the backend's own -diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose -the concrete fix (reshape the seed per the craft rules, lower \`min_ai_score\`, -raise \`max_cost\`, drop a filter) as NEXT STEPS options. +diagnosis), then propose the concrete fix (reshape the seed per the craft +rules, lower \`min_ai_score\`, raise \`max_cost\`, drop a filter) as NEXT STEPS. **Weak batch**: when the BEST delivered \`fit.score\` is under 30, don't present -the table as an answer — open with "weak matches only", show at most the top -3, propose reshaping the seed/filters first. The count was filled with +the table as an answer — open with "weak matches only", show at most the top 3, +propose reshaping the seed/filters first. The count was filled with barely-better-than-random candidates. **Sanity-check every row**: (a) geo — \`city\`/\`region\` must sit inside any diff --git a/packages/core/test/unit/client-queue-deadline.test.ts b/packages/core/test/unit/client-queue-deadline.test.ts new file mode 100644 index 00000000..26304097 --- /dev/null +++ b/packages/core/test/unit/client-queue-deadline.test.ts @@ -0,0 +1,130 @@ +/** + * `timeoutMs` is the caller's TOTAL budget for a call, not its socket budget. + * + * It used to start only when httpsRequest ran — i.e. after the semaphore was + * acquired. Five slow or stalled unrelated requests therefore let a bounded + * call sit in the queue without limit, so even `wait_seconds: 1` could exceed + * its contract by minutes while every individual request looked well-behaved. + * Time spent waiting for a slot is time the caller waited. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { EventEmitter } from "node:events"; + +const seen: Array> = []; +const pending: Array<() => void> = []; + +vi.mock("node:https", () => ({ + default: { + request: (options: Record, cb?: (res: unknown) => void) => { + seen.push(options); + const req = new EventEmitter() as EventEmitter & { + write: () => void; + end: () => void; + destroy: () => void; + }; + req.write = () => {}; + req.destroy = () => {}; + req.end = () => { + pending.push(() => { + const res = new EventEmitter() as EventEmitter & { + statusCode: number; + headers: Record; + }; + res.statusCode = 200; + res.headers = {}; + cb?.(res); + res.emit("data", Buffer.from("{}")); + res.emit("end"); + }); + }; + return req; + }, + }, +})); + +import { LeadbayClient } from "../../src/client.js"; + +const MAX_CONCURRENT = 5; +const newClient = () => new LeadbayClient("https://api-us.leadbay.app", "u.tok", "us"); +const settled = () => new Promise((r) => setImmediate(r)); +const flush = () => pending.splice(0).forEach((f) => f()); +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +beforeEach(() => { + seen.length = 0; + pending.length = 0; +}); + +describe("the request deadline covers the queue wait", () => { + it("expires a queued call instead of stranding it behind stalled traffic", async () => { + const client = newClient(); + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + seen.length = 0; + + // Bounded call, no slot available, and nothing will free one. + const bounded = client.request("GET", "/mcp/jobs/j1", undefined, { timeoutMs: 60 }); + + await expect(bounded).rejects.toMatchObject({ code: "TIMEOUT" }); + // It never reached the wire — the deadline was enforced while queued. + expect(seen).toHaveLength(0); + // And it left the queue clean rather than lingering as a dead waiter. + expect(client._semaphoreState.queued).toBe(0); + expect(client._semaphoreState.active).toBe(MAX_CONCURRENT); + + flush(); + await Promise.all(stalled); + expect(client._semaphoreState.active).toBe(0); + }); + + it("refuses immediately when the budget is already spent", async () => { + const client = newClient(); + await expect( + client.request("GET", "/mcp/jobs/j1", undefined, { timeoutMs: 0 }) + ).rejects.toMatchObject({ code: "TIMEOUT" }); + expect(seen).toHaveLength(0); + expect(client._semaphoreState.active).toBe(0); + }); + + it("charges queue time against the socket budget rather than restarting it", async () => { + const client = newClient(); + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + seen.length = 0; + + const startedAt = Date.now(); + // 300ms total. ~200ms of it will be spent queued; the socket must inherit + // only what is LEFT. + const bounded = client + .request("GET", "/mcp/jobs/j1", undefined, { timeoutMs: 300 }) + .catch((e) => e); + + await wait(200); + // Free every slot. `bounded` acquires and dispatches — and is never + // answered, so its socket deadline decides when it fails. + flush(); + const err = await bounded; + const elapsed = Date.now() - startedAt; + + expect(err).toMatchObject({ code: "TIMEOUT" }); + // A restarted clock would give it a fresh 300ms after acquiring, landing + // near 500ms. Spending the remaining ~100ms lands near 300ms. + expect(elapsed).toBeLessThan(450); + await Promise.all(stalled); + }); + + it("leaves unbounded calls unbounded", async () => { + const client = newClient(); + const p = client.request("GET", "/mcp/jobs/j1"); + await settled(); + expect(seen).toHaveLength(1); + expect(seen[0].signal).toBeUndefined(); + flush(); + await p; + }); +}); diff --git a/packages/mcp/test/audit/cost-cap-rerun-sizing.test.ts b/packages/mcp/test/audit/cost-cap-rerun-sizing.test.ts new file mode 100644 index 00000000..6339bc8f --- /dev/null +++ b/packages/mcp/test/audit/cost-cap-rerun-sizing.test.ts @@ -0,0 +1,46 @@ +/** + * Audit: the max_cost rerun is sized to the shortfall, not the original ask. + * + * A search that stops at its cost cap has delivered PART of the batch. Telling + * the agent only to "raise the cap and continue" makes it re-request the FULL + * original count — a 10-lead ask that delivered 6 asks for 10 more, not 4 — + * and, because `novelty: org` excludes only DELIVERED leads, the rerun also + * pays to re-examine the same examined-and-rejected candidates. Both overspend + * while the action is presented to the user as a continuation. + */ + +import { describe, it, expect } from "vitest"; +import * as Generated from "@leadbay/core/dist/tool-descriptions.generated.js"; + +const G = Generated as unknown as Record; +const D = G.leadbay_find_new_leads; + +// The rerun row, isolated so these assertions cannot be satisfied by wording +// that happens to appear elsewhere in a 16k description. +const ROW = D.split("\n").find((l) => l.includes("stop_reason: max_cost")) ?? ""; + +describe("audit: max_cost rerun sizing", () => { + it("has a rerun row to govern", () => { + expect(ROW, "max_cost NEXT STEPS row missing").not.toBe(""); + }); + + it("sizes the rerun to the remaining gap, not the original count", () => { + expect(ROW).toMatch(/SHORTFALL/); + expect(ROW).toMatch(/items_requested/); + }); + + it("carries the examined-but-rejected ids so the rerun does not re-buy them", () => { + expect(ROW).toMatch(/exclude_lead_ids/); + // Naming WHY: novelty covers delivered leads, so exclusions are the only + // thing that covers the paid misses. + expect(ROW).toMatch(/novelty covers delivered/i); + }); + + it("still requires a new request_id — a same-id resubmit only dedupes onto a live job", () => { + expect(ROW).toMatch(/NEW request_id/); + }); + + it("keeps the tool within its description budget", () => { + expect(D.length).toBeLessThanOrEqual(17000); + }); +}); diff --git a/packages/promptforge/snippets/heuristics/example-lead-craft.md b/packages/promptforge/snippets/heuristics/example-lead-craft.md index 03e08505..833c09b3 100644 --- a/packages/promptforge/snippets/heuristics/example-lead-craft.md +++ b/packages/promptforge/snippets/heuristics/example-lead-craft.md @@ -1,9 +1,9 @@ ### Crafting the `example_lead` seed — the input that decides result quality -The `example_lead` is a FICTIONAL typical ideal customer, embedded and matched -against real registry/website descriptions — which state what a company **IS**, -never what is happening. Write it the same way or the matcher drifts. Every -rule below is measured: +The `example_lead` is a FICTIONAL typical ideal customer, matched against real +registry/website descriptions — which state what a company **IS**, never what +is happening. Write it the same way or the matcher drifts. Every rule below is +measured: 1. **Describe the BUYER, never the seller.** Ask: "would this company write a check to my user?" A seed describing what the user SELLS surfaces their @@ -16,14 +16,13 @@ rule below is measured: model, what they sell or operate, who they serve, observable scale. Write it like the first paragraph of their About-Us page. - STRONG: "Operator of full-service fitness centers offering strength - training areas, group classes and personal training to individual members - across multiple club locations." + areas, group classes and personal training to members across multiple + clubs." - WEAK (generic): "A gym in Texas." - WRONG (seller-side): "Supplier of durable modular flooring for gyms." 4. **No event language.** "hiring", "expanding", "just raised" are not filters — registry descriptions never contain them, so they dilute the - profile. Purchase triggers belong in the org's qualification questions, - which the paid stage scores from fresh research. + profile. Purchase triggers belong in the org's qualification questions. 5. **No meta-markers.** Never "(example)", "(fictional)", "(placeholder)". 6. **Hard constraints go in `filters`, not prose — exact keys:** `sectors: string[]`, `locations: string[]`, `employees_min: number`, @@ -35,6 +34,6 @@ rule below is measured: "gyms that need durable flooring" surfaced flooring VENDORS, 0 delivered. Use `query` only for signal an example can't express. 8. **One seed per buyer archetype.** An ask spanning two segments ("gyms and - warehouses") needs one search each, with its own description and + warehouses") needs one search each with its own description and `request_id` — a blended seed lands between the clusters and matches neither. diff --git a/packages/promptforge/snippets/next-steps/find-new-leads.md b/packages/promptforge/snippets/next-steps/find-new-leads.md index 8ebb7650..a97ca19a 100644 --- a/packages/promptforge/snippets/next-steps/find-new-leads.md +++ b/packages/promptforge/snippets/next-steps/find-new-leads.md @@ -10,6 +10,6 @@ Pick the 2-3 options that match what actually happened — never all seven: | Free run delivered on-profile leads | "Qualify these N against your criteria (paid — `dry_run` first)" | leadbay_qualify_leads(prior_deliveries: {job_id}) | | Delivered leads look right | "Draft outreach for the top ones" | leadbay_prepare_outreach | | Delivered 0 or off-profile | "Reshape the example and retry" (name the fix from funnel + scope_notes) | leadbay_find_new_leads (NEW request_id) | -| Stopped at cost cap (`stop_reason: max_cost`) | "Raise the cap to X and continue" — X in the account's currency per the funnel-line rule, never a hard-coded `$` | leadbay_find_new_leads, NEW request_id + higher max_cost (a same-id re-submit only dedupes onto a LIVE job) | +| Stopped at cost cap (`stop_reason: max_cost`) | "Raise the cap to X and get the remaining N" — X in the account's currency per the funnel-line rule, never a hard-coded `$` | leadbay_find_new_leads, NEW request_id (same-id only dedupes onto a LIVE job) + higher max_cost + `count` = the SHORTFALL (`items_requested` − delivered), not the original + `exclude_lead_ids` = the examined-but-REJECTED ids (novelty covers delivered; these are what it misses — without them the rerun re-buys the same losers) | | Stopped on org quota (`stop_reason: quota`) | "Wait for the reset, or top up" — never a re-run: it cannot clear an org quota and burns a submit slot to stop in the same place | leadbay_account_status, then leadbay_create_topup_link | | User wants these tracked in Leadbay | "Add the keepers to a campaign" | leadbay_create_campaign / leadbay_add_leads_to_campaign | diff --git a/packages/promptforge/snippets/rendering/lead-delivery-table.md b/packages/promptforge/snippets/rendering/lead-delivery-table.md index dc503bcd..08358aad 100644 --- a/packages/promptforge/snippets/rendering/lead-delivery-table.md +++ b/packages/promptforge/snippets/rendering/lead-delivery-table.md @@ -39,7 +39,7 @@ when nothing was delivered. **The funnel line (mandatory, after the table):** -One short line narrating the delivery honestly, built from `funnel` + `cost` + +One short line narrating the delivery honestly, from `funnel` + `cost` + `explain.scope_notes`: > Matched N · examined E · qualified Q · disqualified D → **delivered X of @@ -50,10 +50,10 @@ One short line narrating the delivery honestly, built from `funnel` + `cost` + so `165` renders `1.65`, NEVER `165.00`. Symbol from the account region: US `$`, France `€`, unknown → bare. Never hard-code `$`: it misstates a charge. -"of the Y asked" needs `summary.items_requested`, which submit results carry -but a later `leadbay_lead_job_status` snapshot does not. Without it write -**delivered X** and stop — never back-fill Y from `matched`/`examined` (they -count candidates), never guess it. +"of the Y asked" needs `summary.items_requested`, which submits carry but a +later `leadbay_lead_job_status` snapshot does not. Without it write **delivered +X** and stop — never back-fill Y from `matched`/`examined` (they count +candidates), never guess it. Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` → "ran out of matching candidates", `max_cost` → "hit the cost cap", `quota` → @@ -61,13 +61,12 @@ Plain-word stop reasons: `target_reached` → omit (success), `pool_exhausted` **When `delivered` is 0**: NEVER say just "no results". Render no table; give the funnel line plus the relevant `explain.scope_notes` (the backend's own -diagnosis — vendor-vocabulary queries, pre-screen rejections), then propose -the concrete fix (reshape the seed per the craft rules, lower `min_ai_score`, -raise `max_cost`, drop a filter) as NEXT STEPS options. +diagnosis), then propose the concrete fix (reshape the seed per the craft +rules, lower `min_ai_score`, raise `max_cost`, drop a filter) as NEXT STEPS. **Weak batch**: when the BEST delivered `fit.score` is under 30, don't present -the table as an answer — open with "weak matches only", show at most the top -3, propose reshaping the seed/filters first. The count was filled with +the table as an answer — open with "weak matches only", show at most the top 3, +propose reshaping the seed/filters first. The count was filled with barely-better-than-random candidates. **Sanity-check every row**: (a) geo — `city`/`region` must sit inside any diff --git a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl index 703703c3..f80ae2b9 100644 --- a/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl +++ b/packages/promptforge/tool-descriptions/composite/find-new-leads.md.tmpl @@ -59,10 +59,10 @@ annotations: Submit a net-new lead search: the backend matches an ICP seed against the full company universe, applies hard filters, skips what the org already knows (`novelty: org`), optionally qualifies against the org's own intelligence -(questions, tags, ideal buyer profile — frozen at submit), and optionally -purchases contact channels. Polls up to `wait_seconds` (default 45); a longer -job returns `still_running` + `next_poll` — hand off to -`leadbay_lead_job_status`. Jobs run ≤30 min, results kept 30 days. +(questions, tags, ideal buyer profile — frozen at submit), and optionally buys +contact channels. Polls up to `wait_seconds` (default 45); a longer job returns +`still_running` + `next_poll` — hand off to `leadbay_lead_job_status`. Jobs run +≤30 min, results kept 30 days. **Free vs paid — never spend silently.** Default (`qualify: false`, `channels: []`) is FREE: company profile + fit score + cached research + @@ -87,15 +87,15 @@ and say the tier didn't enforce it. Durable enforcement → **Parameter notes** - `request_id` (REQUIRED) is the retry contract: SAME value retrying the same - ask (same live job, no double spend); NEW value for a changed ask. Derive - from ask + archetype + date: `gyms-dallas-2026-07-28`. + ask (same live job, no double spend); NEW for a changed ask. Derive from ask + + archetype + date: `gyms-dallas-2026-07-28`. - Never lower `min_ai_score` together with `channels` — that buys emails for leads the AI just scored as junk. - `count` ≤ 50; ≤3 active jobs/org; ≤10 submits/hour (429 + Retry-After — wait, don't hammer). -**Read the result honestly** — `funnel` + `explain.scope_notes` tell the -story; zero delivered gets a cause and a next move (rules in RENDERING). +**Read the result honestly** — `funnel` + `explain.scope_notes` tell the story; +zero delivered gets a cause and a next move (rules in RENDERING). --- From 6402ceb61fd695568ad5b6a4b0ecfaaeee1d979f Mon Sep 17 00:00:00 2001 From: Arty Date: Fri, 21 Aug 2026 12:50:05 +0200 Subject: [PATCH 46/46] fix(mcp): split the two budget knobs, and stop stranding a job the user paid for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, all follow-ups on my own work. 1. (P2) The 401 retry ignored the whole-call deadline. Last round I gave it a fresh window because the hosted auth probe needs one — its 250ms backoff outlasts a 200ms probe budget — but that left the implementation incoherent: queue and socket shared a total, the retry did not. The real problem was one knob asked to mean two things. Now `timeoutMs` bounds ONE attempt (what the probe needs) and `totalTimeoutMs` bounds the whole call including backoff and retry (what a job poll needs, so a transient 401 cannot buy it a second full wait_seconds). Either, both, or neither; whichever expires first wins. Job snapshots switched to totalTimeoutMs. 2. (P2) `snapshotBudget` floored a spent budget at 1s, so a final poll ran a full second past the deadline — `wait_seconds: 1` took ~2s — and any fractional wait overran on the very first snapshot. Floor removed, plus a re-check after the sleep, which can itself consume the entire remainder. 3. (P2) A truncated drain on an already-TERMINAL job was unrecoverable. `done` was true so `next_poll` was null, and `next_since` only ever existed inside next_poll — so the rendering rule I added telling the agent to fetch the rest with `since: next_since` named a cursor the response did not contain. Those rows are paid for. All three delivery tools now expose `next_since` at the top level and keep a continuation whenever truncation occurred, with suggested_wait_seconds 0 since it is a page fetch, not a wait. 4. (P2) Post-submit polling failures dropped the job handle. Once the submit returns, a backend-owned job exists and may be spending; job_id is the only route back to it. Only a TIMEOUT on the block-waiting branch preserved it — an abort, a connection reset or a 502 propagated bare, and the zero-wait branch preserved nothing. Both submit paths now go through snapshotAfterSubmit, which attaches the handle to ANY failure. All four mutation-tested. Two of my own assertions were too weak to catch their own bug and were strengthened: "budget <= 1000" cannot detect a floor that IS 1000 (now: budgets must strictly decrease), and the retry-window assertions needed the total/per-attempt split to be observable. Gates: build, typecheck, prompts:check, two consecutive full runs — 1746 tests (core 919, mcp 759, components 47, promptforge 21). Co-Authored-By: Claude Opus 5 --- packages/core/src/client.ts | 92 +++++++---- .../core/src/composite/_mcp-job-helpers.ts | 68 +++++++- packages/core/src/composite/find-new-leads.ts | 52 +++--- .../core/src/composite/lead-job-status.ts | 11 +- packages/core/src/composite/qualify-leads.ts | 55 +++---- .../test/unit/client-queue-deadline.test.ts | 31 +++- .../unit/client-total-budget-retry.test.ts | 101 ++++++++++++ .../unit/composite/job-wait-deadline.test.ts | 14 +- .../composite/post-submit-recovery.test.ts | 151 ++++++++++++++++++ .../truncated-terminal-continuation.test.ts | 115 +++++++++++++ 10 files changed, 590 insertions(+), 100 deletions(-) create mode 100644 packages/core/test/unit/client-total-budget-retry.test.ts create mode 100644 packages/core/test/unit/composite/post-submit-recovery.test.ts create mode 100644 packages/core/test/unit/composite/truncated-terminal-continuation.test.ts diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 85217d9f..60ff8209 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -561,21 +561,38 @@ export class LeadbayClient { // that omits it keeps the unconditional re-acquire, which is what preserves // the balance for the paths that do not track ownership. held?: { value: boolean }, - // Absolute deadline shared by BOTH attempts and the re-acquisition between - // them. `timeoutMs` alone would give the retry a fresh full budget, so a - // 401 could double the wait the caller asked for. - deadlineAt?: number + // Absolute ceiling for the WHOLE call — every phase, retry included. It is + // separate from `timeoutMs` because the two answer different questions: + // `timeoutMs` bounds one attempt (what the hosted auth probe needs, since + // its 250ms 401-backoff outlasts a 200ms probe budget), while this bounds + // what the caller waits in total (what a job snapshot needs, since a 401 + // must not buy the poll a second full wait_seconds). A caller may set + // either, both, or neither. + totalDeadlineAt?: number ): Promise => { - // Throws rather than returning 0: httpsRequest treats a non-positive - // timeoutMs as "no deadline", so passing the spent budget through would - // silently restore the unbounded behaviour this is here to remove. - const firstBudget = (): number | undefined => { - if (deadlineAt === undefined) return timeoutMs; - const left = deadlineAt - Date.now(); + // Budget for a phase starting NOW: the earlier of "one more attempt" and + // "what is left of the whole call". Throws rather than returning 0 — + // httpsRequest reads a non-positive timeout as "no deadline", so passing a + // spent budget through would silently restore unbounded behaviour. + const phaseBudget = (): number | undefined => { + const now = Date.now(); + const perAttempt = timeoutMs !== undefined ? now + timeoutMs : undefined; + const deadline = + totalDeadlineAt === undefined + ? perAttempt + : perAttempt === undefined + ? totalDeadlineAt + : Math.min(perAttempt, totalDeadlineAt); + if (deadline === undefined) return undefined; + const left = deadline - now; if (left <= 0) throw timeoutError(`Request deadline expired: ${method} ${url}`); return left; }; - const res = await httpsRequest(method, url, headers, body, firstBudget(), signal); + const phaseDeadline = (): number | undefined => { + const b = phaseBudget(); + return b === undefined ? undefined : Date.now() + b; + }; + const res = await httpsRequest(method, url, headers, body, phaseBudget(), signal); if (res.status === 401 && method.toUpperCase() === "GET") { // Check BEFORE letting go of the slot: an already-cancelled call that // releases here has to re-queue behind every other waiter just to hand @@ -601,21 +618,17 @@ export class LeadbayClient { // here would leave that caller's `finally` decrementing a slot it never // obtained, drifting the counter permanently — so those paths keep the // unconditional re-acquire instead. - // FRESH window, measured from here — after the backoff, not before it. - // `timeoutMs` is documented as bounding a single attempt, and the - // hosted auth probe depends on that: its 250ms 401-backoff alone - // outlasts a 200ms probe budget, so a deadline that started before the - // sleep would delete the retry rather than bound it. - await this.acquireSemaphore( - held ? signal : undefined, - timeoutMs !== undefined ? Date.now() + timeoutMs : undefined - ); + // Measured from HERE — after the backoff, not before it. A per-attempt + // window that started before the 250ms sleep would already be spent, + // deleting the retry rather than bounding it. The total ceiling still + // applies on top, so a caller that asked for a hard total gets one. + await this.acquireSemaphore(held ? signal : undefined, phaseDeadline()); if (held) held.value = true; } // Don't burn the retry on a call cancelled during the backoff: the caller // is gone, and the retry would only make the wait longer. if (signal?.aborted) return res; - return httpsRequest(method, url, headers, body, timeoutMs, signal); + return httpsRequest(method, url, headers, body, phaseBudget(), signal); } return res; }; @@ -624,10 +637,16 @@ export class LeadbayClient { method: string, path: string, body?: unknown, - // `timeoutMs` bounds a single attempt (each retry gets its own deadline) and + // `timeoutMs` bounds a single ATTEMPT (each retry gets its own window) and // surfaces as a `TIMEOUT`-coded Error — never an auth code, so a caller that // classifies failures reads it as a transient fault. // + // `totalTimeoutMs` bounds the WHOLE call — queue wait, socket, 401 backoff + // and retry together. Callers whose own contract is a total (a job poll + // spending what is left of wait_seconds) pass this; callers that want each + // attempt to get a fair shot (the auth probe, whose backoff outlasts its + // per-attempt budget) pass timeoutMs. Setting both enforces both. + // // Two cancellation scopes, because a paid POST needs half of one: // `signal` — full cancellation. Aborts the queue wait AND the // in-flight socket. Right for reads. @@ -641,6 +660,7 @@ export class LeadbayClient { opts?: { retryOn401?: boolean; timeoutMs?: number; + totalTimeoutMs?: number; signal?: AbortSignal; preSendSignal?: AbortSignal; } @@ -664,13 +684,25 @@ export class LeadbayClient { // wait on unrelated in-flight requests to drain first. A pre-send-only // signal governs the queue wait too — that phase is exactly what it covers. const held = { value: true }; - // Start the clock BEFORE queueing. `timeoutMs` is the caller's total budget - // for this call, and time spent waiting for a slot is time they waited. - const deadlineAt = - opts?.timeoutMs !== undefined ? Date.now() + opts.timeoutMs : undefined; + // Start the clock BEFORE queueing: time spent waiting for a slot is time the + // caller waited, so both bounds must already be running here. + const startedAt = Date.now(); + const totalDeadlineAt = + opts?.totalTimeoutMs !== undefined + ? startedAt + opts.totalTimeoutMs + : undefined; + const phaseDeadlineAt = (): number | undefined => { + const now = Date.now(); + const perAttempt = + opts?.timeoutMs !== undefined ? now + opts.timeoutMs : undefined; + if (totalDeadlineAt === undefined) return perAttempt; + if (perAttempt === undefined) return totalDeadlineAt; + return Math.min(perAttempt, totalDeadlineAt); + }; const remainingBudget = (): number | undefined => { - if (deadlineAt === undefined) return undefined; - const left = deadlineAt - Date.now(); + const deadline = phaseDeadlineAt(); + if (deadline === undefined) return undefined; + const left = deadline - Date.now(); // Never hand back 0: httpsRequest reads a non-positive timeout as "no // deadline at all", which would turn an exhausted budget into an // unbounded request. @@ -679,7 +711,7 @@ export class LeadbayClient { }; await this.acquireSemaphore( opts?.signal ?? opts?.preSendSignal, - deadlineAt + phaseDeadlineAt() ); try { // Last point at which "nothing has been sent" is still true. A submit @@ -706,7 +738,7 @@ export class LeadbayClient { opts?.timeoutMs, opts?.signal, held, - deadlineAt + totalDeadlineAt ) : await httpsRequest( method, diff --git a/packages/core/src/composite/_mcp-job-helpers.ts b/packages/core/src/composite/_mcp-job-helpers.ts index 20d3d6b2..68684243 100644 --- a/packages/core/src/composite/_mcp-job-helpers.ts +++ b/packages/core/src/composite/_mcp-job-helpers.ts @@ -226,7 +226,9 @@ export async function collectJobSnapshot( const remaining = () => deadlineAt - Date.now(); let page = await client.request("GET", qs(since), undefined, { signal, - timeoutMs: remaining(), + // totalTimeoutMs, not timeoutMs: what is left of the wait must cover a 401 + // backoff and its retry too, or a blip buys the poll a second full budget. + totalTimeoutMs: remaining(), }); const items = [...page.items]; // The resumption cursor must survive an empty drain page. Following @@ -256,7 +258,7 @@ export async function collectJobSnapshot( "GET", qs(page.next_since), undefined, - { signal, timeoutMs: remaining() } + { signal, totalTimeoutMs: remaining() } ); items.push(...next.items); pages += 1; @@ -320,12 +322,13 @@ function cancelledError(jobId: string): unknown { } // Each snapshot gets whatever is LEFT of the caller's wait, capped by the -// per-request ceiling. Floored at 1s so a nearly-spent budget still makes a -// real attempt rather than a request born already expired; the loop condition -// is what actually ends the wait, so this floor cannot extend it by more than -// one in-flight read. +// per-request ceiling. NOT floored: a 1s floor let a final poll run a full +// second past an already-spent budget, so `wait_seconds: 1` took ~2s, and any +// fractional wait overran on the very first snapshot. Callers guard against +// polling at all once the budget is gone, so the only job left here is to +// clamp a positive budget. function snapshotBudget(remainingMs: number): number { - return Math.min(SNAPSHOT_TIMEOUT_MS, Math.max(remainingMs, 1000)); + return Math.min(SNAPSHOT_TIMEOUT_MS, Math.max(remainingMs, 1)); } function isTimeout(e: unknown): boolean { @@ -340,11 +343,58 @@ function jobReadTimedOutError(jobId: string, waitSeconds: number): unknown { return { error: true, code: "JOB_READ_TIMEOUT", + // Structured, not just interpolated: a caller recovering programmatically + // should not have to parse the message to find the handle. + job_id: jobId, message: `Job ${jobId} was submitted and is running, but its status could not be read within ${waitSeconds}s.`, hint: `Pass job_id ${jobId} to leadbay_lead_job_status to read it — the job is backend-owned, still running, and its results are kept for 30 days.`, }; } +/** Attach the job handle to ANY post-submit failure. + * + * Once the submit has returned, a backend-owned job exists — it may be + * spending right now — and the job_id is the only way back to it. A polling + * failure that propagates bare (an abort, a connection reset, a 502 from the + * status endpoint) therefore strands paid work: the caller sees an error with + * no handle and cannot poll, cancel, or even find out what it bought. */ +export function jobHandleError(jobId: string, cause: unknown): unknown { + const c = cause as + | { code?: string; message?: string; job_id?: string; hint?: string } + | undefined; + if (c?.job_id === jobId) return cause; // already carries the handle + return { + error: true, + code: c?.code ?? "JOB_READ_FAILED", + job_id: jobId, + message: `Job ${jobId} was submitted and is running, but reading its status failed: ${ + c?.message ?? String(cause) + }`, + hint: `Pass job_id ${jobId} to leadbay_lead_job_status to read it — the job is backend-owned, keeps running whatever happened to this call, and its results are kept for 30 days.`, + }; +} + +/** The post-submit read, for both submit tools. + * + * Wraps the wait/zero-wait branch so no failure path can drop the handle — + * previously only a TIMEOUT on the block-waiting branch preserved it, and the + * zero-wait branch preserved nothing at all. */ +export async function snapshotAfterSubmit( + client: LeadbayClient, + jobId: string, + waitSeconds: number, + ctx?: ToolContext, + itemsRequested?: number +): Promise { + try { + return waitSeconds > 0 + ? await waitForJob(client, jobId, waitSeconds, ctx, itemsRequested) + : await collectJobSnapshot(client, jobId, undefined, undefined, ctx?.signal); + } catch (e) { + throw jobHandleError(jobId, e); + } +} + /** Poll until the job is terminal or `waitSeconds` elapse (0 = single poll). * Fires ctx.progress per poll and respects ctx.signal cancellation. * `since`/`limit` are forwarded to every snapshot so a caller that block-waits @@ -401,6 +451,10 @@ export async function waitForJob( ctx?.signal ); if (ctx?.signal?.aborted) break; + // Re-check AFTER the sleep. The sleep above can consume the entire + // remainder, and polling anyway is how a bounded wait overran: the request + // that follows would be granted a budget the caller no longer has. + if (remainingMsOf() <= 0) break; // A cancel landing mid-flight rejects this request. We already hold a good // snapshot, so return it instead of surfacing an AbortError: the user // cancelled the WAIT, not the job, and the job keeps running backend-side. diff --git a/packages/core/src/composite/find-new-leads.ts b/packages/core/src/composite/find-new-leads.ts index 3c161573..9ac05a80 100644 --- a/packages/core/src/composite/find-new-leads.ts +++ b/packages/core/src/composite/find-new-leads.ts @@ -25,6 +25,7 @@ import { rejectCountryLocations, rejectMalformedExclusions, rejectOversizedExclusions, + snapshotAfterSubmit, splitItems, TERMINAL_JOB_STATES, waitForJob, @@ -383,16 +384,15 @@ export const findNewLeads: Tool = { params.wait_seconds, DEFAULT_WAIT_SECONDS ); - const snapshot = - waitSeconds > 0 - ? await waitForJob(client, submit.job_id, waitSeconds, ctx, params.count) - : await collectJobSnapshot( - client, - submit.job_id, - undefined, - undefined, - ctx?.signal - ); + // Every failure past this point must carry submit.job_id: the job exists + // and may be spending, and this handle is the only way back to it. + const snapshot = await snapshotAfterSubmit( + client, + submit.job_id, + waitSeconds, + ctx, + params.count + ); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); const { leads, skipped } = splitItems(snapshot); @@ -417,21 +417,31 @@ export const findNewLeads: Tool = { leads, skipped, items_truncated: snapshot.items_truncated ?? false, + // Top-level, not only inside next_poll: on a TERMINAL job that truncated, + // next_poll used to be null, so the rendering rule telling the agent to + // fetch the rest with `since: next_since` named a cursor the response did + // not contain. The rows are paid for; the way to reach them cannot be + // conditional on the job still running. + next_since: snapshot.next_since ?? null, cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, still_running: !done, - next_poll: done - ? null - : { - tool: "leadbay_lead_job_status", - job_id: submit.job_id, - // Hand the cursor forward so the follow-up poll continues - // INCREMENTALLY instead of re-reading (and re-rendering) the - // rows already delivered in this response. - since: snapshot.next_since ?? null, - suggested_wait_seconds: 60, - }, + // A finished job can still owe rows: truncation means the drain stopped + // early, so there is a follow-up action even when done is true. It is a + // page fetch, not a wait, hence suggested_wait_seconds 0. + next_poll: + done && !(snapshot.items_truncated ?? false) + ? null + : { + tool: "leadbay_lead_job_status", + job_id: submit.job_id, + // Hand the cursor forward so the follow-up poll continues + // INCREMENTALLY instead of re-reading (and re-rendering) the + // rows already delivered in this response. + since: snapshot.next_since ?? null, + suggested_wait_seconds: done ? 0 : 60, + }, region: client.region, }; }, diff --git a/packages/core/src/composite/lead-job-status.ts b/packages/core/src/composite/lead-job-status.ts index 2bd9cf1b..1b868702 100644 --- a/packages/core/src/composite/lead-job-status.ts +++ b/packages/core/src/composite/lead-job-status.ts @@ -100,15 +100,18 @@ export const leadJobStatus: Tool = { cost: snapshot.cost, explain: snapshot.explain, still_running: !done, - next_poll: done - ? null - : { + // Truncation leaves rows unread even on a finished job, so the follow-up + // action survives `done` — same rule as the two submit tools. + next_poll: + done && !(snapshot.items_truncated ?? false) + ? null + : { tool: "leadbay_lead_job_status", job_id: params.job_id, // Same incremental handoff as the submit tools — following // next_poll without the cursor re-reads the rows just returned. since: snapshot.next_since ?? null, - suggested_wait_seconds: 60, + suggested_wait_seconds: done ? 0 : 60, }, region: client.region, }; diff --git a/packages/core/src/composite/qualify-leads.ts b/packages/core/src/composite/qualify-leads.ts index 1d933187..60256304 100644 --- a/packages/core/src/composite/qualify-leads.ts +++ b/packages/core/src/composite/qualify-leads.ts @@ -22,6 +22,7 @@ import { derivedKey, mockedSubmitPreview, compactBody, + snapshotAfterSubmit, splitItems, TERMINAL_JOB_STATES, waitForJob, @@ -443,22 +444,15 @@ export const qualifyLeads: Tool = { params.wait_seconds, DEFAULT_WAIT_SECONDS ); - const snapshot = - waitSeconds > 0 - ? await waitForJob( - client, - submit.job_id, - waitSeconds, - ctx, - submit.items_requested - ) - : await collectJobSnapshot( - client, - submit.job_id, - undefined, - undefined, - ctx?.signal - ); + // Every failure past this point must carry submit.job_id: the job exists + // and may be spending, and this handle is the only way back to it. + const snapshot = await snapshotAfterSubmit( + client, + submit.job_id, + waitSeconds, + ctx, + submit.items_requested + ); const done = TERMINAL_JOB_STATES.has(snapshot.job.state); @@ -510,21 +504,28 @@ export const qualifyLeads: Tool = { // both split. `items` stays for input-order per-ref mapping. ...splitItems(view), items_truncated: snapshot.items_truncated ?? false, + // Top-level, not only inside next_poll: a TERMINAL job that truncated had + // next_poll null, so the rendering rule pointing at `since: next_since` + // named a cursor the response did not carry. + next_since: snapshot.next_since ?? null, cost: snapshot.cost, estimated_cost: submit.estimated_cost, explain: snapshot.explain, still_running: !done, - next_poll: done - ? null - : { - tool: "leadbay_lead_job_status", - job_id: submit.job_id, - // Hand the cursor forward so the follow-up poll continues - // INCREMENTALLY instead of re-reading (and re-rendering) the - // rows already delivered in this response. - since: snapshot.next_since ?? null, - suggested_wait_seconds: 60, - }, + // A finished job can still owe rows: truncation means the drain stopped + // early, so there is a follow-up action even when done is true. + next_poll: + done && !(snapshot.items_truncated ?? false) + ? null + : { + tool: "leadbay_lead_job_status", + job_id: submit.job_id, + // Hand the cursor forward so the follow-up poll continues + // INCREMENTALLY instead of re-reading (and re-rendering) the + // rows already delivered in this response. + since: snapshot.next_since ?? null, + suggested_wait_seconds: done ? 0 : 60, + }, region: client.region, }; }, diff --git a/packages/core/test/unit/client-queue-deadline.test.ts b/packages/core/test/unit/client-queue-deadline.test.ts index 26304097..ebc9e6c3 100644 --- a/packages/core/test/unit/client-queue-deadline.test.ts +++ b/packages/core/test/unit/client-queue-deadline.test.ts @@ -89,7 +89,7 @@ describe("the request deadline covers the queue wait", () => { expect(client._semaphoreState.active).toBe(0); }); - it("charges queue time against the socket budget rather than restarting it", async () => { + it("charges queue time against a TOTAL budget rather than restarting it", async () => { const client = newClient(); const stalled = Array.from({ length: MAX_CONCURRENT }, () => client.request("GET", "/stalled").catch(() => {}) @@ -98,10 +98,11 @@ describe("the request deadline covers the queue wait", () => { seen.length = 0; const startedAt = Date.now(); - // 300ms total. ~200ms of it will be spent queued; the socket must inherit - // only what is LEFT. + // 300ms TOTAL. ~200ms of it will be spent queued; the socket must inherit + // only what is LEFT. (`timeoutMs` is the other knob — a per-attempt bound — + // and is asserted separately below.) const bounded = client - .request("GET", "/mcp/jobs/j1", undefined, { timeoutMs: 300 }) + .request("GET", "/mcp/jobs/j1", undefined, { totalTimeoutMs: 300 }) .catch((e) => e); await wait(200); @@ -118,6 +119,28 @@ describe("the request deadline covers the queue wait", () => { await Promise.all(stalled); }); + it("bounds the queue wait with the per-attempt knob too", async () => { + // `timeoutMs` deliberately gives each ATTEMPT its own window (the hosted + // auth probe needs that — its 401 backoff outlasts its probe budget). It + // must still bound the QUEUE, or a bounded call is unbounded before it + // ever becomes an attempt. + const client = newClient(); + const stalled = Array.from({ length: MAX_CONCURRENT }, () => + client.request("GET", "/stalled").catch(() => {}) + ); + await settled(); + seen.length = 0; + + await expect( + client.request("GET", "/mcp/jobs/j1", undefined, { timeoutMs: 60 }) + ).rejects.toMatchObject({ code: "TIMEOUT" }); + expect(seen).toHaveLength(0); + expect(client._semaphoreState.queued).toBe(0); + + flush(); + await Promise.all(stalled); + }); + it("leaves unbounded calls unbounded", async () => { const client = newClient(); const p = client.request("GET", "/mcp/jobs/j1"); diff --git a/packages/core/test/unit/client-total-budget-retry.test.ts b/packages/core/test/unit/client-total-budget-retry.test.ts new file mode 100644 index 00000000..492afcab --- /dev/null +++ b/packages/core/test/unit/client-total-budget-retry.test.ts @@ -0,0 +1,101 @@ +/** + * `timeoutMs` and `totalTimeoutMs` answer different questions, and the 401 + * retry is where the difference bites. + * + * timeoutMs — bounds ONE attempt. The hosted auth probe needs this: its + * 250ms 401-backoff outlasts a 200ms probe budget, so a + * retry charged against the first attempt's clock is deleted + * rather than bounded. + * totalTimeoutMs — bounds the WHOLE call, backoff and retry included. A job + * poll needs this: a transient 401 must not buy it a second + * full wait_seconds on top of the one it already spent. + * + * Before they were separated, one knob had to be both, and whichever behaviour + * it picked was wrong for the other caller. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { EventEmitter } from "node:events"; + +let status = 200; +const calls: number[] = []; + +vi.mock("node:https", () => ({ + default: { + request: (_o: Record, cb?: (res: unknown) => void) => { + calls.push(Date.now()); + const s = status; + const req = new EventEmitter() as EventEmitter & { + write: () => void; + end: () => void; + destroy: () => void; + }; + req.write = () => {}; + req.destroy = () => {}; + req.end = () => { + setImmediate(() => { + const res = new EventEmitter() as EventEmitter & { + statusCode: number; + headers: Record; + }; + res.statusCode = s; + res.headers = {}; + cb?.(res); + res.emit("data", Buffer.from("{}")); + res.emit("end"); + }); + }; + return req; + }, + }, +})); + +import { LeadbayClient } from "../../src/client.js"; + +const newClient = () => new LeadbayClient("https://api-us.leadbay.app", "u.tok", "us"); + +beforeEach(() => { + status = 200; + calls.length = 0; +}); + +describe("401 retry and the two budget knobs", () => { + it("a per-attempt budget survives the 250ms backoff and retries", async () => { + status = 401; + // 200ms per attempt, backoff 250ms: the retry only happens if its window + // starts after the sleep. This is the auth-probe contract. + await newClient() + .request("GET", "/users/me", undefined, { timeoutMs: 200 }) + .catch(() => {}); + expect(calls).toHaveLength(2); + }); + + it("a TOTAL budget refuses to fund a second attempt it cannot afford", async () => { + status = 401; + const startedAt = Date.now(); + const err = await newClient() + .request("GET", "/users/me", undefined, { totalTimeoutMs: 200 }) + .catch((e) => e); + + expect(err).toMatchObject({ code: "TIMEOUT" }); + // The first attempt happened; the retry did not, because the 250ms backoff + // already exhausted the caller's whole 200ms. + expect(calls).toHaveLength(1); + // And it did not silently run on to a second full budget. + expect(Date.now() - startedAt).toBeLessThan(400); + }); + + it("honours both at once — whichever expires first wins", async () => { + status = 401; + await newClient() + .request("GET", "/users/me", undefined, { timeoutMs: 200, totalTimeoutMs: 5000 }) + .catch(() => {}); + // Generous total, per-attempt window fresh after the backoff → retry runs. + expect(calls).toHaveLength(2); + }); + + it("leaves a call with neither knob unbounded", async () => { + status = 401; + await newClient().request("GET", "/users/me").catch(() => {}); + expect(calls).toHaveLength(2); + }); +}); diff --git a/packages/core/test/unit/composite/job-wait-deadline.test.ts b/packages/core/test/unit/composite/job-wait-deadline.test.ts index 1d139937..b4414363 100644 --- a/packages/core/test/unit/composite/job-wait-deadline.test.ts +++ b/packages/core/test/unit/composite/job-wait-deadline.test.ts @@ -7,7 +7,7 @@ import { MCP_JOB_POLL, } from "../../../src/composite/_mcp-job-helpers.js"; -type Opts = { signal?: AbortSignal; timeoutMs?: number }; +type Opts = { signal?: AbortSignal; timeoutMs?: number; totalTimeoutMs?: number }; const snapshot = (state: string) => ({ job: { state }, @@ -58,26 +58,26 @@ describe("job snapshots are bounded by the caller's wait budget", () => { expect(opts).toHaveLength(1); // Before the fix this was undefined: wait_seconds was enforced only by the // loop condition, which is not evaluated until the first GET has returned. - expect(opts[0].timeoutMs).toBeDefined(); - expect(opts[0].timeoutMs!).toBeLessThanOrEqual(5000); + expect(opts[0].totalTimeoutMs).toBeDefined(); + expect(opts[0].totalTimeoutMs!).toBeLessThanOrEqual(5000); }); it("never lets a snapshot outlive the requested wait", async () => { const { client, opts } = stubClient(() => snapshot("completed")); await waitForJob(client, "job-1", 1); - expect(opts[0].timeoutMs!).toBeLessThanOrEqual(1000); + expect(opts[0].totalTimeoutMs!).toBeLessThanOrEqual(1000); }); it("caps a generous wait at the per-request ceiling", async () => { const { client, opts } = stubClient(() => snapshot("completed")); await waitForJob(client, "job-1", 600); - expect(opts[0].timeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); + expect(opts[0].totalTimeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); }); it("gives a zero-wait poll a bound of its own", async () => { const { client, opts } = stubClient(() => snapshot("running")); await collectJobSnapshot(client, "job-1"); - expect(opts[0].timeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); + expect(opts[0].totalTimeoutMs).toBe(SNAPSHOT_TIMEOUT_MS); }); it("keeps the job_id when the first read times out, rather than losing a paid job", async () => { @@ -134,7 +134,7 @@ describe("the wait bounds the whole drain, not each page of it", () => { await waitForJob(client, "job-1", 5, undefined, undefined, undefined, 2); expect(opts.length).toBeGreaterThan(1); - const budgets = opts.map((o) => o.timeoutMs!); + const budgets = opts.map((o) => o.totalTimeoutMs!); // STRICTLY decreasing: each page is bounded by what is LEFT, so the drain // as a whole cannot outlast the wait. A per-page budget would hold flat. for (let i = 1; i < budgets.length; i++) { diff --git a/packages/core/test/unit/composite/post-submit-recovery.test.ts b/packages/core/test/unit/composite/post-submit-recovery.test.ts new file mode 100644 index 00000000..af09d875 --- /dev/null +++ b/packages/core/test/unit/composite/post-submit-recovery.test.ts @@ -0,0 +1,151 @@ +/** + * Two ways a paid job could be stranded, and the exhausted-budget overrun. + * + * 1. Once the submit returns, a backend-owned job EXISTS and may be spending. + * Any later failure must still hand back job_id — it is the only route to + * what the user just bought. Previously only a TIMEOUT on the block-waiting + * branch preserved it; an abort, a reset or a 5xx propagated bare, and the + * zero-wait branch preserved nothing at all. + * 2. Truncation means rows were paid for but not read. The rendering rule tells + * the agent to fetch them with `since: next_since` — which the response has + * to actually contain, including on a job that has already finished. + */ +import { describe, it, expect, vi } from "vitest"; +import type { LeadbayClient } from "../../../src/client.js"; +import { + waitForJob, + collectJobSnapshot, + snapshotAfterSubmit, + MCP_JOB_POLL, +} from "../../../src/composite/_mcp-job-helpers.js"; + +const snap = (state: string, extra: Record = {}) => ({ + job: { state }, + items: [], + funnel: {}, + cost: { spent: 0 }, + next_since: null, + ...extra, +}); + +const clientThatThrows = (e: unknown) => + ({ request: async () => { throw e; } }) as unknown as LeadbayClient; + +const coded = (code: string, message: string) => { + const e = new Error(message) as Error & { code?: string }; + e.code = code; + return e; +}; + +describe("post-submit failures keep the job handle", () => { + it.each([ + ["a connection reset", coded("ECONNRESET", "socket hang up")], + ["a backend 5xx", { error: true, code: "UPSTREAM_ERROR", message: "502" }], + ["a cancellation", { error: true, code: "REQUEST_CANCELLED", message: "gone" }], + ["a bare error", new Error("something else")], + ])("attaches job_id when the wait branch hits %s", async (_label, err) => { + const e = await snapshotAfterSubmit(clientThatThrows(err), "job-42", 5).catch((x) => x); + expect(e.job_id).toBe("job-42"); + expect(e.hint).toContain("leadbay_lead_job_status"); + }); + + it("attaches job_id on the ZERO-WAIT branch too", async () => { + // This branch previously propagated every failure bare. + const e = await snapshotAfterSubmit( + clientThatThrows(coded("ECONNRESET", "socket hang up")), + "job-42", + 0 + ).catch((x) => x); + expect(e.job_id).toBe("job-42"); + }); + + it("does not double-wrap an error that already carries the handle", async () => { + const e = await snapshotAfterSubmit( + clientThatThrows(coded("TIMEOUT", "slow")), + "job-42", + 1 + ).catch((x) => x); + expect(e.code).toBe("JOB_READ_TIMEOUT"); + expect(e.job_id).toBe("job-42"); + expect(e.message).not.toContain("reading its status failed"); + }); + + it("passes a successful snapshot straight through", async () => { + const client = { request: async () => snap("completed") } as unknown as LeadbayClient; + await expect(snapshotAfterSubmit(client, "job-42", 0)).resolves.toMatchObject({ + job: { state: "completed" }, + }); + }); +}); + +describe("an exhausted wait budget stops the poll", () => { + const REAL = MCP_JOB_POLL.intervalMs; + + it("does not grant a final snapshot a budget the caller no longer has", async () => { + MCP_JOB_POLL.intervalMs = 5; + let now = 0; + const spy = vi.spyOn(Date, "now").mockImplementation(() => now); + const opts: Array<{ totalTimeoutMs?: number }> = []; + try { + const client = { + request: async (_m: string, _p: string, _b: unknown, o?: { totalTimeoutMs?: number }) => { + opts.push(o ?? {}); + now += 400; // every read eats most of a 1s budget + return snap("running"); + }, + } as unknown as LeadbayClient; + + await waitForJob(client, "job-1", 1); + // The old 1s FLOOR let a final poll run a full second past an already + // spent budget, so `wait_seconds: 1` took ~2s. Asserting "<= 1000" cannot + // catch that — the floor IS 1000. Each budget must match what is actually + // left, so they strictly decrease. + expect(opts.length).toBeGreaterThan(1); + const budgets = opts.map((o) => o.totalTimeoutMs!); + for (let i = 1; i < budgets.length; i++) { + expect(budgets[i]).toBeLessThan(budgets[i - 1]); + } + // 400ms consumed per read out of 1000 → the last one gets ~200, not 1000. + expect(budgets[budgets.length - 1]).toBeLessThanOrEqual(200); + } finally { + spy.mockRestore(); + MCP_JOB_POLL.intervalMs = REAL; + } + }); + + it("does not round a fractional wait up on the first snapshot", async () => { + const opts: Array<{ totalTimeoutMs?: number }> = []; + const client = { + request: async (_m: string, _p: string, _b: unknown, o?: { totalTimeoutMs?: number }) => { + opts.push(o ?? {}); + return snap("completed"); + }, + } as unknown as LeadbayClient; + + await waitForJob(client, "job-1", 0.5); + // A 1s floor turned a 500ms wait into a 1s one. + expect(opts[0].totalTimeoutMs!).toBeLessThanOrEqual(500); + }); +}); + +describe("a truncated drain hands back a usable cursor", () => { + it("keeps next_since on the snapshot when the drain stops early", async () => { + let call = 0; + const client = { + request: async () => { + call++; + // Full page + cursor = more to come. + return { + ...snap("completed"), + items: [{ id: "a" }, { id: "b" }], + next_since: `cur-${call}`, + }; + }, + } as unknown as LeadbayClient; + + const s = await collectJobSnapshot(client, "job-1", undefined, 2, undefined, 1); + expect(s.items_truncated).toBe(true); + // The rendering rule names this cursor; it has to exist. + expect(s.next_since).toBeTruthy(); + }); +}); diff --git a/packages/core/test/unit/composite/truncated-terminal-continuation.test.ts b/packages/core/test/unit/composite/truncated-terminal-continuation.test.ts new file mode 100644 index 00000000..de24ff23 --- /dev/null +++ b/packages/core/test/unit/composite/truncated-terminal-continuation.test.ts @@ -0,0 +1,115 @@ +/** + * A finished job can still owe rows. + * + * When the page drain runs out of budget AFTER the backend job has gone + * terminal, `done` is true — so `next_poll` was null — while `items_truncated` + * is also true. The rendering rule tells the agent to fetch the rest with + * `leadbay_lead_job_status(job_id, since: next_since)`, but neither the cursor + * nor a continuation action was present anywhere in the response. The rows were + * paid for and simply unreachable. + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { mockHttp, resetHttpMock, httpsMockFactory } from "../../harness.js"; +vi.mock("node:https", () => httpsMockFactory()); + +import { LeadbayClient } from "../../../src/client.js"; +import { findNewLeads } from "../../../src/composite/find-new-leads.js"; + +const BASE = "https://api-us.leadbay.app"; +const newClient = () => new LeadbayClient(BASE, "u.tok", "us"); + +beforeEach(() => resetHttpMock()); + +// A page that is FULL and still carries a cursor — the drain's "more to come" +// signal — on a job the backend already marked completed. +const fullPage = (cursor: string) => ({ + job: { state: "completed" }, + items: Array.from({ length: 100 }, (_, i) => ({ + id: `${cursor}-${i}`, + status: "delivered", + company: { name: "Acme" }, + })), + funnel: { delivered: 250 }, + cost: { spent: 0 }, + next_since: cursor, +}); + +describe("truncated + terminal — the rest stays reachable", () => { + it("returns the cursor and a continuation even though done is true", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-t", state: "queued", items_requested: 250 }, + }, + // Every page is full + cursored, so the drain stops on its own budget. + ...Array.from({ length: 30 }, (_, i) => ({ + method: "GET" as const, + path: new RegExp("^/1\\.6/mcp/jobs/job-t"), + status: 200, + body: fullPage(`cur-${i}`), + })), + ]); + + const res: any = await findNewLeads.execute( + newClient(), + { + example_lead: { description: "independent gym" }, + count: 50, + request_id: "trunc-test", + wait_seconds: 0, + } as any, + {} as any + ); + + expect(res.done).toBe(true); + expect(res.items_truncated).toBe(true); + // The two things the rendering rule needs, both previously absent here. + expect(res.next_since).toBeTruthy(); + expect(res.next_poll).not.toBeNull(); + expect(res.next_poll.tool).toBe("leadbay_lead_job_status"); + expect(res.next_poll.job_id).toBe("job-t"); + expect(res.next_poll.since).toBe(res.next_since); + // It is a page fetch, not a wait — nothing is still running. + expect(res.next_poll.suggested_wait_seconds).toBe(0); + }); + + it("still returns no continuation for a clean, complete job", async () => { + mockHttp([ + { + method: "POST", + path: "/1.6/mcp/search", + status: 200, + body: { job_id: "job-c", state: "queued", items_requested: 5 }, + }, + { + method: "GET", + path: new RegExp("^/1\\.6/mcp/jobs/job-c"), + status: 200, + body: { + job: { state: "completed" }, + items: [], + funnel: { delivered: 0 }, + cost: { spent: 0 }, + next_since: null, + }, + }, + ]); + + const res: any = await findNewLeads.execute( + newClient(), + { + example_lead: { description: "independent gym" }, + count: 5, + request_id: "clean-test", + wait_seconds: 0, + } as any, + {} as any + ); + + expect(res.done).toBe(true); + expect(res.items_truncated).toBe(false); + expect(res.next_poll).toBeNull(); + }); +});