diff --git a/bioscancast/stages/extraction/custom_scrapers/cdc_measles.py b/bioscancast/stages/extraction/custom_scrapers/cdc_measles.py new file mode 100644 index 0000000..d048402 --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/cdc_measles.py @@ -0,0 +1,150 @@ +"""Custom scraper for the CDC measles data page (``cdc_measles``). + +The CDC "Measles Cases and Outbreaks" page renders its case/death/hospitalisation +figures client-side from a JSON data endpoint. A static scrape gets the case +*count* from prose (so q14 works) but the **deaths** table cells are empty in the +served HTML — they are JS-injected — so death questions (q16) extracted 0 records. + +The data is published as plain JSON at ``/wcms/vizdata/measles/measles_hosp.json`` +(no browser or Akamai challenge needed), carrying, per year, ``total_cases``, +``total_deaths`` and a ready-made ``deaths_sentence``. We fetch that and render a +compact HTML summary the existing HTML extraction pipeline consumes unchanged, so +the insight stage gets an unambiguous "N confirmed measles deaths in 2026" fact +(and the case count, corroborating the prose figure q14 already uses). + +Returns ``None`` (fall back to the generic fetch) on any failure or in +historical-replay mode — the endpoint carries only current per-year aggregates +with no date history to cut on, so serving it under an ``as_of_date`` would leak. +""" + +from __future__ import annotations + +import html +import json +import logging +from datetime import datetime, timezone + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.fetcher import FetchResult + +logger = logging.getLogger(__name__) + +# The page's client-side data feed. Absolute (host-pinned) because this scraper +# is specific to the CDC measles source; the hub ``url`` passed in is the +# human-facing data-research page. +_DATA_URL = "https://www.cdc.gov/wcms/vizdata/measles/measles_hosp.json" + +# Years to surface, newest first. The current year answers q14/q16; the prior +# year is kept as corroborating context (and a base rate for the death question). +_YEARS = ("2026", "2025") + + +def _first(value) -> str | None: + """CDC wraps each field as a single-element list (e.g. ``["0"]``).""" + if isinstance(value, list): + value = value[0] if value else None + if value is None: + return None + text = str(value).strip() + return text or None + + +def _get_json(url: str, cfg: ExtractionConfig): + try: + from curl_cffi import requests as curl_requests + + resp = curl_requests.get( + url, + timeout=max(cfg.fetch_timeout_seconds, 30.0), + impersonate=cfg.impersonate, + allow_redirects=True, + ) + except Exception as exc: # noqa: BLE001 - network best-effort + logger.info("CDC measles JSON fetch failed for %s: %s", url, exc) + return None + if resp.status_code != 200 or not resp.content: + return None + try: + return resp.json() + except (ValueError, json.JSONDecodeError): + return None + + +def fetch( + url: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + region: str | None = None, + question_text: str | None = None, + json_getter=None, +) -> FetchResult | None: + # Live-only: the feed has no per-date history, so serving it in replay mode + # would leak post-cutoff values. Fall back to the generic (Wayback) path. + if as_of_date is not None: + return None + + cfg = config or ExtractionConfig() + fetched_at = datetime.now(timezone.utc) + + # ``json_getter`` is injectable for no-network tests; defaults to a live + # curl_cffi fetch of the CDC data feed. + getter = json_getter or _get_json + data = getter(_DATA_URL, cfg) + if not isinstance(data, dict): + return None + + blocks: list[str] = [] + for year in _YEARS: + rec = data.get(year) + if not isinstance(rec, dict): + continue + cases = _first(rec.get("total_cases")) + deaths = _first(rec.get("total_deaths")) + deaths_sentence = _first(rec.get("deaths_sentence")) + if cases is None and deaths is None: + continue + parts = [f"

United States measles, {html.escape(year)}

", "

"] + if cases is not None: + parts.append( + f"As of the latest CDC update, a total of {html.escape(cases)} " + f"confirmed measles cases were reported in the United States in " + f"{html.escape(year)}. " + ) + if deaths_sentence is not None: + parts.append(html.escape(deaths_sentence) + " ") + elif deaths is not None: + parts.append( + f"There have been {html.escape(deaths)} confirmed measles deaths " + f"in the United States in {html.escape(year)}. " + ) + parts.append("

") + blocks.append("".join(parts)) + + if not blocks: + return None + + rendered = ( + "" + "CDC — Measles Cases and Outbreaks (United States)" + "

CDC — Measles Cases and Outbreaks (United States)

" + f"

Source: {html.escape(url)} " + f"(data feed {html.escape(_DATA_URL)}), retrieved " + f"{fetched_at.date().isoformat()}.

" + + "".join(blocks) + + "

Note: this custom scraper renders the CDC measles JSON data feed " + "into compact HTML because the page's case/death figures are injected " + "client-side and are absent from the statically served table cells. " + "Counts are cumulative year-to-date totals.

" + "" + ).encode("utf-8") + + return FetchResult( + url=_DATA_URL, + final_url=_DATA_URL, + status_code=200, + content_type="text/html", + content_bytes=rendered, + fetched_at=fetched_at, + error=None, + ) diff --git a/bioscancast/stages/extraction/custom_scrapers/who_h5_hai.py b/bioscancast/stages/extraction/custom_scrapers/who_h5_hai.py new file mode 100644 index 0000000..9f7df30 --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/who_h5_hai.py @@ -0,0 +1,43 @@ +"""Custom scraper for the WHO "Influenza at the human-animal interface" monthly +risk-assessment hub (``who_h5_hai``). + +The monthly-risk-assessment-summary page is an index: it lists dated +*Influenza at the human-animal interface summary and assessment* item pages +(``who.int/publications/m/item/influenza-at-the-human-animal-interface-...``), +each of which links a ``cdn.who.int`` PDF whose tables/prose carry the cumulative +human case counts by A(H5) subtype. The index page itself is context-only, so H5 +human-case questions (q6 case count, q8 predominant subtype) extracted 0 records +from it. + +This resolves the hub to the latest assessment at-or-before the cutoff and returns +that item's PDF, exactly as ``who_cholera`` does for the cholera hub, so the +existing PDF parser extracts the numbers. Falls back (return ``None``) to the +generic fetch of the index page if resolution fails. +""" + +from __future__ import annotations + +from datetime import datetime + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.custom_scrapers._who_hub_common import ( + fetch_who_hub_latest_pdf, +) +from bioscancast.stages.extraction.fetcher import FetchResult + + +def fetch( + url: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + region: str | None = None, + question_text: str | None = None, +) -> FetchResult | None: + # "human-animal" selects the HAI summary items (their slug/text both carry + # "human-animal interface") out of the ~150 mixed publication links on the + # monthly-risk-assessment index, and excludes the FAO/WOAH joint-assessment + # and vaccine-composition items that don't hold the human-case counts. + return fetch_who_hub_latest_pdf( + url, "human-animal", config=config, as_of_date=as_of_date + ) diff --git a/bioscancast/stages/extraction/pipeline.py b/bioscancast/stages/extraction/pipeline.py index f317040..ef904d9 100644 --- a/bioscancast/stages/extraction/pipeline.py +++ b/bioscancast/stages/extraction/pipeline.py @@ -139,9 +139,16 @@ def extract_one(self, filtered_doc: FilteredDocument) -> Document: refiner = self._get_docling_refiner() if refiner is not None: try: + # Match the Docling allowlist against the URL actually + # fetched. Custom hub scrapers (who_cholera, who_h5_hai, ...) + # resolve a landing page to a ``cdn.who.int/.../*.pdf``; the + # allowlist entries target those resolved PDF paths, so + # matching on ``filtered_doc.url`` (the hub) would never fire + # the ``situation-reports`` / ``_sage-`` allowlist for exactly + # the PDFs it exists to catch. parsed = refiner.refine( parsed, - source_url=filtered_doc.url, + source_url=fetch_result.final_url or filtered_doc.url, content=fetch_result.content_bytes, ) except Exception as exc: diff --git a/bioscancast/tests/test_cdc_measles_scraper.py b/bioscancast/tests/test_cdc_measles_scraper.py new file mode 100644 index 0000000..8f8c0a8 --- /dev/null +++ b/bioscancast/tests/test_cdc_measles_scraper.py @@ -0,0 +1,66 @@ +"""Tests for the CDC measles custom scraper (``cdc_measles``). + +No network: the scraper accepts an injected ``json_getter`` returning a fixture +dict shaped like the live ``measles_hosp.json`` feed. Assertions target the +rendered HTML summary (``content_bytes``) that the HTML extraction pipeline then +consumes — the point of the scraper is to surface the JS-injected **deaths** +figure (absent from the statically served page) as extractable prose. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from bioscancast.stages.extraction.custom_scrapers import cdc_measles + +_PAGE = "https://www.cdc.gov/measles/data-research/index.html" + +# Shape mirrors the live feed: each field is a single-element list. +_FEED = { + "2026": { + "total_cases": ["2,170"], + "total_deaths": ["0"], + "deaths_sentence": ["There have been 0 confirmed deaths from measles in 2026."], + }, + "2025": { + "total_cases": ["2,289"], + "total_deaths": ["3"], + "deaths_sentence": ["There were 3 confirmed deaths from measles in 2025."], + }, +} + + +def _getter(feed): + return lambda url, config: feed + + +def _html(result) -> str: + assert result is not None + return result.content_bytes.decode("utf-8") + + +def test_renders_current_year_death_count_as_prose(): + result = cdc_measles.fetch(_PAGE, json_getter=_getter(_FEED)) + html = _html(result) + assert result.content_type == "text/html" + # The death figure that is JS-injected (and thus missing from a static + # scrape) is now present as an unambiguous sentence. + assert "0 confirmed deaths from measles in 2026" in html + # Case count is surfaced too, corroborating the prose figure q14 uses. + assert "2,170 confirmed measles cases" in html + + +def test_historical_mode_returns_none_to_avoid_leakage(): + # The feed carries only current per-year aggregates with no date history, + # so replay mode must fall back rather than serve post-cutoff values. + result = cdc_measles.fetch( + _PAGE, + as_of_date=datetime(2026, 3, 1, tzinfo=timezone.utc), + json_getter=_getter(_FEED), + ) + assert result is None + + +def test_missing_feed_falls_back(): + assert cdc_measles.fetch(_PAGE, json_getter=lambda url, config: None) is None + assert cdc_measles.fetch(_PAGE, json_getter=lambda url, config: {}) is None diff --git a/bioscancast/tests/test_extraction_pipeline.py b/bioscancast/tests/test_extraction_pipeline.py index e256cc0..d65d968 100644 --- a/bioscancast/tests/test_extraction_pipeline.py +++ b/bioscancast/tests/test_extraction_pipeline.py @@ -150,6 +150,53 @@ def test_pdf_extraction_end_to_end(self): assert doc.page_count is not None assert len(doc.chunks) > 0 + def test_refiner_receives_resolved_pdf_url_not_hub(self): + # A hub scraper (who_cholera, who_h5_hai, ...) resolves a landing page + # who.int/ to a cdn.who.int/.../*.pdf. The Docling allowlist targets + # the resolved PDF path, so the refiner must be handed + # ``fetch_result.final_url`` — matching on the hub ``filtered_doc.url`` + # would never fire the allowlist for exactly those scraper PDFs. + pdf_bytes = (FIXTURES / "who_don_sample.pdf").read_bytes() + hub_url = "https://www.who.int/emergencies/situations/cholera-upsurge" + pdf_url = ( + "https://cdn.who.int/media/docs/default-source/documents/" + "emergencies/situation-reports/cholera_update.pdf" + ) + fdoc = _make_filtered_doc( + result_id="r_hub", url=hub_url, domain="who.int", extraction_mode="pdf" + ) + fetch_map = { + hub_url: FetchResult( + url=pdf_url, + final_url=pdf_url, + status_code=200, + content_type="application/pdf", + content_bytes=pdf_bytes, + fetched_at=datetime.now(timezone.utc), + error=None, + ) + } + + seen: dict[str, str] = {} + + class _SpyRefiner: + def refine(self, parsed, *, source_url, content): + seen["source_url"] = source_url + return parsed + + with patch( + "bioscancast.stages.extraction.pipeline.fetch", + side_effect=_fake_fetch_factory(fetch_map), + ), patch.object( + ExtractionPipeline, "_get_docling_refiner", return_value=_SpyRefiner() + ): + pipeline = ExtractionPipeline( + config=ExtractionConfig(enable_docling_refiner=True) + ) + pipeline.run([fdoc]) + + assert seen.get("source_url") == pdf_url + def test_ordering_by_extraction_priority(self): html_bytes = b"

Test content

" fdoc_low = _make_filtered_doc( diff --git a/data/investigations/bfg-evidence-quality-2026-07-05.md b/data/investigations/bfg-evidence-quality-2026-07-05.md new file mode 100644 index 0000000..3f1bc34 --- /dev/null +++ b/data/investigations/bfg-evidence-quality-2026-07-05.md @@ -0,0 +1,265 @@ +# BFG summer-2026 — evidence-coverage audit & remediation + +**Date:** 2026-07-05 · **Branch:** `claude/brave-diffie-2895b6` (based on PR #52 +`feat/bfg-summer-2026-readiness`) · **Mode:** live, evidence-only (`--no-forecast`) + +Instrument: [`scripts/analyze_evidence_coverage.py`](../../scripts/analyze_evidence_coverage.py). +Diagnosis table: [`bfg_evidence_audit_2026-07-05.csv`](bfg_evidence_audit_2026-07-05.csv) +(+ `.json`). All 25 questions run once each in one ~19-min window; total pipeline +cost ≈ **$0.22**; the LLM on-topic judge added **403** gpt-4o-mini calls (≈ $0.03). + +This is Phase 1 (diagnose) of `forecast-evidence-quality-spec.md`. The injected-vs- +organic split is used as the diagnostic instrument, not the deliverable. + +--- + +## Headline + +| Metric | Result | +|---|---| +| Resolution-source dashboard injected | **25/25** — routing is not dropping authoritative sources | +| Evidence-sufficient (high-confidence, correct-scope, correct-basis anchor) | **14/25** | +| Single fragile source (anchor rests on one source) | **7/25** | +| **0 usable records at all** (forecast runs on pure baseline) | **7/25** | + +Classification: **well_supported 7 · dashboard_only 8 · under_supported 10.** + +Two things the audit *rules out* as the dominant problem: + +1. **Routing (#41) is fine in live mode.** Every question injected its resolution + domain (who.int / cdc.gov / paho.org / ecdc.europa.eu / aphis.usda.gov / + polioeradication.org). The single-bucket `route_sources` did not misroute any + of the 25. The additive-injection change #41 proposes is not needed to fix a + coverage gap here (still worth doing for robustness, but it's not load-bearing). +2. **The filter (#13/#44) is mostly fine.** Where the deterministic keyword-overlap + heuristic flagged "on-topic organic dropped by the filter," the gpt-4o-mini judge + rated those pools **0–13% on-topic** on 6 of 7 questions — i.e. generic news + (nypost, abcnews, statnews, motor1), correctly rejected. Only **q15** shows a + genuine authoritative-organic drop (judge: 53% on-topic). + +The actual gaps are downstream: **extraction** and **insight-stage calibration**. + +--- + +## Where forecasts are actually under-supported + +### A. Extraction — dashboard survives but yields 0 records (7 questions) — **highest impact** + +These forecasts have **no current-value evidence at all**; they run on the +retrieval-free baseline. The authoritative dashboard is injected and survives +filtering, but the insight stage extracts nothing usable from it. + +| Q | Question | Injected source that went barren | Root cause | +|---|---|---|---| +| **q9** | US states w/ H5N1 dairy detections | APHIS livestock (Tableau) | **Issue #50** — Tableau behind Akamai; ~492-char stub. Curated-snapshot scraper already drafted in #50. | +| **q6** | H5 human cases (global, window) | WHO avian-influenza landing + monthly HAI risk-assessment | Landing/index pages; the human-case counts live in the HAI report/PDF, not the page body. | +| **q8** | Predominant H5 subtype | same WHO H5 pages | Same as q6; subtype breakdown is in the HAI report. | +| **q16** | US measles **deaths** 2026 | CDC measles data page | Same page yields **case** counts (q14 works) but not the **deaths** figure — extraction-scope gap. | +| **q11** | # PHEICs in effect | general_sources (15 WHO/CDC list/index pages) | "Count items on a list" — pipeline doesn't enumerate list entries. | +| **q24** | Novel-pathogen DON in window | general_sources (15) | Same enumeration gap over the WHO DON list. | +| **q25** | # new WHO DON items in window | general_sources (15) | Same — needs to count DON items in a date window. | + +Two sub-families: **(a) fixable with a targeted scraper** (q9, q6, q8, and likely +q16) — resolve the landing page to the fact-bearing report/PDF, exactly as the +cholera/dengue/polio scrapers already do; **(b) list-enumeration questions** +(q11, q24, q25) — the pipeline has no "count entries on this authoritative list" +capability. That's a larger capability gap, better recommended than hacked in-pass. + +### B. Insight-stage calibration — anchor extracted but not usable (q17) — **Issue #26** + +q17 (global cholera 2026): the correct anchor **114,829 cumulative (Global)** *was* +extracted from the WHO cholera epi-update PDF — but at **confidence 0.5**, while +off-scope regional rows (African Region 61,888; Angola) were extracted at **0.85**. +The high-confidence rows outrank the row the question actually needs. + +This is the exact "confidence miscalibration" failure mode **Issue #26** says to +watch for after the first benchmark. This audit is that benchmark; q17 is the +evidence #26's definition-of-done item 1 requires. Note the anchor *is* in the +evidence digest fed to the forecaster (records aren't confidence-gated before the +digest), so **Phase 3 must check whether the forecast still anchors on 114,829 +despite the low confidence** before concluding a code change is required. + +### C. Robustness — good anchor, single source (7 questions) + +q3, q7, q12, q13, q14, q15, q18, q22 carry a **high-confidence, correct-scope +anchor** (e.g. mpox 180,545 cumulative; dengue-Americas 251,057; measles 2,170; +polio WPV1 7) — but from **one** source, with **no organic corroboration**. The +judge shows *why*: for off-peak counts the organic pool is generic news, so the +authoritative number exists only on the single injected dashboard. These forecasts +are supported but fragile to that one source failing. Corroboration must come from +a **second authoritative source** (another dashboard), not organic news. + +### D. Thin / low-confidence anchor (q19, q20, q23) + +q19 (chikungunya EU/EEA), q20 (Oropouche Americas), q23 (SARS-CoV-2 VOI/VOC): only +a weak dashboard record (conf 0.5–0.7, often a placeholder "1" or an off-scope +cumulative), below the usable bar. q23's "anchor" (775M cumulative COVID cases) is +irrelevant to a variant-designation binary — correctly classified under_supported. + +--- + +## Well-supported (7) — do not regress + +q1, q2, q4, q5 (Bundibugyo Ebola — peak pathogen, rich organic + dashboard), +q10 (PHEIC — organic PHEIC-mention records; *soft* — see caveat), q21 (filovirus +binary), q3 (Ebola PHEIC binary — flagged fragile). Peak pathogens get genuine +organic corroboration; off-peak ones do not. + +**Caveat on categorical "count events" questions (q10, q11, q24, q25):** the +numeric-anchor concept fits case/range questions well but not "how many PHEICs / +DON items" — those need list enumeration. q10 scores well_supported only because it +picked up on-topic PHEIC-mention text, not because it counted PHEICs. Treat q10's +verdict as soft. + +--- + +## Diagnosis → open-issue map + +| Finding | Questions | Issue | How it feeds the issue | +|---|---|---|---| +| Dashboard barren at extraction | q9 | **#50** | Confirms the gap live (0 records); fix = land the drafted curated-snapshot scraper. | +| Anchor extracted but under-confident | q17 | **#26** | The "first benchmark" evidence that confidence miscalibration is real → justifies the refinement pass. | +| Routing never misrouted (25/25) | all | **#41** | Negative result: single-bucket routing isn't the coverage bottleneck in live mode. | +| Generic-news organic dropped correctly | q12,q13,q18,q20,q22,q23 | #13/#44 | Filter is working; judge confirms. No change warranted. | +| WHO H5 / list pages don't yield counts | q6,q8,q11,q24,q25 | (new) | Scraper for HAI report (q6/q8); list-enumeration capability (q11/q24/q25) is net-new. | + +--- + +## Phase-2 remediation plan (prioritized, pending go-ahead) + +Ordered by (impact × safety). Safe levers = **scrapers** (isolated, additive, +per-source — the cholera/dengue/polio pattern). Risky levers = search-weight / +filter-threshold / insight-prompt changes (broad blast radius; the spec and +`findings-issues-3-4-13.md` both warn on these). + +1. **q9 → land the #50 APHIS curated-snapshot scraper.** Isolated, closes/advances + #50. Turns 0 records → 1 clean "N states as of DATE" anchor. *Decision needed:* + snapshot value source (verify current count via web) and refresh model. +2. **q6/q8 → add a WHO HAI scraper** (`who_h5_hai`) that resolves the + avian-influenza monthly-risk-assessment landing to its latest report/PDF with + human H5 case counts + subtype, mirroring `_who_hub_common`. Fixes 2 questions. +3. **q16 → measles-deaths extraction.** Check whether the CDC measles page carries + the death figure and it's an insight-scope miss vs. genuinely absent; add a + corroborating CDC source if needed. +4. **q17 → measure first (Phase 3), then #26.** Confirm whether the forecast + anchors on 114,829 despite conf 0.5. Only if it doesn't, do a scoped + confidence/scope fix under #26 (avoid a broad insight-prompt change mid-audit). +5. **Robustness (q7,q12,q13,q14,q18,q22) → add a second authoritative source** per + pathogen family where one exists (e.g. ECDC/CDC alongside WHO/PAHO). Additive, + low risk. +6. **q11/q24/q25 (list enumeration) → recommend, don't hack.** Net-new capability; + file or extend an issue rather than force it in this pass. + +**Explicitly not doing:** search-stage weight changes, filter-threshold changes, +or forecasting-model tuning (out of scope / broad blast radius). + +## Phase-3 (after remediation) + +Re-run each fixed question **with** the forecast; confirm the distribution anchors +on the correct current value and diverges sensibly from `bioscancast_baseline`. +Regression-guard: re-run a sample of the 7 well-supported questions + the historic +benchmark (`scripts/run_historical_trajectory.py`) to confirm no degradation. + +--- + +# Phase 2 — remediation (outcomes) + +## Environment finding (important): Docling was missing; a real refiner bug + +The Phase-1 sweep ran in a worktree Python that was **missing `docling`** — a +*required* dep (`requirements.txt`: `docling[chunking]>=2.90`). The repo keeps a +separate `.venv-docling`; I completed it (added `tavily`, `pycountry`, +`matplotlib`, `trafilatura`, …) so the pipeline runs with Docling as intended. + +While validating, I found a genuine defect in the extraction pipeline: + +> **Docling table-refiner never fires on custom-scraper-resolved PDFs.** +> `pipeline.py` passed `source_url=filtered_doc.url` (the *hub* URL, e.g. +> `who.int/.../cholera-upsurge`) to `refiner.refine()`, but the Docling allowlist +> entries (`.../situation-reports/`, `.../_sage-`) target the *resolved* PDF URL +> (`cdn.who.int/...`). So the allowlist could never match for exactly the hub +> scrapers (cholera, mpox, HAI, …) it exists to serve. + +**Fix (landed):** [`pipeline.py`](../../bioscancast/stages/extraction/pipeline.py) +now matches on `fetch_result.final_url or filtered_doc.url`. Verified: Docling now +fires on the cholera situation-report and PAHO mpox sitrep PDFs. + +## Code changes landed + +1. **`custom_scrapers/who_h5_hai.py`** (new) — resolves the WHO "human-animal + interface" monthly-risk-assessment hub to its latest assessment PDF, mirroring + `who_cholera`. q6 went 0 → records. **Caveat:** the HAI summary reports the + reporting-period's *events* (this issue: an H9N2 China cluster) and *links to* + the cumulative A(H5N1) human-case total on a **separate** page rather than + stating it — so the scraper is a correct precondition but does not, by itself, + deliver q6/q8's cumulative anchor. +2. **`pipeline.py` refiner-URL fix** (above). +3. **`custom_scrapers/cdc_measles.py`** (new) — the CDC measles page injects its + case/**death** figures client-side; the death table cells are empty in the + statically served HTML (only cases are in prose, which is why q14 worked and + q16 didn't). The data is published as plain JSON at + `/wcms/vizdata/measles/measles_hosp.json` (no browser/Akamai needed); the + scraper fetches it and renders `total_deaths` / `deaths_sentence` / `total_cases` + as clean prose. **q16: 0 records → `0 measles deaths in 2026` @ conf 0.85**, and + it corroborates q14's 2,170 cases. Live-only (returns `None` in replay to avoid + leakage). Tests: `test_cdc_measles_scraper.py`. + +## What Docling did (and didn't) change + +Re-running the PDF questions with Docling + the URL fix **did not change any +classification or anchor**: q17 stays `114,829 cumulative @ conf 0.5`; q13/q18/q22 +keep their high-confidence dashboard anchors. Conclusion: the Phase-1 verdicts are +**robust to the Docling gap** — the cholera/mpox/dengue/polio PDFs carry their key +numbers in prose or simple-enough tables that the base parser already handles. +Docling matters for genuinely hard tables, but it does **not** rescue the gaps below. + +## q16 (measles deaths) — FIXED via the CDC JSON feed + +Initially looked HTML-table-bound (Docling is PDF-only). But the CDC page's data is +served as plain JSON (`measles_hosp.json`) reachable without a browser — so the +`cdc_measles` scraper above resolves it cleanly. This is the APHIS pattern *without* +the Akamai wall. **q16 moved under_supported → high-confidence anchor (0 deaths).** + +## Gaps still NOT scraper/Docling-fixable + +- **q9 (APHIS states):** Tableau-behind-Akamai — **deferred** per decision; needs a + browser package (**issue #50**). +- **q11 / q24 / q25 (list enumeration):** "count PHEICs / DON items" — the pipeline + has no capability to enumerate entries on an authoritative list. Net-new. +- **q6 / q8 (H5):** cumulative total lives on a separate page **and** the Jul–Dec + window had barely opened at run time, so there is little in-window data to anchor. + +# Phase 3 — forecast-quality delta (sample) + +Ran 4 questions **with** the forecast (evidence `bioscancast` vs retrieval-free +`bioscancast_baseline`). The spec's north star — does evidence *change the forecast*? + +| Q | Evidence forecast vs baseline | Read | +|---|---|---| +| **q1** (well-supported, anchor 1,460) | evidence: **0.64** on "1,000–2,499" + 0.36 on "2,500+"; baseline diffuse on low bins | Retrieval **decisively** anchors the forecast on the true current value. ✔ | +| **q17** (anchor 114,829 @ conf 0.5) | evidence shifts mass up to "300k–449k" 0.35 / "450k–599k" 0.27; baseline 0.25/0.35 on the low bins | Low-confidence anchor **is used** — the record reaches the digest regardless of confidence. **q17 needs no insight code change for the forecast to move.** ✔ | +| **q18** (dengue, anchor ~63k–251k) | evidence concentrates on "1–1.9M" 0.43 / "2–3.9M" 0.31 | Anchors sensibly toward the Americas full-year range. ✔ | +| **q16** (remediated — CDC JSON scraper) | evidence: **0.99** on "0" deaths; baseline 0.70 | Post-fix: the new anchor (0 deaths 2026) sharpens the forecast onto the true value. ✔ | +| **q6** (under-supported, ~0 records) | evidence: **0.96** on "0–4"; baseline diffuse | Sparse evidence → **over-confident** forecast. The real risk of under-support. ✘ | + +**Takeaway:** the evidence→forecast path works well where evidence is sufficient +(q1/q17/q18 all diverge sensibly from baseline toward the anchor). The danger is not +that good anchors are ignored — it's that **thin evidence yields over-confident +forecasts** (q6). So the priority is the genuine extraction gaps, not the anchors +that already work. + +# Revised recommendations (prioritized) + +1. **~~HTML-table extraction (q16)~~ — DONE** via the `cdc_measles` JSON scraper. + The general lesson: for JS-rendered gov dashboards, look for the page's JSON + data feed (CDC exposes them openly) before concluding "not extractable." +2. **List-enumeration capability** (q11/q24/q25) — count entries on a WHO DON/PHEIC + list. — *new issue* +3. **q9 APHIS** — proceed on **#50** (curated snapshot or headless) when a browser + dep is acceptable. — *deferred by decision* +4. **q17 / insight confidence calibration (#26)** — *optional*: the forecast already + uses the anchor, so this is a nice-to-have (cleaner digests), not a blocker. +5. **Over-confidence guard for thin evidence** (q6-type) — a forecasting-stage + concern (out of this spec's scope) worth a separate note. +6. **Ship the two landed fixes** (HAI scraper, refiner-URL bug) — the URL fix is a + real correctness win for all hub-scraper PDFs regardless of this round. diff --git a/data/investigations/bfg_evidence_audit_2026-07-05.csv b/data/investigations/bfg_evidence_audit_2026-07-05.csv new file mode 100644 index 0000000..8d7acc6 --- /dev/null +++ b/data/investigations/bfg_evidence_audit_2026-07-05.csv @@ -0,0 +1,26 @@ +qid,run_id,qtype,topic,organic_returned,dashboard_returned,organic_survivors,dashboard_survivors,injected_dash_domains,expected_resolution_domain,resolution_source_injected,pool_overlap_mean,pool_overlap_max,survivor_overlap_mean,llm_pool_ontopic,llm_survivor_ontopic,n_records,n_usable,insight_from_organic,insight_from_dashboard,top_metric_value,top_metric_name,top_count_basis,expected_basis,top_confidence,top_n_sources,top_origin,top_source_url,top_quote,top_scope_ok,anchor_present,evidence_sufficient,single_fragile_source,classification,cause,cause_detail +bfg_q1,20260705_090841,range,bundibugyo ebola (drc-uganda),21,4,4,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.3048,0.4,0.3375,0.0,0.0,7,7,4,3,1460.0,confirmed_cases,cumulative,cumulative,0.85,1,organic,https://www.bbc.com/news/articles/c75ykve4zzxo,"there have been 1,460 confirmed cases in DR Congo",True,True,True,False,well_supported,, +bfg_q2,20260705_090928,range,bundibugyo ebola (drc-uganda),20,4,3,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.2818,0.3636,0.3131,0.0,0.0,3,3,1,2,438.0,deaths,cumulative,cumulative,0.85,1,organic,https://apnews.com/article/ebola-bundibugyo-remdesivir-mbp134-congo-7dd42ecd5ff75a4f1e255db26677a778,"438 have died, WHO Director-General Tedros Adhanom Ghebreyesus said Thursday.",True,True,True,False,well_supported,, +bfg_q3,20260705_091006,binary,bundibugyo ebola (drc-uganda),20,4,2,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.2812,0.4062,0.4062,0.2667,0.5,6,3,4,2,1460.0,confirmed_cases,cumulative,,0.85,1,organic,https://www.bbc.com/news/articles/c75ykve4zzxo,"there have been 1,460 confirmed cases in DR Congo",True,True,True,True,well_supported,robustness,high-confidence anchor present but rests on a single source; add a corroborating source. +bfg_q4,20260705_091046,binary,bundibugyo ebola (drc-uganda),21,4,4,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.3031,0.4146,0.3232,0.2,0.25,9,9,5,4,20.0,confirmed_cases,cumulative,,0.85,1,organic,https://www.bbc.com/news/articles/c75ykve4zzxo,There have also been 20 confirmed cases in Uganda,True,True,True,False,well_supported,, +bfg_q5,20260705_091143,categorical,bundibugyo ebola (drc-uganda),19,4,1,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.2313,0.3158,0.3158,0.5333,1.0,7,6,4,3,1460.0,confirmed_cases,cumulative,,0.85,1,organic,https://www.bbc.com/news/articles/c75ykve4zzxo,"there have been 1,460 confirmed cases in DR Congo",True,True,True,False,well_supported,, +bfg_q6,20260705_091217,range,avian influenza h5 (world),23,4,0,4,"aphis.usda.gov,cdc.gov,who.int",who.int,True,0.2386,0.4,0.0,0.1333,,0,0,0,0,,,,incident,,0,none,,,False,False,False,False,under_supported,extraction,4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q7,20260705_091239,binary,avian influenza h5 (world),27,4,0,4,"aphis.usda.gov,cdc.gov,who.int",who.int,True,0.2008,0.2889,0.0,0.1333,,1,1,0,1,71.0,confirmed_cases,cumulative,,0.85,1,dashboard,https://www.cdc.gov/bird-flu/situation-summary/,Of the 71 total reported human cases of A(H5) bird flu reported in the United States since February 2024,True,True,True,True,dashboard_only,robustness,high-confidence anchor present but rests on a single source; add a corroborating source. +bfg_q8,20260705_091305,categorical,avian influenza h5 (world),35,4,0,4,"aphis.usda.gov,cdc.gov,who.int",who.int,True,0.2276,0.4828,0.0,0.1333,,0,0,0,0,,,,,,0,none,,,False,False,False,False,under_supported,extraction,4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q9,20260705_091328,range,h5n1 (us),32,4,0,4,"aphis.usda.gov,cdc.gov,who.int",aphis.usda.gov,True,0.1861,0.3182,0.0,0.0,,0,0,0,0,,,,cumulative,,0,none,,,False,False,False,False,under_supported,extraction,4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q10,20260705_091405,categorical,pheic (world),45,15,3,15,"africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int",who.int,True,0.303,0.3939,0.3434,0.0667,0.3333,5,4,5,0,1274.0,confirmed_cases,cumulative,,0.85,1,organic,https://www.reuters.com/business/healthcare-pharmaceuticals/congo-says-number-confirmed-ebola-cases-1274-including-360-deaths-2026-06-29/,"confirmed Ebola cases in the country had reached 1,274",True,True,True,False,well_supported,, +bfg_q11,20260705_091506,categorical,pheic (world),44,15,1,15,"africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int",who.int,True,0.2527,0.4706,0.4118,0.0,0.0,0,0,0,0,,,,,,0,none,,,False,False,False,False,under_supported,extraction,15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q12,20260705_091543,binary,mpox (world),45,7,0,7,"cdc.gov,ecdc.europa.eu,mpox-monthly.ecdc.europa.eu,ourworldindata.org,paho.org,who.int,worldhealthorg.shinyapps.io",who.int,True,0.2418,0.3824,0.0,0.0,,22,20,0,22,180545.0,confirmed_cases,cumulative,,0.85,1,dashboard,https://ourworldindata.org/mpox,"cumulative confirmed cases (World): 180,545",True,True,True,False,dashboard_only,search-recall,"7 organic looked on-topic by keyword overlap (e.g. nypost.com) but the LLM judge rates the pool 0% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q13,20260705_091725,range,mpox (world),44,7,0,7,"cdc.gov,ecdc.europa.eu,mpox-monthly.ecdc.europa.eu,ourworldindata.org,paho.org,who.int,worldhealthorg.shinyapps.io",who.int,True,0.2279,0.3514,0.0,0.0,,10,5,0,10,964.0,confirmed_cases,incident,incident,0.85,1,dashboard,https://www.paho.org/en/situation-reports?topic=87192&d%5Bmin%5D=&d%5Bmax%5D=,ten countries have reported a combined 964 mpox cases and one death,True,True,True,False,dashboard_only,search-recall,"4 organic looked on-topic by keyword overlap (e.g. statnews.com) but the LLM judge rates the pool 0% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q14,20260705_091832,range,measles (us),21,2,2,2,"cdc.gov,paho.org",cdc.gov,True,0.3129,0.5238,0.5238,0.2,1.0,3,2,1,2,2170.0,confirmed_cases,cumulative,cumulative,0.85,1,dashboard,https://www.cdc.gov/measles/data-research/index.html,"2,170 confirmed* measles cases were reported in the United States in 2026.",True,True,True,True,dashboard_only,robustness,high-confidence anchor present but rests on a single source; add a corroborating source. +bfg_q15,20260705_091859,binary,measles (us),21,2,0,2,"cdc.gov,paho.org",paho.org,True,0.2063,0.381,0.0,0.5333,,10,4,0,10,285.0,confirmed_cases,cumulative,,0.85,1,dashboard,https://www.cdc.gov/measles/data-research/index.html,a total of 285 measles cases were reported in the U.S.,True,True,True,True,dashboard_only,filter-recall,"4 on-topic organic in pool (max_overlap=0.38, e.g. nypost.com) but 0 survived the filter. LLM judge: pool 53% on-topic." +bfg_q16,20260705_091921,range,measles (us),20,2,0,2,"cdc.gov,paho.org",cdc.gov,True,0.3028,0.5,0.0,0.0667,,0,0,0,0,,,,cumulative,,0,none,,,False,False,False,False,under_supported,extraction,2 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q17,20260705_091942,range,cholera (world),41,1,0,1,who.int,who.int,True,0.2514,0.3846,0.0,0.0,,14,3,0,14,114829.0,confirmed_cases,cumulative,cumulative,0.5,1,dashboard,https://www.who.int/emergencies/situations/cholera-upsurge,a total of 114 829 cholera and AWD cases and 1318 deaths were reported from 23 countries across four WHO regions,True,True,False,True,dashboard_only,insight,"scope-matched anchor present (value=114829.0, basis=cumulative vs expected cumulative, conf=0.5) but below the high-confidence/expected-basis bar; off-scope rows outrank it." +bfg_q18,20260705_092005,range,dengue (americas),34,1,0,1,paho.org,paho.org,True,0.2579,0.3846,0.0,0.0,,14,4,0,14,251057.0,suspected_cases,cumulative,cumulative,0.85,1,dashboard,https://www.paho.org/en/epidemiological-alerts-and-updates,"251,057 suspected dengue cases were reported",True,True,True,True,dashboard_only,search-recall,"13 organic looked on-topic by keyword overlap (e.g. abcnews.com) but the LLM judge rates the pool 0% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q19,20260705_092037,range,chikungunya (eu-eea),29,1,0,1,ecdc.europa.eu,ecdc.europa.eu,True,0.1708,0.2857,0.0,0.0,,1,0,0,1,1.0,confirmed_cases,incident,cumulative,0.5,1,dashboard,https://www.ecdc.europa.eu/en/chikungunya-monthly,"In 2026, Saint Lucia reported a locally acquired CHIKVD case.",False,False,False,False,under_supported,search-recall,organic pool weak (max_overlap=0.29); dashboards carry the question. +bfg_q20,20260705_092052,range,oropouche (americas),38,1,2,1,paho.org,paho.org,True,0.1967,0.3684,0.0526,0.1333,0.0,1,0,0,1,1.0,new_outbreaks_declared,unknown,cumulative,0.7,1,dashboard,https://www.paho.org/en/topics/oropouche-virus-disease,"Desde finales de 2023, se han reportado brotes de la enfermedad por el virus Oropouche en varios países de América del Sur y el Caribe",False,False,False,False,under_supported,search-recall,"7 organic looked on-topic by keyword overlap (e.g. abcnews.com) but the LLM judge rates the pool 13% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q21,20260705_092119,binary,filovirus (world),36,4,2,4,"afro.who.int,cdc.gov,who.int",who.int,True,0.2438,0.4444,0.3472,0.4667,1.0,7,7,3,4,1.0,confirmed_cases,incident,,0.85,1,organic,https://www.reuters.com/business/healthcare-pharmaceuticals/africa-cdc-uganda-health-ministry-confirms-isolated-marburg-case-2026-07-01/,Ugandan health authorities have confirmed an isolated case of Marburg virus disease,True,True,True,False,well_supported,, +bfg_q22,20260705_092157,range,poliovirus (world),41,1,0,1,polioeradication.org,polioeradication.org,True,0.2343,0.3571,0.0,0.0,,2,2,0,2,7.0,confirmed_cases,cumulative,cumulative,0.85,1,dashboard,https://polioeradication.org/about-polio/polio-this-week/,The total number of cases in 2026 is seven.,True,True,True,True,dashboard_only,search-recall,"6 organic looked on-topic by keyword overlap (e.g. nypost.com) but the LLM judge rates the pool 0% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q23,20260705_092215,binary,sars-cov-2 (world),42,3,1,3,"cdc.gov,ourworldindata.org,who.int",who.int,True,0.2519,0.3871,0.2258,0.0,0.0,2,0,0,2,775866783.0,confirmed_cases,cumulative,,0.5,1,dashboard,https://ourworldindata.org/coronavirus,"cumulative confirmed cases (World): 775,866,783",True,False,False,False,under_supported,search-recall,"6 organic looked on-topic by keyword overlap (e.g. motor1.com) but the LLM judge rates the pool 0% on-topic — generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." +bfg_q24,20260705_092256,binary,novel pathogen (world),42,15,0,15,"africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int",who.int,True,0.2546,0.359,0.0,0.3333,,0,0,0,0,,,,,,0,none,,,False,False,False,False,under_supported,extraction,15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). +bfg_q25,20260705_092336,range,who don (world),21,15,3,15,"africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int",who.int,True,0.2711,0.3846,0.3205,0.0,0.0,1,0,1,0,1274.0,confirmed_cases,cumulative,incident,0.85,1,organic,https://www.reuters.com/business/healthcare-pharmaceuticals/congo-says-number-confirmed-ebola-cases-1274-including-360-deaths-2026-06-29/,"confirmed Ebola cases in the country had reached 1,274",False,False,False,False,under_supported,extraction,15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page). diff --git a/data/investigations/bfg_evidence_audit_2026-07-05.json b/data/investigations/bfg_evidence_audit_2026-07-05.json new file mode 100644 index 0000000..4bc036e --- /dev/null +++ b/data/investigations/bfg_evidence_audit_2026-07-05.json @@ -0,0 +1,952 @@ +[ + { + "qid": "bfg_q1", + "run_id": "20260705_090841", + "qtype": "range", + "topic": "bundibugyo ebola (drc-uganda)", + "organic_returned": 21, + "dashboard_returned": 4, + "organic_survivors": 4, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.3048, + "pool_overlap_max": 0.4, + "survivor_overlap_mean": 0.3375, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": 0.0, + "n_records": 7, + "n_usable": 7, + "insight_from_organic": 4, + "insight_from_dashboard": 3, + "top_metric_value": 1460.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.bbc.com/news/articles/c75ykve4zzxo", + "top_quote": "there have been 1,460 confirmed cases in DR Congo", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q2", + "run_id": "20260705_090928", + "qtype": "range", + "topic": "bundibugyo ebola (drc-uganda)", + "organic_returned": 20, + "dashboard_returned": 4, + "organic_survivors": 3, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2818, + "pool_overlap_max": 0.3636, + "survivor_overlap_mean": 0.3131, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": 0.0, + "n_records": 3, + "n_usable": 3, + "insight_from_organic": 1, + "insight_from_dashboard": 2, + "top_metric_value": 438.0, + "top_metric_name": "deaths", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://apnews.com/article/ebola-bundibugyo-remdesivir-mbp134-congo-7dd42ecd5ff75a4f1e255db26677a778", + "top_quote": "438 have died, WHO Director-General Tedros Adhanom Ghebreyesus said Thursday.", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q3", + "run_id": "20260705_091006", + "qtype": "binary", + "topic": "bundibugyo ebola (drc-uganda)", + "organic_returned": 20, + "dashboard_returned": 4, + "organic_survivors": 2, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2812, + "pool_overlap_max": 0.4062, + "survivor_overlap_mean": 0.4062, + "llm_pool_ontopic": 0.2667, + "llm_survivor_ontopic": 0.5, + "n_records": 6, + "n_usable": 3, + "insight_from_organic": 4, + "insight_from_dashboard": 2, + "top_metric_value": 1460.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.bbc.com/news/articles/c75ykve4zzxo", + "top_quote": "there have been 1,460 confirmed cases in DR Congo", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "well_supported", + "cause": "robustness", + "cause_detail": "high-confidence anchor present but rests on a single source; add a corroborating source." + }, + { + "qid": "bfg_q4", + "run_id": "20260705_091046", + "qtype": "binary", + "topic": "bundibugyo ebola (drc-uganda)", + "organic_returned": 21, + "dashboard_returned": 4, + "organic_survivors": 4, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.3031, + "pool_overlap_max": 0.4146, + "survivor_overlap_mean": 0.3232, + "llm_pool_ontopic": 0.2, + "llm_survivor_ontopic": 0.25, + "n_records": 9, + "n_usable": 9, + "insight_from_organic": 5, + "insight_from_dashboard": 4, + "top_metric_value": 20.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.bbc.com/news/articles/c75ykve4zzxo", + "top_quote": "There have also been 20 confirmed cases in Uganda", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q5", + "run_id": "20260705_091143", + "qtype": "categorical", + "topic": "bundibugyo ebola (drc-uganda)", + "organic_returned": 19, + "dashboard_returned": 4, + "organic_survivors": 1, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2313, + "pool_overlap_max": 0.3158, + "survivor_overlap_mean": 0.3158, + "llm_pool_ontopic": 0.5333, + "llm_survivor_ontopic": 1.0, + "n_records": 7, + "n_usable": 6, + "insight_from_organic": 4, + "insight_from_dashboard": 3, + "top_metric_value": 1460.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.bbc.com/news/articles/c75ykve4zzxo", + "top_quote": "there have been 1,460 confirmed cases in DR Congo", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q6", + "run_id": "20260705_091217", + "qtype": "range", + "topic": "avian influenza h5 (world)", + "organic_returned": 23, + "dashboard_returned": 4, + "organic_survivors": 0, + "dashboard_survivors": 4, + "injected_dash_domains": "aphis.usda.gov,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2386, + "pool_overlap_max": 0.4, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.1333, + "llm_survivor_ontopic": null, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": "incident", + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q7", + "run_id": "20260705_091239", + "qtype": "binary", + "topic": "avian influenza h5 (world)", + "organic_returned": 27, + "dashboard_returned": 4, + "organic_survivors": 0, + "dashboard_survivors": 4, + "injected_dash_domains": "aphis.usda.gov,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2008, + "pool_overlap_max": 0.2889, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.1333, + "llm_survivor_ontopic": null, + "n_records": 1, + "n_usable": 1, + "insight_from_organic": 0, + "insight_from_dashboard": 1, + "top_metric_value": 71.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.cdc.gov/bird-flu/situation-summary/", + "top_quote": "Of the 71 total reported human cases of A(H5) bird flu reported in the United States since February 2024", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "robustness", + "cause_detail": "high-confidence anchor present but rests on a single source; add a corroborating source." + }, + { + "qid": "bfg_q8", + "run_id": "20260705_091305", + "qtype": "categorical", + "topic": "avian influenza h5 (world)", + "organic_returned": 35, + "dashboard_returned": 4, + "organic_survivors": 0, + "dashboard_survivors": 4, + "injected_dash_domains": "aphis.usda.gov,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2276, + "pool_overlap_max": 0.4828, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.1333, + "llm_survivor_ontopic": null, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": null, + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q9", + "run_id": "20260705_091328", + "qtype": "range", + "topic": "h5n1 (us)", + "organic_returned": 32, + "dashboard_returned": 4, + "organic_survivors": 0, + "dashboard_survivors": 4, + "injected_dash_domains": "aphis.usda.gov,cdc.gov,who.int", + "expected_resolution_domain": "aphis.usda.gov", + "resolution_source_injected": true, + "pool_overlap_mean": 0.1861, + "pool_overlap_max": 0.3182, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": "cumulative", + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "4 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q10", + "run_id": "20260705_091405", + "qtype": "categorical", + "topic": "pheic (world)", + "organic_returned": 45, + "dashboard_returned": 15, + "organic_survivors": 3, + "dashboard_survivors": 15, + "injected_dash_domains": "africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.303, + "pool_overlap_max": 0.3939, + "survivor_overlap_mean": 0.3434, + "llm_pool_ontopic": 0.0667, + "llm_survivor_ontopic": 0.3333, + "n_records": 5, + "n_usable": 4, + "insight_from_organic": 5, + "insight_from_dashboard": 0, + "top_metric_value": 1274.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.reuters.com/business/healthcare-pharmaceuticals/congo-says-number-confirmed-ebola-cases-1274-including-360-deaths-2026-06-29/", + "top_quote": "confirmed Ebola cases in the country had reached 1,274", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q11", + "run_id": "20260705_091506", + "qtype": "categorical", + "topic": "pheic (world)", + "organic_returned": 44, + "dashboard_returned": 15, + "organic_survivors": 1, + "dashboard_survivors": 15, + "injected_dash_domains": "africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2527, + "pool_overlap_max": 0.4706, + "survivor_overlap_mean": 0.4118, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": 0.0, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": null, + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q12", + "run_id": "20260705_091543", + "qtype": "binary", + "topic": "mpox (world)", + "organic_returned": 45, + "dashboard_returned": 7, + "organic_survivors": 0, + "dashboard_survivors": 7, + "injected_dash_domains": "cdc.gov,ecdc.europa.eu,mpox-monthly.ecdc.europa.eu,ourworldindata.org,paho.org,who.int,worldhealthorg.shinyapps.io", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2418, + "pool_overlap_max": 0.3824, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 22, + "n_usable": 20, + "insight_from_organic": 0, + "insight_from_dashboard": 22, + "top_metric_value": 180545.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://ourworldindata.org/mpox", + "top_quote": "cumulative confirmed cases (World): 180,545", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "dashboard_only", + "cause": "search-recall", + "cause_detail": "7 organic looked on-topic by keyword overlap (e.g. nypost.com) but the LLM judge rates the pool 0% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q13", + "run_id": "20260705_091725", + "qtype": "range", + "topic": "mpox (world)", + "organic_returned": 44, + "dashboard_returned": 7, + "organic_survivors": 0, + "dashboard_survivors": 7, + "injected_dash_domains": "cdc.gov,ecdc.europa.eu,mpox-monthly.ecdc.europa.eu,ourworldindata.org,paho.org,who.int,worldhealthorg.shinyapps.io", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2279, + "pool_overlap_max": 0.3514, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 10, + "n_usable": 5, + "insight_from_organic": 0, + "insight_from_dashboard": 10, + "top_metric_value": 964.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "incident", + "expected_basis": "incident", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.paho.org/en/situation-reports?topic=87192&d%5Bmin%5D=&d%5Bmax%5D=", + "top_quote": "ten countries have reported a combined 964 mpox cases and one death", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "dashboard_only", + "cause": "search-recall", + "cause_detail": "4 organic looked on-topic by keyword overlap (e.g. statnews.com) but the LLM judge rates the pool 0% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q14", + "run_id": "20260705_091832", + "qtype": "range", + "topic": "measles (us)", + "organic_returned": 21, + "dashboard_returned": 2, + "organic_survivors": 2, + "dashboard_survivors": 2, + "injected_dash_domains": "cdc.gov,paho.org", + "expected_resolution_domain": "cdc.gov", + "resolution_source_injected": true, + "pool_overlap_mean": 0.3129, + "pool_overlap_max": 0.5238, + "survivor_overlap_mean": 0.5238, + "llm_pool_ontopic": 0.2, + "llm_survivor_ontopic": 1.0, + "n_records": 3, + "n_usable": 2, + "insight_from_organic": 1, + "insight_from_dashboard": 2, + "top_metric_value": 2170.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.cdc.gov/measles/data-research/index.html", + "top_quote": "2,170 confirmed* measles cases were reported in the United States in 2026.", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "robustness", + "cause_detail": "high-confidence anchor present but rests on a single source; add a corroborating source." + }, + { + "qid": "bfg_q15", + "run_id": "20260705_091859", + "qtype": "binary", + "topic": "measles (us)", + "organic_returned": 21, + "dashboard_returned": 2, + "organic_survivors": 0, + "dashboard_survivors": 2, + "injected_dash_domains": "cdc.gov,paho.org", + "expected_resolution_domain": "paho.org", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2063, + "pool_overlap_max": 0.381, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.5333, + "llm_survivor_ontopic": null, + "n_records": 10, + "n_usable": 4, + "insight_from_organic": 0, + "insight_from_dashboard": 10, + "top_metric_value": 285.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.cdc.gov/measles/data-research/index.html", + "top_quote": "a total of 285 measles cases were reported in the U.S.", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "filter-recall", + "cause_detail": "4 on-topic organic in pool (max_overlap=0.38, e.g. nypost.com) but 0 survived the filter. LLM judge: pool 53% on-topic." + }, + { + "qid": "bfg_q16", + "run_id": "20260705_091921", + "qtype": "range", + "topic": "measles (us)", + "organic_returned": 20, + "dashboard_returned": 2, + "organic_survivors": 0, + "dashboard_survivors": 2, + "injected_dash_domains": "cdc.gov,paho.org", + "expected_resolution_domain": "cdc.gov", + "resolution_source_injected": true, + "pool_overlap_mean": 0.3028, + "pool_overlap_max": 0.5, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0667, + "llm_survivor_ontopic": null, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": "cumulative", + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "2 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q17", + "run_id": "20260705_091942", + "qtype": "range", + "topic": "cholera (world)", + "organic_returned": 41, + "dashboard_returned": 1, + "organic_survivors": 0, + "dashboard_survivors": 1, + "injected_dash_domains": "who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2514, + "pool_overlap_max": 0.3846, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 14, + "n_usable": 3, + "insight_from_organic": 0, + "insight_from_dashboard": 14, + "top_metric_value": 114829.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.5, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.who.int/emergencies/situations/cholera-upsurge", + "top_quote": "a total of 114 829 cholera and AWD cases and 1318 deaths were reported from 23 countries across four WHO regions", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": false, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "insight", + "cause_detail": "scope-matched anchor present (value=114829.0, basis=cumulative vs expected cumulative, conf=0.5) but below the high-confidence/expected-basis bar; off-scope rows outrank it." + }, + { + "qid": "bfg_q18", + "run_id": "20260705_092005", + "qtype": "range", + "topic": "dengue (americas)", + "organic_returned": 34, + "dashboard_returned": 1, + "organic_survivors": 0, + "dashboard_survivors": 1, + "injected_dash_domains": "paho.org", + "expected_resolution_domain": "paho.org", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2579, + "pool_overlap_max": 0.3846, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 14, + "n_usable": 4, + "insight_from_organic": 0, + "insight_from_dashboard": 14, + "top_metric_value": 251057.0, + "top_metric_name": "suspected_cases", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.paho.org/en/epidemiological-alerts-and-updates", + "top_quote": "251,057 suspected dengue cases were reported", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "search-recall", + "cause_detail": "13 organic looked on-topic by keyword overlap (e.g. abcnews.com) but the LLM judge rates the pool 0% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q19", + "run_id": "20260705_092037", + "qtype": "range", + "topic": "chikungunya (eu-eea)", + "organic_returned": 29, + "dashboard_returned": 1, + "organic_survivors": 0, + "dashboard_survivors": 1, + "injected_dash_domains": "ecdc.europa.eu", + "expected_resolution_domain": "ecdc.europa.eu", + "resolution_source_injected": true, + "pool_overlap_mean": 0.1708, + "pool_overlap_max": 0.2857, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 1, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 1, + "top_metric_value": 1.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "incident", + "expected_basis": "cumulative", + "top_confidence": 0.5, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.ecdc.europa.eu/en/chikungunya-monthly", + "top_quote": "In 2026, Saint Lucia reported a locally acquired CHIKVD case.", + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "search-recall", + "cause_detail": "organic pool weak (max_overlap=0.29); dashboards carry the question." + }, + { + "qid": "bfg_q20", + "run_id": "20260705_092052", + "qtype": "range", + "topic": "oropouche (americas)", + "organic_returned": 38, + "dashboard_returned": 1, + "organic_survivors": 2, + "dashboard_survivors": 1, + "injected_dash_domains": "paho.org", + "expected_resolution_domain": "paho.org", + "resolution_source_injected": true, + "pool_overlap_mean": 0.1967, + "pool_overlap_max": 0.3684, + "survivor_overlap_mean": 0.0526, + "llm_pool_ontopic": 0.1333, + "llm_survivor_ontopic": 0.0, + "n_records": 1, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 1, + "top_metric_value": 1.0, + "top_metric_name": "new_outbreaks_declared", + "top_count_basis": "unknown", + "expected_basis": "cumulative", + "top_confidence": 0.7, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://www.paho.org/en/topics/oropouche-virus-disease", + "top_quote": "Desde finales de 2023, se han reportado brotes de la enfermedad por el virus Oropouche en varios pa\u00edses de Am\u00e9rica del Sur y el Caribe", + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "search-recall", + "cause_detail": "7 organic looked on-topic by keyword overlap (e.g. abcnews.com) but the LLM judge rates the pool 13% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q21", + "run_id": "20260705_092119", + "qtype": "binary", + "topic": "filovirus (world)", + "organic_returned": 36, + "dashboard_returned": 4, + "organic_survivors": 2, + "dashboard_survivors": 4, + "injected_dash_domains": "afro.who.int,cdc.gov,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2438, + "pool_overlap_max": 0.4444, + "survivor_overlap_mean": 0.3472, + "llm_pool_ontopic": 0.4667, + "llm_survivor_ontopic": 1.0, + "n_records": 7, + "n_usable": 7, + "insight_from_organic": 3, + "insight_from_dashboard": 4, + "top_metric_value": 1.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "incident", + "expected_basis": null, + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.reuters.com/business/healthcare-pharmaceuticals/africa-cdc-uganda-health-ministry-confirms-isolated-marburg-case-2026-07-01/", + "top_quote": "Ugandan health authorities have confirmed an isolated case of Marburg virus disease", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": false, + "classification": "well_supported", + "cause": null, + "cause_detail": "" + }, + { + "qid": "bfg_q22", + "run_id": "20260705_092157", + "qtype": "range", + "topic": "poliovirus (world)", + "organic_returned": 41, + "dashboard_returned": 1, + "organic_survivors": 0, + "dashboard_survivors": 1, + "injected_dash_domains": "polioeradication.org", + "expected_resolution_domain": "polioeradication.org", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2343, + "pool_overlap_max": 0.3571, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": null, + "n_records": 2, + "n_usable": 2, + "insight_from_organic": 0, + "insight_from_dashboard": 2, + "top_metric_value": 7.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": "cumulative", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://polioeradication.org/about-polio/polio-this-week/", + "top_quote": "The total number of cases in 2026 is seven.", + "top_scope_ok": true, + "anchor_present": true, + "evidence_sufficient": true, + "single_fragile_source": true, + "classification": "dashboard_only", + "cause": "search-recall", + "cause_detail": "6 organic looked on-topic by keyword overlap (e.g. nypost.com) but the LLM judge rates the pool 0% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q23", + "run_id": "20260705_092215", + "qtype": "binary", + "topic": "sars-cov-2 (world)", + "organic_returned": 42, + "dashboard_returned": 3, + "organic_survivors": 1, + "dashboard_survivors": 3, + "injected_dash_domains": "cdc.gov,ourworldindata.org,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2519, + "pool_overlap_max": 0.3871, + "survivor_overlap_mean": 0.2258, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": 0.0, + "n_records": 2, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 2, + "top_metric_value": 775866783.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": null, + "top_confidence": 0.5, + "top_n_sources": 1, + "top_origin": "dashboard", + "top_source_url": "https://ourworldindata.org/coronavirus", + "top_quote": "cumulative confirmed cases (World): 775,866,783", + "top_scope_ok": true, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "search-recall", + "cause_detail": "6 organic looked on-topic by keyword overlap (e.g. motor1.com) but the LLM judge rates the pool 0% on-topic \u2014 generic news, not authoritative coverage. Organic-authority gap; dashboard rightly carries it." + }, + { + "qid": "bfg_q24", + "run_id": "20260705_092256", + "qtype": "binary", + "topic": "novel pathogen (world)", + "organic_returned": 42, + "dashboard_returned": 15, + "organic_survivors": 0, + "dashboard_survivors": 15, + "injected_dash_domains": "africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2546, + "pool_overlap_max": 0.359, + "survivor_overlap_mean": 0.0, + "llm_pool_ontopic": 0.3333, + "llm_survivor_ontopic": null, + "n_records": 0, + "n_usable": 0, + "insight_from_organic": 0, + "insight_from_dashboard": 0, + "top_metric_value": null, + "top_metric_name": null, + "top_count_basis": null, + "expected_basis": null, + "top_confidence": null, + "top_n_sources": 0, + "top_origin": "none", + "top_source_url": null, + "top_quote": null, + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + }, + { + "qid": "bfg_q25", + "run_id": "20260705_092336", + "qtype": "range", + "topic": "who don (world)", + "organic_returned": 21, + "dashboard_returned": 15, + "organic_survivors": 3, + "dashboard_survivors": 15, + "injected_dash_domains": "africacdc.org,cdc.gov,ecdc.europa.eu,empres-i.apps.fao.org,fao.org,gov.uk,healthmap.org,nextstrain.org,paho.org,promedmail.org,wahis.woah.org,who.int", + "expected_resolution_domain": "who.int", + "resolution_source_injected": true, + "pool_overlap_mean": 0.2711, + "pool_overlap_max": 0.3846, + "survivor_overlap_mean": 0.3205, + "llm_pool_ontopic": 0.0, + "llm_survivor_ontopic": 0.0, + "n_records": 1, + "n_usable": 0, + "insight_from_organic": 1, + "insight_from_dashboard": 0, + "top_metric_value": 1274.0, + "top_metric_name": "confirmed_cases", + "top_count_basis": "cumulative", + "expected_basis": "incident", + "top_confidence": 0.85, + "top_n_sources": 1, + "top_origin": "organic", + "top_source_url": "https://www.reuters.com/business/healthcare-pharmaceuticals/congo-says-number-confirmed-ebola-cases-1274-including-360-deaths-2026-06-29/", + "top_quote": "confirmed Ebola cases in the country had reached 1,274", + "top_scope_ok": false, + "anchor_present": false, + "evidence_sufficient": false, + "single_fragile_source": false, + "classification": "under_supported", + "cause": "extraction", + "cause_detail": "15 dashboard(s) survived filtering but produced 0 insight records (non-extractable tracker/index/JS page)." + } +] \ No newline at end of file diff --git a/scripts/_bfg_sweep.sh b/scripts/_bfg_sweep.sh new file mode 100644 index 0000000..1d0600a --- /dev/null +++ b/scripts/_bfg_sweep.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Phase-1 evidence-only sweep across all 25 BFG summer-2026 questions. +# Sequential (shared 24h SQLite search cache dislikes concurrent writers). +# Stamps the run date so the live-drift window is auditable. +set -u +cd "$(dirname "$0")/.." + +QCSV="bioscancast/stages/evaluation/bfg_summer_2026_questions.csv" +OCSV="bioscancast/stages/evaluation/bfg_summer_2026_options.csv" +LOG="${1:-sweep.log}" +STAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +echo "=== BFG evidence-only sweep started ${STAMP} ===" | tee "$LOG" +for i in $(seq 1 25); do + qid="bfg_q${i}" + echo "--- ${qid} $(date -u +%H:%M:%S) ---" | tee -a "$LOG" + python -m bioscancast.main "$qid" --csv "$QCSV" --forecasts-csv "$OCSV" \ + --no-forecast >>"$LOG" 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo " ${qid} OK" | tee -a "$LOG" + else + echo " ${qid} FAILED rc=${rc}" | tee -a "$LOG" + fi +done +echo "=== sweep complete $(date -u +%Y-%m-%dT%H:%M:%SZ) ===" | tee -a "$LOG" diff --git a/scripts/analyze_evidence_coverage.py b/scripts/analyze_evidence_coverage.py new file mode 100644 index 0000000..9b5fa08 --- /dev/null +++ b/scripts/analyze_evidence_coverage.py @@ -0,0 +1,871 @@ +"""Evidence-coverage analyzer for the BFG summer-2026 forecast-quality pass. + +Reads the per-stage artifacts an evidence-only pipeline run writes to +``data/runs/{qid}/{run_id}/`` (search.json, filtered.json, documents.json, +insight.json, manifest.json, optionally forecast.json) and produces, per +question, the diagnosis the spec asks for: + + * evidence sufficiency — is there a usable current-value record, its + count_basis, confidence, source URL + quote (surfaced for spot-check); + * source attribution — organic vs dashboard behind each insight record, + and the number of independent sources behind the top record; + * quantity — organic vs dashboard results returned / surviving; + * quality — deterministic keyword_overlap of the organic pool + vs its survivors, plus (optional) a capped gpt-4o-mini on-topic judge; + * classification+cause — well_supported / dashboard_only / under_supported, + and for weak questions an attributed cause (search-recall / filter-recall / + extraction). + +Provenance chain used for organic-vs-dashboard attribution: + + InsightRecord.sources[].document_id + -> Document.id (documents.json) + -> Document.result_id (FK to FilteredDocument.result_id) + -> SearchResult.id (== result_id; search.json) + -> retrieval_reason == "dashboard_lookup" => dashboard, else organic + +A FilteredDocument is also flagged dashboard when its selection_reasons +contains "dashboard_lookup_bypass". + +Deterministic-only by default (free). Pass --llm-judge to add the on-topic +judge (costs a few gpt-4o-mini calls per question). + +Usage: + python scripts/analyze_evidence_coverage.py --all \ + --out data/investigations/bfg_evidence_audit + python scripts/analyze_evidence_coverage.py bfg_q17 # one question + python scripts/analyze_evidence_coverage.py --run-dir data/runs/bfg_q17/2026... +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +# Reuse the pipeline's own relevance signal so pool/survivor "quality" matches +# what the search and filter stages actually score on. +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from bioscancast.stages.filtering.utils import keyword_overlap_score, tokenize # noqa: E402 + + +DASHBOARD_REASON = "dashboard_lookup" +DASHBOARD_BYPASS = "dashboard_lookup_bypass" + +# Confidence at/above which an extracted record is treated as "high +# confidence" for the evidence-sufficiency bar. +HIGH_CONFIDENCE = 0.6 + +# Below this max keyword-overlap in the organic *pool*, we treat the pool as +# essentially devoid of on-topic organic content (a search-recall failure). +SEARCH_RECALL_POOL_OVERLAP = 0.18 + +# An organic pool result at/above this overlap is "clearly on-topic"; if such a +# result is in the pool but not among survivors, that points at filter-recall. +ON_TOPIC_OVERLAP = 0.30 + +# Question types that resolve to a numeric count/range and therefore need a +# metric-bearing anchor record for evidence sufficiency. +NUMERIC_TYPES = {"range"} + +# Low floor below which a metric record is treated as too weak to be an anchor +# candidate at all. We deliberately keep this well under HIGH_CONFIDENCE so a +# *present-but-under-confident* correct anchor (e.g. q17's Global cumulative +# extracted at 0.5 while off-scope regional rows got 0.85) is still detected — +# the gap between "anchor present" and "anchor present AND high-confidence" is +# itself a finding (an extraction/confidence-calibration cause, not search/filter). +ANCHOR_FLOOR = 0.30 + +# Geography synonym sets for scope-matching a record's ``location`` against the +# geography a range question actually asks about. Substring match on the +# lowercased location, plus token overlap. +GEO_SYNONYMS: dict[str, set[str]] = { + "global": {"global", "world", "worldwide", "globally", "multi-country", + "multicountry", "international", "total", "overall"}, + "us": {"united states", "u.s.", "usa", "us"}, + "americas": {"americas", "region of the americas", "latin america", + "caribbean", "paho", "south america", "central america"}, + "eu_eea": {"eu", "eea", "european union", "europe", "european"}, + "drc_uganda": {"democratic republic", "drc", "congo", "uganda", "combined", + "total"}, +} + +# Authoritative resolution-source domain per question (from the CSV +# resolution_criteria). Used to measure route/coverage: did the pipeline +# actually inject a dashboard on the domain that resolves the question? +# Feeds issue #41 (route_sources single-bucket misroute rate). +EXPECT_RESOLUTION_DOMAIN: dict[str, str] = { + "bfg_q1": "who.int", "bfg_q2": "who.int", "bfg_q3": "who.int", + "bfg_q4": "who.int", "bfg_q5": "who.int", "bfg_q6": "who.int", + "bfg_q7": "who.int", "bfg_q8": "who.int", "bfg_q9": "aphis.usda.gov", + "bfg_q10": "who.int", "bfg_q11": "who.int", "bfg_q12": "who.int", + "bfg_q13": "who.int", "bfg_q14": "cdc.gov", "bfg_q15": "paho.org", + "bfg_q16": "cdc.gov", "bfg_q17": "who.int", "bfg_q18": "paho.org", + "bfg_q19": "ecdc.europa.eu", "bfg_q20": "paho.org", "bfg_q21": "who.int", + "bfg_q22": "polioeradication.org", "bfg_q23": "who.int", + "bfg_q24": "who.int", "bfg_q25": "who.int", +} + +# Expected geography per range question (which scope the anchor must carry). +GEO_EXPECT: dict[str, str] = { + "bfg_q1": "drc_uganda", "bfg_q2": "drc_uganda", "bfg_q6": "global", + "bfg_q9": "us", "bfg_q13": "global", "bfg_q14": "us", "bfg_q16": "us", + "bfg_q17": "global", "bfg_q18": "americas", "bfg_q19": "eu_eea", + "bfg_q20": "americas", "bfg_q22": "global", "bfg_q25": "global", +} + + +def _scope_match(location: Optional[str], geo_key: Optional[str]) -> bool: + """Does this record's location match the question's expected geography?""" + if not geo_key: + return True # no expectation encoded -> don't penalise + loc = (location or "").strip().lower() + if not loc: + return False + syns = GEO_SYNONYMS.get(geo_key, set()) + if any(s in loc for s in syns): + return True + loc_tokens = set(tokenize(loc)) + return bool(loc_tokens & {t for s in syns for t in s.split()}) + +# Per-question hint for the count_basis we'd expect the anchor to carry. +# "cumulative" = running total to date; "incident" = new within a stated +# window. Advisory only — surfaced next to the record's actual basis so a +# mismatch is visible; never used to hard-fail a record. +EXPECTED_BASIS: dict[str, str] = { + "bfg_q1": "cumulative", "bfg_q2": "cumulative", "bfg_q6": "incident", + "bfg_q9": "cumulative", "bfg_q13": "incident", "bfg_q14": "cumulative", + "bfg_q16": "cumulative", "bfg_q17": "cumulative", "bfg_q18": "cumulative", + "bfg_q19": "cumulative", "bfg_q20": "cumulative", "bfg_q22": "cumulative", + "bfg_q25": "incident", +} + + +# -------------------------------------------------------------------------- +# artifact loading +# -------------------------------------------------------------------------- + + +def _load_json(path: Path) -> Any: + if not path.exists(): + return None + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +@dataclass +class RunArtifacts: + qid: str + run_dir: Path + question: dict + search: list[dict] + filtered: list[dict] + documents: list[dict] + insight: dict + manifest: dict + forecast: Optional[dict] + + @property + def records(self) -> list[dict]: + return (self.insight or {}).get("records", []) or [] + + +def load_run(run_dir: Path) -> RunArtifacts: + return RunArtifacts( + qid=run_dir.parent.name, + run_dir=run_dir, + question=_load_json(run_dir / "question.json") or {}, + search=_load_json(run_dir / "search.json") or [], + filtered=_load_json(run_dir / "filtered.json") or [], + documents=_load_json(run_dir / "documents.json") or [], + insight=_load_json(run_dir / "insight.json") or {}, + manifest=_load_json(run_dir / "manifest.json") or {}, + forecast=_load_json(run_dir / "forecast.json"), + ) + + +def latest_run_dir(runs_root: Path, qid: str) -> Optional[Path]: + qdir = runs_root / qid + if not qdir.is_dir(): + return None + runs = [d for d in qdir.iterdir() if d.is_dir()] + if not runs: + return None + # Run dirs are UTC timestamps (YYYYmmdd_HHMMSS) so lexical == chronological. + return sorted(runs, key=lambda d: d.name)[-1] + + +# -------------------------------------------------------------------------- +# query terms (rebuilt from question.json, mirroring +# heuristics.build_query_terms without needing a live ForecastQuestion) +# -------------------------------------------------------------------------- + + +def query_terms(question: dict) -> list[str]: + terms = [question.get("text") or ""] + for key in ("pathogen", "region", "event_type", "resolution_criteria"): + val = question.get(key) + if val: + terms.append(str(val)) + return [t for t in terms if t] + + +# -------------------------------------------------------------------------- +# organic vs dashboard classification + provenance +# -------------------------------------------------------------------------- + + +def _is_dashboard_search(sr: dict) -> bool: + return (sr.get("retrieval_reason") or "") .strip().lower() == DASHBOARD_REASON \ + or (sr.get("engine") or "").lower() == "dashboard" + + +def _is_dashboard_filtered(fd: dict, dashboard_result_ids: set[str]) -> bool: + if DASHBOARD_BYPASS in (fd.get("selection_reasons") or []): + return True + return fd.get("result_id") in dashboard_result_ids + + +def _rel(text: str, terms: list[str]) -> float: + return keyword_overlap_score(text, terms) + + +@dataclass +class Provenance: + """Maps insight records back to organic/dashboard origins.""" + + dashboard_result_ids: set[str] + organic_result_ids: set[str] + # document_id -> ("dashboard"|"organic"|"unknown", source_url, domain) + doc_origin: dict[str, tuple[str, str, str]] + + +def build_provenance(art: RunArtifacts) -> Provenance: + dash_ids = {sr["id"] for sr in art.search if _is_dashboard_search(sr)} + org_ids = {sr["id"] for sr in art.search if not _is_dashboard_search(sr)} + + # Some filtered docs are dashboards only detectable via the bypass reason + # (belt and suspenders with the search-side reason). + for fd in art.filtered: + if DASHBOARD_BYPASS in (fd.get("selection_reasons") or []): + rid = fd.get("result_id") + if rid: + dash_ids.add(rid) + org_ids.discard(rid) + + doc_origin: dict[str, tuple[str, str, str]] = {} + for doc in art.documents: + rid = doc.get("result_id") + if rid in dash_ids: + origin = "dashboard" + elif rid in org_ids: + origin = "organic" + else: + origin = "unknown" + doc_origin[doc.get("id")] = ( + origin, doc.get("source_url") or "", (doc.get("domain") or "").lower() + ) + + return Provenance(dash_ids, org_ids, doc_origin) + + +@dataclass +class RecordProvenance: + record: dict + origins: set[str] = field(default_factory=set) # {"organic","dashboard"} + source_domains: set[str] = field(default_factory=set) + source_urls: list[str] = field(default_factory=list) + + @property + def from_organic(self) -> bool: + return "organic" in self.origins + + @property + def from_dashboard(self) -> bool: + return "dashboard" in self.origins + + @property + def n_independent_sources(self) -> int: + # Independent = distinct source domain behind the record. + return len(self.source_domains) or (1 if self.source_urls else 0) + + +def record_provenance(rec: dict, prov: Provenance) -> RecordProvenance: + rp = RecordProvenance(record=rec) + for src in rec.get("sources", []) or []: + doc_id = src.get("document_id") + url = src.get("source_url") or "" + if url: + rp.source_urls.append(url) + origin, doc_url, domain = prov.doc_origin.get(doc_id, ("unknown", "", "")) + if origin in ("organic", "dashboard"): + rp.origins.add(origin) + dom = domain or _domain_of(url) + if dom: + rp.source_domains.add(dom) + return rp + + +def _domain_of(url: str) -> str: + if not url: + return "" + u = url.split("//", 1)[-1] + return u.split("/", 1)[0].lower().removeprefix("www.") + + +# -------------------------------------------------------------------------- +# usability + classification +# -------------------------------------------------------------------------- + + +def _basis_ok(rec: dict, qid: str) -> bool: + """Record's count_basis matches the expected basis (advisory).""" + exp = EXPECTED_BASIS.get(qid) + if not exp: + return True + return (rec.get("count_basis") or "") == exp + + +def _is_usable(rec: dict, qtype: str, qid: str) -> bool: + """A record is 'usable' as a current-value anchor for this question. + + Range questions require a metric value that clears the anchor floor AND + is scoped to the question's geography (a Global cholera question is not + answered by an Angola row). The strict HIGH_CONFIDENCE bar is applied + separately as ``evidence_sufficient`` so a present-but-under-confident + anchor is still counted as usable-but-weak rather than vanishing. + """ + if qtype in NUMERIC_TYPES: + if rec.get("metric_value") is None: + return False + if (rec.get("confidence") or 0.0) < ANCHOR_FLOOR: + return False + return _scope_match(rec.get("location"), GEO_EXPECT.get(qid)) + # binary / categorical: any high-confidence on-topic record with a metric + # OR a substantive summary counts as usable evidence of current status. + if (rec.get("confidence") or 0.0) < HIGH_CONFIDENCE: + return False + return rec.get("metric_value") is not None or bool(rec.get("summary")) + + +def pick_top_record( + records: list[dict], qtype: str, qid: str, prov: Provenance +) -> Optional[RecordProvenance]: + """Best current-value anchor. For range questions, rank by scope match → + basis match → confidence → corroboration, so a correctly-scoped anchor + beats a higher-confidence off-scope row.""" + if not records: + return None + rps = [record_provenance(r, prov) for r in records] + geo = GEO_EXPECT.get(qid) + + def sort_key(rp: RecordProvenance): + r = rp.record + has_metric = 1 if r.get("metric_value") is not None else 0 + if qtype in NUMERIC_TYPES: + return ( + 1 if _scope_match(r.get("location"), geo) else 0, + 1 if _basis_ok(r, qid) else 0, + has_metric, + r.get("confidence") or 0.0, + rp.n_independent_sources, + 1 if rp.from_organic else 0, + ) + return ( + 0, 0, 0, + r.get("confidence") or 0.0, + rp.n_independent_sources, + 1 if rp.from_organic else 0, + ) + + usable = [rp for rp in rps if _is_usable(rp.record, qtype, qid)] + pool = usable or rps + return sorted(pool, key=sort_key, reverse=True)[0] + + +@dataclass +class Diagnosis: + qid: str + run_id: str + qtype: str + topic: str + # quantity + organic_returned: int + dashboard_returned: int + organic_survivors: int + dashboard_survivors: int + # dashboard routing / coverage (issue #41) + injected_dash_domains: str + expected_resolution_domain: str + resolution_source_injected: bool + # quality + pool_overlap_mean: float + pool_overlap_max: float + survivor_overlap_mean: float + llm_pool_ontopic: Optional[float] + llm_survivor_ontopic: Optional[float] + # evidence / attribution + n_records: int + n_usable: int + insight_from_organic: int + insight_from_dashboard: int + top_metric_value: Optional[float] + top_metric_name: Optional[str] + top_count_basis: Optional[str] + expected_basis: Optional[str] + top_confidence: Optional[float] + top_n_sources: int + top_origin: str + top_source_url: Optional[str] + top_quote: Optional[str] + top_scope_ok: bool + # verdicts + anchor_present: bool # a scope-matched metric record exists (any conf >= floor) + evidence_sufficient: bool # ...AND high-confidence with the right basis (spec headline) + single_fragile_source: bool + classification: str + cause: Optional[str] + cause_detail: str + + def as_row(self) -> dict[str, Any]: + d = dict(self.__dict__) + for k, v in d.items(): + if isinstance(v, float): + d[k] = round(v, 4) + return d + + +def diagnose(art: RunArtifacts, llm_judge=None) -> Diagnosis: + q = art.question + qtype = (q.get("question_type") or _infer_qtype(art)) or "" + terms = query_terms(q) + prov = build_provenance(art) + + # ---- quantity ---- + org_search = [s for s in art.search if not _is_dashboard_search(s)] + dash_search = [s for s in art.search if _is_dashboard_search(s)] + org_filt = [f for f in art.filtered if not _is_dashboard_filtered(f, prov.dashboard_result_ids)] + dash_filt = [f for f in art.filtered if _is_dashboard_filtered(f, prov.dashboard_result_ids)] + + # ---- dashboard routing / coverage (issue #41) ---- + injected_domains = sorted({(s.get("domain") or "").lower() for s in dash_search}) + injected_domains = [d for d in injected_domains if d] + exp_domain = EXPECT_RESOLUTION_DOMAIN.get(art.qid, "") + resolution_injected = any(exp_domain and exp_domain in d for d in injected_domains) + + # ---- quality (deterministic keyword overlap) ---- + def _txt_search(s: dict) -> str: + return f"{s.get('title','')} {s.get('snippet','')} {s.get('domain','')}" + + def _txt_filt(f: dict) -> str: + return f"{f.get('title','')} {f.get('snippet','')} {f.get('domain','')}" + + pool_overlaps = [_rel(_txt_search(s), terms) for s in org_search] + surv_overlaps = [_rel(_txt_filt(f), terms) for f in org_filt] + pool_mean = sum(pool_overlaps) / len(pool_overlaps) if pool_overlaps else 0.0 + pool_max = max(pool_overlaps) if pool_overlaps else 0.0 + surv_mean = sum(surv_overlaps) / len(surv_overlaps) if surv_overlaps else 0.0 + + # ---- optional LLM on-topic judge ---- + llm_pool = llm_surv = None + if llm_judge is not None: + llm_surv = llm_judge.judge_fraction( + q, [(_txt_filt(f), f.get("url", "")) for f in org_filt], label="survivors" + ) + sampled = sorted( + org_search, key=lambda s: s.get("search_stage_score", 0.0), reverse=True + )[:15] + llm_pool = llm_judge.judge_fraction( + q, [(_txt_search(s), s.get("url", "")) for s in sampled], label="pool_top15" + ) + + # ---- records + provenance ---- + records = art.records + rps = [record_provenance(r, prov) for r in records] + usable_rps = [rp for rp in rps if _is_usable(rp.record, qtype, art.qid)] + n_usable = len(usable_rps) + insight_from_organic = sum(1 for rp in rps if rp.from_organic) + insight_from_dashboard = sum(1 for rp in rps if rp.from_dashboard) + + top = pick_top_record(records, qtype, art.qid, prov) + top_scope_ok = bool( + top is not None + and _scope_match(top.record.get("location"), GEO_EXPECT.get(art.qid)) + ) + + # ---- classification ---- + # anchor_present = a usable (scope-matched, floor-clearing) record exists. + # evidence_sufficient (spec headline) is stricter: high-confidence AND the + # expected count_basis. A present-but-under-confident anchor (q17) is + # anchor_present=True, evidence_sufficient=False -> an extraction/insight + # calibration finding rather than a search/filter one. + anchor_present = bool(usable_rps) + high_conf_rps = [ + rp for rp in usable_rps + if (rp.record.get("confidence") or 0.0) >= HIGH_CONFIDENCE + and _basis_ok(rp.record, art.qid) + ] + evidence_sufficient = bool(high_conf_rps) + + if not anchor_present: + classification = "under_supported" + elif any(rp.from_organic for rp in usable_rps): + classification = "well_supported" + else: + classification = "dashboard_only" + + # single fragile source: usable anchors rest on <=1 independent domain + single_fragile = False + if anchor_present: + corr_domains: set[str] = set() + for rp in usable_rps: + corr_domains |= rp.source_domains + single_fragile = len(corr_domains) <= 1 + + # ---- cause attribution for weak/fragile/under-confident questions ---- + cause = None + cause_detail = "" + weak = ( + classification in ("under_supported", "dashboard_only") + or single_fragile + or not evidence_sufficient + ) + if weak: + cause, cause_detail = attribute_cause( + qtype, art.qid, org_search, org_filt, dash_filt, records, prov, + terms, pool_max, anchor_present, evidence_sufficient, top, + llm_pool_ontopic=llm_pool, + ) + + return Diagnosis( + qid=art.qid, + run_id=art.manifest.get("run_id", art.run_dir.name), + qtype=qtype, + topic=str(q.get("pathogen") or "") + (f" ({q.get('region')})" if q.get("region") else ""), + organic_returned=len(org_search), + dashboard_returned=len(dash_search), + organic_survivors=len(org_filt), + dashboard_survivors=len(dash_filt), + injected_dash_domains=",".join(injected_domains), + expected_resolution_domain=exp_domain, + resolution_source_injected=resolution_injected, + pool_overlap_mean=pool_mean, + pool_overlap_max=pool_max, + survivor_overlap_mean=surv_mean, + llm_pool_ontopic=llm_pool, + llm_survivor_ontopic=llm_surv, + n_records=len(records), + n_usable=n_usable, + insight_from_organic=insight_from_organic, + insight_from_dashboard=insight_from_dashboard, + top_metric_value=(top.record.get("metric_value") if top else None), + top_metric_name=(top.record.get("metric_name") if top else None), + top_count_basis=(top.record.get("count_basis") if top else None), + expected_basis=EXPECTED_BASIS.get(art.qid), + top_confidence=(top.record.get("confidence") if top else None), + top_n_sources=(top.n_independent_sources if top else 0), + top_origin=( + ("organic+dashboard" if top.from_organic and top.from_dashboard + else "organic" if top.from_organic + else "dashboard" if top.from_dashboard else "unknown") + if top else "none" + ), + top_source_url=(top.source_urls[0] if top and top.source_urls else None), + top_quote=_first_quote(top.record) if top else None, + top_scope_ok=top_scope_ok, + anchor_present=anchor_present, + evidence_sufficient=evidence_sufficient, + single_fragile_source=single_fragile, + classification=classification, + cause=cause, + cause_detail=cause_detail, + ) + + +def _first_quote(rec: dict) -> Optional[str]: + for src in rec.get("sources", []) or []: + q = src.get("quote") + if q: + return q[:200] + return rec.get("summary") + + +def _infer_qtype(art: RunArtifacts) -> str: + # question.json doesn't carry question_type; leave blank and let the caller + # override from the CSV. Kept for robustness. + return "" + + +def attribute_cause( + qtype, qid, org_search, org_filt, dash_filt, records, prov, terms, pool_max, + anchor_present, evidence_sufficient, top, llm_pool_ontopic=None, +) -> tuple[str, str]: + """Attribute the dominant cause of weak evidence. + + extraction : a dashboard survived but yielded no usable record. + insight : the correct-scope anchor WAS extracted but is under- + confident / off-basis, so it isn't usable at the headline + bar (an insight-stage confidence/scope calibration gap). + search-recall : the organic pool never contained on-topic content. + filter-recall : on-topic organic was in the pool but didn't survive. + robustness : a good high-confidence anchor exists but on a single source. + """ + def _txt_s(s): + return f"{s.get('title','')} {s.get('snippet','')} {s.get('domain','')}" + + def _txt_f(f): + return f"{f.get('title','')} {f.get('snippet','')} {f.get('domain','')}" + + on_topic_pool = [s for s in org_search if _rel(_txt_s(s), terms) >= ON_TOPIC_OVERLAP] + on_topic_surv = [f for f in org_filt if _rel(_txt_f(f), terms) >= ON_TOPIC_OVERLAP] + + dash_doc_ids = { + d for d, (origin, _u, _dm) in prov.doc_origin.items() if origin == "dashboard" + } + records_from_dash = sum( + 1 for rec in records + if any(s.get("document_id") in dash_doc_ids for s in (rec.get("sources") or [])) + ) + dashboard_barren = bool(dash_filt) and records_from_dash == 0 + + # 1. Dashboard survived but produced no records at all → extraction. + if dashboard_barren: + return ( + "extraction", + f"{len(dash_filt)} dashboard(s) survived filtering but produced 0 " + f"insight records (non-extractable tracker/index/JS page).", + ) + + # 2. Correct-scope anchor extracted but not usable at the headline bar → + # insight-stage calibration (present but under-confident / off-basis). + if anchor_present and not evidence_sufficient and qtype in NUMERIC_TYPES: + conf = (top.record.get("confidence") if top else None) + basis = (top.record.get("count_basis") if top else None) + exp = EXPECTED_BASIS.get(qid) + return ( + "insight", + f"scope-matched anchor present (value={top.record.get('metric_value') if top else None}, " + f"basis={basis} vs expected {exp}, conf={conf}) but below the " + f"high-confidence/expected-basis bar; off-scope rows outrank it.", + ) + + # 3. Organic pool had no on-topic content → search-recall. + if pool_max < SEARCH_RECALL_POOL_OVERLAP and not on_topic_pool: + return ( + "search-recall", + f"organic pool max_overlap={pool_max:.2f} (<{SEARCH_RECALL_POOL_OVERLAP}); " + f"~0 clearly-on-topic organic results returned by Tavily.", + ) + + # 4. On-topic organic (by keyword overlap) was in the pool but none + # survived. This is only a genuine filter-recall bug if the dropped + # organic is actually on-topic/authoritative. The LLM judge is the + # arbiter: if it says ~none of the pool is on-topic, keyword overlap + # was fooled by generic news and the true cause is an organic-authority + # gap (search-side), not the filter. + if on_topic_pool and not on_topic_surv: + exemplar = max(on_topic_pool, key=lambda s: _rel(_txt_s(s), terms)) + if llm_pool_ontopic is not None and llm_pool_ontopic < 0.2: + return ( + "search-recall", + f"{len(on_topic_pool)} organic looked on-topic by keyword overlap " + f"(e.g. {exemplar.get('domain')}) but the LLM judge rates the pool " + f"{llm_pool_ontopic:.0%} on-topic — generic news, not authoritative " + f"coverage. Organic-authority gap; dashboard rightly carries it.", + ) + judged = ( + f" LLM judge: pool {llm_pool_ontopic:.0%} on-topic." + if llm_pool_ontopic is not None else + " NOTE: keyword-overlap on-topic != authoritative; run --llm-judge to confirm." + ) + return ( + "filter-recall", + f"{len(on_topic_pool)} on-topic organic in pool (max_overlap=" + f"{_rel(_txt_s(exemplar), terms):.2f}, e.g. {exemplar.get('domain')}) " + f"but 0 survived the filter.{judged}", + ) + + # 5. High-confidence anchor exists but single-sourced → robustness only. + if evidence_sufficient: + return ( + "robustness", + "high-confidence anchor present but rests on a single source; " + "add a corroborating source.", + ) + + return ( + "search-recall", + f"organic pool weak (max_overlap={pool_max:.2f}); dashboards carry the question.", + ) + + +# -------------------------------------------------------------------------- +# optional LLM on-topic judge +# -------------------------------------------------------------------------- + + +class LLMJudge: + """Capped gpt-4o-mini on-topic judge. One call per item, batched small.""" + + def __init__(self, model: str = "gpt-4o-mini", max_items: int = 15): + try: + from dotenv import load_dotenv + load_dotenv() + except ImportError: + pass + from bioscancast.llm.openai_client import OpenAILLMClient + + self._client = OpenAILLMClient() + self._model = model + self._max_items = max_items + self.calls = 0 + + def judge_fraction(self, question: dict, items: list[tuple[str, str]], label: str): + items = items[: self._max_items] + if not items: + return None + on_topic = 0 + qtext = question.get("text", "") + for text, url in items: + verdict = self._judge_one(qtext, text, url) + self.calls += 1 + if verdict: + on_topic += 1 + return on_topic / len(items) + + _SCHEMA = { + "type": "object", + "properties": {"on_topic": {"type": "boolean"}}, + "required": ["on_topic"], + "additionalProperties": False, + } + + def _judge_one(self, qtext: str, item_text: str, url: str) -> bool: + system = ( + "You judge whether a search result is on-topic for a biosecurity " + "forecasting question. on_topic=true only if the result plausibly " + "carries evidence about the exact pathogen/place/metric the " + "question asks about." + ) + user = f"QUESTION: {qtext}\n\nRESULT (url={url}):\n{item_text[:600]}" + try: + resp = self._client.generate_json( + system=system, + user=user, + schema=self._SCHEMA, + model=self._model, + max_tokens=20, + temperature=0.0, + ) + return bool((resp.content or {}).get("on_topic")) + except Exception as exc: # noqa: BLE001 + print(f" ! llm judge error ({url}): {exc}", file=sys.stderr) + return False + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + + +ALL_QIDS = [f"bfg_q{i}" for i in range(1, 26)] + +# question_type per qid, from bfg_summer_2026_questions.csv (question.json +# does not persist it). +QTYPE: dict[str, str] = { + "bfg_q1": "range", "bfg_q2": "range", "bfg_q3": "binary", "bfg_q4": "binary", + "bfg_q5": "categorical", "bfg_q6": "range", "bfg_q7": "binary", + "bfg_q8": "categorical", "bfg_q9": "range", "bfg_q10": "categorical", + "bfg_q11": "categorical", "bfg_q12": "binary", "bfg_q13": "range", + "bfg_q14": "range", "bfg_q15": "binary", "bfg_q16": "range", + "bfg_q17": "range", "bfg_q18": "range", "bfg_q19": "range", + "bfg_q20": "range", "bfg_q21": "binary", "bfg_q22": "range", + "bfg_q23": "binary", "bfg_q24": "binary", "bfg_q25": "range", +} + + +def main(argv: Optional[list[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("qids", nargs="*", help="Question ids (default: all with runs).") + ap.add_argument("--all", action="store_true", help="Analyze all bfg_q1..q25.") + ap.add_argument("--runs-root", default="data/runs") + ap.add_argument("--run-dir", default=None, help="Explicit run dir for one question.") + ap.add_argument("--out", default=None, help="Output basename (writes .csv and .json).") + ap.add_argument("--llm-judge", action="store_true", help="Add gpt-4o-mini on-topic judge.") + args = ap.parse_args(argv if argv is not None else sys.argv[1:]) + + runs_root = Path(args.runs_root) + judge = LLMJudge() if args.llm_judge else None + + run_dirs: list[Path] = [] + if args.run_dir: + run_dirs = [Path(args.run_dir)] + else: + qids = args.qids or (ALL_QIDS if args.all else None) + if qids is None: + # default: every qid that has at least one run + qids = sorted( + (d.name for d in runs_root.iterdir() if d.is_dir()), + key=lambda n: (len(n), n), + ) if runs_root.is_dir() else [] + for qid in qids: + rd = latest_run_dir(runs_root, qid) + if rd is None: + print(f" (no run found for {qid})", file=sys.stderr) + continue + run_dirs.append(rd) + + diags: list[Diagnosis] = [] + for rd in run_dirs: + art = load_run(rd) + # question.json lacks question_type; inject from the CSV-derived map. + if not art.question.get("question_type"): + art.question["question_type"] = QTYPE.get(art.qid, "") + diag = diagnose(art, llm_judge=judge) + diags.append(diag) + _print_diag(diag) + + if args.out and diags: + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + rows = [d.as_row() for d in diags] + with (out.with_suffix(".csv")).open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + w.writeheader() + w.writerows(rows) + with (out.with_suffix(".json")).open("w", encoding="utf-8") as f: + json.dump(rows, f, indent=2) + print(f"\nWrote {out.with_suffix('.csv')} and {out.with_suffix('.json')}") + if judge is not None: + print(f"LLM judge calls: {judge.calls}") + + return 0 + + +def _print_diag(d: Diagnosis) -> None: + flag = { + "well_supported": "OK ", "dashboard_only": "DASH", "under_supported": "WEAK", + }.get(d.classification, "? ") + frag = " [single-fragile]" if d.single_fragile_source else "" + print( + f"[{flag}] {d.qid:8s} {d.qtype:11s} " + f"org {d.organic_survivors}/{d.organic_returned} " + f"dash {d.dashboard_survivors}/{d.dashboard_returned} | " + f"rec={d.n_records} usable={d.n_usable} " + f"(org={d.insight_from_organic} dash={d.insight_from_dashboard}) | " + f"top={d.top_metric_value} [{d.top_count_basis}] conf={d.top_confidence} " + f"src={d.top_n_sources}/{d.top_origin}{frag}" + ) + if d.cause: + print(f" cause={d.cause}: {d.cause_detail}") + + +if __name__ == "__main__": + raise SystemExit(main())