BFG evidence-coverage audit + targeted extraction fixes (q16, refiner-URL, WHO H5 HAI)

bioscancast/stages/extraction/custom_scrapers/cdc_measles.py

@@ -0,0 +1,150 @@
+"""Custom scraper for the CDC measles data page (``cdc_measles``).
+
+The CDC "Measles Cases and Outbreaks" page renders its case/death/hospitalisation
+figures client-side from a JSON data endpoint. A static scrape gets the case
+*count* from prose (so q14 works) but the **deaths** table cells are empty in the
+served HTML — they are JS-injected — so death questions (q16) extracted 0 records.
+
+The data is published as plain JSON at ``/wcms/vizdata/measles/measles_hosp.json``
+(no browser or Akamai challenge needed), carrying, per year, ``total_cases``,
+``total_deaths`` and a ready-made ``deaths_sentence``. We fetch that and render a
+compact HTML summary the existing HTML extraction pipeline consumes unchanged, so
+the insight stage gets an unambiguous "N confirmed measles deaths in 2026" fact
+(and the case count, corroborating the prose figure q14 already uses).
+
+Returns ``None`` (fall back to the generic fetch) on any failure or in
+historical-replay mode — the endpoint carries only current per-year aggregates
+with no date history to cut on, so serving it under an ``as_of_date`` would leak.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import logging
+from datetime import datetime, timezone
+
+from bioscancast.stages.extraction.config import ExtractionConfig
+from bioscancast.stages.extraction.fetcher import FetchResult
+
+logger = logging.getLogger(__name__)
+
+# The page's client-side data feed. Absolute (host-pinned) because this scraper
+# is specific to the CDC measles source; the hub ``url`` passed in is the
+# human-facing data-research page.
+_DATA_URL = "https://www.cdc.gov/wcms/vizdata/measles/measles_hosp.json"
+
+# Years to surface, newest first. The current year answers q14/q16; the prior
+# year is kept as corroborating context (and a base rate for the death question).
+_YEARS = ("2026", "2025")
+
+
+def _first(value) -> str | None:
+    """CDC wraps each field as a single-element list (e.g. ``["0"]``)."""
+    if isinstance(value, list):
+        value = value[0] if value else None
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _get_json(url: str, cfg: ExtractionConfig):
+    try:
+        from curl_cffi import requests as curl_requests
+
+        resp = curl_requests.get(
+            url,
+            timeout=max(cfg.fetch_timeout_seconds, 30.0),
+            impersonate=cfg.impersonate,
+            allow_redirects=True,
+        )
+    except Exception as exc:  # noqa: BLE001 - network best-effort
+        logger.info("CDC measles JSON fetch failed for %s: %s", url, exc)
+        return None
+    if resp.status_code != 200 or not resp.content:
+        return None
+    try:
+        return resp.json()
+    except (ValueError, json.JSONDecodeError):
+        return None
+
+
+def fetch(
+    url: str,
+    *,
+    config: ExtractionConfig | None = None,
+    as_of_date: datetime | None = None,
+    region: str | None = None,
+    question_text: str | None = None,
+    json_getter=None,
+) -> FetchResult | None:
+    # Live-only: the feed has no per-date history, so serving it in replay mode
+    # would leak post-cutoff values. Fall back to the generic (Wayback) path.
+    if as_of_date is not None:
+        return None
+
+    cfg = config or ExtractionConfig()
+    fetched_at = datetime.now(timezone.utc)
+
+    # ``json_getter`` is injectable for no-network tests; defaults to a live
+    # curl_cffi fetch of the CDC data feed.
+    getter = json_getter or _get_json
+    data = getter(_DATA_URL, cfg)
+    if not isinstance(data, dict):
+        return None
+
+    blocks: list[str] = []
+    for year in _YEARS:
+        rec = data.get(year)
+        if not isinstance(rec, dict):
+            continue
+        cases = _first(rec.get("total_cases"))
+        deaths = _first(rec.get("total_deaths"))
+        deaths_sentence = _first(rec.get("deaths_sentence"))
+        if cases is None and deaths is None:
+            continue
+        parts = [f"<h2>United States measles, {html.escape(year)}</h2>", "<p>"]
+        if cases is not None:
+            parts.append(
+                f"As of the latest CDC update, a total of {html.escape(cases)} "
+                f"confirmed measles cases were reported in the United States in "
+                f"{html.escape(year)}. "
+            )
+        if deaths_sentence is not None:
+            parts.append(html.escape(deaths_sentence) + " ")
+        elif deaths is not None:
+            parts.append(
+                f"There have been {html.escape(deaths)} confirmed measles deaths "
+                f"in the United States in {html.escape(year)}. "
+            )
+        parts.append("</p>")
+        blocks.append("".join(parts))
+
+    if not blocks:
+        return None
+
+    rendered = (
+        "<html><head><meta charset='utf-8'>"
+        "<title>CDC — Measles Cases and Outbreaks (United States)</title></head><body>"
+        "<h1>CDC — Measles Cases and Outbreaks (United States)</h1>"
+        f"<p>Source: {html.escape(url)} "
+        f"(data feed {html.escape(_DATA_URL)}), retrieved "
+        f"{fetched_at.date().isoformat()}.</p>"
+        + "".join(blocks)
+        + "<p>Note: this custom scraper renders the CDC measles JSON data feed "
+        "into compact HTML because the page's case/death figures are injected "
+        "client-side and are absent from the statically served table cells. "
+        "Counts are cumulative year-to-date totals.</p>"
+        "</body></html>"
+    ).encode("utf-8")
+
+    return FetchResult(
+        url=_DATA_URL,
+        final_url=_DATA_URL,
+        status_code=200,
+        content_type="text/html",
+        content_bytes=rendered,
+        fetched_at=fetched_at,
+        error=None,
+    )

bioscancast/stages/extraction/custom_scrapers/who_h5_hai.py

@@ -0,0 +1,43 @@
+"""Custom scraper for the WHO "Influenza at the human-animal interface" monthly
+risk-assessment hub (``who_h5_hai``).
+
+The monthly-risk-assessment-summary page is an index: it lists dated
+*Influenza at the human-animal interface summary and assessment* item pages
+(``who.int/publications/m/item/influenza-at-the-human-animal-interface-...``),
+each of which links a ``cdn.who.int`` PDF whose tables/prose carry the cumulative
+human case counts by A(H5) subtype. The index page itself is context-only, so H5
+human-case questions (q6 case count, q8 predominant subtype) extracted 0 records
+from it.
+
+This resolves the hub to the latest assessment at-or-before the cutoff and returns
+that item's PDF, exactly as ``who_cholera`` does for the cholera hub, so the
+existing PDF parser extracts the numbers. Falls back (return ``None``) to the
+generic fetch of the index page if resolution fails.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from bioscancast.stages.extraction.config import ExtractionConfig
+from bioscancast.stages.extraction.custom_scrapers._who_hub_common import (
+    fetch_who_hub_latest_pdf,
+)
+from bioscancast.stages.extraction.fetcher import FetchResult
+
+
+def fetch(
+    url: str,
+    *,
+    config: ExtractionConfig | None = None,
+    as_of_date: datetime | None = None,
+    region: str | None = None,
+    question_text: str | None = None,
+) -> FetchResult | None:
+    # "human-animal" selects the HAI summary items (their slug/text both carry
+    # "human-animal interface") out of the ~150 mixed publication links on the
+    # monthly-risk-assessment index, and excludes the FAO/WOAH joint-assessment
+    # and vaccine-composition items that don't hold the human-case counts.
+    return fetch_who_hub_latest_pdf(
+        url, "human-animal", config=config, as_of_date=as_of_date
+    )

bioscancast/stages/extraction/pipeline.py

@@ -139,9 +139,16 @@ def extract_one(self, filtered_doc: FilteredDocument) -> Document:
             refiner = self._get_docling_refiner()
             if refiner is not None:
                 try:
+                    # Match the Docling allowlist against the URL actually
+                    # fetched. Custom hub scrapers (who_cholera, who_h5_hai, ...)
+                    # resolve a landing page to a ``cdn.who.int/.../*.pdf``; the
+                    # allowlist entries target those resolved PDF paths, so
+                    # matching on ``filtered_doc.url`` (the hub) would never fire
+                    # the ``situation-reports`` / ``_sage-`` allowlist for exactly
+                    # the PDFs it exists to catch.
                     parsed = refiner.refine(
                         parsed,
-                        source_url=filtered_doc.url,
+                        source_url=fetch_result.final_url or filtered_doc.url,
                         content=fetch_result.content_bytes,
                     )
                 except Exception as exc:

bioscancast/tests/test_cdc_measles_scraper.py

@@ -0,0 +1,66 @@
+"""Tests for the CDC measles custom scraper (``cdc_measles``).
+
+No network: the scraper accepts an injected ``json_getter`` returning a fixture
+dict shaped like the live ``measles_hosp.json`` feed. Assertions target the
+rendered HTML summary (``content_bytes``) that the HTML extraction pipeline then
+consumes — the point of the scraper is to surface the JS-injected **deaths**
+figure (absent from the statically served page) as extractable prose.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from bioscancast.stages.extraction.custom_scrapers import cdc_measles
+
+_PAGE = "https://www.cdc.gov/measles/data-research/index.html"
+
+# Shape mirrors the live feed: each field is a single-element list.
+_FEED = {
+    "2026": {
+        "total_cases": ["2,170"],
+        "total_deaths": ["0"],
+        "deaths_sentence": ["There have been 0 confirmed deaths from measles in 2026."],
+    },
+    "2025": {
+        "total_cases": ["2,289"],
+        "total_deaths": ["3"],
+        "deaths_sentence": ["There were 3 confirmed deaths from measles in 2025."],
+    },
+}
+
+
+def _getter(feed):
+    return lambda url, config: feed
+
+
+def _html(result) -> str:
+    assert result is not None
+    return result.content_bytes.decode("utf-8")
+
+
+def test_renders_current_year_death_count_as_prose():
+    result = cdc_measles.fetch(_PAGE, json_getter=_getter(_FEED))
+    html = _html(result)
+    assert result.content_type == "text/html"
+    # The death figure that is JS-injected (and thus missing from a static
+    # scrape) is now present as an unambiguous sentence.
+    assert "0 confirmed deaths from measles in 2026" in html
+    # Case count is surfaced too, corroborating the prose figure q14 uses.
+    assert "2,170 confirmed measles cases" in html
+
+
+def test_historical_mode_returns_none_to_avoid_leakage():
+    # The feed carries only current per-year aggregates with no date history,
+    # so replay mode must fall back rather than serve post-cutoff values.
+    result = cdc_measles.fetch(
+        _PAGE,
+        as_of_date=datetime(2026, 3, 1, tzinfo=timezone.utc),
+        json_getter=_getter(_FEED),
+    )
+    assert result is None
+
+
+def test_missing_feed_falls_back():
+    assert cdc_measles.fetch(_PAGE, json_getter=lambda url, config: None) is None
+    assert cdc_measles.fetch(_PAGE, json_getter=lambda url, config: {}) is None

bioscancast/tests/test_extraction_pipeline.py

@@ -150,6 +150,53 @@ def test_pdf_extraction_end_to_end(self):
         assert doc.page_count is not None
         assert len(doc.chunks) > 0
 
+    def test_refiner_receives_resolved_pdf_url_not_hub(self):
+        # A hub scraper (who_cholera, who_h5_hai, ...) resolves a landing page
+        # who.int/<hub> to a cdn.who.int/.../*.pdf. The Docling allowlist targets
+        # the resolved PDF path, so the refiner must be handed
+        # ``fetch_result.final_url`` — matching on the hub ``filtered_doc.url``
+        # would never fire the allowlist for exactly those scraper PDFs.
+        pdf_bytes = (FIXTURES / "who_don_sample.pdf").read_bytes()
+        hub_url = "https://www.who.int/emergencies/situations/cholera-upsurge"
+        pdf_url = (
+            "https://cdn.who.int/media/docs/default-source/documents/"
+            "emergencies/situation-reports/cholera_update.pdf"
+        )
+        fdoc = _make_filtered_doc(
+            result_id="r_hub", url=hub_url, domain="who.int", extraction_mode="pdf"
+        )
+        fetch_map = {
+            hub_url: FetchResult(
+                url=pdf_url,
+                final_url=pdf_url,
+                status_code=200,
+                content_type="application/pdf",
+                content_bytes=pdf_bytes,
+                fetched_at=datetime.now(timezone.utc),
+                error=None,
+            )
+        }
+
+        seen: dict[str, str] = {}
+
+        class _SpyRefiner:
+            def refine(self, parsed, *, source_url, content):
+                seen["source_url"] = source_url
+                return parsed
+
+        with patch(
+            "bioscancast.stages.extraction.pipeline.fetch",
+            side_effect=_fake_fetch_factory(fetch_map),
+        ), patch.object(
+            ExtractionPipeline, "_get_docling_refiner", return_value=_SpyRefiner()
+        ):
+            pipeline = ExtractionPipeline(
+                config=ExtractionConfig(enable_docling_refiner=True)
+            )
+            pipeline.run([fdoc])
+
+        assert seen.get("source_url") == pdf_url
+
     def test_ordering_by_extraction_priority(self):
         html_bytes = b"<html><body><p>Test content</p></body></html>"
         fdoc_low = _make_filtered_doc(