diff --git a/bioscancast/datasets/sources.yaml b/bioscancast/datasets/sources.yaml index 8dd4e52..d50713f 100644 --- a/bioscancast/datasets/sources.yaml +++ b/bioscancast/datasets/sources.yaml @@ -4,6 +4,16 @@ general_sources: url: "https://www.who.int/emergencies/disease-outbreak-news" geography: "Global" + - id: "who_pheic_list" + name: "WHO IHR Emergency Committees (PHEIC status)" + url: "https://www.who.int/teams/ihr/ihr-emergency-committees" + geography: "Global" + + - id: "who_variant_tracking" + name: "WHO Tracking SARS-CoV-2 variants" + url: "https://www.who.int/activities/tracking-SARS-CoV-2-variants" + geography: "Global" + - id: "cdc_outbreaks" name: "CDC Current Outbreaks" url: "https://www.cdc.gov/outbreaks/index.html" @@ -73,11 +83,21 @@ specific_pathogen_sources: url: "https://www.who.int/teams/global-influenza-programme/avian-influenza" geography: "Global" + - id: "who_h5_hai" + name: "WHO Influenza at the human-animal interface (monthly risk assessment)" + url: "https://www.who.int/teams/global-influenza-programme/avian-influenza/monthly-risk-assessment-summary" + geography: "Global" + - id: "cdc_h5n1" name: "CDC H5N1 Situation Summary" url: "https://www.cdc.gov/bird-flu/situation-summary/" geography: "Global" + - id: "usda_aphis_livestock" + name: "USDA APHIS HPAI Confirmed Cases in Livestock" + url: "https://www.aphis.usda.gov/livestock-poultry-disease/avian/avian-influenza/hpai-detections/hpai-confirmed-cases-livestock" + geography: "United States" + covid-19: - id: "ourworldindata_covid" name: "Our World in Data - COVID-19" @@ -87,6 +107,10 @@ specific_pathogen_sources: name: "CDC COVID-19" url: "https://www.cdc.gov/covid/php/surveillance/index.html" geography: "Global" + - id: "who_variant_tracking" + name: "WHO Tracking SARS-CoV-2 variants" + url: "https://www.who.int/activities/tracking-SARS-CoV-2-variants" + geography: "Global" influenza: - id: "who_flunet" @@ -101,14 +125,24 @@ specific_pathogen_sources: hemorrhagic: ebola: + - id: "who_ebola_drc" + name: "WHO Ebola outbreak DRC 2026 (Bundibugyo virus) event page" + url: "https://www.who.int/emergencies/situations/ebola-outbreak---drc-2026" + geography: "Global" + + - id: "who_ebola_don" + name: "WHO Disease Outbreak News - Ebola (Bundibugyo) DRC & Uganda" + url: "https://www.who.int/emergencies/disease-outbreak-news/item/2026-DON602" + geography: "Global" + - id: "who_ebola" name: "WHO Ebola" - url: "https://www.afro.who.int/health-topics/ebola-virus-disease" + url: "https://www.afro.who.int/health-topics/ebola-disease" geography: "Global" - id: "cdc_ebola" name: "CDC Ebola" - url: "https://www.cdc.gov/ebola/index.html" + url: "https://www.cdc.gov/ebola/about/index.html" geography: "Global" marburg: @@ -125,8 +159,8 @@ specific_pathogen_sources: pox_re_emerging_viruses: mpox: - id: "who_mpox" - name: "WHO MPOX" - url: "https://www.who.int/health-topics/mpox" + name: "WHO Mpox outbreak (multi-country)" + url: "https://www.who.int/emergencies/situations/mpox-outbreak" geography: "Global" - id: "ourworldindata_mpox" @@ -135,8 +169,8 @@ specific_pathogen_sources: geography: "Global" - id: "cdc_mpox" - name: "CDC MPOX" - url: "https://www.cdc.gov/mpox/data-research/index.html" + name: "CDC Mpox Situation Summary" + url: "https://www.cdc.gov/monkeypox/situation-summary/index.html" geography: "Global" - id: "ecdc_mpox" @@ -159,11 +193,77 @@ specific_pathogen_sources: url: "https://www.paho.org/en/situation-reports?topic=87192&d%5Bmin%5D=&d%5Bmax%5D=" geography: "Americas" - vector_borne: [] + vector_borne: + dengue: + - id: "paho_dengue" + name: "PAHO Epidemiological Update - Dengue in the Americas" + url: "https://www.paho.org/en/epidemiological-alerts-and-updates" + geography: "Americas" + + chikungunya: + - id: "ecdc_chikungunya" + name: "ECDC Chikungunya worldwide overview (monthly)" + url: "https://www.ecdc.europa.eu/en/chikungunya-monthly" + geography: "Europe" + + oropouche: + - id: "paho_oropouche" + name: "PAHO Oropouche virus disease" + url: "https://www.paho.org/en/topics/oropouche-virus-disease" + geography: "Americas" + + enteric: + cholera: + - id: "who_cholera" + name: "WHO Cholera upsurge (multi-country external situation reports)" + url: "https://www.who.int/emergencies/situations/cholera-upsurge" + geography: "Global" + + vaccine_preventable: + measles: + - id: "cdc_measles" + name: "CDC Measles Cases and Outbreaks (data & research)" + url: "https://www.cdc.gov/measles/data-research/index.html" + geography: "United States" + + - id: "paho_measles" + name: "PAHO Measles" + url: "https://www.paho.org/en/topics/measles" + geography: "Americas" + + polio: + - id: "gpei_polio" + name: "GPEI Polio this week" + url: "https://polioeradication.org/about-polio/polio-this-week/" + geography: "Global" animal_spillover: nipah: - id: "who_nipah" name: "WHO Nipah" url: "https://www.who.int/health-topics/nipah-virus-infection" - geography: "Global" \ No newline at end of file + geography: "Global" + + # Avian influenza is zoonotic, so the source router often classifies H5 + # questions as animal_spillover rather than respiratory. Mirror the H5N1 + # sources here so either route injects the right dashboards. + h5n1: + - id: "who_h5n1" + name: "WHO H5N1 Situation Updates" + url: "https://www.who.int/teams/global-influenza-programme/avian-influenza" + geography: "Global" + + - id: "who_h5_hai" + name: "WHO Influenza at the human-animal interface (monthly risk assessment)" + url: "https://www.who.int/teams/global-influenza-programme/avian-influenza/monthly-risk-assessment-summary" + geography: "Global" + + - id: "cdc_h5n1" + name: "CDC H5N1 Situation Summary" + url: "https://www.cdc.gov/bird-flu/situation-summary/" + geography: "Global" + + - id: "usda_aphis_livestock" + name: "USDA APHIS HPAI Confirmed Cases in Livestock" + url: "https://www.aphis.usda.gov/livestock-poultry-disease/avian/avian-influenza/hpai-detections/hpai-confirmed-cases-livestock" + geography: "United States" diff --git a/bioscancast/stages/evaluation/bfg_summer_2026_options.csv b/bioscancast/stages/evaluation/bfg_summer_2026_options.csv new file mode 100644 index 0000000..5ae02cc --- /dev/null +++ b/bioscancast/stages/evaluation/bfg_summer_2026_options.csv @@ -0,0 +1,97 @@ +question_id;forecast_source;forecast_version;option;probability +bfg_q1;bins;1;less than 250;0 +bfg_q1;bins;1;250 to 499;0 +bfg_q1;bins;1;500 to 999;0 +bfg_q1;bins;1;1,000 to 2,499;0 +bfg_q1;bins;1;2,500 or more;0 +bfg_q2;bins;1;100 or fewer;0 +bfg_q2;bins;1;101 to 250;0 +bfg_q2;bins;1;251 to 500;0 +bfg_q2;bins;1;501 to 1,000;0 +bfg_q2;bins;1;more than 1,000;0 +bfg_q3;bins;1;YES;0 +bfg_q3;bins;1;NO;0 +bfg_q4;bins;1;YES;0 +bfg_q4;bins;1;NO;0 +bfg_q5;bins;1;Vaccine candidate enters clinical trial in affected countries;0 +bfg_q5;bins;1;Investigational therapeutic administered under expanded access or MEURI;0 +bfg_q5;bins;1;Both occur;0 +bfg_q5;bins;1;Neither occurs;0 +bfg_q6;bins;1;0 to 4;0 +bfg_q6;bins;1;5 to 14;0 +bfg_q6;bins;1;15 to 29;0 +bfg_q6;bins;1;30 to 59;0 +bfg_q6;bins;1;60 or more;0 +bfg_q7;bins;1;YES;0 +bfg_q7;bins;1;NO;0 +bfg_q8;bins;1;H5N1;0 +bfg_q8;bins;1;H5N5;0 +bfg_q8;bins;1;Another H5Nx;0 +bfg_q8;bins;1;No human H5 cases reported in the period;0 +bfg_q9;bins;1;19 or fewer;0 +bfg_q9;bins;1;20;0 +bfg_q9;bins;1;21 to 22;0 +bfg_q9;bins;1;23 to 25;0 +bfg_q9;bins;1;26 or more;0 +bfg_q10;bins;1;0;0 +bfg_q10;bins;1;1;0 +bfg_q10;bins;1;2 or more;0 +bfg_q11;bins;1;1;0 +bfg_q11;bins;1;2;0 +bfg_q11;bins;1;3;0 +bfg_q11;bins;1;4 or more;0 +bfg_q12;bins;1;YES;0 +bfg_q12;bins;1;NO;0 +bfg_q13;bins;1;fewer than 5,000;0 +bfg_q13;bins;1;5,000 to 14,999;0 +bfg_q13;bins;1;15,000 to 29,999;0 +bfg_q13;bins;1;30,000 to 59,999;0 +bfg_q13;bins;1;60,000 or more;0 +bfg_q14;bins;1;under 2,500;0 +bfg_q14;bins;1;2,500 to 3,499;0 +bfg_q14;bins;1;3,500 to 4,999;0 +bfg_q14;bins;1;5,000 to 7,499;0 +bfg_q14;bins;1;7,500 or more;0 +bfg_q15;bins;1;YES;0 +bfg_q15;bins;1;NO;0 +bfg_q16;bins;1;0;0 +bfg_q16;bins;1;1;0 +bfg_q16;bins;1;2 to 3;0 +bfg_q16;bins;1;4 to 6;0 +bfg_q16;bins;1;7 or more;0 +bfg_q17;bins;1;under 150k;0 +bfg_q17;bins;1;150k to 299k;0 +bfg_q17;bins;1;300k to 449k;0 +bfg_q17;bins;1;450k to 599k;0 +bfg_q17;bins;1;600k or more;0 +bfg_q18;bins;1;under 1M;0 +bfg_q18;bins;1;1 to 1.9M;0 +bfg_q18;bins;1;2 to 3.9M;0 +bfg_q18;bins;1;4 to 6.9M;0 +bfg_q18;bins;1;7M or more;0 +bfg_q19;bins;1;0;0 +bfg_q19;bins;1;1;0 +bfg_q19;bins;1;2 to 3;0 +bfg_q19;bins;1;4 to 5;0 +bfg_q19;bins;1;6 or more;0 +bfg_q20;bins;1;8 or fewer;0 +bfg_q20;bins;1;9 to 11;0 +bfg_q20;bins;1;12 to 14;0 +bfg_q20;bins;1;15 to 17;0 +bfg_q20;bins;1;18 or more;0 +bfg_q21;bins;1;YES;0 +bfg_q21;bins;1;NO;0 +bfg_q22;bins;1;10 or fewer;0 +bfg_q22;bins;1;11 to 25;0 +bfg_q22;bins;1;26 to 50;0 +bfg_q22;bins;1;51 to 99;0 +bfg_q22;bins;1;100 or more;0 +bfg_q23;bins;1;YES;0 +bfg_q23;bins;1;NO;0 +bfg_q24;bins;1;YES;0 +bfg_q24;bins;1;NO;0 +bfg_q25;bins;1;10 or fewer;0 +bfg_q25;bins;1;11 to 20;0 +bfg_q25;bins;1;21 to 30;0 +bfg_q25;bins;1;31 to 45;0 +bfg_q25;bins;1;46 or more;0 diff --git a/bioscancast/stages/evaluation/bfg_summer_2026_questions.csv b/bioscancast/stages/evaluation/bfg_summer_2026_questions.csv new file mode 100644 index 0000000..f978b36 --- /dev/null +++ b/bioscancast/stages/evaluation/bfg_summer_2026_questions.csv @@ -0,0 +1,26 @@ +question_id;topic;question_text;question_type;resolution_criteria;created_date;question_status;resolved_option;comparison_to_outcome;takeaways;relevant_links +bfg_q1;Bundibugyo Ebola (DRC-Uganda);How many cumulative laboratory-confirmed Bundibugyo virus disease cases (DRC and Uganda combined) will be in WHO reporting as of October 31, 2026?;range;The latest confirmed-case total in the WHO report covering on or nearest before October 31, 2026. Source: WHO Ebola (DRC 2026) situation reports and Disease Outbreak News.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q2;Bundibugyo Ebola (DRC-Uganda);How many cumulative reported deaths (confirmed plus probable) in the Bundibugyo outbreak will WHO report as of December 31, 2026?;range;The death total in the WHO report covering on or nearest before December 31, 2026. Source: WHO situation reports.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q3;Bundibugyo Ebola (DRC-Uganda);Will the Bundibugyo virus disease PHEIC still be in effect on December 31, 2026?;binary;TRUE if the PHEIC has not been terminated by December 31, 2026. Source: WHO IHR Emergency Committee statements and the Director-General PHEIC determinations.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/health-topics/international-health-regulations +bfg_q4;Bundibugyo Ebola (DRC-Uganda);Will at least one laboratory-confirmed Bundibugyo case linked to this outbreak be reported in a country other than DRC or Uganda on or before December 31, 2026?;binary;TRUE on the first confirmed exported or third-country case attributed to this outbreak, by December 31, 2026. Source: WHO Disease Outbreak News.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q5;Bundibugyo Ebola (DRC-Uganda);Which Bundibugyo-relevant medical countermeasure milestone will be reached first during the outbreak window by December 31, 2026?;categorical;The first option publicly documented by December 31, 2026, or both if both occur in the window. Source: WHO, ICTRP and other trial registries, partner announcements.;46218;unresolved;TBD;TBD;TBD;https://trialsearch.who.int +bfg_q6;Avian influenza H5 (World);How many human H5 infections (any neuraminidase subtype) will be reported to WHO globally between July 1 and December 31, 2026?;range;Count of newly reported human H5 cases with onset or report date inside the window, as of December 31, 2026. Source: WHO Influenza at the human-animal interface assessments and Disease Outbreak News.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/teams/global-influenza-programme/avian-influenza/monthly-risk-assessment-summary +bfg_q7;Avian influenza H5 (World);Will WHO or a national public health authority confirm at least one cluster meeting its stated criteria for sustained human-to-human transmission of an H5 virus on or before December 31, 2026?;binary;TRUE on a WHO or national-authority statement reporting confirmed onward human-to-human spread beyond a single generation by December 31, 2026. Source: WHO.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/teams/global-influenza-programme/avian-influenza +bfg_q8;Avian influenza H5 (World);What will be the predominant H5 subtype among human cases reported globally in the second half of 2026, as of December 31, 2026?;categorical;The subtype with the most reported human cases in the window, as of December 31, 2026. Source: WHO.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/teams/global-influenza-programme/avian-influenza/monthly-risk-assessment-summary +bfg_q9;H5N1 (US);How many US states will have cumulative H5N1 dairy-cattle detections by December 31, 2026?;range;Cumulative count of US states with H5N1 dairy-cattle detections as of December 31, 2026. Source: USDA APHIS livestock detections.;46218;unresolved;TBD;TBD;TBD;https://www.aphis.usda.gov/livestock-poultry-disease/avian/avian-influenza/hpai-detections/hpai-confirmed-cases-livestock +bfg_q10;PHEIC (World);How many new PHEICs will be declared for events not already under a PHEIC as of July 1, 2026, during July 1 to December 31, 2026?;categorical;Count of new PHEIC determinations by the Director-General in the window, as of December 31, 2026. Source: WHO.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/health-topics/international-health-regulations +bfg_q11;PHEIC (World);How many PHEICs will be in effect on December 31, 2026?;categorical;Count of PHEICs in effect on December 31, 2026. Source: WHO.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/health-topics/international-health-regulations +bfg_q12;Mpox (World);Will WHO declare a new PHEIC for mpox at any point between July 1 and December 31, 2026?;binary;TRUE on a fresh mpox PHEIC determination in the window, by December 31, 2026. The prior mpox PHEIC was lifted September 5, 2025. Source: WHO.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/situation-reports +bfg_q13;Mpox (World);How many cumulative human mpox cases will be reported to WHO globally between July 1 and December 31, 2026?;range;New confirmed cases reported in the window, as of December 31, 2026. Bins to be locked at launch. Source: WHO multi-country mpox external situation reports.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/situation-reports +bfg_q14;Measles (US);How many US confirmed measles cases will be reported in 2026, as of December 31, 2026?;range;US confirmed measles cases in 2026 per the CDC measles tracker, as of December 31, 2026. Source: CDC.;46218;unresolved;TBD;TBD;TBD;https://www.cdc.gov/measles/data-research/index.html +bfg_q15;Measles (US);Will the US lose measles elimination status on or before December 31, 2026?;binary;TRUE if PAHO or WHO declares the US has lost measles elimination status on or before December 31, 2026. Source: PAHO.;46218;unresolved;TBD;TBD;TBD;https://www.paho.org/en/topics/measles +bfg_q16;Measles (US);How many US measles-associated deaths will be reported in 2026, as of December 31, 2026?;range;US measles-associated deaths in 2026, as of December 31, 2026. Source: CDC.;46218;unresolved;TBD;TBD;TBD;https://www.cdc.gov/measles/data-research/index.html +bfg_q17;Cholera (World);How many global cholera cases will be reported in 2026, as of December 31, 2026?;range;Global cholera cases reported in 2026, as of December 31, 2026. Bins to be locked at launch. Source: WHO multi-country cholera epidemiological update.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q18;Dengue (Americas);How many dengue suspected cases will be reported in the Americas in 2026, as of December 31, 2026?;range;Dengue suspected cases in the Americas in 2026, as of December 31, 2026. Bins to be locked at launch. Source: PAHO dengue situation reports.;46218;unresolved;TBD;TBD;TBD;https://www3.paho.org/data/index.php/en/topics/indicadores-dengue.html +bfg_q19;Chikungunya (EU-EEA);How many EU/EEA countries will report locally acquired autochthonous chikungunya in 2026, as of December 31, 2026?;range;EU/EEA countries reporting locally acquired chikungunya in 2026, as of December 31, 2026. Source: ECDC.;46218;unresolved;TBD;TBD;TBD;https://www.ecdc.europa.eu/en/chikungunya-monthly +bfg_q20;Oropouche (Americas);How many Americas countries or territories will report confirmed Oropouche in 2026, as of December 31, 2026?;range;Americas countries or territories reporting confirmed Oropouche in 2026, as of December 31, 2026. Source: PAHO.;46218;unresolved;TBD;TBD;TBD;https://www.paho.org/en/epidemiological-alerts-and-updates +bfg_q21;Filovirus (World);Will WHO declare a new filovirus outbreak distinct from the ongoing Bundibugyo event between July 1 and December 31, 2026?;binary;TRUE if WHO declares a new Marburg or Ebola-species outbreak not linked to the current Bundibugyo event, by December 31, 2026. Source: WHO Disease Outbreak News.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q22;Poliovirus (World);How many wild poliovirus type 1 cases will be reported globally in 2026, as of December 31, 2026?;range;Wild poliovirus type 1 cases reported globally in 2026, as of December 31, 2026. Bins to be locked at launch. Source: GPEI polioeradication.org.;46218;unresolved;TBD;TBD;TBD;https://polioeradication.org/polio-today/polio-now/this-week/ +bfg_q23;SARS-CoV-2 (World);Will WHO designate a new SARS-CoV-2 variant of interest or variant of concern in the second half of 2026?;binary;TRUE on any new VOI or VOC designation between July 1 and December 31, 2026. Source: WHO variant tracking.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/activities/tracking-SARS-CoV-2-variants +bfg_q24;Novel pathogen (World);Will WHO issue a Disease Outbreak News item for an outbreak attributed to a pathogen newly identified in humans between July 1 and December 31, 2026?;binary;TRUE if WHO issues a DON item for an outbreak attributed to a pathogen newly identified in humans as a novel species, strain, or reassortant by December 31, 2026. Committee-adjudicated. Source: WHO DON.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news +bfg_q25;WHO DON (World);How many new WHO Disease Outbreak News items will be published between July 1 and December 31, 2026?;range;Number of new WHO Disease Outbreak News items published between July 1 and December 31, 2026, as of December 31, 2026. Source: WHO DON list.;46218;unresolved;TBD;TBD;TBD;https://www.who.int/emergencies/disease-outbreak-news diff --git a/bioscancast/stages/extraction/custom_scrapers/_who_hub_common.py b/bioscancast/stages/extraction/custom_scrapers/_who_hub_common.py new file mode 100644 index 0000000..e7bc7ff --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/_who_hub_common.py @@ -0,0 +1,150 @@ +"""Shared WHO situation-hub scraper core. + +Several WHO emergency hubs (cholera upsurge, mpox outbreak, ...) render as a +landing page that lists dated *epidemiological update* / *situation report* +item pages under ``who.int/publications/m/item/...``. The landing page itself +carries only narrative context, not the current case/death totals (issue: those +questions extracted 0 records because the injected hub is context-only), while +each dated item page links a ``cdn.who.int/.../situation-reports/*.pdf`` whose +tables hold the cumulative figures. + +This module resolves a hub URL to the latest dated item at-or-before the cutoff +and returns that item's PDF bytes, so the existing PDF parser + Docling table +refiner (the ``situation-reports`` path is on the Docling allowlist) extract the +numbers unchanged. Each WHO source gets a thin ``custom_scrapers/.py`` that +delegates here with its hub keyword. +""" + +from __future__ import annotations + +import logging +import re +from datetime import datetime, timezone +from typing import Optional +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup +from curl_cffi import requests as curl_requests + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.fetcher import FetchResult + +logger = logging.getLogger(__name__) + +_MONTHS = { + "jan": 1, "january": 1, "feb": 2, "february": 2, "mar": 3, "march": 3, + "apr": 4, "april": 4, "may": 5, "jun": 6, "june": 6, "jul": 7, "july": 7, + "aug": 8, "august": 8, "sep": 9, "sept": 9, "september": 9, "oct": 10, + "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12, +} + +# "30 June 2026" / "5 Sept 2026" +_DATE_RE = re.compile( + r"\b(\d{1,2})\s+" + r"(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|" + r"Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)" + r"\s+(\d{4})\b", + flags=re.IGNORECASE, +) + + +def _parse_date(text: str) -> Optional[datetime]: + m = _DATE_RE.search(text or "") + if not m: + return None + month = _MONTHS.get(m.group(2).lower()) + if month is None: + return None + try: + return datetime(int(m.group(3)), month, int(m.group(1)), tzinfo=timezone.utc) + except ValueError: + return None + + +def _get(url: str, cfg: ExtractionConfig) -> Optional[curl_requests.Response]: + try: + return curl_requests.get( + url, + timeout=max(cfg.fetch_timeout_seconds, 30.0), + impersonate=cfg.impersonate, + allow_redirects=True, + ) + except Exception as exc: # noqa: BLE001 - network best-effort + logger.info("WHO hub fetch failed for %s: %s", url, exc) + return None + + +def fetch_who_hub_latest_pdf( + url: str, + keyword: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + **_ignored, +) -> FetchResult | None: + """Resolve a WHO hub to its latest dated item PDF at-or-before the cutoff. + + Returns a ``FetchResult`` holding PDF bytes, or ``None`` to fall back to the + generic fetch of the hub itself. + """ + cfg = config or ExtractionConfig() + fetched_at = datetime.now(timezone.utc) + cutoff = as_of_date or fetched_at + kw = keyword.lower() + + hub = _get(url, cfg) + if hub is None or hub.status_code != 200: + return None + + soup = BeautifulSoup(hub.text, "html.parser") + + # Collect dated item pages whose link text mentions the keyword. + candidates: list[tuple[datetime, str]] = [] + for a in soup.find_all("a", href=True): + item_url = urljoin(url, a["href"]) + if "/publications/" not in item_url: + continue + text = a.get_text(" ", strip=True) + if kw not in text.lower() and kw not in item_url.lower(): + continue + dt = _parse_date(text) or _parse_date(item_url.replace("-", " ")) + if dt is None or dt > cutoff: + continue + candidates.append((dt, item_url)) + + if not candidates: + return None + + # Newest item at-or-before the cutoff. + candidates.sort(key=lambda c: c[0], reverse=True) + _, item_url = candidates[0] + + item = _get(item_url, cfg) + if item is None or item.status_code != 200: + return None + isoup = BeautifulSoup(item.text, "html.parser") + + pdf_url = None + for a in isoup.find_all("a", href=True): + cand = urljoin(item_url, a["href"]) + if ".pdf" in cand.lower(): + pdf_url = cand + break + if not pdf_url: + return None + + pdf = _get(pdf_url, cfg) + if pdf is None or pdf.status_code != 200 or not pdf.content: + return None + if len(pdf.content) > cfg.fetch_max_bytes: + return None + + return FetchResult( + url=pdf_url, + final_url=str(pdf.url), + status_code=pdf.status_code, + content_type="application/pdf", + content_bytes=pdf.content, + fetched_at=fetched_at, + error=None, + ) diff --git a/bioscancast/stages/extraction/custom_scrapers/gpei_polio.py b/bioscancast/stages/extraction/custom_scrapers/gpei_polio.py new file mode 100644 index 0000000..efc58a0 --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/gpei_polio.py @@ -0,0 +1,66 @@ +"""Custom scraper for the GPEI "Polio this week" page (``gpei_polio``). + +The page carries the current wild-poliovirus (WPV1) case narrative and a +year-to-date country table, but the generic HTML parser drops the main content +container (extracting only ~1.4k chars of chrome), so polio questions got 0 +records. This fetches the page, strips nav/script/style, and re-renders the +visible text as minimal HTML the existing parser consumes cleanly — the same +"render to clean HTML" approach the OWID scrapers use. +""" + +from __future__ import annotations + +import html as html_lib +from datetime import datetime, timezone + +from bs4 import BeautifulSoup +from curl_cffi import requests as curl_requests + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.fetcher import FetchResult + +_STRIP_TAGS = ("script", "style", "noscript", "form") + + +def fetch( + url: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + region: str | None = None, + question_text: str | None = None, +) -> FetchResult | None: + cfg = config or ExtractionConfig() + fetched_at = datetime.now(timezone.utc) + try: + resp = curl_requests.get( + url, timeout=max(cfg.fetch_timeout_seconds, 30.0), + impersonate=cfg.impersonate, allow_redirects=True, + ) + except Exception: # noqa: BLE001 - network best-effort + return None + if resp.status_code != 200 or not resp.text: + return None + + soup = BeautifulSoup(resp.text, "html.parser") + for tag in soup(list(_STRIP_TAGS)): + tag.decompose() + text = soup.get_text("\n", strip=True) + # Keep substantive lines (the WPV1/cVDPV narrative + country table rows); + # drop one/two-word nav fragments. + lines = [ln.strip() for ln in text.splitlines() if len(ln.strip()) > 3] + if sum(len(ln) for ln in lines) < 500: + return None # nothing useful recovered; fall back to generic fetch + + paragraphs = "".join(f"

{html_lib.escape(ln)}

" for ln in lines) + rendered = ( + "GPEI Polio this week" + f"

GPEI Polio this week

Source: {html_lib.escape(url)}

" + f"{paragraphs}" + ).encode("utf-8") + + return FetchResult( + url=url, final_url=str(resp.url), status_code=200, + content_type="text/html", content_bytes=rendered, + fetched_at=fetched_at, error=None, + ) diff --git a/bioscancast/stages/extraction/custom_scrapers/paho_dengue.py b/bioscancast/stages/extraction/custom_scrapers/paho_dengue.py new file mode 100644 index 0000000..53777c1 --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/paho_dengue.py @@ -0,0 +1,91 @@ +"""Custom scraper for PAHO dengue (``paho_dengue``). + +The PAHO dengue landing/arbo-portal pages are dashboard gateways that carry only +programme prose, so dengue questions extracted 0 records. PAHO publishes dated +"Epidemiological Update - Dengue in the Americas Region" documents on its +epidemiological-alerts-and-updates listing; each links a PDF whose tables hold +the Americas cumulative suspected/confirmed case totals. This resolves the +listing to the latest dengue update at-or-before the cutoff and returns its PDF. + +Reuses the tested primitives from :mod:`paho_mpox` (curl fetch, date parsing, +PDF-link extraction) so only the listing-filter differs. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from urllib.parse import urljoin + +from bs4 import BeautifulSoup + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.custom_scrapers.paho_mpox import ( + _extract_pdf_url, + _fetch_bytes, + _parse_date, +) +from bioscancast.stages.extraction.fetcher import FetchResult + + +def _collect_dengue_documents(listing_html: str, listing_url: str) -> list[tuple[datetime, str]]: + """Return (date, document_url) for dated dengue updates on the listing.""" + soup = BeautifulSoup(listing_html, "html.parser") + out: list[tuple[datetime, str]] = [] + for a in soup.find_all("a", href=True): + abs_url = urljoin(listing_url, a["href"]) + if "/documents/" not in abs_url: + continue + text = a.get_text(" ", strip=True) + if "dengue" not in (text + abs_url).lower(): + continue + dt = _parse_date(text) or _parse_date(abs_url.replace("-", " ")) + if dt is not None: + out.append((dt, abs_url)) + return out + + +def fetch( + url: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + region: str | None = None, + question_text: str | None = None, +) -> FetchResult | None: + cfg = config or ExtractionConfig() + fetched_at = datetime.now(timezone.utc) + cutoff = as_of_date or fetched_at + + listing_bytes, listing_final, _s, _ct, err = _fetch_bytes(url, cfg) + if err is not None or listing_bytes is None: + return None + docs = _collect_dengue_documents( + listing_bytes.decode("utf-8", errors="replace"), listing_final + ) + docs = [(d, u) for d, u in docs if d <= cutoff] + if not docs: + return None + + docs.sort(key=lambda item: item[0], reverse=True) + _, doc_url = docs[0] + + doc_bytes, doc_final, _s2, _ct2, err2 = _fetch_bytes(doc_url, cfg) + if err2 is not None or doc_bytes is None: + return None + pdf_url = _extract_pdf_url(doc_bytes.decode("utf-8", errors="replace"), doc_final) + if not pdf_url: + return None + + pdf_bytes, pdf_final, pdf_status, pdf_ct, pdf_err = _fetch_bytes(pdf_url, cfg) + if pdf_err is not None or pdf_bytes is None: + return None + + return FetchResult( + url=pdf_url, + final_url=pdf_final, + status_code=pdf_status, + content_type=pdf_ct or "application/pdf", + content_bytes=pdf_bytes, + fetched_at=fetched_at, + error=None, + ) diff --git a/bioscancast/stages/extraction/custom_scrapers/who_cholera.py b/bioscancast/stages/extraction/custom_scrapers/who_cholera.py new file mode 100644 index 0000000..87e0932 --- /dev/null +++ b/bioscancast/stages/extraction/custom_scrapers/who_cholera.py @@ -0,0 +1,30 @@ +"""Custom scraper for the WHO cholera upsurge hub (``who_cholera``). + +The hub page carries only narrative context, so cholera questions extracted 0 +records from it. This resolves the hub to the latest *Multi-country outbreak of +cholera, epidemiological update* PDF at-or-before the cutoff, whose tables carry +the global cumulative case/death totals. +""" + +from __future__ import annotations + +from datetime import datetime + +from bioscancast.stages.extraction.config import ExtractionConfig +from bioscancast.stages.extraction.custom_scrapers._who_hub_common import ( + fetch_who_hub_latest_pdf, +) +from bioscancast.stages.extraction.fetcher import FetchResult + + +def fetch( + url: str, + *, + config: ExtractionConfig | None = None, + as_of_date: datetime | None = None, + region: str | None = None, + question_text: str | None = None, +) -> FetchResult | None: + return fetch_who_hub_latest_pdf( + url, "cholera", config=config, as_of_date=as_of_date + ) diff --git a/bioscancast/stages/searching/source_lookup.py b/bioscancast/stages/searching/source_lookup.py index af5c109..e399a22 100644 --- a/bioscancast/stages/searching/source_lookup.py +++ b/bioscancast/stages/searching/source_lookup.py @@ -42,6 +42,15 @@ "coronavirus": "covid-19", "bird flu": "h5n1", "avian flu": "h5n1", + "avian influenza": "h5n1", + "h5": "h5n1", + "h5nx": "h5n1", + "bundibugyo": "ebola", + "sudan virus": "ebola", + "filovirus": "ebola", + "poliovirus": "polio", + "wild poliovirus": "polio", + "wpv1": "polio", }