diff --git a/docs/research/2026-10-05-district-data-source-verification.md b/docs/research/2026-10-05-district-data-source-verification.md index 38b5f9d..87a9f5b 100644 --- a/docs/research/2026-10-05-district-data-source-verification.md +++ b/docs/research/2026-10-05-district-data-source-verification.md @@ -62,8 +62,12 @@ Census states that the 2024 vintage contains the 119th Congressional District and 2024 state-legislative plans and that legal boundaries/names are as of 2024-01-01. The official directory members use `tl_2024__cd119.zip`, `tl_2024__sldu.zip`, and -`tl_2024__sldl.zip`. File sizes are publisher-listed and are -re-probed by the normal `tiger-bulk-preview` capacity gate before transfer. +`tl_2024__sldl.zip`. The publisher indexes contain **56 CD119 +archives, 52 SLDU archives, and 50 SLDL archives (158 total)**. These counts are +now a fail-closed completeness contract for the frozen 2024 package: discovery +still comes from Census, but a partial directory response cannot silently become +an approved manifest. File sizes are publisher-listed and are re-probed by the +normal `tiger-bulk-preview` capacity gate before transfer. The 2024 record layouts make the join contract explicit: diff --git a/src/opendiscourse_research/ingestion/tiger_bulk.py b/src/opendiscourse_research/ingestion/tiger_bulk.py index a69cd69..1b315c3 100644 --- a/src/opendiscourse_research/ingestion/tiger_bulk.py +++ b/src/opendiscourse_research/ingestion/tiger_bulk.py @@ -39,6 +39,14 @@ ("SLDL", "sldl"), ) +# Verified against the frozen official TIGER2024 publisher indexes on +# 2026-10-05. These are package completeness checks, not guessed jurisdiction +# lists: discovery still reads the Census directory, but a partial response +# may not silently become an approved manifest. +_POLITICAL_EXPECTED_MEMBER_COUNTS = { + 2024: {"cd119": 56, "sldu": 52, "sldl": 50}, +} + def _zcta_layer(year: int) -> str: if year < _ZCTA_CUTOVER_YEAR: @@ -98,6 +106,22 @@ def _national_core_artifacts(year: int) -> list[dict[str, Any]]: ] +def _require_political_member_count( + year: int, kind: str, count: int, index_url: str +) -> None: + """Fail closed when a reviewed frozen publisher index is incomplete.""" + expected = _POLITICAL_EXPECTED_MEMBER_COUNTS.get(year, {}).get(kind) + if expected is None: + raise ValueError( + f"No reviewed TIGER member-count contract exists for {year} {kind}" + ) + if count != expected: + raise ValueError( + f"Census {year} {kind} manifest at {index_url} returned {count} " + f"ZIP members; expected reviewed publisher count {expected}" + ) + + def discover_political_tiger_artifacts(year: int) -> list[dict[str, Any]]: """Enumerate exactly the political ZIPs Census publishes for one reviewed vintage. @@ -125,10 +149,9 @@ def discover_political_tiger_artifacts(year: int) -> list[dict[str, Any]]: match = pattern.fullmatch(filename) if match: matches.append((url, filename, match.group("state"))) - if not matches: - raise ValueError( - f"Census published no {kind} ZIP members at {index_url}" - ) + _require_political_member_count( + year, kind, len(matches), index_url + ) for url, filename, state_fips in sorted(matches): artifacts.append( { diff --git a/tests/test_census_bulk.py b/tests/test_census_bulk.py index 3c2320c..fa54138 100644 --- a/tests/test_census_bulk.py +++ b/tests/test_census_bulk.py @@ -30,6 +30,7 @@ from opendiscourse_research.ingestion.pep_bulk import build_pep_bulk_plan from opendiscourse_research.ingestion.pep_load import _scope as pep_scope from opendiscourse_research.ingestion.tiger_bulk import ( + _require_political_member_count, build_tiger_bulk_plan, tiger_layers, ) @@ -197,6 +198,21 @@ def test_tiger_plan_requires_exact_package(self) -> None: with self.assertRaisesRegex(ValueError, "exactly"): build_tiger_bulk_plan("test", []) + def test_tiger_political_manifest_count_fails_closed_when_partial(self) -> None: + _require_political_member_count( + 2024, + "cd119", + 56, + "https://www2.census.gov/geo/tiger/TIGER2024/CD/", + ) + with self.assertRaisesRegex(ValueError, "expected reviewed publisher count 56"): + _require_political_member_count( + 2024, + "cd119", + 55, + "https://www2.census.gov/geo/tiger/TIGER2024/CD/", + ) + def test_tiger_political_plan_is_bounded_to_119th_and_2024_sld(self) -> None: artifacts = [ {