From 53942993a8af3e341ffc77d2699cc7f258de3abf Mon Sep 17 00:00:00 2001 From: Codewriter90x Date: Thu, 30 Jul 2026 14:48:23 +0200 Subject: [PATCH] Resolve reliability and adoption audit backlog --- .github/ISSUE_TEMPLATE/data_correction.yml | 2 +- .github/ISSUE_TEMPLATE/missing-coordinate.yml | 2 +- .github/ISSUE_TEMPLATE/wrong-locality.yml | 2 +- .github/ISSUE_TEMPLATE/wrong-postal-code.yml | 2 +- .github/workflows/data-pipeline.yml | 17 +- .github/workflows/pages.yml | 8 + .github/workflows/release.yml | 42 +- .github/workflows/source-freshness.yml | 68 ++ .gitignore | 2 + DATASET_PIPELINE.md | 22 +- DATA_LICENSE.md | 2 +- LICENSE | 35 +- NOTICE.md | 2 +- README.md | 12 +- ROADMAP.md | 6 +- SCHEMA.md | 5 + docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md | 79 +++ docs/TYPED_CONTRACT_MIGRATION.md | 28 + pyproject.toml | 20 +- release/v2.0.0.md | 4 +- reports/determinism.json | 1 + reports/reconciliation-backlog.json | 625 ++++++++++++++++++ requirements.txt | 1 + schemas/italian_locations-v4.schema.json | 102 +++ scripts/build_dataset.py | 15 +- scripts/build_geography.py | 97 ++- scripts/build_pages.py | 121 +--- scripts/build_release.py | 1 - scripts/check_determinism.py | 2 +- scripts/dataset_common.py | 3 +- scripts/export_formats.py | 6 +- scripts/export_sql.py | 1 - scripts/export_typed_json.py | 113 ++++ scripts/legacy_comparison.py | 1 - scripts/normalize_legacy.py | 27 +- scripts/pages_model.py | 108 +++ scripts/project_metadata.py | 1 - scripts/reconcile_sources.py | 3 +- scripts/reconciliation_backlog.py | 124 ++++ scripts/source_data.py | 1 - scripts/typed_contract.py | 33 + scripts/update_sources.py | 174 +++++ scripts/validate_dataset.py | 3 +- scripts/validate_milestone1.py | 1 - scripts/validate_release.py | 1 - scripts/validate_site.py | 158 +++++ scripts/validators/common.py | 1 - scripts/validators/coordinates.py | 1 - scripts/validators/formats.py | 2 +- scripts/validators/provenance.py | 19 +- scripts/validators/schema.py | 1 - scripts/validators/territory.py | 1 + site/assets/app.js | 73 +- site/assets/styles.css | 15 + site/index.html | 13 +- tests/test_coordinates.py | 1 - tests/test_integrity.py | 17 +- tests/test_pages.py | 20 +- tests/test_quality_gate.py | 1 - tests/test_release.py | 3 +- tests/test_schema.py | 1 - tests/test_source_updates.py | 78 +++ tests/test_typed_contract.py | 63 ++ 63 files changed, 2172 insertions(+), 221 deletions(-) create mode 100644 .github/workflows/source-freshness.yml create mode 100644 docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md create mode 100644 docs/TYPED_CONTRACT_MIGRATION.md create mode 100644 reports/reconciliation-backlog.json create mode 100644 schemas/italian_locations-v4.schema.json create mode 100644 scripts/export_typed_json.py create mode 100644 scripts/pages_model.py create mode 100644 scripts/reconciliation_backlog.py create mode 100644 scripts/typed_contract.py create mode 100644 scripts/update_sources.py create mode 100644 scripts/validate_site.py create mode 100644 tests/test_source_updates.py create mode 100644 tests/test_typed_contract.py diff --git a/.github/ISSUE_TEMPLATE/data_correction.yml b/.github/ISSUE_TEMPLATE/data_correction.yml index 4870aed..dd9bc71 100644 --- a/.github/ISSUE_TEMPLATE/data_correction.yml +++ b/.github/ISSUE_TEMPLATE/data_correction.yml @@ -13,7 +13,7 @@ body: id: location_id attributes: label: Identificativo del record - description: location_id oppure legacy_uuid. + description: location_id del record corrente. placeholder: IT-COM-058091 validations: required: true diff --git a/.github/ISSUE_TEMPLATE/missing-coordinate.yml b/.github/ISSUE_TEMPLATE/missing-coordinate.yml index 52c169a..de40242 100644 --- a/.github/ISSUE_TEMPLATE/missing-coordinate.yml +++ b/.github/ISSUE_TEMPLATE/missing-coordinate.yml @@ -14,7 +14,7 @@ body: id: location_id attributes: label: Identificativo - description: "`location_id` o `legacy_uuid`." + description: "`location_id` del record corrente." validations: required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/wrong-locality.yml b/.github/ISSUE_TEMPLATE/wrong-locality.yml index 7e4d16d..ff04c08 100644 --- a/.github/ISSUE_TEMPLATE/wrong-locality.yml +++ b/.github/ISSUE_TEMPLATE/wrong-locality.yml @@ -14,7 +14,7 @@ body: id: location_id attributes: label: Identificativo - description: "`location_id` o `legacy_uuid` del record." + description: "`location_id` del record corrente." placeholder: IT-LOC-e6b0bc3e-1acf-5509-8923-238575fc9025 validations: required: true diff --git a/.github/ISSUE_TEMPLATE/wrong-postal-code.yml b/.github/ISSUE_TEMPLATE/wrong-postal-code.yml index 15a809c..94f2b49 100644 --- a/.github/ISSUE_TEMPLATE/wrong-postal-code.yml +++ b/.github/ISSUE_TEMPLATE/wrong-postal-code.yml @@ -14,7 +14,7 @@ body: id: location_id attributes: label: Identificativo - description: "`location_id` o `legacy_uuid`." + description: "`location_id` del record corrente." validations: required: true - type: input diff --git a/.github/workflows/data-pipeline.yml b/.github/workflows/data-pipeline.yml index 2d6c89c..905037a 100644 --- a/.github/workflows/data-pipeline.yml +++ b/.github/workflows/data-pipeline.yml @@ -35,13 +35,19 @@ jobs: - name: Lint Python sources and tests run: ruff check scripts tests - - name: Type-check the validation boundary + - name: Type-check all maintained Python scripts run: mypy - - name: Enforce validator test coverage + - name: Enforce maintained-script test coverage run: | coverage run -m unittest discover -s tests + coverage run --append scripts/build_dataset.py + coverage run --append scripts/build_pages.py + coverage run --append scripts/build_release.py + coverage run --append scripts/validate_release.py + coverage run --append scripts/build_geography.py --check coverage report + coverage xml python-compatibility: name: Python ${{ matrix.python-version }} @@ -109,9 +115,15 @@ jobs: - name: Test Pages search and map logic run: node --test tests/pages_core.test.mjs + - name: Verify committed geographic base + run: python scripts/build_geography.py --check + - name: Build deterministic GitHub Pages site run: python scripts/build_pages.py + - name: Validate Pages accessibility, links and budgets + run: python scripts/validate_site.py dist/pages + - name: Build versioned release assets run: python scripts/build_release.py @@ -135,6 +147,7 @@ jobs: reports/release-diff.json reports/build-metadata.json reports/legacy-comparison.json + reports/reconciliation-backlog.json reports/export-manifest.json reports/determinism.json reports/quality-validation.json diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index d257023..d19121c 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -8,6 +8,7 @@ on: - "data/italian_locations.csv" - "project.json" - "scripts/build_pages.py" + - "scripts/build_geography.py" - "scripts/project_metadata.py" - "site/**" - "tests/pages_core.test.mjs" @@ -21,6 +22,7 @@ on: - "data/italian_locations.csv" - "project.json" - "scripts/build_pages.py" + - "scripts/build_geography.py" - "scripts/project_metadata.py" - "site/**" - "tests/pages_core.test.mjs" @@ -61,9 +63,15 @@ jobs: - name: Test deterministic Pages build run: python -m unittest discover -s tests -p "test_pages.py" -v + - name: Verify committed geographic base + run: python scripts/build_geography.py --check + - name: Build GitHub Pages artifact run: python scripts/build_pages.py --output _site + - name: Validate accessibility, local links and performance budgets + run: python scripts/validate_site.py _site + - name: Configure GitHub Pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7d28b7..b3f4ec1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,8 +19,18 @@ jobs: - name: Check out tagged commit uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false + - name: Require tagged commit to be integrated in the default branch + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + git fetch --no-tags origin "${DEFAULT_BRANCH}" + git merge-base --is-ancestor \ + "${GITHUB_SHA}" \ + "origin/${DEFAULT_BRANCH}" + - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -104,7 +114,7 @@ jobs: ! gh release view "${GITHUB_REF_NAME}" --repo "${GITHUB_REPOSITORY}" - - name: Create GitHub release + - name: Create draft GitHub release env: GH_TOKEN: ${{ github.token }} run: | @@ -113,6 +123,7 @@ jobs: gh release create "${GITHUB_REF_NAME}" \ --repo "${GITHUB_REPOSITORY}" \ --verify-tag \ + --draft \ --prerelease \ --title "Italian Cities ${GITHUB_REF_NAME}" \ --notes-file "release/${GITHUB_REF_NAME}.md" @@ -120,11 +131,12 @@ jobs: gh release create "${GITHUB_REF_NAME}" \ --repo "${GITHUB_REPOSITORY}" \ --verify-tag \ + --draft \ --title "Italian Cities ${GITHUB_REF_NAME}" \ --notes-file "release/${GITHUB_REF_NAME}.md" fi - - name: Upload immutable release assets + - name: Upload release assets to the draft env: GH_TOKEN: ${{ github.token }} run: | @@ -139,3 +151,29 @@ jobs: "dist/${GITHUB_REF_NAME}/italian_locations.sql" \ "dist/${GITHUB_REF_NAME}/SHA256SUMS" \ --repo "${GITHUB_REPOSITORY}" + + - name: Verify draft assets before publication + env: + GH_TOKEN: ${{ github.token }} + run: | + expected_assets="$( + find "dist/${GITHUB_REF_NAME}" -maxdepth 1 -type f \ + -exec basename {} \; | + LC_ALL=C sort + )" + published_assets="$( + gh release view "${GITHUB_REF_NAME}" \ + --repo "${GITHUB_REPOSITORY}" \ + --json assets \ + --jq '.assets[].name' | + LC_ALL=C sort + )" + test "${expected_assets}" = "${published_assets}" + + - name: Publish the complete release + env: + GH_TOKEN: ${{ github.token }} + run: >- + gh release edit "${GITHUB_REF_NAME}" + --repo "${GITHUB_REPOSITORY}" + --draft=false diff --git a/.github/workflows/source-freshness.yml b/.github/workflows/source-freshness.yml new file mode 100644 index 0000000..9597c61 --- /dev/null +++ b/.github/workflows/source-freshness.yml @@ -0,0 +1,68 @@ +name: Source freshness + +on: + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + check: + name: Check canonical upstream sources + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + + - name: Install runtime dependencies + run: python -m pip install -r requirements.txt + + - name: Compare declared snapshots with upstream artifacts + run: >- + python scripts/update_sources.py + --check + --report "${RUNNER_TEMP}/source-freshness.json" + + - name: Preserve freshness evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: source-freshness + path: ${{ runner.temp }}/source-freshness.json + retention-days: 30 + + - name: Open one tracking issue when an update is available + env: + GH_TOKEN: ${{ github.token }} + REPORT: ${{ runner.temp }}/source-freshness.json + run: | + updates="$(python -c 'import json, os; print(", ".join(json.load(open(os.environ["REPORT"]))["updates_available"]))')" + if [ -z "${updates}" ]; then + exit 0 + fi + existing="$( + gh issue list \ + --repo "${GITHUB_REPOSITORY}" \ + --state open \ + --search '"Upstream source update available" in:title' \ + --json number \ + --jq 'length' + )" + if [ "${existing}" -eq 0 ]; then + gh issue create \ + --repo "${GITHUB_REPOSITORY}" \ + --title "Upstream source update available" \ + --label "source-update" \ + --body "The scheduled freshness check detected changes for: ${updates}. Review the workflow artifact and use scripts/update_sources.py to stage a dated snapshot. No source or generated dataset was modified automatically." + fi diff --git a/.gitignore b/.gitignore index b520462..c8c9b26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ sources/cache/* !sources/cache/.gitkeep +.DS_Store +.venv/ __pycache__/ *.pyc *.tmp diff --git a/DATASET_PIPELINE.md b/DATASET_PIPELINE.md index 0779a35..70207e7 100644 --- a/DATASET_PIPELINE.md +++ b/DATASET_PIPELINE.md @@ -72,6 +72,8 @@ canonica. `scripts/export_sql.py` crea lo script SQLite-compatible. - `reports/build-metadata.json`: fonti, qualità, readiness e statistiche; - `reports/legacy-comparison.json`: confronto storico limitato e deterministico; - `reports/release-diff.json`: confronto logico con v1.1.0; +- `reports/reconciliation-backlog.json`: segmenti da revisionare per regione, + provincia e codice territoriale sorgente; - `reports/export-manifest.json`: digest degli export; - `reports/determinism.json`: firme di due build; - `reports/quality-validation.json`: gate strutturali separati dalla readiness. @@ -114,7 +116,25 @@ a commit SHA immutabili. ## Aggiornamento fonti -1. acquisire un nuovo snapshot dalla stessa origine autorizzata; +Controllare senza modificare il repository: + +```bash +python scripts/update_sources.py --check +``` + +Per preparare esplicitamente un nuovo snapshot in un percorso datato: + +```bash +python scripts/update_sources.py \ + --download geonames_postal_codes \ + --output sources/cache/IT-new.zip +``` + +Lo script rifiuta di sovrascrivere file esistenti e non modifica il manifest. +Il workflow settimanale `Source freshness` apre una sola issue quando rileva +un checksum upstream differente; non esegue commit né rigenera il dataset. + +1. acquisire o preparare un nuovo snapshot dalla stessa origine autorizzata; 2. conservarlo sotto un percorso datato; 3. aggiornare manifest, checksum, timestamp e riferimento; 4. modificare la trasformazione se il formato cambia; diff --git a/DATA_LICENSE.md b/DATA_LICENSE.md index d9d223c..defbad8 100644 --- a/DATA_LICENSE.md +++ b/DATA_LICENSE.md @@ -1,6 +1,6 @@ # Dataset licensing -Il candidato v2 combina esclusivamente due fonti canoniche attribuite: +La release v2 combina esclusivamente due fonti canoniche attribuite: | Componente | Licenza | | --- | --- | diff --git a/LICENSE b/LICENSE index 3aa992d..f74ed29 100644 --- a/LICENSE +++ b/LICENSE @@ -1,28 +1,37 @@ -Italian Cities licensing notice +Italian Cities multi-license notice Repository code and original documentation ------------------------------------------ -The repository's original code and documentation are offered under CC0 1.0 -Universal to the extent that the repository owner holds the applicable rights. -The complete legal text is available at: +The repository's original code and documentation are dedicated under CC0 1.0 +Universal to the extent that the repository owner holds the applicable +rights. The complete legal text is available at: LICENSES/CC0-1.0.txt Dataset and third-party inputs ------------------------------ -The CC0 dedication above does not apply to third-party source material or -prove that legacy geographic and postal values may be relicensed under CC0. -The canonical dataset combines: +The CC0 dedication above does not apply to third-party source material or to +the canonical dataset derived from it. Italian Cities v2 combines: -- legacy values whose upstream provenance and rights are not fully known; -- ISTAT-derived classifications and codes that require attribution under - CC BY 4.0. +- ISTAT municipality classifications and codes, licensed under CC BY 4.0; +- GeoNames Postal Codes for Italy, licensed under CC BY 4.0. -The dataset is therefore distributed with provisional licensing status and -must not be represented as entirely CC0. Read DATA_LICENSE.md, NOTICE.md and -DATA_SOURCES.md before reuse or redistribution. +Reuse of the canonical dataset therefore requires attribution to both ISTAT +and GeoNames under CC BY 4.0. It must not be represented as entirely CC0. +Read DATA_LICENSE.md, NOTICE.md and DATA_SOURCES.md before reuse or +redistribution. + +Historical material +------------------- + +Files under legacy/ are retained only as a historical archive and bounded +comparison input. They do not contribute to the v2 canonical dataset, +derived formats, GitHub Pages or release assets. Their upstream provenance +and rights have not been independently established by the current pipeline; +their presence does not extend the repository CC0 dedication to their +contents. No warranty ----------- diff --git a/NOTICE.md b/NOTICE.md index 8c22ffb..c5251b6 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -13,6 +13,6 @@ determined or estimated algorithmically; the source `accuracy` field is preserved by this project. Historical files under `legacy/` do not contribute to the v2 canonical -dataset, derived formats, release candidate or GitHub Pages data. They are +dataset, derived formats, published release or GitHub Pages data. They are used only for a bounded investigative comparison, which cannot establish their original provenance. diff --git a/README.md b/README.md index 0a5b03e..738fd59 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,11 @@ coordinate, costruito con una pipeline **clean room**. Il tag e la prerelease [`v2.0.0`](https://github.com/Codewriter90x/Italian_Cities/releases/tag/v2.0.0) -sono pubblicati con asset immutabili e checksum SHA-256. +sono pubblicati con tag protetto, attestazioni e checksum SHA-256. Gli asset +della v2.0.0 non sono retroattivamente immutabili; il workflow applica il +processo draft–upload–publish alle release future. L'impostazione GitHub +**Immutable releases** è attiva dal 30 luglio 2026 e si applica soltanto alle +release pubblicate dopo l'attivazione. ## Fonti canoniche @@ -191,11 +195,14 @@ anche senza CAP. ## Licenze e attribuzione +- Il codice e la documentazione originali del repository sono dedicati CC0 + 1.0 nei limiti dei diritti detenuti. - © Istituto nazionale di statistica (ISTAT), dati riutilizzati secondo CC BY 4.0. - GeoNames Postal Codes, © GeoNames, CC BY 4.0, . +Il dataset canonico derivato non è CC0 e richiede entrambe le attribuzioni. GeoNames non è una fonte ufficiale di Poste Italiane. Vedere [`DATA_LICENSE.md`](DATA_LICENSE.md), [`NOTICE.md`](NOTICE.md) e [`DATA_SOURCES.md`](DATA_SOURCES.md). @@ -206,3 +213,6 @@ Ogni aggiornamento richiede uno snapshot immutabile, checksum, licenza, attribuzione, build deterministica e una PR verde. Per una correzione aprire un [issue form](https://github.com/Codewriter90x/Italian_Cities/issues/new/choose) indicando record, fonte, data e licenza. + +Un esempio completo del processo è disponibile in +[`docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md`](docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md). diff --git a/ROADMAP.md b/ROADMAP.md index 21153fd..42926ff 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,7 +4,7 @@ Italian Cities evolves by improving provenance and reproducibility before claiming broader coverage. The roadmap is public, but dates are intentionally not promised until the required sources and licences are available. -## Current: v2.0.0 clean-room prerelease candidate +## Current: v2.0.0 clean-room prerelease The current line replaces the unresolved legacy source with: @@ -18,8 +18,8 @@ The current line replaces the unresolved legacy source with: - searchable GitHub Pages with shareable filters, ISTAT regional boundaries and an accessible coverage table. -It remains a prerelease because GeoNames is not Poste Italiane and CAP and -coordinates are not operationally certified. +It is published as a prerelease because GeoNames is not Poste Italiane and +CAP and coordinates are not operationally certified. ## Next: v2.1.0 — reviewed reconciliation diff --git a/SCHEMA.md b/SCHEMA.md index aea4d82..d7d580c 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -125,3 +125,8 @@ foglio `Dataset Info` con attribuzione e warning. CSV, JSON, XLSX, SQLite e SQL devono avere lo stesso ordine di campi, record e digest semantico. Due build consecutive devono essere identiche. + +La tabella e gli asset v2 originali restano invariati. Una preview separata +del contratto tipizzato della prossima major può essere generata senza +sovrascriverli; è descritta in +[`docs/TYPED_CONTRACT_MIGRATION.md`](docs/TYPED_CONTRACT_MIGRATION.md). diff --git a/docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md b/docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md new file mode 100644 index 0000000..a2b6546 --- /dev/null +++ b/docs/SOURCE_BACKED_CORRECTION_EXAMPLE.md @@ -0,0 +1,79 @@ +# Example: source-backed data correction + +Generated CSV, JSON, XLSX, SQLite and SQL files are never edited directly. +This example shows the evidence and review trail required for a correction. +It is procedural and does not assert that a real Italian Cities record is +wrong. + +## 1. Identify the canonical record + +Record the stable identifier and current values from the generated CSV: + +```text +location_id: IT-COM-000000 +field: postal_code +current value: 00000 +proposed value: 00001 +``` + +Do not use row numbers: sorting and source snapshots can change them. + +## 2. Attach reusable evidence + +The issue must provide: + +- a public source URL or exact official publication reference; +- the source publisher; +- its reference date; +- the licence and required attribution; +- the exact source record, page or table supporting the change; +- whether the old value is wrong, obsolete or merely incomplete. + +A search result, personal observation or another generated aggregator is not +sufficient evidence. Public Nominatim must not be queried systematically. + +## 3. Reproduce the correction in the pipeline + +The pull request updates a declared source snapshot or a narrowly scoped, +reviewable transformation. It must not patch `data/*.csv` directly. + +For an upstream snapshot change: + +```bash +python scripts/update_sources.py \ + --download geonames_postal_codes \ + --output sources/cache/IT-YYYY-MM-DD.zip +``` + +After inspecting the staged file, update `sources/manifest.json` explicitly +with its dated path, SHA-256, reference date, licence and attribution. Then: + +```bash +python scripts/build_dataset.py +python scripts/check_determinism.py +python scripts/validate_dataset.py +``` + +## 4. Review the complete impact + +The pull request must explain: + +- which source records changed; +- all generated rows affected by the transformation; +- changes in `reports/release-diff.json`; +- changes in `reports/reconciliation-backlog.json`; +- whether coverage or operational-readiness claims changed. + +A correction is rejected if it silently promotes an ambiguous match, removes +provenance, changes unrelated records or cannot be reproduced from the +declared evidence. + +## 5. Acceptance checklist + +- [ ] issue form includes source, date, licence and attribution; +- [ ] no generated file was manually edited; +- [ ] source checksum validation passes; +- [ ] schema, integrity, territory and coordinate tests pass; +- [ ] two builds are deterministic; +- [ ] all export formats remain semantically equivalent; +- [ ] reviewer can trace the canonical row back to source record IDs. diff --git a/docs/TYPED_CONTRACT_MIGRATION.md b/docs/TYPED_CONTRACT_MIGRATION.md new file mode 100644 index 0000000..b4ce68d --- /dev/null +++ b/docs/TYPED_CONTRACT_MIGRATION.md @@ -0,0 +1,28 @@ +# Typed export contract migration + +The published v2.0.0 assets remain byte-for-byte unchanged. In schema 3.0.0, +CSV, JSON, SQLite and SQL expose source columns as strings and use an empty +string for missing values. + +The next major schema is designed as `4.0.0`: + +| Field | Schema 3 | Schema 4 | +| --- | --- | --- | +| `latitude`, `longitude` | string, `""` when missing | number or `null` | +| `coordinate_accuracy` | string, `""` when missing | integer or `null` | +| identifiers and CAP | string | string | + +Consumers can test the preview without changing committed release assets: + +```bash +python scripts/export_typed_json.py \ + --output dist/italian_locations.typed.json \ + --sqlite-output dist/italian_locations.typed.sqlite +``` + +The preview SQLite table uses `REAL`, `INTEGER` and `NULL` directly. The +original v2 JSON, SQLite table and SQL script stay unchanged. + +The default JSON/table contract will switch only in a new major release. That +release must include a migration note, cross-format type assertions and a new +immutable release bundle; v2 assets must never be overwritten. diff --git a/pyproject.toml b/pyproject.toml index 310a877..c977ffa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,23 +3,17 @@ target-version = "py311" line-length = 88 [tool.ruff.lint] -select = ["E4", "E7", "E9", "F"] +select = ["B", "C4", "E4", "E7", "E9", "F", "I", "SIM", "UP"] [tool.ruff.lint.per-file-ignores] "tests/*.py" = ["E402"] [tool.mypy] python_version = "3.11" -files = [ - "scripts/build_dataset.py", - "scripts/legacy_comparison.py", - "scripts/reconcile_sources.py", - "scripts/source_data.py", - "scripts/validate_dataset.py", - "scripts/validators", -] +files = ["scripts"] check_untyped_defs = true disallow_untyped_defs = true +ignore_missing_imports = true no_implicit_optional = true strict_equality = true warn_redundant_casts = true @@ -29,9 +23,13 @@ follow_imports = "silent" [tool.coverage.run] branch = true -source = ["scripts/validators"] +source = ["scripts"] +omit = [ + "scripts/normalize_legacy.py", + "scripts/validate_milestone1.py", +] [tool.coverage.report] -fail_under = 75 +fail_under = 80 show_missing = true skip_covered = true diff --git a/release/v2.0.0.md b/release/v2.0.0.md index 3dabf2b..0fa4928 100644 --- a/release/v2.0.0.md +++ b/release/v2.0.0.md @@ -18,7 +18,7 @@ GeoNames coordinates may be estimated algorithmically; the original `structural_quality` is `passed`, while `operational_data_readiness` is `experimental_non_official`. -## Candidate statistics +## Published statistics - 7,894 ISTAT municipalities; - 18,415 GeoNames source records; @@ -55,4 +55,4 @@ The prepared bundle contains: - `italian_locations.sql` - `SHA256SUMS` -Published after explicit maintainer approval. +Published on 30 July 2026 after explicit maintainer approval. diff --git a/reports/determinism.json b/reports/determinism.json index 249c1ae..995ed5f 100644 --- a/reports/determinism.json +++ b/reports/determinism.json @@ -17,6 +17,7 @@ "localities_csv": "8071405388fdf96f4f46013d3ff83e3fd4768ea05d77629f6fdc99ab1c950747", "municipalities_csv": "905af234a403be1750d6e697de04d812766ceb829a62aceaa8796970b461bfd6", "postal_codes_csv": "2059b5e36af6a21a6f73efd36945b67185500b5e2c15eb599c0fd1a8139efe9a", + "reconciliation_backlog": "8c3a13633cb1d21362581bb64239ccf90d6113b819d6d495bb95909c8081eca8", "release_diff_report": "ed57759aabe4e79b6c7542eccaaa27248a9c39d74e5ca7edfb666901fd73d610" }, "semantic_sha256": { diff --git a/reports/reconciliation-backlog.json b/reports/reconciliation-backlog.json new file mode 100644 index 0000000..995f252 --- /dev/null +++ b/reports/reconciliation-backlog.json @@ -0,0 +1,625 @@ +{ + "dataset_version": "v2.0.0", + "missing_municipality_match_by_region": [ + { + "name": "Sardegna", + "records": 103 + }, + { + "name": "Calabria", + "records": 100 + }, + { + "name": "Lombardia", + "records": 79 + }, + { + "name": "Trentino-Alto Adige/Südtirol", + "records": 32 + }, + { + "name": "Piemonte", + "records": 22 + }, + { + "name": "Veneto", + "records": 19 + }, + { + "name": "Emilia-Romagna", + "records": 12 + }, + { + "name": "Marche", + "records": 6 + }, + { + "name": "Friuli-Venezia Giulia", + "records": 5 + }, + { + "name": "Toscana", + "records": 5 + }, + { + "name": "Liguria", + "records": 4 + }, + { + "name": "Sicilia", + "records": 3 + }, + { + "name": "Puglia", + "records": 2 + }, + { + "name": "Abruzzo", + "records": 1 + }, + { + "name": "Campania", + "records": 1 + }, + { + "name": "Lazio", + "records": 1 + }, + { + "name": "Umbria", + "records": 1 + } + ], + "policy": { + "automatic_fuzzy_promotion": false, + "evidence_required": [ + "source", + "reference_date", + "license", + "reproducible_transformation" + ], + "public_nominatim_bulk_geocoding": false + }, + "review_order": [ + { + "priority": 1, + "reason": "Historical source codes need explicit aliases.", + "segment": "noncurrent_source_province_codes" + }, + { + "priority": 2, + "reason": "Municipalities remain canonical but lack CAP/coordinates.", + "segment": "municipalities_without_geonames_postal_match" + }, + { + "priority": 3, + "reason": "Review high-volume segments using licensed evidence.", + "segment": "highest_unreconciled_regions" + } + ], + "schema_version": "3.0.0", + "status": "review_required", + "summary": { + "municipalities": 7894, + "municipalities_without_geonames_postal_match": 396, + "noncurrent_source_province_codes": [ + "SU" + ], + "reconciliation_outcomes": { + "unmatched_no_parent": 10270 + }, + "unreconciled_localities": 10270 + }, + "unreconciled_by_province": [ + { + "name": "Bolzano/Bozen (BZ)", + "records": 639 + }, + { + "name": "Trento (TN)", + "records": 301 + }, + { + "name": "Salerno (SA)", + "records": 249 + }, + { + "name": "Modena (MO)", + "records": 241 + }, + { + "name": "Reggio Calabria (RC)", + "records": 240 + }, + { + "name": "Perugia (PG)", + "records": 232 + }, + { + "name": "Firenze (FI)", + "records": 199 + }, + { + "name": "Udine (UD)", + "records": 185 + }, + { + "name": "Messina (ME)", + "records": 184 + }, + { + "name": "Parma (PR)", + "records": 176 + }, + { + "name": "Bologna (BO)", + "records": 173 + }, + { + "name": "Treviso (TV)", + "records": 168 + }, + { + "name": "Brescia (BS)", + "records": 167 + }, + { + "name": "Cosenza (CS)", + "records": 167 + }, + { + "name": "Lucca (LU)", + "records": 158 + }, + { + "name": "Verona (VR)", + "records": 157 + }, + { + "name": "Roma (RM)", + "records": 155 + }, + { + "name": "Cuneo (CN)", + "records": 147 + }, + { + "name": "Torino (TO)", + "records": 146 + }, + { + "name": "Sassari (SS)", + "records": 142 + }, + { + "name": "L'Aquila (AQ)", + "records": 141 + }, + { + "name": "Vicenza (VI)", + "records": 136 + }, + { + "name": "Grosseto (GR)", + "records": 133 + }, + { + "name": "Venezia (VE)", + "records": 133 + }, + { + "name": "Caserta (CE)", + "records": 131 + }, + { + "name": "Genova (GE)", + "records": 125 + }, + { + "name": "Arezzo (AR)", + "records": 122 + }, + { + "name": "Napoli (NA)", + "records": 122 + }, + { + "name": "Pisa (PI)", + "records": 122 + }, + { + "name": "Belluno (BL)", + "records": 121 + }, + { + "name": "Teramo (TE)", + "records": 119 + }, + { + "name": "Reggio nell'Emilia (RE)", + "records": 117 + }, + { + "name": "Padova (PD)", + "records": 113 + }, + { + "name": "Pistoia (PT)", + "records": 111 + }, + { + "name": "Milano (MI)", + "records": 110 + }, + { + "name": "Forlì-Cesena (FC)", + "records": 108 + }, + { + "name": "Mantova (MN)", + "records": 105 + }, + { + "name": "Pesaro e Urbino (PU)", + "records": 105 + }, + { + "name": "Siena (SI)", + "records": 101 + }, + { + "name": "Ancona (AN)", + "records": 99 + }, + { + "name": "Piacenza (PC)", + "records": 99 + }, + { + "name": "Ferrara (FE)", + "records": 98 + }, + { + "name": "Bergamo (BG)", + "records": 96 + }, + { + "name": "Como (CO)", + "records": 95 + }, + { + "name": "Rieti (RI)", + "records": 95 + }, + { + "name": "La Spezia (SP)", + "records": 93 + }, + { + "name": "Ravenna (RA)", + "records": 92 + }, + { + "name": "Terni (TR)", + "records": 91 + }, + { + "name": "Monza e della Brianza (MB)", + "records": 90 + }, + { + "name": "Frosinone (FR)", + "records": 87 + }, + { + "name": "Massa-Carrara (MS)", + "records": 87 + }, + { + "name": "Varese (VA)", + "records": 87 + }, + { + "name": "Avellino (AV)", + "records": 86 + }, + { + "name": "Alessandria (AL)", + "records": 84 + }, + { + "name": "Catania (CT)", + "records": 82 + }, + { + "name": "Pordenone (PN)", + "records": 81 + }, + { + "name": "Catanzaro (CZ)", + "records": 80 + }, + { + "name": "Sondrio (SO)", + "records": 77 + }, + { + "name": "Rovigo (RO)", + "records": 76 + }, + { + "name": "Sud Sardegna (SU)", + "records": 76 + }, + { + "name": "Latina (LT)", + "records": 75 + }, + { + "name": "Nuoro (NU)", + "records": 74 + }, + { + "name": "Palermo (PA)", + "records": 70 + }, + { + "name": "Trapani (TP)", + "records": 70 + }, + { + "name": "Asti (AT)", + "records": 67 + }, + { + "name": "Lecce (LE)", + "records": 66 + }, + { + "name": "Potenza (PZ)", + "records": 65 + }, + { + "name": "Rimini (RN)", + "records": 63 + }, + { + "name": "Chieti (CH)", + "records": 62 + }, + { + "name": "Macerata (MC)", + "records": 62 + }, + { + "name": "Livorno (LI)", + "records": 61 + }, + { + "name": "Pavia (PV)", + "records": 61 + }, + { + "name": "Valle d'Aosta/Vallée d'Aoste (AO)", + "records": 61 + }, + { + "name": "Cremona (CR)", + "records": 59 + }, + { + "name": "Vibo Valentia (VV)", + "records": 59 + }, + { + "name": "Verbano-Cusio-Ossola (VB)", + "records": 57 + }, + { + "name": "Savona (SV)", + "records": 53 + }, + { + "name": "Bari (BA)", + "records": 48 + }, + { + "name": "Biella (BI)", + "records": 48 + }, + { + "name": "Ascoli Piceno (AP)", + "records": 47 + }, + { + "name": "Lecco (LC)", + "records": 47 + }, + { + "name": "Imperia (IM)", + "records": 44 + }, + { + "name": "Pescara (PE)", + "records": 43 + }, + { + "name": "Viterbo (VT)", + "records": 42 + }, + { + "name": "Benevento (BN)", + "records": 38 + }, + { + "name": "Foggia (FG)", + "records": 36 + }, + { + "name": "Prato (PO)", + "records": 36 + }, + { + "name": "Gorizia (GO)", + "records": 35 + }, + { + "name": "Novara (NO)", + "records": 34 + }, + { + "name": "Oristano (OR)", + "records": 31 + }, + { + "name": "Fermo (FM)", + "records": 30 + }, + { + "name": "Isernia (IS)", + "records": 27 + }, + { + "name": "Vercelli (VC)", + "records": 27 + }, + { + "name": "Brindisi (BR)", + "records": 22 + }, + { + "name": "Lodi (LO)", + "records": 22 + }, + { + "name": "Taranto (TA)", + "records": 21 + }, + { + "name": "Cagliari (CA)", + "records": 20 + }, + { + "name": "Siracusa (SR)", + "records": 20 + }, + { + "name": "Trieste (TS)", + "records": 20 + }, + { + "name": "Crotone (KR)", + "records": 19 + }, + { + "name": "Matera (MT)", + "records": 17 + }, + { + "name": "Agrigento (AG)", + "records": 16 + }, + { + "name": "Ragusa (RG)", + "records": 16 + }, + { + "name": "Campobasso (CB)", + "records": 11 + }, + { + "name": "Enna (EN)", + "records": 9 + }, + { + "name": "Caltanissetta (CL)", + "records": 5 + }, + { + "name": "Barletta-Andria-Trani (BT)", + "records": 2 + } + ], + "unreconciled_by_region": [ + { + "name": "Emilia-Romagna", + "records": 1167 + }, + { + "name": "Toscana", + "records": 1130 + }, + { + "name": "Lombardia", + "records": 1016 + }, + { + "name": "Trentino-Alto Adige/Südtirol", + "records": 940 + }, + { + "name": "Veneto", + "records": 904 + }, + { + "name": "Campania", + "records": 626 + }, + { + "name": "Piemonte", + "records": 610 + }, + { + "name": "Calabria", + "records": 565 + }, + { + "name": "Sicilia", + "records": 472 + }, + { + "name": "Lazio", + "records": 454 + }, + { + "name": "Abruzzo", + "records": 365 + }, + { + "name": "Marche", + "records": 343 + }, + { + "name": "Sardegna", + "records": 343 + }, + { + "name": "Umbria", + "records": 323 + }, + { + "name": "Friuli-Venezia Giulia", + "records": 321 + }, + { + "name": "Liguria", + "records": 315 + }, + { + "name": "Puglia", + "records": 195 + }, + { + "name": "Basilicata", + "records": 82 + }, + { + "name": "Valle d'Aosta/Vallée d'Aoste", + "records": 61 + }, + { + "name": "Molise", + "records": 38 + } + ] +} diff --git a/requirements.txt b/requirements.txt index dd5bc83..ead0044 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ XlsxWriter==3.2.9 +truststore==0.10.4 diff --git a/schemas/italian_locations-v4.schema.json b/schemas/italian_locations-v4.schema.json new file mode 100644 index 0000000..7f18a21 --- /dev/null +++ b/schemas/italian_locations-v4.schema.json @@ -0,0 +1,102 @@ +{ + "$id": "https://codewriter90x.github.io/Italian_Cities/schemas/italian_locations-v4.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "fields": { + "items": { + "type": "string" + }, + "type": "array" + }, + "metadata": { + "additionalProperties": true, + "properties": { + "contract_status": { + "const": "next_major_preview" + }, + "schema_version": { + "const": "4.0.0" + } + }, + "required": [ + "schema_version", + "contract_status" + ], + "type": "object" + }, + "rows": { + "items": { + "$ref": "#/$defs/location" + }, + "type": "array" + } + }, + "required": [ + "metadata", + "fields", + "rows" + ], + "title": "Italian Cities typed JSON preview", + "type": "object", + "$defs": { + "location": { + "additionalProperties": false, + "properties": { + "candidate_municipality_ids": { "type": "string" }, + "coordinate_accuracy": { "type": ["integer", "null"] }, + "coordinate_verification": { "type": "string" }, + "country_code": { "type": "string" }, + "country_name": { "type": "string" }, + "latitude": { "type": ["number", "null"] }, + "location_id": { "type": "string" }, + "location_kind": { "type": "string" }, + "location_postal_id": { "type": "string" }, + "longitude": { "type": ["number", "null"] }, + "municipality_istat_code": { "type": "string" }, + "name": { "type": "string" }, + "normalized_name": { "type": "string" }, + "parent_municipality_id": { "type": "string" }, + "postal_code": { "type": "string" }, + "postal_code_status": { "type": "string" }, + "province_code": { "type": "string" }, + "province_name": { "type": "string" }, + "reconciliation_confidence": { "type": "string" }, + "reconciliation_method": { "type": "string" }, + "reconciliation_outcome": { "type": "string" }, + "region_name": { "type": "string" }, + "source_ids": { "type": "string" }, + "source_record_ids": { "type": "string" }, + "source_reference_dates": { "type": "string" } + }, + "required": [ + "location_postal_id", + "location_id", + "name", + "normalized_name", + "location_kind", + "municipality_istat_code", + "parent_municipality_id", + "candidate_municipality_ids", + "postal_code", + "postal_code_status", + "province_code", + "province_name", + "region_name", + "country_code", + "country_name", + "latitude", + "longitude", + "coordinate_verification", + "coordinate_accuracy", + "reconciliation_outcome", + "reconciliation_method", + "reconciliation_confidence", + "source_ids", + "source_record_ids", + "source_reference_dates" + ], + "type": "object" + } + } +} diff --git a/scripts/build_dataset.py b/scripts/build_dataset.py index fb72937..5c6b7c3 100644 --- a/scripts/build_dataset.py +++ b/scripts/build_dataset.py @@ -35,6 +35,7 @@ STRUCTURAL_QUALITY, ) from reconcile_sources import reconcile +from reconciliation_backlog import build_backlog from source_data import ( DEFAULT_GEONAMES, DEFAULT_ISTAT, @@ -44,11 +45,11 @@ validate_declared_sources, ) - DEFAULT_PREVIOUS_RELEASE = ROOT / PREVIOUS_RELEASE["canonical_path"] DEFAULT_DIFF_REPORT = REPORTS_DIR / "release-diff.json" DEFAULT_LEGACY_REPORT = REPORTS_DIR / "legacy-comparison.json" DEFAULT_BUILD_REPORT = REPORTS_DIR / "build-metadata.json" +DEFAULT_RECONCILIATION_REPORT = REPORTS_DIR / "reconciliation-backlog.json" def parse_args() -> argparse.Namespace: @@ -69,6 +70,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--build-report", type=Path, default=DEFAULT_BUILD_REPORT ) + parser.add_argument( + "--reconciliation-report", + type=Path, + default=DEFAULT_RECONCILIATION_REPORT, + ) parser.add_argument( "--no-exports", action="store_true", @@ -222,6 +228,13 @@ def build_all(args: argparse.Namespace) -> dict[str, Any]: row["coordinate_accuracy"] or "missing" for row in unique_location_rows ) + write_json( + args.reconciliation_report, + build_backlog( + cast(list[dict[str, str]], model["municipalities"]), + cast(list[dict[str, str]], model["localities"]), + ), + ) build_report = { "dataset_version": DATASET_VERSION, "schema_version": SCHEMA_VERSION, diff --git a/scripts/build_geography.py b/scripts/build_geography.py index 646f20b..84e0674 100644 --- a/scripts/build_geography.py +++ b/scripts/build_geography.py @@ -11,12 +11,6 @@ from pathlib import Path from typing import Any -import shapefile -from pyproj import Transformer -from shapely.geometry import mapping, shape -from shapely.ops import transform - - ROOT = Path(__file__).resolve().parents[1] DEFAULT_SOURCE = ROOT / "sources" / "cache" / "Limiti01012026_g.zip" DEFAULT_OUTPUT = ROOT / "site" / "assets" / "italy-regions.geojson" @@ -61,6 +55,11 @@ def extract_region_layer(source: Path, destination: Path) -> Path: def build_geography(source: Path, output: Path) -> dict[str, Any]: + import shapefile + from pyproj import Transformer + from shapely.geometry import mapping, shape + from shapely.ops import transform + source = source.resolve() output = output.resolve() if not source.is_file(): @@ -130,23 +129,97 @@ def build_geography(source: Path, output: Path) -> dict[str, Any]: } +def check_geography( + source: Path, + output: Path, + *, + require_rebuild: bool = False, +) -> dict[str, Any]: + if not output.is_file(): + raise FileNotFoundError(f"committed geography is missing: {output}") + payload = json.loads(output.read_text(encoding="utf-8")) + if payload.get("type") != "FeatureCollection": + raise ValueError(f"{output}: expected a GeoJSON FeatureCollection") + provenance = payload.get("source", {}) + expected_provenance = { + "reference_date": SOURCE_REFERENCE_DATE, + "url": SOURCE_URL, + "archive_sha256": SOURCE_SHA256, + "layer": f"{SHAPEFILE_ROOT}.shp", + "license": "CC BY 4.0", + } + for field, expected in expected_provenance.items(): + if provenance.get(field) != expected: + raise ValueError( + f"{output}: source.{field} is {provenance.get(field)!r}; " + f"expected {expected!r}" + ) + features = payload.get("features") + if not isinstance(features, list) or len(features) != 20: + raise ValueError(f"{output}: expected exactly 20 region features") + region_codes = [ + feature.get("properties", {}).get("region_code") + for feature in features + ] + if region_codes != sorted(region_codes) or len(set(region_codes)) != 20: + raise ValueError(f"{output}: region codes must be unique and sorted") + + rebuild_status = "skipped_cache_not_present" + if source.is_file(): + try: + with tempfile.TemporaryDirectory() as directory: + candidate = Path(directory) / output.name + build_geography(source, candidate) + if candidate.read_bytes() != output.read_bytes(): + raise ValueError( + f"{output}: committed geography differs from a clean rebuild" + ) + rebuild_status = "passed" + except ModuleNotFoundError: + if require_rebuild: + raise + rebuild_status = "skipped_geography_dependencies_not_installed" + return { + "status": "passed", + "output": str(output.resolve().relative_to(ROOT)), + "features": len(features), + "sha256": sha256(output), + "source_rebuild": rebuild_status, + } + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--check", + action="store_true", + help=( + "Validate the committed GeoJSON and compare it with a clean " + "rebuild when the declared source archive is available." + ), + ) + parser.add_argument( + "--require-rebuild", + action="store_true", + help="Fail unless --check can rebuild from the declared source archive.", + ) return parser.parse_args() def main() -> None: args = parse_args() - print( - json.dumps( - build_geography(args.source, args.output), - ensure_ascii=False, - indent=2, - sort_keys=True, + report = ( + check_geography( + args.source, + args.output, + require_rebuild=args.require_rebuild, ) + if args.check + else build_geography(args.source, args.output) ) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) if __name__ == "__main__": diff --git a/scripts/build_pages.py b/scripts/build_pages.py index 5f1efe0..e943a2f 100644 --- a/scripts/build_pages.py +++ b/scripts/build_pages.py @@ -4,17 +4,22 @@ from __future__ import annotations import argparse -import csv import hashlib import html import json import re import shutil -import unicodedata from pathlib import Path from typing import Any from urllib.parse import quote_plus +from pages_model import ( + WEB_FIELDS, + calculate_stats, + format_integer, + read_locations, + slugify, +) from project_metadata import ( BUILD_DATE, DATASET_VERSION, @@ -26,7 +31,6 @@ STRUCTURAL_QUALITY, ) - REPOSITORY_ROOT = Path(__file__).resolve().parents[1] SOURCE_SITE = REPOSITORY_ROOT / "site" CANONICAL_DATA = REPOSITORY_ROOT / "data" / "italian_locations.csv" @@ -40,26 +44,6 @@ f"{REPOSITORY_URL}/releases/download/{RELEASE_VERSION}" ) -WEB_FIELDS = ( - "location_id", - "name", - "normalized_name", - "location_kind", - "municipality_istat_code", - "postal_code", - "postal_code_status", - "province_code", - "province_name", - "region_name", - "latitude", - "longitude", - "coordinate_verification", - "coordinate_accuracy", - "reconciliation_outcome", - "reconciliation_confidence", -) - - def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -68,89 +52,6 @@ def sha256(path: Path) -> str: return digest.hexdigest() -def read_locations(path: Path) -> list[dict[str, Any]]: - with path.open(encoding="utf-8", newline="") as stream: - reader = csv.DictReader(stream) - missing = set(WEB_FIELDS) - set(reader.fieldnames or []) - if missing: - raise ValueError(f"Canonical dataset is missing fields: {sorted(missing)}") - - rows: list[dict[str, Any]] = [] - for source in reader: - latitude = source["latitude"].strip() - longitude = source["longitude"].strip() - if bool(latitude) != bool(longitude): - raise ValueError( - f"Coordinate pair is incomplete for {source['location_id']}" - ) - - row = {field: source[field] for field in WEB_FIELDS} - row["latitude"] = float(latitude) if latitude else None - row["longitude"] = float(longitude) if longitude else None - rows.append(row) - - return rows - - -def calculate_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: - unique_locations = { - row["location_id"]: row for row in reversed(rows) - } - location_rows = list(unique_locations.values()) - with_coordinates = [ - row - for row in location_rows - if row["latitude"] is not None and row["longitude"] is not None - ] - latitudes = [row["latitude"] for row in with_coordinates] - longitudes = [row["longitude"] for row in with_coordinates] - - return { - "total_locations": len(location_rows), - "postal_code_relations": len(rows), - "municipalities": sum( - row["location_kind"] == "municipality" for row in location_rows - ), - "unclassified_localities": sum( - row["location_kind"] != "municipality" for row in location_rows - ), - "unique_postal_codes": len( - {row["postal_code"] for row in rows if row["postal_code"]} - ), - "with_coordinates": len(with_coordinates), - "missing_coordinates": len(location_rows) - len(with_coordinates), - "geonames_place_match_coordinates": sum( - row["coordinate_verification"] == "geonames_place_match" - for row in location_rows - ), - "geonames_estimated_coordinates": sum( - row["coordinate_verification"] == "geonames_estimated" - for row in location_rows - ), - "missing_postal_code_municipalities": sum( - row["postal_code_status"] == "missing" for row in rows - ), - "provinces": len({row["province_code"] for row in rows}), - "regions": len({row["region_name"] for row in rows}), - "bounds": { - "min_latitude": min(latitudes), - "max_latitude": max(latitudes), - "min_longitude": min(longitudes), - "max_longitude": max(longitudes), - }, - } - - -def format_integer(value: int) -> str: - return f"{value:,}".replace(",", ".") - - -def slugify(value: str) -> str: - normalized = unicodedata.normalize("NFKD", value) - ascii_value = normalized.encode("ascii", "ignore").decode("ascii") - return re.sub(r"[^a-z0-9]+", "-", ascii_value.casefold()).strip("-") - - def release_asset_url(name: str) -> str: return f"{RELEASE_DOWNLOAD_ROOT}/{name}" @@ -234,6 +135,7 @@ def structured_data(stats: dict[str, Any]) -> str: ), ("italian_locations.sqlite", "application/vnd.sqlite3"), ("italian_locations.sql", "application/sql"), + ("SHA256SUMS", "text/plain"), ) ], }, @@ -427,14 +329,17 @@ def build_information_pages( non ufficiale.

Download della release {version}

-

Gli asset pubblicati sono immutabili e accompagnati da checksum - SHA-256.

+

Gli asset sono accompagnati da checksum SHA-256. La v2.0.0 + pubblicata precede l'abilitazione dell'immutabilità GitHub del 30 + luglio 2026; le release future usano il flusso + draft–upload–publish.

CSV JSON XLSX SQLite SQL + SHA256SUMS

Documentazione

diff --git a/scripts/build_release.py b/scripts/build_release.py index 7c9c80a..9773c04 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -17,7 +17,6 @@ from export_sql import export_sql from project_metadata import DATASET_VERSION - RELEASE_VERSION = DATASET_VERSION DEFAULT_OUTPUT = ROOT / "dist" / RELEASE_VERSION RELEASE_ASSETS = { diff --git a/scripts/check_determinism.py b/scripts/check_determinism.py index 740d6ee..12b044c 100644 --- a/scripts/check_determinism.py +++ b/scripts/check_determinism.py @@ -21,7 +21,6 @@ ) from project_metadata import QUALITY_GATE_VERSION - DEFAULT_REPORT = REPORTS_DIR / "determinism.json" BYTE_STABLE_PATHS = { "municipalities_csv": GENERATED_PATHS["municipalities"], @@ -33,6 +32,7 @@ "build_metadata": REPORTS_DIR / "build-metadata.json", "legacy_comparison": REPORTS_DIR / "legacy-comparison.json", "release_diff_report": REPORTS_DIR / "release-diff.json", + "reconciliation_backlog": REPORTS_DIR / "reconciliation-backlog.json", "export_manifest": REPORTS_DIR / "export-manifest.json", } diff --git a/scripts/dataset_common.py b/scripts/dataset_common.py index 32350a8..ddb46f2 100644 --- a/scripts/dataset_common.py +++ b/scripts/dataset_common.py @@ -9,9 +9,8 @@ import os import re import unicodedata +from collections.abc import Iterable from pathlib import Path -from typing import Iterable - ROOT = Path(__file__).resolve().parents[1] DATA_DIR = ROOT / "data" diff --git a/scripts/export_formats.py b/scripts/export_formats.py index f108161..68be847 100644 --- a/scripts/export_formats.py +++ b/scripts/export_formats.py @@ -4,17 +4,16 @@ from __future__ import annotations import argparse -from datetime import datetime, timezone import json import os import re import sqlite3 import tempfile import zipfile +from datetime import UTC, datetime from pathlib import Path import xlsxwriter - from dataset_common import ( GENERATED_PATHS, ITALIAN_LOCATION_FIELDS, @@ -34,7 +33,6 @@ version_number, ) - FIXED_XLSX_TIMESTAMP = f"{BUILD_DATE}T00:00:00Z" DEFAULT_REPORT = REPORTS_DIR / "export-manifest.json" GEONAMES_ATTRIBUTION = ( @@ -227,7 +225,7 @@ def export_xlsx( "author": "Codewriter90x", "comments": "Generated; do not edit manually.", "created": datetime.fromisoformat(BUILD_DATE).replace( - tzinfo=timezone.utc + tzinfo=UTC ), } ) diff --git a/scripts/export_sql.py b/scripts/export_sql.py index 72c045c..375f49d 100644 --- a/scripts/export_sql.py +++ b/scripts/export_sql.py @@ -15,7 +15,6 @@ ) from project_metadata import DATASET_VERSION, SCHEMA_VERSION, version_number - DEFAULT_OUTPUT = Path("dist") / DATASET_VERSION / "italian_locations.sql" INSERT_BATCH_SIZE = 250 diff --git a/scripts/export_typed_json.py b/scripts/export_typed_json.py new file mode 100644 index 0000000..58b4723 --- /dev/null +++ b/scripts/export_typed_json.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Build the opt-in typed JSON contract planned for the next major release.""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +from pathlib import Path + +from dataset_common import ( + GENERATED_PATHS, + ITALIAN_LOCATION_FIELDS, + read_csv_rows, + write_json, +) +from project_metadata import DATASET_VERSION +from typed_contract import ( + TYPED_CONTRACT_VERSION, + sqlite_column_definition, + typed_row, +) + + +def export_typed_json(canonical: Path, output: Path) -> dict[str, object]: + rows = read_csv_rows(canonical, ITALIAN_LOCATION_FIELDS) + payload = { + "metadata": { + "source_dataset_version": DATASET_VERSION, + "schema_version": TYPED_CONTRACT_VERSION, + "contract_status": "next_major_preview", + "null_policy": "JSON null for missing numeric values", + }, + "fields": list(ITALIAN_LOCATION_FIELDS), + "rows": [typed_row(row) for row in rows], + } + write_json(output, payload) + return { + "status": "exported", + "schema_version": TYPED_CONTRACT_VERSION, + "rows": len(rows), + "output": str(output), + } + + +def export_typed_sqlite(canonical: Path, output: Path) -> dict[str, object]: + rows = read_csv_rows(canonical, ITALIAN_LOCATION_FIELDS) + output.parent.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(output) + try: + columns = ", ".join( + sqlite_column_definition(field) + for field in ITALIAN_LOCATION_FIELDS + ) + connection.execute( + f'CREATE TABLE "italian_locations" ({columns}, ' + 'PRIMARY KEY ("location_postal_id")) WITHOUT ROWID' + ) + quoted_fields = ", ".join( + f'"{field}"' for field in ITALIAN_LOCATION_FIELDS + ) + placeholders = ", ".join("?" for _ in ITALIAN_LOCATION_FIELDS) + converted_rows = [typed_row(row) for row in rows] + connection.executemany( + f'INSERT INTO "italian_locations" ({quoted_fields}) ' + f"VALUES ({placeholders})", + [ + tuple(row[field] for field in ITALIAN_LOCATION_FIELDS) + for row in converted_rows + ], + ) + connection.commit() + finally: + connection.close() + return { + "status": "exported", + "schema_version": TYPED_CONTRACT_VERSION, + "rows": len(rows), + "output": str(output), + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--canonical", + type=Path, + default=GENERATED_PATHS["italian_locations"], + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--sqlite-output", + type=Path, + help="Optional typed SQLite preview output.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + report: dict[str, object] = { + "json": export_typed_json(args.canonical, args.output) + } + if args.sqlite_output: + report["sqlite"] = export_typed_sqlite( + args.canonical, + args.sqlite_output, + ) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/legacy_comparison.py b/scripts/legacy_comparison.py index a1f40b0..ff70ae3 100644 --- a/scripts/legacy_comparison.py +++ b/scripts/legacy_comparison.py @@ -10,7 +10,6 @@ from dataset_common import normalize_name, sha256_file - LEGACY_FIELDS = ( "legacy_uuid", "name", diff --git a/scripts/normalize_legacy.py b/scripts/normalize_legacy.py index e8bb1b3..0fb1131 100644 --- a/scripts/normalize_legacy.py +++ b/scripts/normalize_legacy.py @@ -16,7 +16,6 @@ from collections import Counter from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] DEFAULT_LEGACY = ROOT / "legacy/2023-05-02-original/Italian Cities.csv" DEFAULT_ISTAT = ( @@ -107,13 +106,13 @@ def load_istat_municipalities(path: Path) -> dict[tuple[str, str], dict[str, str ] sheet_root = ET.fromstring(archive.read("xl/worksheets/sheet1.xml")) - parsed_rows: list[list[str | None]] = [] - for row in sheet_root.findall(".//x:sheetData/x:row", ISTAT_NS): - values: list[str | None] = [None] * 27 - for cell in row.findall("x:c", ISTAT_NS): + parsed_rows: list[list[str]] = [] + for sheet_row in sheet_root.findall(".//x:sheetData/x:row", ISTAT_NS): + values = [""] * 27 + for cell in sheet_row.findall("x:c", ISTAT_NS): value_node = cell.find("x:v", ISTAT_NS) - if value_node is None: - value = None + if value_node is None or value_node.text is None: + value = "" elif cell.get("t") == "s": value = shared_strings[int(value_node.text)] else: @@ -122,14 +121,14 @@ def load_istat_municipalities(path: Path) -> dict[tuple[str, str], dict[str, str parsed_rows.append(values) municipalities: dict[tuple[str, str], dict[str, str]] = {} - for row in parsed_rows[1:]: + for parsed_row in parsed_rows[1:]: record = { - "istat_code": row[4] or "", - "bilingual_name": row[5] or "", - "italian_name": row[6] or "", - "region_name": row[10] or "", - "province_name": row[11] or "", - "province_code": row[14] or "", + "istat_code": parsed_row[4], + "bilingual_name": parsed_row[5], + "italian_name": parsed_row[6], + "region_name": parsed_row[10], + "province_name": parsed_row[11], + "province_code": parsed_row[14], } for name in (record["italian_name"], record["bilingual_name"]): if name: diff --git a/scripts/pages_model.py b/scripts/pages_model.py new file mode 100644 index 0000000..295e5cd --- /dev/null +++ b/scripts/pages_model.py @@ -0,0 +1,108 @@ +"""Data loading and aggregation for the static Pages build.""" + +from __future__ import annotations + +import csv +import re +import unicodedata +from pathlib import Path +from typing import Any + +WEB_FIELDS = ( + "location_id", + "name", + "normalized_name", + "location_kind", + "municipality_istat_code", + "postal_code", + "postal_code_status", + "province_code", + "province_name", + "region_name", + "latitude", + "longitude", + "coordinate_verification", + "coordinate_accuracy", + "reconciliation_outcome", + "reconciliation_confidence", +) + + +def read_locations(path: Path) -> list[dict[str, Any]]: + with path.open(encoding="utf-8", newline="") as stream: + reader = csv.DictReader(stream) + missing = set(WEB_FIELDS) - set(reader.fieldnames or []) + if missing: + raise ValueError( + f"Canonical dataset is missing fields: {sorted(missing)}" + ) + + rows: list[dict[str, Any]] = [] + for source in reader: + latitude = source["latitude"].strip() + longitude = source["longitude"].strip() + if bool(latitude) != bool(longitude): + raise ValueError( + f"Coordinate pair is incomplete for {source['location_id']}" + ) + row = {field: source[field] for field in WEB_FIELDS} + row["latitude"] = float(latitude) if latitude else None + row["longitude"] = float(longitude) if longitude else None + rows.append(row) + return rows + + +def calculate_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: + unique_locations = {row["location_id"]: row for row in reversed(rows)} + location_rows = list(unique_locations.values()) + with_coordinates = [ + row + for row in location_rows + if row["latitude"] is not None and row["longitude"] is not None + ] + latitudes = [row["latitude"] for row in with_coordinates] + longitudes = [row["longitude"] for row in with_coordinates] + return { + "total_locations": len(location_rows), + "postal_code_relations": len(rows), + "municipalities": sum( + row["location_kind"] == "municipality" for row in location_rows + ), + "unclassified_localities": sum( + row["location_kind"] != "municipality" for row in location_rows + ), + "unique_postal_codes": len( + {row["postal_code"] for row in rows if row["postal_code"]} + ), + "with_coordinates": len(with_coordinates), + "missing_coordinates": len(location_rows) - len(with_coordinates), + "geonames_place_match_coordinates": sum( + row["coordinate_verification"] == "geonames_place_match" + for row in location_rows + ), + "geonames_estimated_coordinates": sum( + row["coordinate_verification"] == "geonames_estimated" + for row in location_rows + ), + "missing_postal_code_municipalities": sum( + row["postal_code_status"] == "missing" for row in rows + ), + "provinces": len({row["province_code"] for row in rows}), + "regions": len({row["region_name"] for row in rows}), + "bounds": { + "min_latitude": min(latitudes), + "max_latitude": max(latitudes), + "min_longitude": min(longitudes), + "max_longitude": max(longitudes), + }, + } + + +def format_integer(value: int) -> str: + return f"{value:,}".replace(",", ".") + + +def slugify(value: str) -> str: + normalized = unicodedata.normalize("NFKD", value) + ascii_value = normalized.encode("ascii", "ignore").decode("ascii") + return re.sub(r"[^a-z0-9]+", "-", ascii_value.casefold()).strip("-") diff --git a/scripts/project_metadata.py b/scripts/project_metadata.py index 4304d7a..b5b0ee9 100644 --- a/scripts/project_metadata.py +++ b/scripts/project_metadata.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] PROJECT_METADATA_PATH = ROOT / "project.json" SEMVER = re.compile(r"^v\d+\.\d+\.\d+$") diff --git a/scripts/reconcile_sources.py b/scripts/reconcile_sources.py index 44b034b..60aa015 100644 --- a/scripts/reconcile_sources.py +++ b/scripts/reconcile_sources.py @@ -6,11 +6,10 @@ import hashlib import uuid from collections import Counter, defaultdict -from typing import Iterable +from collections.abc import Iterable from dataset_common import normalize_name - LOCATION_NAMESPACE = uuid.uuid5( uuid.NAMESPACE_URL, "https://github.com/Codewriter90x/Italian_Cities/v2/location", diff --git a/scripts/reconciliation_backlog.py b/scripts/reconciliation_backlog.py new file mode 100644 index 0000000..037957e --- /dev/null +++ b/scripts/reconciliation_backlog.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Build a deterministic, evidence-first reconciliation backlog.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +from dataset_common import ( + GENERATED_PATHS, + LOCALITY_FIELDS, + MUNICIPALITY_FIELDS, + REPORTS_DIR, + read_csv_rows, + write_json, +) +from project_metadata import DATASET_VERSION, SCHEMA_VERSION + +DEFAULT_OUTPUT = REPORTS_DIR / "reconciliation-backlog.json" + + +def ranked(counter: Counter[str]) -> list[dict[str, object]]: + return [ + {"name": name, "records": count} + for name, count in sorted( + counter.items(), + key=lambda item: (-item[1], item[0]), + ) + ] + + +def build_backlog( + municipalities: list[dict[str, str]], + localities: list[dict[str, str]], +) -> dict[str, object]: + municipality_province_codes = { + row["province_code"] for row in municipalities + } + noncurrent_codes = sorted( + { + row["province_code"] + for row in localities + if row["province_code"] not in municipality_province_codes + } + ) + missing_postal = [ + row + for row in municipalities + if row["coordinate_verification"] == "missing" + ] + outcomes = Counter(row["reconciliation_outcome"] for row in localities) + return { + "dataset_version": DATASET_VERSION, + "schema_version": SCHEMA_VERSION, + "status": "review_required", + "policy": { + "automatic_fuzzy_promotion": False, + "public_nominatim_bulk_geocoding": False, + "evidence_required": [ + "source", + "reference_date", + "license", + "reproducible_transformation", + ], + }, + "summary": { + "municipalities": len(municipalities), + "unreconciled_localities": len(localities), + "municipalities_without_geonames_postal_match": len(missing_postal), + "noncurrent_source_province_codes": noncurrent_codes, + "reconciliation_outcomes": dict(sorted(outcomes.items())), + }, + "unreconciled_by_region": ranked( + Counter(row["region_name"] for row in localities) + ), + "unreconciled_by_province": ranked( + Counter( + f"{row['province_name']} ({row['province_code']})" + for row in localities + ) + ), + "missing_municipality_match_by_region": ranked( + Counter(row["region_name"] for row in missing_postal) + ), + "review_order": [ + { + "priority": 1, + "segment": "noncurrent_source_province_codes", + "reason": "Historical source codes need explicit aliases.", + }, + { + "priority": 2, + "segment": "municipalities_without_geonames_postal_match", + "reason": "Municipalities remain canonical but lack CAP/coordinates.", + }, + { + "priority": 3, + "segment": "highest_unreconciled_regions", + "reason": "Review high-volume segments using licensed evidence.", + }, + ], + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + report = build_backlog( + read_csv_rows(GENERATED_PATHS["municipalities"], MUNICIPALITY_FIELDS), + read_csv_rows(GENERATED_PATHS["localities"], LOCALITY_FIELDS), + ) + write_json(args.output, report) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/source_data.py b/scripts/source_data.py index e2d1613..df3fdc5 100644 --- a/scripts/source_data.py +++ b/scripts/source_data.py @@ -15,7 +15,6 @@ from dataset_common import ROOT, SOURCE_MANIFEST, normalize_name, sha256_file - DEFAULT_ISTAT = ( ROOT / "sources/snapshots/istat/Elenco-comuni-italiani-2026-02-21.xlsx" diff --git a/scripts/typed_contract.py b/scripts/typed_contract.py new file mode 100644 index 0000000..dcf9774 --- /dev/null +++ b/scripts/typed_contract.py @@ -0,0 +1,33 @@ +"""Typed next-major representation for consumer-facing exports.""" + +from __future__ import annotations + +from typing import Any + +from dataset_common import ITALIAN_LOCATION_FIELDS + +TYPED_CONTRACT_VERSION = "4.0.0" +NULLABLE_NUMBER_FIELDS = frozenset({"latitude", "longitude"}) +NULLABLE_INTEGER_FIELDS = frozenset({"coordinate_accuracy"}) + + +def typed_row(row: dict[str, str]) -> dict[str, Any]: + result: dict[str, Any] = {} + for field in ITALIAN_LOCATION_FIELDS: + value = row[field] + if field in NULLABLE_NUMBER_FIELDS: + result[field] = float(value) if value else None + elif field in NULLABLE_INTEGER_FIELDS: + result[field] = int(value) if value else None + else: + result[field] = value + return result + + +def sqlite_column_definition(field: str) -> str: + quoted = f'"{field}"' + if field in NULLABLE_NUMBER_FIELDS: + return f"{quoted} REAL" + if field in NULLABLE_INTEGER_FIELDS: + return f"{quoted} INTEGER" + return f"{quoted} TEXT NOT NULL" diff --git a/scripts/update_sources.py b/scripts/update_sources.py new file mode 100644 index 0000000..95d370b --- /dev/null +++ b/scripts/update_sources.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Check canonical upstream snapshots and stage explicit source downloads.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import ssl +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, BinaryIO +from urllib.request import Request, urlopen + +import truststore +from dataset_common import ROOT, SOURCE_MANIFEST, sha256_file, write_json +from source_data import CANONICAL_SOURCE_IDS, load_manifest, source_by_id + +USER_AGENT = ( + "Italian_Cities source freshness checker " + "(https://github.com/Codewriter90x/Italian_Cities)" +) +TLS_CONTEXT = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + +def download(url: str, destination: Path) -> dict[str, str]: + request = Request(url, headers={"User-Agent": USER_AGENT}) + digest = hashlib.sha256() + with urlopen( + request, + timeout=90, + context=TLS_CONTEXT, + ) as response: # noqa: S310 + last_modified = response.headers.get("Last-Modified", "") + etag = response.headers.get("ETag", "") + with destination.open("wb") as output: + stream: BinaryIO = response + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + output.write(chunk) + if destination.stat().st_size == 0: + raise ValueError(f"{url}: upstream returned an empty artifact") + return { + "sha256": digest.hexdigest(), + "last_modified": last_modified, + "etag": etag, + } + + +def inspect_source(source: dict[str, Any]) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="italian-cities-source-") as folder: + candidate = Path(folder) / "candidate" + remote = download(str(source["url"]), candidate) + declared_sha256 = str(source["sha256"]) + return { + "id": source["id"], + "url": source["url"], + "declared_path": source["path"], + "declared_sha256": declared_sha256, + "remote_sha256": remote["sha256"], + "remote_last_modified": remote["last_modified"], + "remote_etag": remote["etag"], + "status": ( + "current" + if remote["sha256"] == declared_sha256 + else "update_available" + ), + } + + +def check_sources(manifest_path: Path = SOURCE_MANIFEST) -> dict[str, Any]: + manifest = load_manifest(manifest_path) + sources = [ + inspect_source(source_by_id(manifest, source_id)) + for source_id in CANONICAL_SOURCE_IDS + ] + changed = [source["id"] for source in sources if source["status"] != "current"] + return { + "checked_at": datetime.now(UTC).isoformat(), + "manifest": str(manifest_path.resolve().relative_to(ROOT)), + "status": "updates_available" if changed else "current", + "updates_available": changed, + "sources": sources, + } + + +def stage_download( + source_id: str, + output: Path, + manifest_path: Path = SOURCE_MANIFEST, +) -> dict[str, Any]: + manifest = load_manifest(manifest_path) + if source_id not in CANONICAL_SOURCE_IDS: + raise ValueError( + f"{source_id!r} is not a canonical source: {CANONICAL_SOURCE_IDS}" + ) + if output.exists(): + raise FileExistsError( + f"refusing to overwrite {output}; choose a new dated path" + ) + output.parent.mkdir(parents=True, exist_ok=True) + source = source_by_id(manifest, source_id) + with tempfile.NamedTemporaryFile( + prefix=f"{source_id}-", + dir=output.parent, + delete=False, + ) as handle: + temporary = Path(handle.name) + try: + remote = download(str(source["url"]), temporary) + os.replace(temporary, output) + finally: + if temporary.exists(): + temporary.unlink() + return { + "status": "staged", + "source_id": source_id, + "output": str(output), + "sha256": sha256_file(output), + "remote_last_modified": remote["last_modified"], + "remote_etag": remote["etag"], + "next_steps": [ + "inspect the staged artifact", + "update sources/manifest.json explicitly", + "rebuild and review every generated diff", + ], + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument( + "--check", + action="store_true", + help="Download canonical upstream artifacts to a temporary directory.", + ) + action.add_argument( + "--download", + choices=CANONICAL_SOURCE_IDS, + metavar="SOURCE_ID", + help="Stage one upstream artifact without editing the manifest.", + ) + parser.add_argument("--manifest", type=Path, default=SOURCE_MANIFEST) + parser.add_argument( + "--output", + type=Path, + help="New, non-existing path required by --download.", + ) + parser.add_argument( + "--report", + type=Path, + help="Optional JSON report path.", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.download: + if args.output is None: + raise SystemExit("--output is required with --download") + report = stage_download(args.download, args.output, args.manifest) + else: + report = check_sources(args.manifest) + if args.report: + write_json(args.report, report) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_dataset.py b/scripts/validate_dataset.py index afeb8f7..c6ebdab 100644 --- a/scripts/validate_dataset.py +++ b/scripts/validate_dataset.py @@ -4,8 +4,8 @@ from __future__ import annotations import argparse -from collections import Counter import json +from collections import Counter from pathlib import Path from check_determinism import DEFAULT_REPORT as DETERMINISM_REPORT @@ -51,7 +51,6 @@ validate_territories, ) - DEFAULT_REPORT = REPORTS_DIR / "quality-validation.json" QUALITY_CHECK_NAMES = ( "schema_columns", diff --git a/scripts/validate_milestone1.py b/scripts/validate_milestone1.py index c00a56d..8aa087e 100644 --- a/scripts/validate_milestone1.py +++ b/scripts/validate_milestone1.py @@ -30,7 +30,6 @@ normalize_name, ) - ROOT = Path(__file__).resolve().parents[1] DEFAULT_REPORT = ROOT / "reports/milestone1-validation.json" diff --git a/scripts/validate_release.py b/scripts/validate_release.py index 4b4a92b..15e3684 100644 --- a/scripts/validate_release.py +++ b/scripts/validate_release.py @@ -25,7 +25,6 @@ ) from project_metadata import DATASET_VERSION, SCHEMA_VERSION - EXPECTED_ASSETS = {*RELEASE_ASSETS, SQL_ASSET, CHECKSUM_ASSET} diff --git a/scripts/validate_site.py b/scripts/validate_site.py new file mode 100644 index 0000000..38c374c --- /dev/null +++ b/scripts/validate_site.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Validate built Pages links, accessibility basics and performance budgets.""" + +from __future__ import annotations + +import argparse +import json +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urlparse + +MAX_HOMEPAGE_BYTES = 30_000 +MAX_SEARCH_DATA_BYTES = 4_200_000 +MAX_FIRST_PARTY_CODE_BYTES = 80_000 + + +class PageInspector(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.ids: list[str] = [] + self.links: list[str] = [] + self.controls: list[tuple[str, str]] = [] + self.label_targets: set[str] = set() + self.images_without_alt: list[str] = [] + self.button_texts: list[str] = [] + self._current_button: list[str] | None = None + + def handle_starttag( + self, + tag: str, + attrs: list[tuple[str, str | None]], + ) -> None: + values = dict(attrs) + element_id = values.get("id") + if element_id: + self.ids.append(element_id) + if tag == "a" and values.get("href"): + self.links.append(values["href"] or "") + if tag in {"input", "select", "textarea"} and element_id: + self.controls.append((tag, element_id)) + if tag == "label" and values.get("for"): + self.label_targets.add(values["for"] or "") + if tag == "img" and "alt" not in values: + self.images_without_alt.append(values.get("src", "") or "") + if tag == "button": + self._current_button = [] + + def handle_data(self, data: str) -> None: + if self._current_button is not None: + self._current_button.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag == "button" and self._current_button is not None: + self.button_texts.append("".join(self._current_button).strip()) + self._current_button = None + + +def resolve_local_link(site: Path, page: Path, href: str) -> Path | None: + parsed = urlparse(href) + if parsed.scheme or parsed.netloc or href.startswith(("mailto:", "#")): + return None + path = unquote(parsed.path) + if path.startswith("/Italian_Cities/"): + target = site / path.removeprefix("/Italian_Cities/") + elif path.startswith("/"): + return None + else: + target = page.parent / path + if not path or path.endswith("/"): + target /= "index.html" + return target.resolve() + + +def validate_site(site: Path) -> dict[str, object]: + site = site.resolve() + errors: list[str] = [] + html_files = sorted(site.rglob("*.html")) + if not html_files: + errors.append("site contains no HTML pages") + for page in html_files: + inspector = PageInspector() + inspector.feed(page.read_text(encoding="utf-8")) + duplicate_ids = sorted( + identifier + for identifier in set(inspector.ids) + if inspector.ids.count(identifier) > 1 + ) + if duplicate_ids: + errors.append(f"{page}: duplicate ids {duplicate_ids}") + missing_labels = sorted( + control_id + for _, control_id in inspector.controls + if control_id not in inspector.label_targets + ) + if missing_labels: + errors.append(f"{page}: controls without labels {missing_labels}") + if inspector.images_without_alt: + errors.append( + f"{page}: images without alt {inspector.images_without_alt}" + ) + if any(not text for text in inspector.button_texts): + errors.append(f"{page}: button without accessible text") + for href in inspector.links: + target = resolve_local_link(site, page, href) + if target is not None and not target.is_file(): + errors.append(f"{page}: broken local link {href!r}") + + homepage = site / "index.html" + locations = site / "assets/locations.json" + code_bytes = sum( + path.stat().st_size + for path in (site / "assets").iterdir() + if path.suffix in {".css", ".js", ".mjs"} + ) + budgets = { + "homepage_bytes": homepage.stat().st_size, + "search_data_bytes": locations.stat().st_size, + "first_party_code_bytes": code_bytes, + } + limits = { + "homepage_bytes": MAX_HOMEPAGE_BYTES, + "search_data_bytes": MAX_SEARCH_DATA_BYTES, + "first_party_code_bytes": MAX_FIRST_PARTY_CODE_BYTES, + } + for name, value in budgets.items(): + if value > limits[name]: + errors.append(f"{name} exceeds budget: {value} > {limits[name]}") + app_source = (site / "assets/app.js").read_text(encoding="utf-8") + if "IntersectionObserver" not in app_source: + errors.append("search data is not guarded by lazy loading") + if 'id="map-fallback-body"' not in homepage.read_text(encoding="utf-8"): + errors.append("canvas map lacks the accessible tabular fallback") + + report: dict[str, object] = { + "status": "passed" if not errors else "failed", + "pages": len(html_files), + "budgets": budgets, + "limits": limits, + "errors": errors, + } + if errors: + raise ValueError(json.dumps(report, ensure_ascii=False, indent=2)) + return report + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("site", type=Path) + return parser.parse_args() + + +def main() -> None: + report = validate_site(parse_args().site) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/validators/common.py b/scripts/validators/common.py index 9da97fd..f9483e0 100644 --- a/scripts/validators/common.py +++ b/scripts/validators/common.py @@ -5,7 +5,6 @@ import csv from pathlib import Path - QualityChecks = dict[str, dict[str, object]] diff --git a/scripts/validators/coordinates.py b/scripts/validators/coordinates.py index 585b2da..1b55279 100644 --- a/scripts/validators/coordinates.py +++ b/scripts/validators/coordinates.py @@ -7,7 +7,6 @@ from validators.common import QualityChecks, add_quality_error - ITALY_BOUNDS = { "latitude_min": 35.0, "latitude_max": 48.0, diff --git a/scripts/validators/formats.py b/scripts/validators/formats.py index 42f154f..4f712b6 100644 --- a/scripts/validators/formats.py +++ b/scripts/validators/formats.py @@ -23,8 +23,8 @@ SCHEMA_VERSION, STRUCTURAL_QUALITY, ) -from validators.common import QualityChecks, add_quality_error +from validators.common import QualityChecks, add_quality_error SHEET_NS = {"x": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} DOC_REL_NS = { diff --git a/scripts/validators/provenance.py b/scripts/validators/provenance.py index 9f0a495..a7d1504 100644 --- a/scripts/validators/provenance.py +++ b/scripts/validators/provenance.py @@ -4,7 +4,6 @@ from validators.common import QualityChecks, add_quality_error - CANONICAL_SOURCES = { "istat_municipalities", "geonames_postal_codes", @@ -69,11 +68,13 @@ def validate_verification_and_provenance( "provenance_completeness", f"{label}: ambiguous match lacks candidate ids", ) - if row.get("reconciliation_outcome") == "unmatched_no_parent": - if row.get("parent_municipality_id"): - add_quality_error( - errors, - checks, - "clean_room_isolation", - f"{label}: unmatched locality has a parent municipality", - ) + if ( + row.get("reconciliation_outcome") == "unmatched_no_parent" + and row.get("parent_municipality_id") + ): + add_quality_error( + errors, + checks, + "clean_room_isolation", + f"{label}: unmatched locality has a parent municipality", + ) diff --git a/scripts/validators/schema.py b/scripts/validators/schema.py index 786be96..c8da088 100644 --- a/scripts/validators/schema.py +++ b/scripts/validators/schema.py @@ -6,7 +6,6 @@ from validators.common import QualityChecks, add_quality_error - POSTAL_CODE_STATUSES = { "geonames_matched", "geonames_ambiguous", diff --git a/scripts/validators/territory.py b/scripts/validators/territory.py index 1bdfb00..d1b33dd 100644 --- a/scripts/validators/territory.py +++ b/scripts/validators/territory.py @@ -5,6 +5,7 @@ import re from dataset_common import normalize_name + from validators.common import QualityChecks, add_quality_error diff --git a/site/assets/app.js b/site/assets/app.js index 19dfb65..d74c220 100644 --- a/site/assets/app.js +++ b/site/assets/app.js @@ -9,9 +9,11 @@ import { const state = { rows: [], + mapRows: [], boundaries: null, stats: null, selected: null, + loadPromise: null, filters: { query: "", province: "", @@ -28,6 +30,8 @@ const elements = { kind: document.querySelector("#kind"), coordinateStatus: document.querySelector("#coordinate-status"), reset: document.querySelector("#reset-search"), + copyFilterLink: document.querySelector("#copy-filter-link"), + copyStatus: document.querySelector("#copy-status"), resultCount: document.querySelector("#result-count"), results: document.querySelector("#result-list"), canvas: document.querySelector("#coverage-map"), @@ -50,6 +54,11 @@ function populateStats(stats) { } function populateProvinces(rows) { + const currentProvinceCodes = new Set( + rows + .filter((row) => row.location_kind === "municipality") + .map((row) => row.province_code), + ); const provinces = [ ...new Map( rows.map((row) => [ @@ -62,7 +71,9 @@ function populateProvinces(rows) { for (const [code, label] of provinces) { const option = document.createElement("option"); option.value = code; - option.textContent = label; + option.textContent = currentProvinceCodes.has(code) + ? label + : `${label} · sigla sorgente non corrente`; elements.province.append(option); } } @@ -233,7 +244,7 @@ function drawMap() { const bounds = state.stats.bounds; drawBoundaries(context, width, height, bounds); - for (const row of state.rows) { + for (const row of state.mapRows) { if (row.latitude === null || row.longitude === null) continue; const point = projectCoordinates( row.longitude, @@ -304,6 +315,31 @@ function updateFilters() { window.history.replaceState(null, "", url); } +async function copyCurrentFilterLink() { + const url = window.location.href; + try { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(url); + } else { + const temporary = document.createElement("textarea"); + temporary.value = url; + temporary.setAttribute("readonly", ""); + temporary.style.position = "fixed"; + temporary.style.opacity = "0"; + document.body.append(temporary); + temporary.select(); + if (!document.execCommand("copy")) { + throw new Error("copy command unavailable"); + } + temporary.remove(); + } + elements.copyStatus.textContent = "Link dei filtri copiato."; + } catch { + elements.copyStatus.textContent = + "Copia non disponibile: seleziona l’indirizzo dalla barra del browser."; + } +} + function resetSearch() { elements.form.reset(); state.selected = null; @@ -337,6 +373,11 @@ async function loadDataset() { boundaryResponse.json(), ]); state.rows = inflateRows(payload.fields, payload.rows); + state.mapRows = [ + ...new Map( + [...state.rows].reverse().map((row) => [row.location_id, row]), + ).values(), + ]; state.stats = payload.stats; state.boundaries = boundaries; populateStats(payload.stats); @@ -357,13 +398,39 @@ async function loadDataset() { } } +function ensureDatasetLoaded() { + if (!state.loadPromise) { + elements.status.textContent = "Caricamento del dataset…"; + elements.status.dataset.state = "loading"; + state.loadPromise = loadDataset(); + } + return state.loadPromise; +} + elements.form.addEventListener("input", updateFilters); +elements.form.addEventListener("focusin", ensureDatasetLoaded, { once: true }); elements.form.addEventListener("submit", (event) => event.preventDefault()); elements.reset.addEventListener("click", resetSearch); +elements.copyFilterLink.addEventListener("click", copyCurrentFilterLink); window.addEventListener("resize", drawMap); window.addEventListener("popstate", () => { applyFiltersFromUrl(); renderResults(); }); -loadDataset(); +if (window.location.search || ["#search", "#map"].includes(window.location.hash)) { + ensureDatasetLoaded(); +} else if ("IntersectionObserver" in window) { + const observer = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + observer.disconnect(); + ensureDatasetLoaded(); + } + }, + { rootMargin: "240px" }, + ); + observer.observe(elements.form); +} else { + ensureDatasetLoaded(); +} diff --git a/site/assets/styles.css b/site/assets/styles.css index 626f169..5420c3c 100644 --- a/site/assets/styles.css +++ b/site/assets/styles.css @@ -347,6 +347,13 @@ a:focus-visible { margin-block: 1.25rem 0.75rem; } +.search-actions { + display: flex; + flex-wrap: wrap; + gap: 0.9rem; + justify-content: flex-end; +} + .search-toolbar button { padding: 0; color: var(--cyan); @@ -355,6 +362,14 @@ a:focus-visible { cursor: pointer; } +.copy-status { + min-height: 1.5rem; + margin: 0; + color: var(--muted); + font-size: 0.82rem; + text-align: right; +} + .result-list { display: grid; gap: 0.6rem; diff --git a/site/index.html b/site/index.html index dfe7627..2f483fb 100644 --- a/site/index.html +++ b/site/index.html @@ -213,8 +213,14 @@

Cerca comuni, CAP e province

Caricamento… - +
+ + +
+

    @@ -305,6 +311,9 @@

    Scarica CSV, JSON, XLSX, SQLite o SQL.

    SQL {{DATASET_VERSION}} + + SHA256SUMS + Note della release @@ -352,7 +361,7 @@

    Come usare correttamente Italian Cities

    - Caricamento del dataset… + La ricerca interattiva verrà caricata quando raggiungi questa sezione.

    diff --git a/tests/test_coordinates.py b/tests/test_coordinates.py index bd9a28a..00d7c7a 100644 --- a/tests/test_coordinates.py +++ b/tests/test_coordinates.py @@ -6,7 +6,6 @@ from collections import Counter from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) diff --git a/tests/test_integrity.py b/tests/test_integrity.py index 4560f5a..2176e07 100644 --- a/tests/test_integrity.py +++ b/tests/test_integrity.py @@ -7,13 +7,14 @@ import unittest from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from build_dataset import build_clean_room # noqa: E402 from check_determinism import ( # noqa: E402 DEFAULT_REPORT as DETERMINISM_REPORT, +) +from check_determinism import ( collect_signatures, ) from dataset_common import ( # noqa: E402 @@ -28,6 +29,7 @@ record_digest, ) from legacy_comparison import build_legacy_comparison # noqa: E402 +from reconciliation_backlog import build_backlog # noqa: E402 from source_data import ( # noqa: E402 DEFAULT_GEONAMES, DEFAULT_LEGACY, @@ -189,6 +191,19 @@ def test_export_manifest_uses_semantic_sqlite_digest(self) -> None: sqlite_output["semantic_sha256"], ) + def test_reconciliation_backlog_is_evidence_first(self) -> None: + report = build_backlog(self.municipalities, self.localities) + self.assertEqual(10270, report["summary"]["unreconciled_localities"]) + self.assertEqual( + 396, + report["summary"]["municipalities_without_geonames_postal_match"], + ) + self.assertIn( + "SU", + report["summary"]["noncurrent_source_province_codes"], + ) + self.assertFalse(report["policy"]["automatic_fuzzy_promotion"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pages.py b/tests/test_pages.py index 75cf633..fc23801 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -8,12 +8,12 @@ import unittest from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from build_pages import WEB_FIELDS, build_site # noqa: E402 from project_metadata import DATASET_VERSION # noqa: E402 +from validate_site import validate_site # noqa: E402 def directory_hashes(root: Path) -> dict[str, str]: @@ -27,7 +27,7 @@ def directory_hashes(root: Path) -> dict[str, str]: class GitHubPagesBuildTests(unittest.TestCase): def test_build_is_complete_and_deterministic(self) -> None: - with tempfile.TemporaryDirectory() as first_dir: + with tempfile.TemporaryDirectory() as first_dir: # noqa: SIM117 with tempfile.TemporaryDirectory() as second_dir: first = Path(first_dir) / "site" second = Path(second_dir) / "site" @@ -84,6 +84,7 @@ def test_site_declares_seo_release_and_clean_room_contracts(self) -> None: "GeoNames, CC BY 4.0", "v2.0.0 · prerelease pubblicata", "releases/download/v2.0.0/italian_locations.csv", + "releases/download/v2.0.0/SHA256SUMS", "releases/tag/v2.0.0", "Confini regionali generalizzati ISTAT", 'rel="canonical"', @@ -113,7 +114,12 @@ def test_site_declares_seo_release_and_clean_room_contracts(self) -> None: if item["@type"] == "Dataset" ) self.assertEqual("2.0.0", dataset["version"]) - self.assertEqual(5, len(dataset["distribution"])) + self.assertEqual(6, len(dataset["distribution"])) + self.assertIn('id="copy-filter-link"', source) + self.assertNotIn( + "Gli asset pubblicati sono immutabili", + source, + ) def test_sitemap_and_indexable_information_pages(self) -> None: with tempfile.TemporaryDirectory() as temporary_dir: @@ -152,6 +158,14 @@ def test_committed_geographic_base_has_official_provenance(self) -> None: self.assertEqual(20, len(payload["features"])) self.assertEqual("CC BY 4.0", payload["source"]["license"]) + def test_site_accessibility_links_and_budgets(self) -> None: + with tempfile.TemporaryDirectory() as temporary_dir: + output = Path(temporary_dir) / "site" + build_site(output) + report = validate_site(output) + self.assertEqual("passed", report["status"]) + self.assertEqual(25, report["pages"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_quality_gate.py b/tests/test_quality_gate.py index 8bc6700..2f6dfab 100644 --- a/tests/test_quality_gate.py +++ b/tests/test_quality_gate.py @@ -5,7 +5,6 @@ import unittest from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) diff --git a/tests/test_release.py b/tests/test_release.py index 7d4a6a2..e844eb8 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -5,7 +5,6 @@ import unittest from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) @@ -22,8 +21,8 @@ sha256_file, ) from export_sql import export_sql # noqa: E402 -from validate_release import EXPECTED_ASSETS, validate_release # noqa: E402 from project_metadata import DATASET_VERSION # noqa: E402 +from validate_release import EXPECTED_ASSETS, validate_release # noqa: E402 class ReleaseTests(unittest.TestCase): diff --git a/tests/test_schema.py b/tests/test_schema.py index aef0b61..a3a76ef 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -6,7 +6,6 @@ import unittest from pathlib import Path - ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) diff --git a/tests/test_source_updates.py b/tests/test_source_updates.py new file mode 100644 index 0000000..67596f7 --- /dev/null +++ b/tests/test_source_updates.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import hashlib +import io +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from update_sources import download, inspect_source, stage_download # noqa: E402 + + +class FakeResponse(io.BytesIO): + def __init__(self, payload: bytes) -> None: + super().__init__(payload) + self.headers = { + "Last-Modified": "Thu, 30 Jul 2026 00:00:00 GMT", + "ETag": '"test"', + } + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *args: object) -> None: + self.close() + + +class SourceUpdateTests(unittest.TestCase): + def test_download_records_hash_and_http_metadata(self) -> None: + payload = b"upstream artifact" + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "source.bin" + with patch( + "update_sources.urlopen", + return_value=FakeResponse(payload), + ): + report = download("https://example.test/source", destination) + self.assertEqual(payload, destination.read_bytes()) + self.assertEqual(hashlib.sha256(payload).hexdigest(), report["sha256"]) + self.assertEqual('"test"', report["etag"]) + + def test_inspection_distinguishes_current_and_changed(self) -> None: + payload = b"declared" + source = { + "id": "example", + "url": "https://example.test/source", + "path": "source.bin", + "sha256": hashlib.sha256(payload).hexdigest(), + } + with patch( + "update_sources.urlopen", + return_value=FakeResponse(payload), + ): + self.assertEqual("current", inspect_source(source)["status"]) + with patch( + "update_sources.urlopen", + return_value=FakeResponse(b"changed"), + ): + self.assertEqual( + "update_available", + inspect_source(source)["status"], + ) + + def test_staging_never_overwrites_an_existing_file(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "existing.zip" + output.write_bytes(b"keep") + with self.assertRaisesRegex(FileExistsError, "refusing to overwrite"): + stage_download("geonames_postal_codes", output) + self.assertEqual(b"keep", output.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_typed_contract.py b/tests/test_typed_contract.py new file mode 100644 index 0000000..4bd86f1 --- /dev/null +++ b/tests/test_typed_contract.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from dataset_common import GENERATED_PATHS # noqa: E402 +from export_typed_json import ( # noqa: E402 + export_typed_json, + export_typed_sqlite, +) + + +class TypedContractTests(unittest.TestCase): + def test_preview_uses_numbers_and_null_without_changing_v2(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "typed.json" + export_typed_json(GENERATED_PATHS["italian_locations"], output) + payload = json.loads(output.read_text(encoding="utf-8")) + rows = payload["rows"] + present = next(row for row in rows if row["latitude"] is not None) + missing = next(row for row in rows if row["latitude"] is None) + self.assertIsInstance(present["latitude"], float) + self.assertIsInstance(present["longitude"], float) + self.assertIsInstance(present["coordinate_accuracy"], int) + self.assertIsNone(missing["longitude"]) + self.assertIsNone(missing["coordinate_accuracy"]) + self.assertEqual("4.0.0", payload["metadata"]["schema_version"]) + + def test_typed_sqlite_preview_uses_real_integer_and_null(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "typed.sqlite" + export_typed_sqlite( + GENERATED_PATHS["italian_locations"], + output, + ) + connection = sqlite3.connect(output) + try: + types = connection.execute( + 'SELECT typeof(latitude), typeof(longitude), ' + 'typeof(coordinate_accuracy) ' + 'FROM "italian_locations" ' + 'WHERE latitude IS NOT NULL LIMIT 1' + ).fetchone() + missing = connection.execute( + 'SELECT latitude, longitude, coordinate_accuracy ' + 'FROM "italian_locations" ' + 'WHERE coordinate_verification = "missing" LIMIT 1' + ).fetchone() + finally: + connection.close() + self.assertEqual(("real", "real", "integer"), types) + self.assertEqual((None, None, None), missing) + + +if __name__ == "__main__": + unittest.main()