From f511762caa921e92f0f90904553e1c2864dd20f3 Mon Sep 17 00:00:00 2001 From: raffmont Date: Sat, 29 Aug 2026 12:37:42 +0200 Subject: [PATCH] feat: add DataTiles revision 8 and Store platform --- CHANGELOG.md | 7 + CITATION.cff | 2 +- README.md | 48 ++ demo/from-gaeta-to-maratea/CITATION.cff | 2 +- demo/from-gaeta-to-maratea/README.md | 16 + .../fair-publication-profile.json | 25 + demo/from-gaeta-to-maratea/runtime-lock.json | 2 +- docs/README.md | 13 +- docs/conventions.md | 43 ++ docs/data-licensing.md | 19 + docs/data_sources_and_citation.md | 175 +++++ docs/digital-signatures.md | 119 ++++ docs/drm-and-commercial-licensing.md | 95 +++ docs/fair-by-design.md | 6 + docs/fair-publication-profile.md | 65 ++ docs/images/demo/README.md | 2 +- docs/import-utilities.md | 81 +++ docs/migrations/v4-to-v5.sql | 62 ++ docs/migrations/v5-to-v6.sql | 29 + docs/migrations/v6-to-v7.sql | 24 + docs/migrations/v7-to-v8.sql | 16 + docs/playground.md | 12 + docs/provenance.md | 17 + docs/semantic-variables.md | 85 +++ docs/specification-revision-5-addendum.md | 7 + docs/specification-revision-6-addendum.md | 45 ++ docs/specification-revision-7-addendum.md | 36 + docs/specification-revision-8-addendum.md | 21 + docs/specification.md | 24 +- docs/store-api.md | 114 ++++ docs/store-pwa.md | 104 +++ docs/tutorial/README.md | 9 +- docs/tutorial/examples/build_tutorial.py | 20 +- docs/tutorial/lesson-10-datatiles-store.md | 134 ++++ .../lesson-6-conventions-and-utils.md | 160 +++++ .../lesson-7-fair-provenance-and-licensing.md | 89 +++ .../lesson-8-cryptographic-integrity.md | 71 ++ docs/tutorial/lesson-9-commercial-drm.md | 39 ++ docs/white-paper-data-sources-addendum.md | 20 + docs/white-paper.md | 47 ++ docs/zarr-source-profile.md | 78 +++ pyproject.toml | 15 +- src/datatiles/__init__.py | 2 +- src/datatiles/cli.py | 30 +- src/datatiles/drm.py | 358 ++++++++++ src/datatiles/drm_cli.py | 32 + src/datatiles/fair.py | 89 +++ src/datatiles/integrity.py | 376 ++++++++++ src/datatiles/integrity_cli.py | 99 +++ src/datatiles/migration_v5_to_v6.sql | 29 + src/datatiles/migration_v6_to_v7.sql | 24 + src/datatiles/migration_v7_to_v8.sql | 16 + src/datatiles/profile-demo.html | 9 +- src/datatiles/schema.sql | 116 +++- src/datatiles/semantic.py | 71 ++ src/datatiles/server.py | 15 +- src/datatiles/static_demo.py | 7 +- src/datatiles/store.py | 346 +++++++++- store/AGENTS.md | 19 + store/LICENSE.md | 206 ++++++ store/README.md | 19 + store/config.py | 36 + store/datatiles_store/__init__.py | 5 + store/datatiles_store/admin_ext.py | 83 +++ store/datatiles_store/agreements.py | 75 ++ store/datatiles_store/app.py | 641 ++++++++++++++++++ store/datatiles_store/auth_ext.py | 125 ++++ store/datatiles_store/catalog.py | 146 ++++ store/datatiles_store/commerce.py | 47 ++ store/datatiles_store/db.py | 94 +++ store/datatiles_store/mailer.py | 15 + store/datatiles_store/models.py | 253 +++++++ store/datatiles_store/payments.py | 46 ++ store/datatiles_store/security.py | 83 +++ store/datatiles_store/settings.py | 66 ++ store/datatiles_store/static/app.css | 2 + store/datatiles_store/static/explorer.js | 1 + .../datatiles_store/static/icons/icon-192.png | Bin 0 -> 1051 bytes .../datatiles_store/static/icons/icon-512.png | Bin 0 -> 3412 bytes .../static/manifest.webmanifest | 1 + .../datatiles_store/static/service-worker.js | 1 + .../datatiles_store/templates/agreement.html | 24 + store/datatiles_store/templates/base.html | 28 + store/datatiles_store/templates/catalog.html | 12 + store/datatiles_store/templates/checkout.html | 1 + .../templates/configuration.html | 1 + store/datatiles_store/templates/detail.html | 22 + store/datatiles_store/templates/groups.html | 1 + store/datatiles_store/templates/help.html | 1 + store/datatiles_store/templates/library.html | 1 + store/datatiles_store/templates/login.html | 5 + store/datatiles_store/templates/register.html | 1 + store/datatiles_store/templates/users.html | 1 + store/docs/README.md | 15 + store/docs/agreements.md | 5 + store/docs/api.md | 13 + store/docs/authentication.md | 29 + store/docs/configuration.md | 26 + store/docs/index.md | 17 + store/docs/installation.md | 27 + store/docs/management.md | 28 + store/docs/payments.md | 26 + store/docs/security.md | 9 + store/docs/smtp.md | 7 + store/docs/troubleshooting.md | 17 + store/docs/versioning-and-updates.md | 7 + store/pyproject.toml | 26 + store/run.py | 5 + store/tests/test_catalog.py | 56 ++ store/tests/test_commerce.py | 20 + store/tests/test_policy_models.py | 40 ++ tests/test_drm.py | 29 + tests/test_fair.py | 17 + tests/test_import_utils.py | 88 +++ tests/test_integrity.py | 91 +++ tests/test_semantic.py | 23 + tests/test_store.py | 7 +- utils/README.md | 83 +++ utils/common.py | 472 +++++++++++++ utils/grib2datatiles.py | 77 +++ utils/netcdf2datatiles.py | 56 ++ utils/zarr2datatiles.py | 115 ++++ 122 files changed, 7068 insertions(+), 42 deletions(-) create mode 100644 demo/from-gaeta-to-maratea/fair-publication-profile.json create mode 100644 docs/conventions.md create mode 100644 docs/data-licensing.md create mode 100644 docs/data_sources_and_citation.md create mode 100644 docs/digital-signatures.md create mode 100644 docs/drm-and-commercial-licensing.md create mode 100644 docs/fair-publication-profile.md create mode 100644 docs/import-utilities.md create mode 100644 docs/migrations/v4-to-v5.sql create mode 100644 docs/migrations/v5-to-v6.sql create mode 100644 docs/migrations/v6-to-v7.sql create mode 100644 docs/migrations/v7-to-v8.sql create mode 100644 docs/provenance.md create mode 100644 docs/semantic-variables.md create mode 100644 docs/specification-revision-5-addendum.md create mode 100644 docs/specification-revision-6-addendum.md create mode 100644 docs/specification-revision-7-addendum.md create mode 100644 docs/specification-revision-8-addendum.md create mode 100644 docs/store-api.md create mode 100644 docs/store-pwa.md create mode 100644 docs/tutorial/lesson-10-datatiles-store.md create mode 100644 docs/tutorial/lesson-6-conventions-and-utils.md create mode 100644 docs/tutorial/lesson-7-fair-provenance-and-licensing.md create mode 100644 docs/tutorial/lesson-8-cryptographic-integrity.md create mode 100644 docs/tutorial/lesson-9-commercial-drm.md create mode 100644 docs/white-paper-data-sources-addendum.md create mode 100644 docs/zarr-source-profile.md create mode 100644 src/datatiles/drm.py create mode 100644 src/datatiles/drm_cli.py create mode 100644 src/datatiles/fair.py create mode 100644 src/datatiles/integrity.py create mode 100644 src/datatiles/integrity_cli.py create mode 100644 src/datatiles/migration_v5_to_v6.sql create mode 100644 src/datatiles/migration_v6_to_v7.sql create mode 100644 src/datatiles/migration_v7_to_v8.sql create mode 100644 src/datatiles/semantic.py create mode 100644 store/AGENTS.md create mode 100644 store/LICENSE.md create mode 100644 store/README.md create mode 100644 store/config.py create mode 100644 store/datatiles_store/__init__.py create mode 100644 store/datatiles_store/admin_ext.py create mode 100644 store/datatiles_store/agreements.py create mode 100644 store/datatiles_store/app.py create mode 100644 store/datatiles_store/auth_ext.py create mode 100644 store/datatiles_store/catalog.py create mode 100644 store/datatiles_store/commerce.py create mode 100644 store/datatiles_store/db.py create mode 100644 store/datatiles_store/mailer.py create mode 100644 store/datatiles_store/models.py create mode 100644 store/datatiles_store/payments.py create mode 100644 store/datatiles_store/security.py create mode 100644 store/datatiles_store/settings.py create mode 100644 store/datatiles_store/static/app.css create mode 100644 store/datatiles_store/static/explorer.js create mode 100644 store/datatiles_store/static/icons/icon-192.png create mode 100644 store/datatiles_store/static/icons/icon-512.png create mode 100644 store/datatiles_store/static/manifest.webmanifest create mode 100644 store/datatiles_store/static/service-worker.js create mode 100644 store/datatiles_store/templates/agreement.html create mode 100644 store/datatiles_store/templates/base.html create mode 100644 store/datatiles_store/templates/catalog.html create mode 100644 store/datatiles_store/templates/checkout.html create mode 100644 store/datatiles_store/templates/configuration.html create mode 100644 store/datatiles_store/templates/detail.html create mode 100644 store/datatiles_store/templates/groups.html create mode 100644 store/datatiles_store/templates/help.html create mode 100644 store/datatiles_store/templates/library.html create mode 100644 store/datatiles_store/templates/login.html create mode 100644 store/datatiles_store/templates/register.html create mode 100644 store/datatiles_store/templates/users.html create mode 100644 store/docs/README.md create mode 100644 store/docs/agreements.md create mode 100644 store/docs/api.md create mode 100644 store/docs/authentication.md create mode 100644 store/docs/configuration.md create mode 100644 store/docs/index.md create mode 100644 store/docs/installation.md create mode 100644 store/docs/management.md create mode 100644 store/docs/payments.md create mode 100644 store/docs/security.md create mode 100644 store/docs/smtp.md create mode 100644 store/docs/troubleshooting.md create mode 100644 store/docs/versioning-and-updates.md create mode 100644 store/pyproject.toml create mode 100644 store/run.py create mode 100644 store/tests/test_catalog.py create mode 100644 store/tests/test_commerce.py create mode 100644 store/tests/test_policy_models.py create mode 100644 tests/test_drm.py create mode 100644 tests/test_fair.py create mode 100644 tests/test_import_utils.py create mode 100644 tests/test_integrity.py create mode 100644 tests/test_semantic.py create mode 100644 utils/README.md create mode 100644 utils/common.py create mode 100644 utils/grib2datatiles.py create mode 100644 utils/netcdf2datatiles.py create mode 100644 utils/zarr2datatiles.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ece3467..acba1d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.19.0 — 2026-08-29 + +- advances the DataTiles schema to revision 8 with CF-first semantic variables, FAIR publication metadata, provenance, rights, optional integrity signatures, optional protected distribution, and immutable release identity; +- adds deterministic NetCDF, GRIB2, and Zarr scientific import utilities that preserve numeric DNT1 values and record source and conversion provenance; +- adds the optional Flask/SQLAlchemy DataTiles Store PWA, API, authentication, agreement enforcement, provider-neutral commerce, user libraries, and update discovery; +- expands the normative specification, migration addenda, source-citation register, white paper, tutorials, and regression coverage for revisions 4 through 8. + ## 0.10.0 — 2026-08-27 - adds a command-line static-demo exporter, a Safari-compatible client-side numeric/vector portrayal, the full scientific playground toolset (relief, contours, smart labels, profile, compound query, and interactive 3D mesh), embedded checksum-identified data for direct `file://` use, an offline OpenLayers bundle, and an optional loopback-only launcher; diff --git a/CITATION.cff b/CITATION.cff index c4508ff..1726ede 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,7 +5,7 @@ type: software authors: - family-names: Montella given-names: Raffaele -version: 0.10.0 +version: 0.19.0 date-released: 2026-08-27 license: Apache-2.0 keywords: diff --git a/README.md b/README.md index a605329..0ca3162 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ The spatial row follows MBTiles/TMS convention. Use `--xyz` on `put` and `get` t Version 0.10 adds conservative physical-table MBTiles fallback, a self-sufficient implementation specification, and an onboard edge-intelligence manifesto/white paper. It also includes mixed raster/vector content profiles, a tested five-lesson zero-to-hero curriculum, FAIR-by-design publication profile, OpenLayers scientific playground, comprehensive quality suite, protected CI/CD release path, interval axes, PROV-inspired provenance, scientific CRS records, bounded numeric-array decoding, OpenAPI description, and read-only OGC-style access. +## NetCDF, GRIB, and Zarr import utilities + +Optional import utilities under `utils/` convert local or URL-identified NetCDF, GRIB2, and Zarr rectilinear grids into semantic DNT1 DataTiles. They preserve CF Standard Names when present, add GRIB2 crosswalk identifiers when available, retain source identity plus format-appropriate checksum provenance, and never pre-render scientific arrays as imagery. Install with `python -m pip install -e '.[utils]'`; see `docs/import-utilities.md`. + ## Raster and vector content - Numeric raster matrices use the dependency-free DNT1 encoding and retain dtype, shape, byte order, nodata, scale, offset, unit, and compression. @@ -69,6 +73,18 @@ The release workspace may also contain `dist/from-gaeta-to-maratea.datatiles` an Repository guidance is in [`AGENTS.md`](AGENTS.md), and the Markdown license notice is in [`LICENSE.md`](LICENSE.md). The complete Apache-2.0 legal text remains in `LICENSE`. +## Zarr source ingestion + +DataTiles 0.13 adds `utils/zarr2datatiles.py` for Zarr v2/v3 stores. Local directory stores use the canonical `zarr-tree-sha256-v1` store digest; remote stores must be bound to an immutable snapshot with an authoritative checksum. Groups, consolidated metadata policy, fsspec storage backends, CF semantic mapping, explicit source/output rights, and credential-safe provenance are documented in `docs/zarr-source-profile.md`. + +## Optional cryptographic integrity and signatures + +DataTiles schema revision 6 can cryptographically bind the final logical scientific object to a signer without making signatures mandatory. `datatiles-integrity` creates deterministic SHA-256 logical manifests and optional Ed25519 signatures, either embedded or detached. Verification explicitly distinguishes mathematical validity from trust in the signer: an embedded public key is not, by itself, an authenticated publisher identity. The dependency-free core remains unchanged; install `.[integrity]` only when signing/verification is required. See `docs/digital-signatures.md`. + +## FAIR scientific publication + +DataTiles 0.12 treats FAIR, provenance, and licensing as enforceable publication evidence. Published objects separate persistent identity, DataCite 4.7 citation metadata, W3C PROV-aligned lineage, SPDX rights for the generated dataset/metadata and each source, cryptographic source identity, and external repository evidence. `fair_report(strict_publication=True)` is a release gate; it does not claim that FAIRness proves scientific validity or navigational fitness. See `docs/fair-publication-profile.md`, `docs/provenance.md`, and `docs/data-licensing.md`. + ## Quality assurance and releases Pull requests run the complete suite on Python 3.10–3.13, validate playground JavaScript, build and install the distributions, and repeat the deterministic scientific-fixture build. Version tags additionally produce checksums, a GitHub build-provenance attestation, a GitHub Release, and—after the protected environment gate—a PyPI Trusted Publishing deployment. See [`docs/testing-and-release.md`](docs/testing-and-release.md) for the local protocol and repository configuration. @@ -104,6 +120,24 @@ The [onboard intelligence white paper](docs/white-paper.md) presents DataTiles a New users can follow the [five-lesson DataTiles zero-to-hero tutorial](docs/tutorial/README.md), which combines formal discussion with a fully offline mixed raster/vector laboratory dataset. +## Data acknowledgement and citation — required reading + +Data Tiles demo derivatives are multi-source scientific products. Every map, MBTiles file, figure, service, paper, and release must visibly credit every dataset that actually contributed cells or features and must ship its machine-readable provenance. The MIT software licence does not relicense any input data. +The authoritative, source-by-source register is docs/data_sources_and_citation.md. It gives full bibliographic citations, mandatory map acknowledgements, licences, persistent identifiers, coverage/resolution limitations, frozen-manifest locations, and a FAIR release checklist. + +For the current From Gaeta to Maratea production build, acknowledge: + +JammeGaia22/MGDS bathymetry: Foglini, F., Tonielli, R., & Rovere, M. (2024), Multi-Resolution bathymetry grids of the Naples and Pozzuoli Gulf and Amalfi Coastal Area, Jamme_Gaia22 (2022), https://doi.org/10.60521/331667, CC BY 4.0. +EMODnet fallback bathymetry: EMODnet Bathymetry Consortium (2024), EMODnet Digital Bathymetry (DTM 2024), https://doi.org/10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1, CC BY 4.0. EMODnet is used only where JammeGaia22 contains no finite measurement. +Land/coastline: GSHHG 2.3.7 (Wessel & Smith, 1996, https://doi.org/10.1029/96JB00104) and, when enabled, S2Coast-2023 (Duan et al., https://doi.org/10.5281/zenodo.17092775, CC BY 4.0). These remain separate topology and high-water-line facts. +Navigation and land context: © OpenStreetMap contributors, ODbL 1.0, https://www.openstreetmap.org/copyright. +The required short-form map credit is: + +Bathymetry: Foglini, Tonielli & Rovere (2024), JammeGaia22/MGDS, doi:10.60521/331667; EMODnet Bathymetry Consortium DTM 2024, doi:10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1. Land/coastline: GSHHG 2.3.7 and S2Coast-2023. Context: © OpenStreetMap contributors, ODbL 1.0. Not for navigation. +Do not list GMRT, GEBCO, EMODnet thematic products, or ISPRA as contributors unless the specific run manifest proves that they were used. Citation is source-specific evidence, not a generic project boilerplate. + +> **Software-licence consistency check.** The current public DataTiles repository declares Apache-2.0 in `LICENSE`. The preceding requested wording says MIT. Before publication, the README wording and `LICENSE` MUST agree; regardless of which software licence is adopted, it does not relicense input data. + ## How to cite DataTiles Research using the software SHOULD cite the released software described by [`CITATION.cff`](CITATION.cff). Also cite the paper(s) that support the part of the scientific lineage or application you use: @@ -121,3 +155,17 @@ This repository is a working reference implementation and a draft format specifi ## License Apache-2.0. + +## Optional commercial DRM + +DataTiles can optionally package a finalized lawful commercial product as an encrypted `.dtpkg` with recipient-specific issuer-signed licence grants. The portable profile uses AES-256-GCM, X25519/HKDF key wrapping, Ed25519 issuer signatures, and W3C ODRL 2.2 policy metadata. DRM is a technical access-control mechanism only: it does not relicense source data, remove attribution obligations, establish ownership, prove scientific validity, or make a product suitable for navigation. See `docs/drm-and-commercial-licensing.md`. + +## DataTiles Store PWA + +The optional `/store` application is a Python/Flask progressive web application backed by SQLAlchemy. It indexes machine-readable metadata from stored DataTiles files into a separate application database, supports full catalog search, authenticated browsing, role/group based authorization, an interactive chart-explorer-style preview of declared portrayal slices, and authorized downloads of the original DataTiles files. The bootstrap `admin` user belongs to the `administrators` group with the `admin` role; its initial password is explicitly configured in `store/config.py` and must be changed/secret-managed for production. The PWA never treats numeric scientific matrices as imagery and its service worker does not cache protected APIs, map payloads, or downloads. See `docs/store-pwa.md`. Managed authentication, verified-email self-registration, Google OIDC, Microsoft Entra ID tenants, generic OAuth2/OIDC, SMTP, public callback URL, and agreement settings are administrator-configurable in the PWA and API. Complete Store help is maintained under `store/docs/` and rendered by the PWA Help section. + +## Release versioning and optional Store payments + +Schema revision 8 defines `DataTiles-Release-Versioning-1`: a release can carry stable product identity, a human version label, a monotonically increasing release sequence, timestamp, predecessor, release-notes URI, and update-discovery URI. Published versioned releases are immutable; corrections and updates are new DataTiles objects with new checksums and signatures where used. + +The optional Store 0.4 adds a provider-neutral payment interface, with PayPal Orders v2 as the reference adapter, plus release-specific purchase entitlements, exact download history, a user library, and update notifications for users who previously acquired older sequences. Payment is disabled by default and is legally independent from data-licence acceptance: paying for a product never creates rights that the publisher does not possess. See `store/docs/payments.md` and `store/docs/versioning-and-updates.md`. diff --git a/demo/from-gaeta-to-maratea/CITATION.cff b/demo/from-gaeta-to-maratea/CITATION.cff index cd11d47..8f36068 100644 --- a/demo/from-gaeta-to-maratea/CITATION.cff +++ b/demo/from-gaeta-to-maratea/CITATION.cff @@ -5,6 +5,6 @@ type: software authors: - family-names: Montella given-names: Raffaele -version: 0.10.0 +version: 0.19.0 date-released: 2026-08-27 license: Apache-2.0 diff --git a/demo/from-gaeta-to-maratea/README.md b/demo/from-gaeta-to-maratea/README.md index 08b5d61..3316ade 100644 --- a/demo/from-gaeta-to-maratea/README.md +++ b/demo/from-gaeta-to-maratea/README.md @@ -89,3 +89,19 @@ After `make all`, run `datatiles-serve work/gaeta-to-maratea.datatiles --port 80 EMODnet is acquired at 2,754 × 1,167 regional cells; the compositing grid is 5,508 × 2,334 (nominal 50 m target spacing) so measured Jamme detail is not first collapsed to the coarser fallback grid. Storage extends through Web Mercator zoom 12. The client requests a bounded 128 × 128 map portrayal and a separate 96 × 72 analytic surface. The source-coverage layer makes every Jamme/EMODnet decision inspectable. Contour density is scale-dependent and remains limited by the selected source resolution; no new soundings are invented. The shelter field is a finite north-west ray test over the bathymetry water/land mask. It is a reproducible exposure proxy, not a wind-wave or atmospheric model, and must not support navigational or safety decisions. + +## FAIR, provenance, and rights gate + +The demo is also a publication-evidence exercise. Every upstream dataset must be represented by a checksum-identified provenance entity and an explicit source-rights record; the generated DataTiles dataset and its metadata have separate rights. The build must never infer permission from successful HTTP access. Before a scholarly release, replace illustrative identifiers with a registered PID, export DataCite 4.7 metadata and the W3C PROV-aligned graph, and archive the strict FAIR report with catalogue, landing-page, and metadata-retention evidence. `fair-publication-profile.json` lists the required evidence. The navigation warning remains independent of FAIR status: FAIRness does not establish hydrographic authority or safety fitness. + +## Source-specific citation gate + +`docs/data_sources_and_citation.md` is the authoritative citation register. A production run MUST record whether each acquired source actually contributed cells/features. Visible map/release acknowledgements MUST be checked against that frozen manifest: JammeGaia22 is primary bathymetry; EMODnet DTM 2024 is credited only when fallback cells are used; GSHHG topology and optional S2Coast-2023 high-water-line facts remain distinct; OpenStreetMap context retains ODbL attribution. Do not credit GMRT, GEBCO, EMODnet thematic products, ISPRA, or other candidates unless the run evidence proves contribution. + +## Optional signed release + +After the final source-contribution manifest, citations, rights, provenance, and FAIR checks are frozen, the demo MAY be signed with `datatiles-integrity`. Do not auto-sign intermediate NetCDF/GRIB/Zarr imports. The release should publish the detached signature and independently authenticated public key. A valid signature does not change the `Not for navigation` status. + +## Optional commercial edition + +The reference demo is not automatically converted into a proprietary product. A commercial edition may be created only if the frozen contribution manifest and every upstream licence permit that use. If enabled, register product/ODRL metadata in the inner DataTiles object, complete all acknowledgements, sign the frozen plaintext release if desired, and only then create `.dtpkg` packages and recipient licences. `Not for navigation` remains mandatory unless an independent competent authority establishes otherwise. diff --git a/demo/from-gaeta-to-maratea/fair-publication-profile.json b/demo/from-gaeta-to-maratea/fair-publication-profile.json new file mode 100644 index 0000000..8011d9a --- /dev/null +++ b/demo/from-gaeta-to-maratea/fair-publication-profile.json @@ -0,0 +1,25 @@ +{ + "profile": "DataTiles FAIR scholarly demonstration", + "purpose": "Template only; replace illustrative identifiers with registered publication evidence", + "required_evidence": [ + "primary PID", + "dataset rights", + "metadata rights", + "source rights per provenance entity", + "source SHA-256 locks", + "W3C PROV-aligned graph", + "DataCite 4.7 metadata export", + "catalogue registration", + "landing page", + "metadata retention policy", + "scientific QA and limitations" + ], + "warning": "Demo outputs are research/visualization artifacts and are not for navigation.", + "optional_integrity_evidence": [ + "DataTiles-Integrity-Manifest-1", + "Ed25519 detached/embedded signature", + "independently authenticated public key or certificate", + "optional Sigstore/timestamp/transparency evidence" + ], + "integrity_warning": "Signature validity does not establish scientific validity, legal compliance, hydrographic authority, or navigation fitness." +} diff --git a/demo/from-gaeta-to-maratea/runtime-lock.json b/demo/from-gaeta-to-maratea/runtime-lock.json index bb88e20..e405309 100644 --- a/demo/from-gaeta-to-maratea/runtime-lock.json +++ b/demo/from-gaeta-to-maratea/runtime-lock.json @@ -4,5 +4,5 @@ "zlib": "1.3.2", "numpy": "2.3.5", "pillow": "12.3.0", - "datatiles": "0.10.0" + "datatiles": "0.19.0" } diff --git a/docs/README.md b/docs/README.md index 76f324b..9e49e03 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,16 +10,27 @@ The [figure provenance register](figures/README.md) identifies the source docume | [MBTiles fallback](mbtiles-fallback.md) | selected-slice compatibility and physical-table export for conservative OpenLayers adapters | | [Onboard intelligence white paper](white-paper.md) | manifesto, AI evidence contract, practical marine/automotive architecture, and safety boundaries | | [Design evaluation](design-evaluation.md) | concise definition, audit findings, corrections, and compatibility conclusion | -| [Zero-to-hero tutorial](tutorial/README.md) | five theory-led lessons with executable raster/vector laboratories | +| [Zero-to-hero tutorial](tutorial/README.md) | seven theory-led lessons with executable raster/vector laboratories | | [FAIR by design](fair-by-design.md) | operational mapping from each FAIR principle to DataTiles evidence and publication gates | +| [FAIR scientific publication profile](fair-publication-profile.md) | normative-supporting FAIR gates, DataCite 4.7 alignment, and release evidence | +| [Scientific provenance](provenance.md) | W3C PROV-aligned entities, activities, agents, lineage, and audit rules | +| [Data licensing and rights](data-licensing.md) | SPDX expressions, source/output/metadata rights separation, attribution, and access conditions | +| [Cryptographic integrity and digital signatures](digital-signatures.md) | optional SHA-256 logical manifests, Ed25519 signatures, trust policy, offline verification, and Sigstore/in-toto interoperability | +| [Data sources and citation](data_sources_and_citation.md) | authoritative source-by-source citations, acknowledgements, licences, frozen-manifest rules, and FAIR release checklist | | [Architecture](architecture.md) | component boundaries and design rationale | | [Numeric tiles](numeric-tiles.md) | DNT1 encoding and physical-value semantics | | [Vector tiles](vector-tiles.md) | multidimensional MVT/GeoJSON profiles and MBTiles projection | | [HTTP/OGC API](ogc-api.md) | resource model and interoperability scope | | [Playground](playground.md) | OpenLayers scientific operations and limitations | +| [DataTiles Store PWA](store-pwa.md) | Flask/SQLAlchemy institutional catalog, metadata search, role-based access, interactive preview, licence/safety acceptance, API-first CRUD, and authorized downloads | +| [DataTiles Store API v1](store-api.md) | Bearer authentication, agreement workflow, catalog/search, preview/download, managers CRUD, identity administration, and audit endpoints | | [Testing and release](testing-and-release.md) | local suite, CI gates, protected delivery, and release checklist | | [Profile demo](profile-demo.md) | two-point transect algorithm and evidence | | [Reproducibility](reproducibility.md) | exact reconstruction of the Naples reference object | | [Replicability](replicability.md) | protocol for independent regions, sources, variables, and implementations | | [Getting started](getting-started.md) | concise installation and API tutorial | +| [Interoperability conventions](conventions.md) | CF-first variable semantics, units, CRS/datum, source identity, dimensions, and portrayal boundaries | +| [Import utilities](import-utilities.md) | NetCDF/GRIB/Zarr local-or-URL ingestion, semantic mapping, numeric tiling, and provenance | +| [Zarr source profile](zarr-source-profile.md) | normative Zarr v2/v3 source identity, checksums, groups, remote stores, FAIR provenance and rights | | [References](references.md) | research papers, standards, vocabularies, and source records | +| [Commercial DRM](drm-and-commercial-licensing.md) | optional protected distribution, ODRL policy, recipient licences | diff --git a/docs/conventions.md b/docs/conventions.md new file mode 100644 index 0000000..a240f4a --- /dev/null +++ b/docs/conventions.md @@ -0,0 +1,43 @@ +# DataTiles interoperability conventions + +This document defines the interoperability conventions used by the semantic-variable extension and the NetCDF/GRIB/Zarr import utilities. The container specification remains authoritative for byte-level and schema behavior; these conventions define how common scientific concepts should be represented so independently developed software can discover and compare them. + +## Variable identity + +A DataTiles coordinate named `variable` is a producer-facing token used in multidimensional addresses. It is not, by itself, the scientific definition of the quantity. Every token should resolve through the semantic variable registry. + +CF Standard Names are the preferred canonical vocabulary when a valid CF concept exists. A producer-local variable without a valid CF name must use an explicitly different vocabulary such as `source`; software must never manufacture a CF name merely to satisfy validation. + +External identifiers are crosswalks. WMO GRIB2 discipline/category/parameter, GRIB short names, SeaDataNet/NERC vocabulary URIs, GCMD identifiers and provider-specific codes can all coexist for one semantic variable. + +## Units + +Units remain explicit at both semantic-registry and numeric-payload levels. A CF Standard Name does not justify discarding the source unit. Validation against a pinned CF table can verify the declared CF canonical unit; physical equivalence between alternate units requires a UDUNITS-capable validation profile rather than string guessing. + +## Vertical reference and CRS + +Quantities involving depth, height or elevation must preserve the relevant vertical reference. Names such as `sea_floor_depth_below_geoid` and `sea_floor_depth_below_mean_sea_level` are not interchangeable. CRS/datum information remains first-class metadata even when the CF name carries part of that meaning. + +Spatial tiles are stored using MBTiles/TMS row convention. XYZ is an interface convention and is converted at the boundary. Import utilities target Web Mercator (`EPSG:3857`) and must declare their resampling method. + +## Time and other scientific dimensions + +Time, pressure, depth/elevation, ensemble, model run, scenario, band and similar axes remain typed DataTiles dimensions. They must not be hidden in filenames or overloaded into variable names. Source dimensions that are not common to every imported variable may be optional, allowing heterogeneous variables to coexist in one container. + +## Numeric data versus portrayal + +Scientific numeric arrays are encoded as DNT1 and declared as `application/vnd.datatiles.numeric`. PNG, JPEG and WebP are portrayals, not substitutes for numeric evidence. Any portrayal used for MBTiles compatibility should be generated as a documented, provenance-linked derivation. + +## Source identity and provenance + +A source can be local or URL-identified. Single-file NetCDF/GRIB imports retain the original identifier and SHA-256 of the exact bytes used. Zarr requires a store-aware identity rule because it is normally a set of named objects: local stores use the canonical `zarr-tree-sha256-v1` digest defined in `zarr-source-profile.md`, while remote stores require an authoritative checksum for an immutable published store/snapshot. A transport location is provenance, not a semantic variable identifier. Reproducible workflows should prefer immutable/versioned source identifiers and independently published checksums. + +## Import algorithm declaration + +Conversion is a scientific transformation. The importer must expose the source grid assumptions, target CRS/grid, resampling algorithm, tile size, bounding region, selected variables and source checksum. Unsupported geometry must be rejected rather than silently approximated. + +The initial utilities support rectilinear one-dimensional latitude/longitude grids and nearest-neighbour resampling to Web Mercator tile pixel centers. Curvilinear grids, rotated grids, projected native grids and conservative area remapping require separate declared algorithms. + +## Integrity convention + +Checksums identify source objects and reproducible derivations; optional revision-6 signatures attest a canonical final DataTiles manifest. Do not use a signature as a replacement for source checksums, provenance, semantic names, units, CRS/datum, citation, or rights. The native integrity profile is defined in `digital-signatures.md`. diff --git a/docs/data-licensing.md b/docs/data-licensing.md new file mode 100644 index 0000000..85955b1 --- /dev/null +++ b/docs/data-licensing.md @@ -0,0 +1,19 @@ +# Data licensing and rights + +DataTiles never infers legal permission from technical accessibility. A URL that returns bytes establishes transport, not a licence. + +Each imported source MUST have a `source` rights record bound to its provenance entity. Each publishable object MUST separately declare `dataset` and `metadata` rights. Records contain an SPDX expression, authoritative terms URI, attribution, access classification, optional rights holder and the scope to which the terms apply. + +The converter utilities therefore require `--source-license`, `--source-license-uri`, `--source-attribution`, `--dataset-license`, and `--dataset-license-uri`. This forces the operator to make the legal basis explicit before ingestion. Use `LicenseRef-*` for proprietary, bespoke, or otherwise non-SPDX terms; do not guess a standard licence. + +Licensing compatibility is a legal/policy determination and is deliberately not automated by DataTiles. The software can validate that records exist and are syntactically machine actionable; it cannot decide whether combining two sources is legally permissible. Publication workflows SHOULD include a human rights review and retain its decision outside credentials/secrets. + +Attribution displayed in a playground is a convenience, not the sole legal record. The complete source-rights manifest travels with the DataTiles object and publication evidence. + +## Signatures do not alter rights + +Digital signatures attest integrity/authenticity; they do not grant copyright permission, sublicense upstream datasets, or resolve licence compatibility. Rights evaluation must be complete before a release is signed, and any later rights correction produces a new manifest and therefore requires a new signature. + +## Commercial licensing and DRM + +Commercial DRM may enforce an access grant only after the publisher has established the legal right to commercialize every contributing source. W3C ODRL policy and a DataTiles commercial licence are additional machine-readable/access-control layers; they do not replace SPDX/source rights records and cannot narrow attribution or other mandatory upstream notices invisibly. diff --git a/docs/data_sources_and_citation.md b/docs/data_sources_and_citation.md new file mode 100644 index 0000000..62f7139 --- /dev/null +++ b/docs/data_sources_and_citation.md @@ -0,0 +1,175 @@ +# Data sources, acknowledgement, licensing, and citation + +## Status and scope + +This document is the authoritative source-by-source citation register for DataTiles demonstration derivatives, especially the **From Gaeta to Maratea** production workflow. It is part of the release evidence, not optional narrative documentation. + +A source MUST be credited only when the frozen run manifest and provenance graph prove that the source contributed cells, features, topology, or contextual data to the released derivative. Conversely, every source that actually contributed MUST appear in the human-readable acknowledgement and machine-readable provenance/rights records. + +Data licensing is independent of software licensing. The repository's current public `LICENSE` is Apache-2.0; no software licence grants rights in third-party input data. If the repository software licence changes, this document and the README statement MUST be reconciled in the same release. + +## Mandatory release rule + +Every map, MBTiles export, DataTiles container, static playground, figure, service, paper, archive, and release derived from the demo MUST: + +1. identify the exact frozen source objects used by the run; +2. preserve source URI/PID, cryptographic identity or documented immutable-store identity, retrieval time, licence and required attribution; +3. record the transformation activity, software/version, parameters, CRS/datum handling, resampling/masking/fallback decisions, and generated-object identity; +4. visibly acknowledge every source that contributed to the displayed or distributed result; +5. distribute machine-readable provenance and rights with the artifact; +6. avoid crediting sources that were merely candidates, configured fallbacks, documentation references, or unused downloads; and +7. retain the **Not for navigation** warning for the research/demo products. + +The frozen manifest is the evidentiary boundary. A generic project-level acknowledgement MUST NOT override it. + +## Current From Gaeta to Maratea production sources + +### 1. Primary bathymetry — JammeGaia22 / MGDS + +**Role.** Primary finite bathymetric measurements wherever the selected JammeGaia22 grids contain usable values. + +**Bibliographic citation.** Foglini, F., Tonielli, R., & Rovere, M. (2024). *Multi-Resolution bathymetry grids of the Naples and Pozzuoli Gulf and the Amalfi Coastal Area collected during cruise Jamme_Gaia22, 2022*. Marine Geoscience Data System (MGDS). https://doi.org/10.60521/331667 + +**Persistent identifier.** DOI `10.60521/331667`. + +**Licence.** Creative Commons Attribution 4.0 International (CC BY 4.0). + +**Required acknowledgement.** `Bathymetry: Foglini, Tonielli & Rovere (2024), JammeGaia22/MGDS, doi:10.60521/331667.` + +**Scientific limitations.** This is multi-resolution multibeam bathymetry rather than a uniform-resolution navigation surface. MGDS describes GeoTIFF grids whose nominal resolutions vary with depth (2, 5, 15, 20, 30 and 40 m in the published products). Users MUST preserve the source grid identity and applicable resolution in provenance. The dataset is scientific bathymetry and is not an official nautical chart. + +**Release evidence.** The frozen manifest MUST name the exact downloaded MGDS object(s), SHA-256 digest(s), source DOI, retrieval timestamp, spatial coverage used, and transformation activity that produced DataTiles cells. + +### 2. Bathymetric fallback — EMODnet Digital Bathymetry DTM 2024 + +**Role.** Fallback bathymetry **only where JammeGaia22 contains no finite measurement**. The fallback decision is a transformation rule and MUST be recorded; it must not overwrite finite JammeGaia22 values. + +**Bibliographic citation.** EMODnet Bathymetry Consortium (2024). *EMODnet Digital Bathymetry (DTM 2024)*. https://doi.org/10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1 + +**Persistent identifier.** DOI `10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1`. + +**Licence for this project profile.** CC BY 4.0; the frozen acquisition metadata/terms retained by the release remain authoritative for the exact downloaded object. + +**Required acknowledgement when it contributed.** `EMODnet Bathymetry Consortium DTM 2024, doi:10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1.` + +**Scientific limitations.** EMODnet DTM is a harmonised composite product assembled from heterogeneous source datasets. Resolution, survey age, source quality, vertical reference and uncertainty can vary spatially. It MUST NOT be represented as equivalent to the local JammeGaia22 multibeam measurements. The DataTiles source-coverage variable MUST retain which source supplied each output cell. + +### 3. Land topology / coastline — GSHHG 2.3.7 + +**Role.** Independent land/ocean topology and land mask. It is not bathymetry and MUST remain a separate fact from high-water-line products. + +**Citation.** Wessel, P., & Smith, W. H. F. (1996). A global, self-consistent, hierarchical, high-resolution shoreline database. *Journal of Geophysical Research*, 101(B4), 8741–8743. https://doi.org/10.1029/96JB00104 + +**Dataset/version.** GSHHG 2.3.7 (15 June 2017). + +**Licence.** GNU Lesser General Public License (as stated by NOAA/NCEI for GSHHG 2.3.7). Preserve the exact distributed notice with frozen source material. + +**Required acknowledgement.** `Land/coastline: GSHHG 2.3.7 (Wessel & Smith, 1996, doi:10.1029/96JB00104).` + +**Limitations.** GSHHG is a global hierarchical shoreline/topology database in WGS84 with multiple resolution levels. The production manifest MUST identify the resolution level actually used (the reference workflow uses full resolution where specified) and any clipping/generalisation performed. + +### 4. Optional high-water-line fact — S2Coast-2023 + +**Role.** Optional Sentinel-2-derived coastline/high-water-line evidence when explicitly enabled. It MUST remain distinguishable from GSHHG topology and from bathymetry. + +**Dataset citation.** Duan, Y., Sanchez-Azofeifa, A., Chen, C., Tian, B., Li, X., Sengupta, D., & Zhou, Y. *S2Coast-2023: The First Global 10-Meter Resolution Coastline Dataset Derived from Enhanced Sentinel-2 Composite Imagery Using Google Earth Engine*. Zenodo. https://doi.org/10.5281/zenodo.17092775 + +**Associated article.** Duan, Y. et al. (2026). S2Coast-2023: The first global 10-meter resolution coastline dataset derived from enhanced Sentinel-2 composite imagery using Google Earth Engine. *Remote Sensing of Environment*, 334, 115186. https://doi.org/10.1016/j.rse.2025.115186 + +**Licence.** CC BY 4.0 for the project source profile; retain the licence metadata of the exact frozen Zenodo object. + +**Required acknowledgement when enabled and contributing.** `S2Coast-2023 (Duan et al., doi:10.5281/zenodo.17092775), CC BY 4.0.` + +**Limitations.** Nominal 10 m global coastline derived from annual Sentinel-2 imagery. It is a remotely sensed coastline fact with its own temporal/reference assumptions; it is not interchangeable with GSHHG topology or a hydrographic chart coastline. + +### 5. Navigation and land context — OpenStreetMap + +**Role.** Stored navigation/land contextual features actually obtained from OpenStreetMap, including the demo's stored seamark/context vectors where present. + +**Attribution.** `© OpenStreetMap contributors`. + +**Licence.** Open Data Commons Open Database License (ODbL) 1.0. + +**Licence/copyright page.** https://www.openstreetmap.org/copyright + +**Required acknowledgement when OSM data contributed.** `Context: © OpenStreetMap contributors, ODbL 1.0.` + +**Licence note.** Publicly used produced works must provide attribution and make users aware that OpenStreetMap data are available under the ODbL. Distribution of a derivative database may trigger ODbL share-alike obligations. Release engineering MUST evaluate the actual distributed database/product rather than assuming that a software licence controls OSM-derived data. + +## Required short-form map credit + +For the current production build, when all named sources actually contributed, use: + +> Bathymetry: Foglini, Tonielli & Rovere (2024), JammeGaia22/MGDS, doi:10.60521/331667; EMODnet Bathymetry Consortium DTM 2024, doi:10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1. Land/coastline: GSHHG 2.3.7 and S2Coast-2023. Context: © OpenStreetMap contributors, ODbL 1.0. Not for navigation. + +The renderer/release process MUST remove a source from this credit when the run manifest proves that it contributed nothing. In particular, S2Coast-2023 must not be named when disabled or non-contributing, and EMODnet must not be named when no fallback cells were used. + +## Sources that MUST NOT be credited without evidence of use + +Do **not** list GMRT, GEBCO, EMODnet thematic products, ISPRA, or any other candidate/reference source as a contributor unless the specific frozen run manifest proves that its data contributed cells or features to the released product. A bibliography may discuss related datasets as background, but that is distinct from a data acknowledgement. + +Citation is source-specific evidence, not generic project boilerplate. + +## Frozen-manifest requirements + +Every production release SHOULD archive a manifest under the release evidence tree (for the reference demo, under `demo/from-gaeta-to-maratea/dist/` or the immutable evidence bundle produced by the build). At minimum, each source record MUST contain: + +- stable source key and human title; +- source role (`primary_bathymetry`, `fallback_bathymetry`, `land_topology`, `high_water_line`, `context`, etc.); +- PID/DOI and canonical landing URI where available; +- exact acquired object URI without credentials; +- retrieval timestamp; +- SHA-256 or the format-specific immutable-store identity defined by the DataTiles source profile; +- media/format and source version; +- licence identifier, licence URI, attribution text and notices; +- spatial/temporal coverage and resolution relevant to the run; +- transformation activity identifier; +- number/extent of output cells or features contributed; and +- an explicit `contributed: true|false` outcome determined by the build. + +The manifest itself MUST be checksummed and referenced by the generated DataTiles provenance graph. A source with `contributed: false` remains useful acquisition evidence but MUST NOT appear in contributor acknowledgements. + +## Machine-readable provenance and rights + +The DataTiles object and release package MUST preserve, for every contributing source: + +- a provenance entity carrying its PID/URI and cryptographic identity; +- a source-rights record separate from the generated dataset and metadata rights; +- the generating activity and its parameters; +- `used` / `wasDerivedFrom` relationships connecting output variables/tiles to inputs; +- source-coverage evidence for conditional fusion such as JammeGaia22 → EMODnet fallback; +- creator/provider attribution and citation metadata; and +- the generated object's own persistent identifier when published. + +Human-visible credits and machine-readable provenance are complementary; neither substitutes for the other. + +## FAIR release checklist + +Before a scholarly or public demo release, verify all of the following: + +- [ ] Every contributing source has a PID or stable URI and immutable object identity. +- [ ] Every contributing source has explicit licence/rights metadata and required attribution. +- [ ] No licence is inferred merely from successful download/access. +- [ ] Source and output licences are represented separately. +- [ ] The frozen manifest records `contributed` status and quantitative contribution evidence. +- [ ] Conditional fallback/fusion rules are explicit and reproducible. +- [ ] CF/controlled semantic names, units, CRS and vertical datum are retained where applicable. +- [ ] Machine-readable provenance links each derived product to the exact source objects and transformation activities. +- [ ] Human-visible acknowledgements are generated/checked against the frozen manifest. +- [ ] Non-contributing candidate sources are absent from contributor acknowledgements. +- [ ] DataCite/citation metadata includes appropriate related identifiers for contributing datasets. +- [ ] The release archive contains the manifest, provenance, rights, checksums, citation register and software/environment identity. +- [ ] `Not for navigation` is visible on the demo/map artifacts. +- [ ] Repository software licensing and data licensing are not conflated. + +## Verification sources for this register + +The citations and licence statements above should be periodically rechecked against the authoritative provider records before publication. At the time this register was prepared, the controlling public records were MGDS DOI `10.60521/331667`, the EMODnet Bathymetry DTM 2024 citation page, NOAA/NCEI GSHHG 2.3.7 documentation, Zenodo DOI `10.5281/zenodo.17092775`, and the OpenStreetMap copyright/licensing page. + +## Optional signed-release checklist + +When the release policy requires cryptographic integrity, sign only after the frozen run manifest has established actual source contribution and the visible acknowledgements have been derived from it. Archive the DataTiles object, detached signature envelope, trusted public key/certificate reference, and any external timestamp/transparency evidence. A signature never licenses a source and never changes which sources must be cited. + +## Commercial products remain source-specific + +Selling or encrypting a product does not change source attribution. The frozen contribution manifest remains authoritative: every source that actually contributed cells/features must be cited according to its licence, and a source must not be listed merely because it was a candidate. Commercialization must be blocked when a contributing source's terms do not permit the intended use. diff --git a/docs/digital-signatures.md b/docs/digital-signatures.md new file mode 100644 index 0000000..76ad9c8 --- /dev/null +++ b/docs/digital-signatures.md @@ -0,0 +1,119 @@ +# Cryptographic integrity and digital signatures + +DataTiles supports **optional** digital signatures as an integrity and authenticity layer. Signatures complement, but do not replace, checksums, provenance, persistent identifiers, licences, repository preservation, or scientific validation. + +## Threat model and claims + +A cryptographic hash answers: *are these logical DataTiles contents identical to the contents that produced this digest?* A digital signature additionally answers: *was that digest signed by the holder of a particular private key?* Neither statement, by itself, establishes who the human or institution behind the key is. Trust requires an independently authenticated public key, certificate, institutional key registry, Sigstore identity, or another trust policy. + +The native DataTiles profile signs a canonical **logical SQLite manifest**, not raw database bytes. This deliberately survives SQLite page layout, VACUUM, journaling, and other byte-level storage changes that do not alter the represented DataTiles content. It detects changes to signed tables, values, schema definitions, views, and triggers. The two integrity/signature tables are excluded to avoid circular self-signing. + +## Native profile + +- manifest profile: `DataTiles-Integrity-Manifest-1` +- canonicalization: `datatiles-sqlite-logical-v1` +- digest: SHA-256 +- native signature: Ed25519 +- signature envelope: `DataTiles-Ed25519-Signature-1` +- payload media type: `application/vnd.openfairwind.datatiles.integrity-manifest.v1+json` + +Ed25519 is specified by RFC 8032 and is included by NIST FIPS 186-5. The native signer is an optional dependency so the dependency-free DataTiles core remains usable without cryptography packages. + +## What is signed + +The manifest records schema revision and, for every non-integrity user table, a normalized schema digest, row count, and deterministic digest of the complete row multiset. Views and triggers are also schema-hashed. Each SQLite value is type-tagged before hashing; REAL values are represented by their IEEE-754 binary64 bit pattern and BLOB values by their exact bytes. + +Rows are independently SHA-256 hashed and externally sorted in bounded chunks before the table digest is produced. Consequently database row order is irrelevant, duplicate multiplicity is retained, and large tile tables do not require all row hashes in memory. + +`datatiles_integrity_manifests` and `datatiles_signatures` are excluded from the signed domain. Adding or removing signatures therefore does not invalidate the scientific object they attest. + +## Key generation and signing + +Install the optional cryptographic support: + +```bash +python -m pip install -e '.[integrity]' +``` + +Generate an Ed25519 key pair: + +```bash +datatiles-integrity generate-key \ + --private-key release-signing-key.pem \ + --public-key release-signing-key.pub.pem +``` + +The private key is PKCS#8 PEM and should be protected as a secret. The tool attempts to create it with owner-only permissions. Do not commit private keys to Git, release archives, containers, notebooks, or DataTiles files. + +Sign a final container and also emit a detached envelope: + +```bash +datatiles-integrity sign product.datatiles \ + --private-key release-signing-key.pem \ + --signer 'Ocean Data Laboratory' \ + --detached product.datatiles.sig.json +``` + +The signature may be stored inside the DataTiles object, emitted as a detached JSON envelope, or both. Detached signatures are useful for read-only repositories and archival systems. + +## Verification and trust + +Cryptographic verification with only the public key embedded in a signature proves self-consistency, **not signer trust**. A scholarly verification should supply a public key obtained independently: + +```bash +datatiles-integrity verify product.datatiles \ + --detached product.datatiles.sig.json \ + --public-key trusted-release-key.pub.pem +``` + +For automated release gates, pin the expected key identifier too: + +```bash +datatiles-integrity verify product.datatiles \ + --signature-id urn:uuid:... \ + --public-key trusted-release-key.pub.pem \ + --require-key-id sha256:... +``` + +Verification reports separately: + +- cryptographic validity of the signature; +- whether a trusted external key was supplied; +- whether the current DataTiles logical contents match the signed manifest; +- signer/key metadata and the signed root digest. + +Software MUST NOT convert “signature valid using its embedded key” into “trusted publisher”. + +## Provenance integration + +A signature can reference a `datatiles_provenance_agents` record through `signer_agent_id`. The agent identity should use an institutional URI, ORCID, ROR, or another persistent identifier when appropriate. The cryptographic key identifier and the scholarly agent identifier are separate facts and MUST NOT be conflated. + +Signing is a provenance activity performed **after** the final scientific build, rights evaluation, citation resolution, and release metadata freeze. NetCDF, GRIB2, and Zarr import utilities therefore do not auto-sign intermediate outputs. Release automation should sign only after all intended mutations have completed. + +## Sigstore and in-toto interoperability + +The native profile is offline-first. Publication pipelines MAY additionally publish a Sigstore bundle or an in-toto/DSSE attestation referencing the DataTiles manifest root. Such external evidence can provide certificate identity, transparency-log inclusion, and/or trusted timestamps. It must be recorded as verification material or publication evidence; it does not change the native DataTiles manifest digest. + +A Sigstore bundle should be archived with the release rather than requiring future network access for basic DataTiles verification. Keyless signatures require identity-policy verification (issuer and subject), not merely mathematical signature checking. + +## FAIR and preservation + +Digital signatures strengthen integrity and provenance evidence, particularly reproducibility and chain-of-custody assertions, but they are not a FAIR principle by themselves and remain optional. `fair_report()` must never fail solely because a DataTiles object is unsigned. A release profile may establish a stricter institutional policy requiring a trusted signature. + +For long-term preservation, archive together: + +1. the DataTiles object; +2. detached signature envelope(s), if used; +3. trusted public key/certificate or durable reference to it; +4. any Sigstore bundle, RFC 3161 timestamp, or transparency evidence; +5. PID/DataCite metadata, source manifests, citations, rights, and provenance. + +Key rotation is expected. Historical signatures must remain verifiable with archived historical public keys. Revocation affects trust policy, not the immutable mathematical fact that a given key produced a signature. + +## Non-goals + +A valid signature does not certify hydrographic authority, navigation safety, correctness of a model, correctness of source data, licence compatibility, or fitness for purpose. It proves integrity/authenticity only within the verifier's explicit trust policy. + +## Sign before protecting + +When both integrity signatures and DRM are used, finalize and sign the plaintext DataTiles object first, then create the protected `.dtpkg`. The DRM header binds to the plaintext SHA-256. This makes the signed scientific object stable and allows authorized recipients to verify the recovered DataTiles independently of the transport/access-control wrapper. diff --git a/docs/drm-and-commercial-licensing.md b/docs/drm-and-commercial-licensing.md new file mode 100644 index 0000000..bbf9de2 --- /dev/null +++ b/docs/drm-and-commercial-licensing.md @@ -0,0 +1,95 @@ +# Optional DRM and commercial licensing + +DataTiles supports an optional protected-distribution profile for institutions and companies that are legally entitled to sell proprietary cartographic or scientific products. DRM is a distribution control, not a substitute for copyright ownership, source licensing, attribution, provenance, FAIR metadata, or scientific validation. + +## Separation of concerns + +A DataTiles object remains an ordinary revision-7 SQLite scientific object. Publishers finalize its semantics, provenance, rights, citations, QA, and optional digital signatures first. Commercial distribution then wraps that immutable file in `DataTiles-Protected-Distribution-1` (`.dtpkg`). Authorized recipients decrypt the package back to the exact original bytes. + +The legal licence and technical entitlement are distinct. `datatiles_rights` and human-readable terms state the legal basis. W3C ODRL 2.2 can encode machine-readable permissions, prohibitions, constraints, and duties. The DRM licence token only conveys an issuer-signed access grant and a recipient-wrapped content-encryption key; it does not create rights the publisher does not already possess. + +## Cryptographic profile + +- content encryption: AES-256-GCM; +- package nonce: fresh random 96-bit nonce per protected package; +- recipient key agreement: X25519; +- key derivation: HKDF-SHA-256 with product-specific context; +- content-key wrapping: AES-256-GCM; +- licence issuer signature: Ed25519; +- product and key identifiers: SHA-256 based identifiers; +- public policy model: W3C ODRL 2.2. + +Private content keys and recipient private keys MUST NOT be embedded in the DataTiles object, package header, provenance graph, demo, playground, source repository, or licence token. Publisher content-key files are generated with restrictive filesystem permissions and should normally be moved into a KMS/HSM or equivalent institutional secret-management system. + +## Package and licence binding + +Each protected package carries a public, authenticated header containing the product identifier, edition, issuer, terms URI, optional licence-service URI, encryption profile, creation time, original DataTiles SHA-256, and original size. The header is authenticated as AES-GCM associated data. + +Each licence is bound to: + +- the exact `product_id`; +- the exact plaintext DataTiles SHA-256; +- a recipient X25519 public-key identifier; +- an issuer Ed25519 key identifier; +- optional validity start/end times; +- an explicit list of technical permissions; +- an optional ODRL 2.2 policy document. + +The content key is never sent in plaintext. It is wrapped to the recipient public key using an ephemeral X25519 key and HKDF-derived wrapping key. A copied licence therefore does not unlock the product for a different recipient key. + +## Threat and trust model + +This profile protects data at rest and in distribution and provides cryptographic evidence of issuer authorization. It cannot prevent an authorized user or compromised client from copying plaintext after decryption, photographing/rendering displayed information, or extracting data from a process with sufficient privileges. Stronger controls may use secure enclaves, platform key stores, HSM-backed device identities, remote attestation, or online authorization, but these are deployment profiles rather than requirements of the portable DataTiles format. + +The issuer public key must be authenticated independently. A mathematically valid licence signed by an unknown key is not evidence that a particular company or institution authorized access. + +## Commercial-law and upstream-rights rule + +A publisher MUST NOT commercialize a derivative merely because DataTiles can encrypt it. Every contributing source must permit the intended commercial use and redistribution, all attribution/share-alike/database-right obligations must be satisfied, and incompatible upstream terms must block the commercial release. DRM MUST NOT be used to remove, conceal, contradict, or technically frustrate licence notices that must remain available to recipients. + +## Workflow + +1. Build the DataTiles object from lawful sources. +2. Preserve source citations, provenance, source licences, and commercial-use evidence. +3. Register the commercial product and ODRL policy in the inner DataTiles object. +4. Complete scientific QA and FAIR checks. +5. Optionally sign the canonical DataTiles integrity manifest. +6. Freeze the release. +7. Protect the exact frozen file into `.dtpkg` and securely retain the generated content key. +8. Generate or obtain each customer's X25519 public key. +9. Issue an issuer-signed licence for that recipient and product. +10. Deliver the package plus licence; distribute the issuer public key through an authenticated channel. + +## CLI example + +```bash +python -m pip install -e '.[drm,integrity]' + +datatiles-drm protect product.datatiles product.dtpkg \ + --content-key-file product.cek.json \ + --product-id urn:example:charts:med-2026 \ + --issuer 'Example Hydrographic Publisher' \ + --terms-uri https://example.org/licence/med-2026 + +datatiles-drm generate-recipient-key \ + --private-key customer.key \ + --public-key customer.pub.pem + +datatiles-drm issue-license product.dtpkg \ + --content-key-file product.cek.json \ + --recipient-public-key customer.pub.pem \ + --issuer-private-key publisher-ed25519.pem \ + --issuer 'Example Hydrographic Publisher' \ + --recipient-id customer-123 \ + --permission read \ + --output customer-123.license.json + +datatiles-drm decrypt product.dtpkg product.datatiles \ + --license customer-123.license.json \ + --recipient-private-key customer.key \ + --issuer-public-key publisher-ed25519.pub.pem +``` + +## FAIR compatibility + +FAIR does not mean free-of-charge or unrestricted. Restricted/proprietary DataTiles can remain FAIR when metadata, identifiers, access conditions, provenance, vocabulary semantics, and licence terms are sufficiently machine-readable and persistent. Public discovery metadata should remain available even when data access requires purchase or authorization. DRM status must never be reported as a FAIR score, quality score, hydrographic certification, or navigation-safety claim. diff --git a/docs/fair-by-design.md b/docs/fair-by-design.md index 2bcb562..3af73ce 100644 --- a/docs/fair-by-design.md +++ b/docs/fair-by-design.md @@ -33,3 +33,9 @@ FAIR access does not imply frictionless or anonymous download. A provider may re ## FAIR maturity FAIRness should be reported as individual test outcomes, not a single opaque score. Passing container checks demonstrates machine actionability but cannot establish that an external PID is registered, a repository will retain metadata, or a scientific vocabulary is governed. Those repository-level assertions require publication evidence and periodic audit. + +## Academic publication profile + +The publication gate is strengthened by `fair-publication-profile.md`. DataTiles separates persistent identifiers, DataCite 4.7 citation metadata, W3C PROV-aligned lineage, SPDX rights, and external repository evidence. Dataset, metadata, and every upstream source MUST have distinct machine-actionable rights records. A public URL is never interpreted as permission. A strict publication report MUST fail when PID, catalogue registration, landing-page evidence, metadata-retention evidence, source rights, source lineage, or community semantics are absent. + +FAIR reports expose principle-level outcomes and repository-dependent assertions; they MUST NOT report an unexplained scalar FAIR score. diff --git a/docs/fair-publication-profile.md b/docs/fair-publication-profile.md new file mode 100644 index 0000000..34dd03f --- /dev/null +++ b/docs/fair-publication-profile.md @@ -0,0 +1,65 @@ +# DataTiles FAIR scientific publication profile + +## Status and normative intent + +This profile operationalizes the FAIR Guiding Principles for DataTiles scientific objects. It does **not** equate FAIR with open access, quality, truth, or fitness for a safety-critical purpose. The original FAIR principles require persistent identity and rich metadata (F1-F4), standardized access (A1-A2), formal shared semantics and qualified links (I1-I3), and explicit licence, provenance, and community standards (R1-R1.3). DataTiles therefore treats FAIRness as verifiable evidence, not a marketing score. + +Normative references are Wilkinson et al. (2016), W3C PROV-O/PROV-DM, DataCite Metadata Schema 4.7 (2026), SPDX 3.0.1 license expressions, CF Standard Names for scientific quantities, EPSG/OGC CRS identifiers, and domain vocabularies such as NERC NVS where applicable. + +## Publication invariants + +A DataTiles object intended for scholarly publication MUST satisfy all locally testable requirements below and MUST carry external publication evidence for repository-level assertions. + +1. **Identity.** The published object/revision has one explicit primary globally unique persistent identifier. DOI is RECOMMENDED for citable research datasets; Handle, ARK or another governed PID is acceptable. A mutable project URL is not a substitute for a versioned PID. +2. **Metadata.** Title, creators/contributors, publisher, publication year, spatial/temporal coverage, variables, units, CRS/datum, resolution, uncertainty/quality limitations, content profiles and version are machine readable. A DataCite 4.7-shaped export is available without claiming registration. +3. **Semantics.** Scientific variables use CF Standard Names when a suitable term exists, canonical units are declared, and external vocabulary mappings are retained. No local name is silently promoted to a CF term. +4. **Provenance.** Sources are immutable entities identified by URI/PID plus cryptographic digest where bytes are available. Transformations are activities with software identity/version, parameters and timestamps. Agents are explicit. Relations map to W3C PROV semantics. Tile lineage points to source/derived entities. +5. **Rights.** Dataset rights, metadata rights and every source dataset's rights are distinct records. SPDX expressions are used for machine-actionable licence identity; authoritative licence/terms URIs, attribution text, rights holder where known, and access-rights classification are retained. Public HTTP access MUST NOT be interpreted as permission to redistribute. +6. **Integrity and reproducibility.** Source locks, runtime locks, deterministic parameters, artifact manifests and checksums are retained. A checksum proves byte identity, not scientific validity. +7. **Accessibility.** The PID/landing page uses an open standardized protocol. Authentication/authorization may be required. Metadata persistence after data withdrawal is a repository obligation and requires evidence such as a preservation/retention policy. +8. **Indexing.** Catalogue/registry deposit is externally evidenced. The SQLite file cannot prove F4 by itself. +9. **Qualified links.** Source datasets, software, papers, previous/new versions, supplements and derived objects use typed related identifiers aligned with DataCite relation types. +10. **Limitations.** Known uncertainty, exclusions, transformations, resolution limits, licensing restrictions and safety boundaries remain visible in documentation and machine-readable metadata. + +## FAIR report semantics + +`fair_report(strict_publication=False)` evaluates facts the container can establish and labels repository-dependent checks as external. `strict_publication=True` makes missing catalogue registration, landing-page evidence and metadata-retention evidence publication blockers. A report MUST expose each principle-level result; implementations MUST NOT collapse FAIRness into an unexplained percentage. + +## Licensing model + +DataTiles distinguishes at least three legal layers: + +- **software licence**: governs the DataTiles implementation, not automatically the data; +- **dataset licence**: governs the generated scientific object; +- **source licences**: govern each upstream source and may impose attribution, share-alike, database-right or redistribution obligations. + +Metadata has its own rights record. CC0-1.0 is RECOMMENDED for newly authored machine-readable metadata where the publisher has authority to dedicate it, but DataTiles MUST NOT override third-party rights embedded in metadata or attribution requirements inherited from sources. + +Composite rights use valid SPDX expressions such as `CC-BY-4.0`, `ODbL-1.0`, or `(LicenseRef-A AND LicenseRef-B)`. Custom/non-SPDX terms use a stable `LicenseRef-*` plus the authoritative terms URI. The dependency-free core validates expression syntax; CI/publication SHOULD validate identifiers against a pinned SPDX License List release. + +## Provenance model + +The internal graph is PROV-aligned: + +- `datatiles_provenance_entities` -> `prov:Entity`; +- `datatiles_provenance_activities` -> `prov:Activity`; +- `datatiles_provenance_agents` -> `prov:Agent`; +- `wasGeneratedBy`, `used`, `wasDerivedFrom`, `wasAttributedTo`, `wasAssociatedWith`, `specializationOf` retain their W3C PROV meanings. + +A conversion SHOULD create a source entity, conversion activity, generated dataset entity, software agent and responsible agent, then connect them with `used`, `wasGeneratedBy`, `wasAssociatedWith`, `wasAttributedTo`, and `wasDerivedFrom`. Parameters MUST include resampling, CRS transformation, masking, classification/fusion rules and all values capable of changing scientific output. + +## DataCite alignment + +DataTiles 0.12 targets DataCite Metadata Schema 4.7 for publication export. The export is a candidate deposit record, not proof of DOI registration. `datatiles_related_identifiers` retains typed scholarly links. DataCite 4.7 added RAiD and SWHID identifier types and a general `Other` relation type; use more specific relation types whenever possible. + +## Minimum release evidence + +A scholarly release SHOULD archive: the `.datatiles` object; SHA-256 manifest; source lock; source-rights manifest; runtime/software lock; configuration; PROV export; DataCite metadata export; validation/FAIR report; citation metadata; quality-control results; and, where redistribution permits, immutable raw inputs. If raw bytes cannot legally be redistributed, preserve checksums, PIDs/request identity, rights metadata and a lawful reacquisition procedure instead. + +## Optional cryptographic release evidence + +A DataTiles release MAY include a revision-6 integrity manifest and one or more trusted digital signatures. Signature absence MUST NOT make the FAIR report fail: FAIRness and cryptographic authenticity are distinct properties. When signatures are used, archive the detached envelope, independently authenticated public key/certificate, signer PROV-agent binding, and any timestamp/transparency evidence with the publication package. + +## Restricted and commercial access + +FAIR does not require that every dataset be free or anonymously downloadable. A restricted/proprietary DataTiles release may use the revision-7 DRM profile while retaining public persistent identifiers, rich metadata, source-specific citation, provenance, interoperable semantics, and explicit access conditions. DRM presence or absence is not a FAIR criterion by itself. diff --git a/docs/images/demo/README.md b/docs/images/demo/README.md index bd08292..38f8276 100644 --- a/docs/images/demo/README.md +++ b/docs/images/demo/README.md @@ -5,7 +5,7 @@ These screenshots document an executed DataTiles demonstration; they are not sto ## Capture environment and inputs — 2026-08-28 - viewport: 1280 × 720 pixels; -- DataTiles version: 0.10.0; +- DataTiles version: 0.19.0; - publication extent: `12.85–15.71851° E, 39.99852–41.21408° N`; - generated container SHA-256: `b0f9732ce43a1b16729978b9ec8feabf82d513e1d8d71405893b1c828b4f55e2`; - source-lock SHA-256: `20effbe5c4612492c0a1ca3a1575216f3d2bce5e5178ef56d1c49c84616308b7`; diff --git a/docs/import-utilities.md b/docs/import-utilities.md new file mode 100644 index 0000000..c89e471 --- /dev/null +++ b/docs/import-utilities.md @@ -0,0 +1,81 @@ +# NetCDF, GRIB, and Zarr import utilities + +DataTiles treats NetCDF, GRIB, and Zarr as **source encodings**, not as the semantic identity of a quantity. The utilities under `utils/` translate those encodings into the DataTiles multidimensional model while retaining source identity, variable semantics, units, non-spatial coordinates, CRS and provenance. + +## Input identifiers + +A source argument may be a local filesystem path, a `file:` URI, or an HTTP(S) URL. URL sources are materialized to a temporary file because GRIB readers require seekable access and because byte-level SHA-256 provenance should be computed consistently. The converter records both the original source identifier and the checksum. It does not treat a downloaded temporary filename as scientific identity. + +## NetCDF mapping + +For each selected NetCDF data variable: + +- the local NetCDF variable name becomes the DataTiles variable token; +- a CF `standard_name`, when present, becomes the canonical semantic name under vocabulary `CF`; +- `units` becomes the registered canonical/unit declaration and DNT1 tile unit; +- `long_name` remains descriptive metadata, never a substitute for a controlled standard name; +- non-spatial dimensions such as `time`, `depth`, `pressure`, `ensemble` or `scenario` become typed DataTiles dimensions; +- one-dimensional latitude/longitude coordinates define the source rectilinear grid. + +If `standard_name` is absent, the converter registers the local token under vocabulary `source`. This preserves discoverability without inventing a CF Standard Name. + +## GRIB2 mapping + +GRIB is decoded through `cfgrib`/ecCodes. The decoded variable follows the same CF mapping as NetCDF. When cfgrib exposes GRIB metadata, the converter also records external identifiers: + +- `WMO-GRIB2`: `discipline.parameterCategory.parameterNumber`; +- `GRIB-shortName`: the producer/ecCodes short name; +- `GRIB2-variable`: the decoded xarray variable name. + +These identifiers are crosswalks. They do not replace the canonical CF semantic identity when a CF Standard Name is available. + +## Zarr mapping + +Zarr import uses xarray's Zarr backend and follows the same CF semantic mapping as NetCDF. Zarr format 2 and 3 are supported when the installed xarray/zarr stack supports them. Local directory stores receive a deterministic `zarr-tree-sha256-v1` digest over the complete object tree; remote Zarr stores require an authoritative `--source-sha256` for an immutable published store or snapshot. This distinction is normative: a multi-object store is not misrepresented as a single file. + +The importer supports `--group`, `--consolidated`, `--zarr-format`, and repeatable fsspec `--storage-option` settings. Storage-option values can contain credentials and therefore are never persisted; provenance records only the option names. Signed or credential-bearing access URLs require a separate stable `--provenance-uri`, preventing secrets from entering the evidence graph. See `zarr-source-profile.md` for the complete ingestion contract. + +## Spatial transformation + +The utility profile is deliberately conservative. It supports rectilinear 1-D latitude/longitude source grids. Target pixels are Web Mercator tile pixel centers and source values are selected using nearest-neighbour sampling. Data are stored as DNT1 numeric arrays and written through the XYZ interface; DataTiles persists the corresponding TMS row. + +This is an explicit algorithm, not a hidden portrayal step. The tile content schema records `resampling=nearest`, `grid=WebMercatorQuad`, tile size and source kind. Future higher-order or conservative remapping methods must declare different algorithm identifiers and parameters. + +The utilities reject rather than guess for unsupported curvilinear/2-D coordinates, antimeridian-crossing bboxes, and non-rectilinear grids. + +## Resource and safety bounds + +`--max-tiles` limits accidental expansion at inappropriate zooms. `--bbox` can constrain the import domain. URL access has a timeout and is streamed to disk while hashing instead of being accumulated in memory. DNT1 itself retains its normal element and header bounds. + +## Examples + +```bash +python utils/netcdf2datatiles.py input.nc output.datatiles \ + --variable sea_floor_depth_below_geoid --zoom 8 +``` + +```bash +python utils/grib2datatiles.py https://example.org/model.grib2 output.datatiles \ + --filter-by-keys typeOfLevel=surface --variable t2m --zoom 6 +``` + +```bash +python utils/zarr2datatiles.py ./ocean.zarr output.datatiles \ + --variable depth --zoom 7 +``` + +Validate semantic registration after import: + +```bash +datatiles validate output.datatiles --require-variable-semantics +``` + +For strict CF-table membership validation, additionally provide the pinned official CF XML table used by the publication workflow. + +## Signing imported products + +The NetCDF, GRIB2, and Zarr utilities intentionally do not sign during ingestion. Import output is normally intermediate scientific evidence that may still undergo semantic curation, fusion, portrayal preparation, citation resolution, QA, and rights review. Sign the final frozen DataTiles object with `datatiles-integrity` only after those steps. This avoids producing apparently authoritative signatures over incomplete intermediate products. + +## DRM is not an ingestion step + +NetCDF, GRIB2, and Zarr importers do not encrypt or issue commercial licences. Source ingestion must preserve upstream rights and provenance first. DRM is applied only to a finalized lawful release after scientific QA, citation/licence review, FAIR metadata completion, and optional integrity signing. diff --git a/docs/migrations/v4-to-v5.sql b/docs/migrations/v4-to-v5.sql new file mode 100644 index 0000000..227ed43 --- /dev/null +++ b/docs/migrations/v4-to-v5.sql @@ -0,0 +1,62 @@ +-- DataTiles schema revision 5: FAIR publication identity and rights. +-- Revision 5 makes rights machine-actionable and separates object identifiers, +-- related identifiers, and publication evidence from generic MBTiles metadata. + +CREATE TABLE datatiles_identifiers ( + identifier_id INTEGER PRIMARY KEY, + scheme TEXT NOT NULL, + identifier TEXT NOT NULL, + uri TEXT, + is_primary INTEGER NOT NULL DEFAULT 0 CHECK(is_primary IN (0,1)), + UNIQUE(scheme,identifier) +); +CREATE UNIQUE INDEX datatiles_one_primary_identifier ON datatiles_identifiers(is_primary) WHERE is_primary=1; + +CREATE TABLE datatiles_related_identifiers ( + scheme TEXT NOT NULL, + identifier TEXT NOT NULL, + relation_type TEXT NOT NULL, + uri TEXT, + resource_type TEXT, + relation_information TEXT, + PRIMARY KEY(scheme,identifier,relation_type) +) WITHOUT ROWID; + +CREATE TABLE datatiles_rights ( + rights_id INTEGER PRIMARY KEY, + scope TEXT NOT NULL CHECK(scope IN ('dataset','metadata','source','portrayal','software','other')), + license_expression TEXT NOT NULL, + license_uri TEXT, + rights_holder TEXT, + rights_holder_uri TEXT, + attribution_text TEXT, + copyright_notice TEXT, + access_rights TEXT NOT NULL DEFAULT 'open' CHECK(access_rights IN ('open','embargoed','restricted','closed')), + source_entity_id TEXT REFERENCES datatiles_provenance_entities(entity_id) ON DELETE CASCADE, + applies_to TEXT, + UNIQUE(scope,source_entity_id,license_expression,applies_to) +); +CREATE INDEX datatiles_rights_scope ON datatiles_rights(scope,access_rights); + +CREATE TABLE datatiles_fair_agents ( + agent_id TEXT NOT NULL REFERENCES datatiles_provenance_agents(agent_id) ON DELETE CASCADE, + role TEXT NOT NULL, + sequence INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(agent_id,role) +) WITHOUT ROWID; + +CREATE TABLE datatiles_publication_evidence ( + evidence_type TEXT NOT NULL, + uri TEXT NOT NULL, + checked_at TEXT, + checksum_algorithm TEXT, + checksum TEXT, + notes TEXT, + PRIMARY KEY(evidence_type,uri) +) WITHOUT ROWID; + +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:fair_profile','FAIR-Guiding-Principles-2016'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:provenance_profile','W3C-PROV-2013'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:metadata_profile','DataCite-4.7'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:license_profile','SPDX-3.0.1-expression'); +PRAGMA user_version = 5; diff --git a/docs/migrations/v5-to-v6.sql b/docs/migrations/v5-to-v6.sql new file mode 100644 index 0000000..02850c8 --- /dev/null +++ b/docs/migrations/v5-to-v6.sql @@ -0,0 +1,29 @@ +CREATE TABLE datatiles_integrity_manifests ( + manifest_id TEXT PRIMARY KEY, + profile TEXT NOT NULL, + canonicalization TEXT NOT NULL, + hash_algorithm TEXT NOT NULL CHECK(hash_algorithm = 'sha256'), + root_sha256 TEXT NOT NULL UNIQUE CHECK(length(root_sha256) = 64), + manifest_json TEXT NOT NULL, + created_at TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE datatiles_signatures ( + signature_id TEXT PRIMARY KEY, + manifest_id TEXT NOT NULL REFERENCES datatiles_integrity_manifests(manifest_id) ON DELETE CASCADE, + signature_scheme TEXT NOT NULL, + signature_encoding TEXT NOT NULL, + signature BLOB NOT NULL, + key_id TEXT NOT NULL, + public_key BLOB, + signer_agent_id TEXT REFERENCES datatiles_provenance_agents(agent_id) ON DELETE SET NULL, + signed_at TEXT NOT NULL, + verification_material_json TEXT, + UNIQUE(manifest_id, signature_scheme, key_id, signature) +) WITHOUT ROWID; +CREATE INDEX datatiles_signatures_manifest ON datatiles_signatures(manifest_id); +CREATE INDEX datatiles_signatures_key ON datatiles_signatures(key_id); + +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:integrity_profile','DataTiles-Integrity-Manifest-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:signature_profile','DataTiles-Ed25519-Signature-1'); +PRAGMA user_version = 6; diff --git a/docs/migrations/v6-to-v7.sql b/docs/migrations/v6-to-v7.sql new file mode 100644 index 0000000..5f24756 --- /dev/null +++ b/docs/migrations/v6-to-v7.sql @@ -0,0 +1,24 @@ +CREATE TABLE datatiles_commercial_products ( + product_id TEXT PRIMARY KEY, + edition TEXT, + issuer TEXT NOT NULL, + issuer_uri TEXT, + terms_uri TEXT NOT NULL, + license_service_uri TEXT, + protection_profile TEXT NOT NULL DEFAULT 'DataTiles-Protected-Distribution-1', + created_at TEXT NOT NULL, + metadata_json TEXT +) WITHOUT ROWID; + +CREATE TABLE datatiles_drm_policies ( + product_id TEXT NOT NULL REFERENCES datatiles_commercial_products(product_id) ON DELETE CASCADE, + policy_id TEXT NOT NULL, + policy_profile TEXT NOT NULL DEFAULT 'W3C-ODRL-2.2', + policy_json TEXT NOT NULL, + PRIMARY KEY(product_id, policy_id) +) WITHOUT ROWID; +CREATE INDEX datatiles_drm_policy_product ON datatiles_drm_policies(product_id); + +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:drm_profile','DataTiles-Protected-Distribution-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:rights_policy_profile','W3C-ODRL-2.2'); +PRAGMA user_version = 7; diff --git a/docs/migrations/v7-to-v8.sql b/docs/migrations/v7-to-v8.sql new file mode 100644 index 0000000..2d8386a --- /dev/null +++ b/docs/migrations/v7-to-v8.sql @@ -0,0 +1,16 @@ +CREATE TABLE datatiles_release ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), + product_id TEXT NOT NULL, + version TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK(sequence >= 1), + released_at TEXT NOT NULL, + previous_version TEXT, + previous_identifier TEXT, + release_notes_uri TEXT, + update_uri TEXT, + CHECK(product_id <> '' AND trim(product_id) = product_id), + CHECK(version <> '' AND trim(version) = version) +); +CREATE UNIQUE INDEX datatiles_release_product_version ON datatiles_release(product_id, version); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:versioning_profile','DataTiles-Release-Versioning-1'); +PRAGMA user_version = 8; diff --git a/docs/playground.md b/docs/playground.md index 1d39912..bbb2125 100644 --- a/docs/playground.md +++ b/docs/playground.md @@ -63,3 +63,15 @@ The rendering pipeline is therefore `DNT1 arrays → sampled numeric surface → *Figure 9. Executed `0 < depth < 200` m and sand-or-mud query with 820 highlighted analysis cells. The result is a request-time GeoJSON derivation and not a stored portrayal.* For development, build the demo, run `datatiles-serve work/gaeta-to-maratea.datatiles --port 8080`, and visit `http://127.0.0.1:8080/playground`. The OpenLayers dependency is version-pinned. A production deployment should self-host the pinned asset, apply CSP and request rate limits, cache derived responses by canonical query plus dataset checksum, and execute expensive processing through a bounded worker service. + +## FAIR evidence panel + +The live playground now exposes the container's principle-level FAIR report and rights-record count, with machine-readable `/fair`, `/rights`, `/provenance`, and `/datacite` resources. This is intentionally adjacent to the scientific visualization: a user must be able to inspect lineage and reuse conditions without leaving the object. The panel never converts FAIRness into a safety or scientific-validity badge. Static exports embed the same rights, PROV, DataCite candidate metadata, primary identifier and FAIR report in `data/manifest.json`. + +## Cryptographic integrity evidence + +The playground may display the number of recorded signature records via `/integrity`. This is disclosure, not verification or trust. Full manifest computation is available explicitly at `/integrity/manifest` because it can require reading every scientific tile. Private-key signing is intentionally absent from the HTTP service. + +## Commercial/DRM disclosure + +An authorized plaintext playground may expose non-secret commercial product metadata and ODRL policy. A public storefront/demo should disclose product identity, issuer, terms URI, access status, and required source acknowledgements without exposing protected scientific payloads or key material. DRM status is not a safety or quality badge. diff --git a/docs/provenance.md b/docs/provenance.md new file mode 100644 index 0000000..a04531b --- /dev/null +++ b/docs/provenance.md @@ -0,0 +1,17 @@ +# Scientific provenance + +DataTiles provenance follows the W3C PROV entity-activity-agent model and keeps tile-level lineage. Provenance must answer: **what bytes or observations were used, who/what acted, what transformation occurred, with which parameters/software, and what was generated?** + +Source entities SHOULD have stable PID/URI, release/version, SHA-256 (or stronger approved digest), acquisition time, request identity and source-kind attributes. Activities SHOULD record start/end timestamps, software package/version or SWHID, configuration digest, resampling/interpolation, CRS operations, masks, quality filters and deterministic/random seeds. Agents SHOULD use ORCID for persons and ROR or another governed organizational identifier where available. + +The provenance graph must distinguish evidence from assertion. A checksum verifies bytes. A citation identifies intellectual origin. A licence states terms. A quality report supports scientific validity. None substitutes for the others. + +`prov_json()` exports the graph with explicit W3C PROV namespace mappings. For publication, validate the graph for dangling identifiers, source coverage, generated-entity links and activity-agent associations, then archive the export beside the artifact manifest. + +## Signing as a provenance activity + +Signing occurs only after the final reproducible build is frozen. The signer key identifier and scholarly PROV agent are separate identifiers. A valid signature may establish that a key attested a particular immutable manifest; attribution to a person or organization requires an external trust binding. Key rotation and revocation policy belong to the publishing institution and must not rewrite historical provenance. + +## DRM provenance boundary + +The commercial product ID, edition, issuer, terms URI, and public ODRL policy may be provenance-relevant and are part of the signed inner DataTiles object. Content keys, recipient private keys, licence-signing private keys, payment data, bearer tokens, and other secrets are operational security material and MUST NOT enter the scholarly provenance graph. diff --git a/docs/semantic-variables.md b/docs/semantic-variables.md new file mode 100644 index 0000000..abf8d25 --- /dev/null +++ b/docs/semantic-variables.md @@ -0,0 +1,85 @@ +# Semantic variable interoperability profile + +## Status + +Proposed additive extension for DataTiles schema revision 4. + +## Goal + +A DataTiles consumer must be able to discover a physical quantity without knowing a producer-specific coordinate value such as `depth`, `DEPTH`, `h`, or `bathymetry`. + +The `variable` scientific dimension remains part of tile addressing. A separate semantic registry maps each local variable token to a controlled semantic identity. + +## Canonical vocabulary + +For geophysical and environmental physical quantities, DataTiles uses **CF Standard Names** as the preferred canonical semantic vocabulary. The registry records the vocabulary and version explicitly; examples in this profile target CF Standard Name Table v94 (2026-06-09). + +A registered CF variable SHOULD include the CF canonical unit. Actual numeric payload units may be physically equivalent rather than textually identical. Core DataTiles does not claim UDUNITS dimensional-equivalence validation because the project keeps its core dependency-free. + +DataTiles MUST NOT invent a CF standard name. If no suitable CF name exists, the producer should use an explicitly named external vocabulary or propose a new CF name through the CF process. + +## External identifiers + +Additional identifiers are first-class crosswalks, not replacements for the canonical semantic identity. Typical schemes include: + +- `WMO-GRIB2` +- `SeaDataNet-P01` +- `NERC-NVS` +- `GCMD` +- provider-specific namespaces + +Each mapping stores a scheme, identifier, optional scheme version, and optional persistent URI. + +## Example: bathymetry + +Local tile coordinate: + +```text +variable=bathymetry +``` + +Semantic registry entry: + +```json +{ + "name": "bathymetry", + "standard_name": "sea_floor_depth_below_geoid", + "standard_name_vocabulary": "CF", + "standard_name_vocabulary_version": "94", + "canonical_unit": "m", + "long_name": "Sea floor depth below geoid" +} +``` + +CF distinguishes this quantity from sea-floor depth below mean sea level, a geopotential datum, a reference ellipsoid, or the instantaneous sea surface. The datum/reference surface therefore remains semantically significant and must not be collapsed into a generic `depth` concept. + +## Enforcement policy + +`metadata['datatiles:variable_semantics']` is one of: + +- `required`: every point value of a text `variable` dimension must have a registry entry before tiles using that value may be written; validation also rejects unregistered values. +- `recommended`: legacy-compatible mode. Unregistered values are permitted, but semantic validation may report them when strict validation is requested. + +New schema-revision-4 containers SHOULD default to `required`. Migration from revision 3 SHOULD set `recommended` so old containers remain readable and writable until curated. + +## Offline CF validation + +The core implementation must not download vocabularies implicitly. A strict validator accepts a local, pinned official CF standard-name-table XML snapshot. It checks that the standard name exists and, when `canonical_unit` is supplied, that it matches the CF table's canonical-unit string. + +This exact-string canonical-unit check is intentionally narrower than UDUNITS equivalence. DataTiles must not claim full CF unit conformance without a real UDUNITS-compatible implementation. + +## Discovery + +Consumers should be able to query by semantic identity: + +```text +standard_name=sea_floor_depth_below_geoid +``` + +and receive the matching local variable(s), external identifiers, and coordinate-set IDs. The HTTP service should expose the same registry under: + +```text +/collections/{collectionId}/variables +``` + +This enables independent software to discover compatible DataTiles datasets without prior knowledge of producer-local variable names. diff --git a/docs/specification-revision-5-addendum.md b/docs/specification-revision-5-addendum.md new file mode 100644 index 0000000..e45380d --- /dev/null +++ b/docs/specification-revision-5-addendum.md @@ -0,0 +1,7 @@ +# DataTiles schema revision 5 — semantic, FAIR, rights, and source-ingestion addendum + +This addendum accompanies the extension bundle and records normative changes layered on the repository's DataTiles 1.0-draft revision-3 specification. The application script migrates schema revisions 3→4→5 and updates the repository specification with the Zarr ingestion profile. Until these changes are merged into one consolidated upstream specification, a conformance statement for this extension MUST cite both the base DataTiles 1.0-draft specification and this revision-5 addendum. + +Revision 4 adds the canonical semantic-variable registry and external identifier crosswalks. Revision 5 adds persistent identifiers, related identifiers, structured rights/licensing, FAIR publication evidence, DataCite-shaped citation metadata, and strengthened W3C PROV-aligned publication records. The scientific import profile comprises NetCDF, GRIB2, and Zarr source encodings. + +For Zarr specifically, the normative source contract is `zarr-source-profile.md`. Implementations MUST distinguish a multi-object store digest from a single-file SHA-256, MUST bind remote stores to immutable/versioned evidence, MUST preserve scientific semantics and source/output rights, MUST record the conversion parameters needed to reproduce tiling, and MUST NOT persist credentials or infer reuse permission from network accessibility. diff --git a/docs/specification-revision-6-addendum.md b/docs/specification-revision-6-addendum.md new file mode 100644 index 0000000..3715531 --- /dev/null +++ b/docs/specification-revision-6-addendum.md @@ -0,0 +1,45 @@ +# DataTiles 1.0-draft — schema revision 6 addendum: cryptographic integrity + +This addendum extends schema revision 5 with optional integrity manifests and digital signatures. All revision-5 scientific, semantic, FAIR, provenance, licensing, and MBTiles-compatibility requirements remain unchanged. + +## New tables + +`datatiles_integrity_manifests` stores canonical logical manifests. Each manifest declares its profile, canonicalization, hash algorithm, SHA-256 root, canonical JSON representation, and creation time. + +`datatiles_signatures` stores one or more signatures over a recorded manifest. Native signatures use Ed25519 and may embed the raw public key required for self-contained mathematical verification. `signer_agent_id` MAY link the signature to a PROV agent. `verification_material_json` MAY retain additional certificate, transparency-log, timestamp, or external trust material. + +The signature tables are intentionally outside the canonical signed domain. Signature insertion/removal MUST NOT change the manifest root of otherwise identical scientific content. + +## Canonical integrity manifest + +Conforming implementations of `DataTiles-Integrity-Manifest-1` MUST: + +1. hash the logical SQLite object rather than raw database bytes; +2. include the DataTiles schema revision; +3. include every user table except `datatiles_integrity_manifests` and `datatiles_signatures`; +4. include normalized schema definitions for those tables and definitions for user views/triggers; +5. preserve SQLite value types and exact BLOB bytes in row canonicalization; +6. preserve the IEEE-754 binary64 representation of REAL values; +7. make table row order irrelevant while preserving duplicate multiplicity; +8. use SHA-256 for row, table, schema, and root digests; +9. serialize manifest JSON with UTF-8, lexicographically sorted object keys, no insignificant whitespace, and no NaN/Infinity JSON numbers. + +The normative implementation profile is documented in `digital-signatures.md`. + +## Signature profile + +`DataTiles-Ed25519-Signature-1` signs the complete canonical manifest JSON, including its verified `root_sha256`. The signature record MUST identify its algorithm and key identifier. The key identifier is `sha256:` followed by the lowercase SHA-256 digest of the 32-byte raw Ed25519 public key. + +Verification MUST distinguish mathematical validity from trust. An embedded public key is sufficient for cryptographic verification but is not, by itself, evidence that the key belongs to a claimed person or institution. + +## Migration + +Revision 5 containers migrate to revision 6 by adding the two tables and profile metadata. Existing content remains unsigned. Migration MUST NOT invent signatures, signer identities, timestamps, or trust assertions. + +## HTTP and playground + +Servers MAY expose read-only integrity/signature metadata. They MUST NOT expose private keys and SHOULD NOT provide remote signing endpoints. Because full manifest recomputation may require reading every tile, interactive services SHOULD avoid implicit full verification on every request; verification belongs in release/validation workflows or explicitly requested operations. + +## FAIR relationship + +Digital signatures are optional enhanced evidence. Unsigned DataTiles can still satisfy the FAIR publication profile. A FAIR report MAY disclose signature presence, but MUST NOT conflate a signature with FAIRness, scientific validity, legal compliance, or safety certification. diff --git a/docs/specification-revision-7-addendum.md b/docs/specification-revision-7-addendum.md new file mode 100644 index 0000000..40e90e8 --- /dev/null +++ b/docs/specification-revision-7-addendum.md @@ -0,0 +1,36 @@ +# DataTiles 1.0-draft — schema revision 7 addendum + +## Commercial product metadata + +Revision 7 adds `datatiles_commercial_products` and `datatiles_drm_policies`. These tables are part of the canonical scientific object and therefore are covered by `DataTiles-Integrity-Manifest-1` when the object is signed. + +`datatiles_commercial_products` identifies a commercial edition, publisher/issuer, human-readable terms URI, optional licence-service URI, and protection profile. `datatiles_drm_policies` stores machine-readable policy documents; the default policy profile is W3C ODRL 2.2. + +These tables MUST NOT contain plaintext content-encryption keys, customer private keys, passwords, payment credentials, access tokens, or other secrets. + +## Protected distribution + +`DataTiles-Protected-Distribution-1` is an optional outer distribution format and does not replace the SQLite DataTiles format. A `.dtpkg` contains a public canonical JSON header plus AES-256-GCM ciphertext of one complete finalized DataTiles file and the GCM authentication tag. The public header is authenticated additional data and binds ciphertext to the product identity and plaintext SHA-256. + +Authorized decryption MUST reproduce the exact original DataTiles bytes and MUST verify the stored SHA-256 before releasing the result as valid. + +## Licence grants + +`DataTiles-Commercial-License-1` is an issuer-signed JSON envelope. The native portable profile uses Ed25519 signatures and X25519 recipient keys. The content key is wrapped with a product-bound HKDF-SHA-256 derived key and AES-256-GCM. + +Implementations MUST distinguish: + +1. cryptographic validity of the issuer signature; +2. external trust in the issuer public key; +3. recipient-key binding; +4. validity-period enforcement; +5. product/hash binding; +6. legal rights expressed by the dataset licence and ODRL/human-readable terms. + +A valid DRM licence MUST NOT be interpreted as evidence that upstream data may lawfully be sold or redistributed. + +## Services + +The standard read-only DataTiles HTTP service MAY expose non-secret commercial product metadata and ODRL policies from an authorized plaintext DataTiles object. It MUST NOT expose publisher content keys, recipient private keys, licence signing keys, or a standard endpoint that bypasses protected-distribution authorization. + +Online purchase, payment, account management, revocation, floating licences, device activation, and remote attestation are deployment-specific services outside the normative portable profile. diff --git a/docs/specification-revision-8-addendum.md b/docs/specification-revision-8-addendum.md new file mode 100644 index 0000000..b1e119f --- /dev/null +++ b/docs/specification-revision-8-addendum.md @@ -0,0 +1,21 @@ +# DataTiles schema revision 8 addendum: release versioning + +## Status + +This addendum defines the normative `DataTiles-Release-Versioning-1` profile. A DataTiles file represents one immutable scientific release. Versioning identifies successive releases of the same logical product; it does not mutate an old signed release in place. + +## Stable product identity and release identity + +A revision-8 container MAY contain exactly one row in `datatiles_release`. When present, `product_id`, `version`, `sequence`, and `released_at` are REQUIRED. `product_id` MUST remain stable across releases of the same logical product and SHOULD be a persistent URI/URN/DOI concept identifier under publisher control. `version` is the human-facing release label and MAY use Semantic Versioning, calendar versioning, an institutional scheme, or another documented scheme. + +`sequence` is the normative ordering field. It MUST be a positive integer, MUST increase for each published successor of the same `product_id`, and MUST NOT be reused for a different release. Consumers MUST NOT infer ordering from lexical comparison of `version`. `released_at` MUST be an RFC 3339 timestamp. + +Optional `previous_version` and `previous_identifier` fields link the release to its predecessor. `release_notes_uri` SHOULD identify a human-readable changelog. `update_uri` MAY identify a catalogue/API endpoint where successors can be discovered. + +## Immutability + +A new scientific release MUST be represented by a new DataTiles object with a new checksum and, where signing is used, a new integrity manifest/signature. Replacing bytes underneath an already published version is non-conforming. Corrected products MUST increment `sequence` and use a new release label. + +## Provenance, citation, and Store update discovery + +Version metadata complements, but never replaces, provenance, persistent identifiers, rights, source citation, and integrity evidence. A citation SHOULD identify the exact release used. A Store MAY notify users who accessed an older sequence when a larger sequence for the same `product_id` is published. Such notification does not by itself grant entitlement to the new release. diff --git a/docs/specification.md b/docs/specification.md index bf898ea..8cc5402 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -1,4 +1,4 @@ -# DataTiles 1.0-draft specification (schema revision 3) +# DataTiles 1.0-draft specification (schema revision 8) ## Status, audience, and reading convention @@ -36,7 +36,7 @@ A partial implementation MUST NOT claim the class whose requirements it omits. - The file MUST be SQLite 3 and SHOULD use `.datatiles`. - `PRAGMA application_id` MUST equal hexadecimal `0x44415441` (`DATA`). -- `PRAGMA user_version` MUST equal `3`. +- `PRAGMA user_version` MUST equal `8`. - Foreign keys MUST be enabled by writers and validators. - Text is UTF-8. JSON is UTF-8 and MUST reject NaN and infinities. - The spatial tile matrix is Web Mercator; rows stored in the database use TMS orientation. @@ -380,3 +380,23 @@ An implementation MUST NOT guess payload type from BLOB signatures when a conten ## 18. Evolution Readers MUST reject unsupported schema revisions rather than infer them. A migration MUST be transactional, deterministic, documented, and preserve scientific identity. Existing columns MUST NOT be reinterpreted. Future revisions may define non-Gregorian calendars, content deduplication, tile checksums, dimension overviews, additional vector encodings, and further OGC API profiles. + + +## Zarr source-ingestion profile + +A conforming `zarr2datatiles` source utility MUST follow `zarr-source-profile.md`. Zarr is a multi-object N-dimensional store rather than a single-file checksum domain. Local directory stores MUST use the canonical `zarr-tree-sha256-v1` digest defined there; remote stores MUST be bound to an immutable store/snapshot by an authoritative checksum supplied by the publication workflow. The importer MUST preserve the source identifier, checksum algorithm/value, CF semantics when present, dimensions, units, declared resampling, source/output rights, and tile-level lineage. It MUST NOT persist backend credentials or infer data licensing from transport accessibility. Zarr format 2 and 3 MAY be accepted; any required format, group, consolidated-metadata policy, and non-secret storage-option keys MUST be recorded as conversion parameters. Scientific arrays MUST remain numeric DNT1 evidence rather than being silently converted to portrayal imagery. + + +## Cryptographic integrity profile + +Schema revision 6 adds optional canonical integrity manifests and digital signatures. Implementations claiming the native profile MUST follow `digital-signatures.md` and `specification-revision-6-addendum.md`. The signed subject is the canonical logical DataTiles object, not raw SQLite file bytes. The native signature algorithm is Ed25519 over `DataTiles-Integrity-Manifest-1`; SHA-256 remains the digest primitive. Signature metadata MUST distinguish cryptographic validity from trust in signer identity. Signature presence is optional and MUST NOT be presented as proof of FAIRness, scientific correctness, legal compliance, hydrographic authority, or navigation safety. Servers MUST NOT expose private-key signing endpoints as part of the standard DataTiles read service. + + +## Optional commercial protected distribution + +Schema revision 7 adds commercial-product and machine-readable policy metadata. Implementations supporting commercial protected distribution MUST follow `drm-and-commercial-licensing.md` and `specification-revision-7-addendum.md`. `DataTiles-Protected-Distribution-1` is an optional outer encrypted distribution package; after authorized decryption the payload MUST be the exact ordinary DataTiles SQLite object. W3C ODRL 2.2 is the default machine-readable rights-policy model. Technical DRM grants MUST NOT be interpreted as source relicensing, scientific certification, hydrographic authority, FAIR certification, or navigation safety. Secrets MUST remain outside the DataTiles metadata/provenance graph and standard read-only service. + + +## Release versioning + +Schema revision 8 defines `DataTiles-Release-Versioning-1`. A published DataTiles object MAY declare one `datatiles_release` record containing stable `product_id`, human `version`, monotonically increasing integer `sequence`, RFC 3339 `released_at`, and optional predecessor/release-notes/update links. Consumers MUST use `sequence`, not lexical `version`, to order releases. Published versions are immutable; a correction is a new DataTiles object with a larger sequence, new checksum, and new signature where signing is used. See `specification-revision-8-addendum.md`. diff --git a/docs/store-api.md b/docs/store-api.md new file mode 100644 index 0000000..3ce8d0b --- /dev/null +++ b/docs/store-api.md @@ -0,0 +1,114 @@ +# DataTiles Store API v1 + +The Store is API-first. Human HTML/PWA pages are convenience clients; third-party applications should integrate through `/api/v1`. API authorization resolves to the same SQLAlchemy `User -> Group -> Role -> Permission` graph used by browser sessions. + +## Authentication + +Obtain an opaque Bearer token: + +```http +POST /api/v1/auth/token +Content-Type: application/json + +{"username":"alice","password":"...","name":"desktop client","expires_in":2592000} +``` + +Use the returned token as: + +```http +Authorization: Bearer dts_... +``` + +The Store persists only a SHA-256 hash and short non-secret prefix, never the bearer secret itself. Tokens can be listed, created, and revoked through `/api/v1/auth/tokens`. Revocation or user deactivation stops future authorization. TLS is mandatory for production API use. + +## Agreement workflow + +Catalog metadata can be discovered without accepting a product licence when the user's role permits catalog view. Payload access is different: portrayal tiles and file download require a current acceptance for the exact release. + +```http +GET /api/v1/catalog/42/agreement +``` + +returns the file SHA-256, structured rights records, canonical licence fingerprint, safety agreement version/text and current acceptance status. + +Acceptance requires an explicit JSON body: + +```http +POST /api/v1/catalog/42/agreement/accept +Authorization: Bearer ... +Content-Type: application/json + +{"accept_license":true,"accept_safety":true} +``` + +The acceptance is invalid for access purposes if the DataTiles SHA-256, extracted rights fingerprint, or configured safety-agreement version changes. A gated endpoint returns HTTP `428 Precondition Required` with an `agreement_url` when a current acceptance is missing. + +## Catalog and search + +```text +GET /api/v1/catalog?q=sea_floor_depth_below_geoid +GET /api/v1/catalog/{id} +``` + +Search uses metadata automatically extracted from the indexed DataTiles objects: title/description, MBTiles metadata, CF standard names, rights/licences, provenance identifiers and labels, bounds and other catalog facts. It does not full-text-index raw scientific cell values. + +## Preview and download + +```text +GET /api/v1/catalog/{id}/tiles/{z}/{x}/{y} +GET /api/v1/catalog/{id}/download +``` + +Both require appropriate role permission and current agreement acceptance. The portrayal endpoint returns only recognized portrayal image payloads from the selected compatibility slice; numeric DNT1 arrays are not converted heuristically. Responses are marked private/no-store where appropriate. + +## DataTiles CRUD + +Users in the bootstrapped `managers` group inherit the `catalog_manager` role and can perform Store-level file CRUD: + +```text +POST /api/v1/catalog multipart `file`; optional `filename` +GET /api/v1/catalog/{id} read indexed metadata +PUT /api/v1/catalog/{id}/file multipart replacement; optional `filename` +PATCH /api/v1/catalog/{id} JSON rename: {"filename":"new.datatiles"} +DELETE /api/v1/catalog/{id} +POST /api/v1/catalog/scan +``` + +Create/update first validates the candidate as a readable DataTiles/MBTiles-compatible SQLite file. Replacement is atomic and re-indexes metadata and SHA-256. A new checksum necessarily invalidates prior access acceptance. These endpoints never edit released scientific metadata in place. + +## User/group administration + +Administrative clients have API counterparts for identity management: + +```text +GET/POST /api/v1/users +PATCH/DELETE /api/v1/users/{id} +GET/POST /api/v1/groups +PATCH/DELETE /api/v1/groups/{id} +``` + +The `administrators` and `managers` bootstrap groups are protected from API deletion. Applications should assign permissions through groups/roles rather than username-specific policy. + +## Audit + +```text +GET /api/v1/audit?limit=100 +``` + +returns operational events such as agreement acceptance, download, catalog create/update/rename/delete, scans, token issuance/revocation and administrative changes. Audit retention and client metadata collection must follow the operator's privacy and records policy. The Store audit is not a substitute for W3C PROV evidence inside DataTiles. + +## Discovery and errors + +```text +GET /api/v1 +GET /api/v1/openapi.json +GET /healthz +``` + +Common API status semantics are `401` authentication required, `403` insufficient role permission, `404` absent asset, `409` state conflict, `415` selected slice is not portrayal imagery, and `428` current agreement acceptance required. + +The reference OpenAPI document is machine-readable discovery metadata. Production deployments may generate a more detailed contract from the same endpoint set and should version incompatible API changes under a new API prefix. + +## Authentication/configuration additions + +API v1 also exposes authentication-provider discovery, managed-account registration/email verification, administrator configuration, and help documentation. Interactive Google/Microsoft/generic OIDC authorization uses browser redirects but resolves to the same Store identity and authorization model. See `store/docs/api.md` and `store/docs/authentication.md`. diff --git a/docs/store-pwa.md b/docs/store-pwa.md new file mode 100644 index 0000000..0d6c404 --- /dev/null +++ b/docs/store-pwa.md @@ -0,0 +1,104 @@ +# DataTiles Store: institutional catalog and PWA + +The optional `/store` application turns a directory of DataTiles releases into a searchable institutional catalog without changing the scientific objects themselves. It is implemented in Python/Flask and uses SQLAlchemy 2.x for all application-database persistence. + +## Architectural separation + +There are two databases with deliberately different purposes: + +1. **DataTiles containers** are immutable or publication-controlled SQLite scientific objects. The store opens them read-only to extract metadata and to serve a selected compatibility portrayal. +2. **The Store database** is an application database managed through SQLAlchemy. It contains users, groups, roles and a denormalized searchable catalog index. It does not become part of the scholarly DataTiles provenance chain. + +This separation avoids accidental modification of published checksums/signatures and permits the catalog index to be rebuilt at any time from authoritative DataTiles files. + +## Catalog discovery and indexing + +A scan walks `CATALOG_DIR`, opens candidate SQLite files through SQLAlchemy in read-only/query-only mode, and extracts ordinary MBTiles metadata plus DataTiles-specific metadata when present: schema revision, semantic variables/CF names, structured rights, provenance entities/activities, integrity signatures, commercial-product metadata, bounds and zoom ranges. Search operates over this extracted index, not over raw tile payloads. + +The catalog records a SHA-256 of each stored file for operational identity. This is separate from revision-6 logical integrity manifests and signatures, whose semantics remain authoritative for signed scholarly releases. + +## Explorer + +The map explorer follows familiar chart-browser interaction patterns: pan/zoom, coverage fitting, layer enable/disable, location inspection, product metadata, variables, provenance, rights and downloads. C-MAP Chart Explorer is a qualitative interaction reference only; its datasets, cartographic symbols, assets and pixels are not copied. + +The preview endpoint consumes the selected four-column `tiles` compatibility view and translates web XYZ `y` to stored TMS rows. Only recognized PNG, JPEG and WebP portrayal payloads are displayed. Numeric DNT1 arrays are never heuristically treated as images. A non-image compatibility slice therefore remains discoverable/downloadable but requires an explicit portrayal profile to appear as a map overlay. + +## Authentication and authorization + +The authorization graph is: + +```text +User -> Group -> Role -> Permission +``` + +The built-in roles are `viewer`, `downloader`, `catalog_manager`, and `admin`. The configuration file defines the bootstrap administrator username/password/group/role. On first application initialization, the `admin` user is created, its configured password is hashed, it is placed in the `administrators` group, and that group receives the `admin` role. + +No application route should infer authorization directly from a username. Route protection uses permissions derived from group-role membership. + +## PWA and confidentiality + +The service worker caches only the static app shell. It explicitly bypasses `/api/*` and file-download responses. Consequently, enabling PWA installation does not automatically create uncontrolled offline copies of protected datasets. + +A production deployment should use HTTPS, a strong random Flask secret, secure cookies, a reverse proxy, database backups, rate limiting at the proxy/gateway, and external secret management. The repository's `config.py` contains an explicit bootstrap password setting as requested, but deployments should override it from a secret manager and rotate it after initialization. + +## FAIR, licences, signatures and DRM + +Authentication and download authorization do not change dataset rights. The application surfaces source-specific acknowledgements, licensing and provenance extracted from each DataTiles release. A restricted catalog can still support FAIR discovery when public metadata/PIDs are published appropriately, but this specific PWA defaults to authenticated catalog access. + +Digital signatures continue to attest the inner scientific object; the Store index is disposable and unsigned. DRM content keys and customer secrets do not belong in the Store catalog database. A commercial deployment can connect the Store to a separate entitlement/KMS service, but that is an integration profile rather than part of the portable Store reference implementation. + +## Versioned licence acceptance and safety/no-liability agreement + +Data availability is not consent. Before a user or API client may retrieve portrayal tiles or download the underlying DataTiles object, the Store requires a current acceptance record tied to four facts: the authenticated user, the catalog item, the exact DataTiles SHA-256, and the current agreement identity. The agreement identity includes the structured rights/licence records extracted from the DataTiles release plus the configured safety/no-liability agreement version. + +The mandatory safety text states that the product is **not suitable for navigation**, is provided **AS IS / AS AVAILABLE**, is not an official chart/ENC/ECDIS or certified navigation aid, requires independent verification against authoritative sources, and limits responsibility of distributors/licensors/contributors/institutions/software providers/service operators to the maximum extent permitted by applicable law. This reference language is a technical default, not jurisdiction-specific legal advice; commercial operators must have counsel review the actual contract and consumer/business-law requirements of their distribution jurisdictions. + +An acceptance row preserves the licence snapshot, file checksum, safety text/version, timestamp, user identity, interaction source (`web` or `api`), and optionally bounded client IP/user-agent evidence. If the file is replaced, its rights change, or the configured safety version changes, the old acceptance remains in the audit history but no longer unlocks preview/download. + +This separation is important academically and legally: accepting the Store agreement cannot create rights absent from upstream licences, waive non-waivable statutory rights, certify data quality, or make a scientific portrayal navigationally authoritative. + +## API-first interaction contract + +Every material Store interaction is exposed under `/api/v1`, including authentication/token lifecycle, catalog search/detail, agreement retrieval and acceptance, portrayal tiles, downloads, catalog rescans, DataTiles CRUD, user/group administration and audit retrieval. Browser routes are a human UI over the same authorization and data model; third-party applications use opaque Bearer tokens associated with an ordinary Store user and therefore receive the same group/role permissions. + +The primary endpoints are: + +```text +POST /api/v1/auth/token +GET /api/v1/auth/me +GET /api/v1/auth/tokens +POST /api/v1/auth/tokens +DELETE /api/v1/auth/tokens/{token_id} + +GET /api/v1/catalog?q=... +POST /api/v1/catalog +GET /api/v1/catalog/{id} +PUT /api/v1/catalog/{id}/file +PATCH /api/v1/catalog/{id} +DELETE /api/v1/catalog/{id} +POST /api/v1/catalog/scan + +GET /api/v1/catalog/{id}/agreement +POST /api/v1/catalog/{id}/agreement/accept +GET /api/v1/catalog/{id}/tiles/{z}/{x}/{y} +GET /api/v1/catalog/{id}/download + +GET/POST/PATCH/DELETE /api/v1/users... +GET/POST/PATCH/DELETE /api/v1/groups... +GET /api/v1/audit +GET /api/v1/openapi.json +``` + +API write endpoints accept Bearer authentication and are CSRF-exempt because they do not rely on ambient browser cookies for third-party authorization. Browser form writes remain CSRF protected. + +## Managers and Store-level CRUD + +The bootstrap process creates a `managers` group (configurable with `MANAGERS_GROUP`) and assigns it the `catalog_manager` role. Members can create/upload, read, replace/update, rename and delete DataTiles files. These are Store-level file operations: update means atomic replacement by another valid DataTiles/MBTiles-compatible SQLite file, not mutation of the signed scientific object in place. + +On replacement, the new file is opened read-only through SQLAlchemy and metadata extraction must succeed before it replaces the published file. A new SHA-256 and metadata index are generated. Because acceptance is checksum-bound, the new release immediately requires renewed licence/safety acceptance. + +Destructive operations and agreement acceptance are recorded in the Store audit table. Audit records are operational evidence and do not replace the W3C PROV graph embedded in the scientific DataTiles object. + +## Authentication and infrastructure integration + +The Store supports managed password accounts, optional verified-email self-registration, Google OpenID Connect, Microsoft Entra ID tenants, and a generic OAuth2/OpenID Connect institutional provider. All identities resolve to the same SQLAlchemy user/group/role model. Runtime authentication and SMTP settings are administered from the PWA **Configuration** section by administrators and through the administrator API. Complete operator documentation is installed under `store/docs/` and rendered by the PWA **Help** section. diff --git a/docs/tutorial/README.md b/docs/tutorial/README.md index 54ee9ff..73dd039 100644 --- a/docs/tutorial/README.md +++ b/docs/tutorial/README.md @@ -1,4 +1,4 @@ -# DataTiles zero to hero in five lessons +# DataTiles zero to hero in ten lessons This course develops DataTiles from first principles and ends with a reproducible mixed raster/vector publication. It treats DataTiles as a scientific data model rather than merely a file-manipulation library. Every lesson contains theory, a hands-on laboratory, observable checks, and questions for critical reflection. @@ -11,6 +11,11 @@ This course develops DataTiles from first principles and ends with a reproducibl | [3. Raster matrices and vector features](lesson-3-raster-vector.md) | Coverage and feature models, content profiles, encoding semantics | Build one container containing DNT1 and tiled GeoJSON | | [4. Querying and deriving live products](lesson-4-query-api.md) | Exact selection, sampling, interpolation boundaries, OGC API patterns | Serve and query point, profile, surface, contour, and content resources | | [5. FAIR and reproducible publication](lesson-5-fair-publication.md) | Identity, provenance, CRS, licences, reproducibility versus replicability | Validate and prepare a citable research artifact | +| [6. Conventions and source utilities](lesson-6-conventions-and-utils.md) | CF semantic identity, NetCDF/GRIB crosswalks, source provenance, declared resampling | Import local or URL-identified scientific grids into semantic DNT1 DataTiles | +| [7. FAIR evidence, provenance, and licensing](lesson-7-fair-provenance-and-licensing.md) | FAIR gates, W3C PROV, SPDX rights, DataCite 4.7, publication evidence | Prepare a defensible scholarly release | +| [8. Cryptographic integrity and signed releases](lesson-8-cryptographic-integrity.md) | canonical logical manifests, Ed25519 signatures, trust, detached envelopes, release verification | Sign and verify a final scholarly DataTiles release | +| [9. Commercial distribution and DRM](lesson-9-commercial-drm.md) | ODRL policy, encrypted `.dtpkg`, recipient licences, key management | Distribute lawful proprietary DataTiles products | +| [10. DataTiles Store PWA](lesson-10-datatiles-store.md) | Flask, SQLAlchemy, PWA, metadata indexing, RBAC, catalog search, interactive exploration | Publish an authenticated institutional DataTiles catalog | ## Prerequisites and setup @@ -36,6 +41,6 @@ The builder is part of the tested repository. It creates two DNT1 matrices and o ## Evidence of completion -A learner completing the course should be able to explain why the `tiles` view is necessary but insufficient, derive a canonical coordinate-set key, distinguish a numeric raster matrix from a portrayal, justify a vector content profile, query exact coordinates without silently interpolating, interpret the FAIR report conservatively, and reproduce the tutorial artifact from source. +A learner completing the course should be able to explain why the `tiles` view is necessary but insufficient, derive a canonical coordinate-set key, distinguish a numeric raster matrix from a portrayal, justify a vector content profile, query exact coordinates without silently interpolating, interpret the FAIR report conservatively, and reproduce the tutorial artifact from source, and explain how NetCDF/CF and GRIB2 source metadata map into the DataTiles semantic-variable convention. The tutorial dataset is for education only and must not be used for navigation. diff --git a/docs/tutorial/examples/build_tutorial.py b/docs/tutorial/examples/build_tutorial.py index 4ce3caf..0683624 100644 --- a/docs/tutorial/examples/build_tutorial.py +++ b/docs/tutorial/examples/build_tutorial.py @@ -27,6 +27,13 @@ def build(path: Path) -> None: store.add_dimension("release","text",axis="O",description="Dataset release identifier") store.add_crs("horizontal",authority="EPSG",code="3857",uri="http://www.opengis.net/def/crs/EPSG/0/3857") store.add_crs("vertical",uri="https://vocab.example.org/datum/tutorial-lat",wkt2="VERTCRS[\"Tutorial depth relative to LAT, positive down\"]") + for name,standard_name,unit in ( + ("depth_below_lat_m","sea_floor_depth_below_sea_surface","m"), + ("seafloor_class","sea_floor_lithology","1"), + ("depth_portrayal","sea_floor_depth_below_sea_surface","m"), + ("survey_stations","platform_id",None), + ): + store.add_variable(name,standard_name,canonical_unit=unit) depth=[5.0,8.0,12.0,20.0,6.0,9.0,14.0,24.0,7.0,10.0,16.0,28.0,8.0,11.0,18.0,32.0] seabed=[4,4,3,6,4,4,3,6,4,3,3,6,5,5,6,6] @@ -53,17 +60,26 @@ def build(path: Path) -> None: encoding="GeoJSON",schema={"geometryTypes":["Point"],"fields":{"name":"String","depth_m":"Number"}}) store.add_provenance_agent("agent:tutorial-author","DataTiles tutorial",agent_type="software") + store.assign_fair_agent_role("agent:tutorial-author","creator") store.add_provenance_activity("activity:tutorial-build","deterministic-build","Build tutorial raster and vector tiles", software="datatiles",parameters={"grid":[4,4],"zoom":0}) source_bytes=json.dumps({"depth":depth,"seabed":seabed,"observations":observations},sort_keys=True,separators=(",",":")).encode() - store.add_provenance_entity("entity:tutorial-source","source","Embedded tutorial observations", + store.add_provenance_entity("entity:tutorial-source","dataset","Embedded tutorial observations", checksum_algorithm="SHA-256",checksum=hashlib.sha256(source_bytes).hexdigest()) - store.add_provenance_entity("entity:tutorial-container","dataset","Tutorial DataTiles container") + store.add_provenance_entity("entity:tutorial-container","collection","Tutorial DataTiles container") store.add_provenance_relation("activity:tutorial-build","used","entity:tutorial-source") store.add_provenance_relation("entity:tutorial-container","wasGeneratedBy","activity:tutorial-build") store.add_provenance_relation("activity:tutorial-build","wasAssociatedWith","agent:tutorial-author") for variable in ("depth_below_lat_m","seafloor_class","depth_portrayal","survey_stations"): store.link_tile_provenance(0,0,0,coordinates(variable),"entity:tutorial-container") + store.add_identifier("URN","urn:uuid:4ea0e809-599d-5f2b-9669-75cd132b9997", + uri="https://example.org/datatiles/tutorial",primary=True) + for scope in ("dataset","metadata"): + store.add_rights(scope,"CC-BY-4.0",license_uri="https://creativecommons.org/licenses/by/4.0/", + rights_holder="DataTiles tutorial",attribution_text="DataTiles tutorial") + store.add_rights("source","CC-BY-4.0",license_uri="https://creativecommons.org/licenses/by/4.0/", + rights_holder="DataTiles tutorial",attribution_text="DataTiles tutorial", + source_entity_id="entity:tutorial-source") metadata={ "description":"Self-contained raster and vector dataset for the DataTiles zero-to-hero course.", diff --git a/docs/tutorial/lesson-10-datatiles-store.md b/docs/tutorial/lesson-10-datatiles-store.md new file mode 100644 index 0000000..8c46199 --- /dev/null +++ b/docs/tutorial/lesson-10-datatiles-store.md @@ -0,0 +1,134 @@ +# Lesson 10 — Publish a DataTiles catalog with the Store PWA + +This lesson turns a directory of finalized DataTiles products into an authenticated searchable catalog and interactive browser. + +## 1. Install the Store + +From the repository root: + +```bash +cd store +python -m pip install -e . +``` + +Edit `store/config.py`. At minimum replace `SECRET_KEY` and `ADMIN_PASSWORD`, and point `CATALOG_DIR` to the directory containing released DataTiles files. + +## 2. Understand the trust boundary + +The Store SQLAlchemy database is not a scientific database. It contains identity/authorization state and a rebuildable metadata search index. Published `.datatiles` files remain the authoritative research objects and are opened read-only. + +This matters for revision-6 signatures: rescanning the Store must not mutate a signed file and must not change its canonical manifest. + +## 3. Start the PWA + +```bash +python run.py +``` + +Open `http://localhost:8080` and sign in with the configured bootstrap administrator. On the first initialization the account is assigned through: + +```text +admin user -> administrators group -> admin role -> all permissions +``` + +## 4. Add products + +Copy finalized DataTiles files under `CATALOG_DIR` and choose **Rescan catalog**, or upload a file while authenticated with `catalog.manage` permission. + +The indexer extracts metadata, CF semantic variables, rights/licence records, provenance summaries, bounds, zoom levels, signature records and public commercial metadata. Search therefore works for queries such as a DOI, `sea_floor_depth_below_geoid`, a rights holder, an SPDX licence expression or a source label. + +## 5. Explore a product + +Select **Explore**. The browser provides pan/zoom, coverage fitting, a portrayal-layer toggle, variables, rights/citation, provenance and metadata panels. + +DataTiles stores TMS rows while web clients use XYZ. The Store converts only at the HTTP interface. A PNG/JPEG/WebP selected compatibility slice is displayable. A DNT1 scientific matrix is not guessed into colors: prepare/select a declared portrayal profile when a preview is required. + +The permanent **Not for navigation** notice remains visible for the reference workflow. + +## 6. Add users and groups + +The built-in roles provide progressively broader permissions. Create groups such as `researchers`, `customers`, or `publishers`, assign the appropriate roles, then create users in those groups. Do not create authorization checks based on hard-coded usernames. + +## 7. Deploy as a PWA safely + +The service worker makes the user interface installable but does not cache APIs, map payloads or file downloads. In production place Flask behind HTTPS, enable secure session cookies, keep the secret key and bootstrap password in a secret manager, and use a production WSGI server. + +## 8. Commercial products + +The Store is not the revision-7 DRM entitlement server. For proprietary products, keep content keys and customer private material in a dedicated KMS/licence service. Catalog metadata may advertise the product, issuer and terms, while actual authorization/decryption remains an independently secured subsystem. + +## 9. Publication checklist + +Before exposing any product, verify that its source-specific citation, rights, FAIR metadata, provenance and optional signature evidence are complete. Download authorization cannot cure a missing attribution or incompatible upstream licence, and an interactive preview does not certify the data for navigation. + +## 10. Require explicit licence and safety acceptance + +Open a product as a user with download permission. The Store shows the rights extracted from the DataTiles release together with the configured **not suitable for navigation / AS IS / limitation-of-responsibility** agreement. Preview and download remain unavailable until both checkboxes are explicitly accepted. + +The acceptance is version-specific. Replace the file with a different release and observe that the old acceptance no longer authorizes access because the SHA-256 changed. The same happens when embedded rights change or the operator advances `SAFETY_AGREEMENT_VERSION`. + +A third-party client performs the same workflow: + +```bash +TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/token \ + -H 'Content-Type: application/json' \ + -d '{"username":"customer","password":"a-long-customer-password"}' | jq -r .access_token) + +curl -H "Authorization: Bearer $TOKEN" \ + http://localhost:8080/api/v1/catalog/1/agreement + +curl -X POST -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"accept_license":true,"accept_safety":true}' \ + http://localhost:8080/api/v1/catalog/1/agreement/accept + +curl -L -H "Authorization: Bearer $TOKEN" \ + -o product.datatiles \ + http://localhost:8080/api/v1/catalog/1/download +``` + +## 11. Use the managers group for DataTiles CRUD + +The Store automatically creates the configured `managers` group with the `catalog_manager` role. Add a publishing user to this group. That user can upload a new DataTiles product, replace an existing release atomically, rename the stored file, or delete it. + +Equivalent API calls use `POST /api/v1/catalog`, `PUT /api/v1/catalog/{id}/file`, `PATCH /api/v1/catalog/{id}`, and `DELETE /api/v1/catalog/{id}`. The API enforces the same permissions as the browser. + +Do not use CRUD to rewrite the scientific SQLite object in place. Build/finalize/sign a new release first, then replace the catalog asset. This preserves the distinction between scholarly provenance and distribution operations. + +## 12. Treat the API as the integration contract + +Use `/api/v1/openapi.json` for machine discovery. Third-party portals, desktop GIS/MFD applications, institutional repositories and commercial entitlement frontends should integrate through the API rather than scraping HTML. Bearer tokens resolve to ordinary users and inherit exactly the roles granted through their groups. + +The Store audit endpoint records operational actions such as agreement acceptance, downloads, uploads, replacements and deletes. Keep this audit database protected and retained according to institutional privacy/records policies; it is not a replacement for source provenance inside DataTiles. + +## Identity integration and administrator configuration + +The Store can use managed passwords, verified-email registration, Google OpenID Connect, Microsoft Entra ID tenants, or a generic OAuth2/OpenID Connect provider. Open the administrator **Configuration** section to enable providers, enter client credentials, configure the public callback URL and SMTP delivery, and choose whether new-user registration is allowed. All external identities still map into the same Store groups/roles; SSO is authentication, not automatic administrative authorization. + +The complete operational manual is available inside the PWA **Help** section from the Markdown sources under `store/docs/`. Use `/api/v1/configuration`, `/api/v1/auth/providers`, `/api/v1/register`, and `/api/v1/help` when integrating infrastructure or third-party clients. + +## 13. Publish versioned releases + +Schema revision 8 adds `DataTiles-Release-Versioning-1`. Each release may declare a stable `product_id`, a human-facing `version`, a monotonically increasing integer `sequence`, `released_at`, and optional predecessor/release-notes/update links. Use `sequence` for ordering; do not sort arbitrary version strings lexically. + +A published version is immutable. Corrections and updates are new DataTiles files with a larger sequence, a new checksum, and a new signature when cryptographic signing is used. The Store indexes release metadata and can identify successors without rewriting historical files. + +## 14. Configure optional payments + +Open **Configuration → Commerce / PayPal** as an administrator. Payment is disabled by default. Enable commerce, select the provider, and configure the PayPal reference adapter in sandbox mode first. The PayPal implementation uses server-side OAuth credentials and Orders v2; client secrets never enter DataTiles files or browser JavaScript. + +Managers can mark a catalog release as purchase-required and assign price/currency. Licence/safety acceptance and payment entitlement are separate gates: payment never overrides an upstream licence or acknowledgement obligation. + +Third-party clients can discover the provider and create/capture checkout transactions using: + +```text +GET /api/v1/payments/providers +POST /api/v1/catalog/{id}/checkout +POST /api/v1/payments/{transaction_id}/capture +``` + +## 15. Use My Library and update notifications + +Every successful Store download is recorded with the exact release and SHA-256. Completed paid transactions create release-specific purchase records. Users can inspect both histories in **My Library** or with `GET /api/v1/library`. + +When a higher `sequence` of the same stable `product_id` is indexed, users who previously purchased or downloaded an older release can receive an update notification. The notification is only discovery evidence: it does not automatically transfer a paid entitlement to the newer release. diff --git a/docs/tutorial/lesson-6-conventions-and-utils.md b/docs/tutorial/lesson-6-conventions-and-utils.md new file mode 100644 index 0000000..d9a36c7 --- /dev/null +++ b/docs/tutorial/lesson-6-conventions-and-utils.md @@ -0,0 +1,160 @@ +# Lesson 6 — DataTiles conventions and source utilities + +## Learning goals + +This chapter connects the DataTiles container model to existing scientific conventions. At the end you should be able to explain why a local variable name is insufficient for interoperability, map NetCDF/CF, GRIB2, and Zarr metadata into DataTiles, import local or URL-identified sources, and verify that imported numeric tiles remain data rather than pre-rendered cartography. + +## 1. Convention before conversion + +A file format answers *how bytes are encoded*. A convention or controlled vocabulary answers *what a quantity means*. A NetCDF variable called `depth`, a GRIB short name, and a provider field called `z` may refer to the same physical quantity—or to materially different vertical references. DataTiles therefore separates the local variable token used in coordinates from the semantic variable registry. + +The preferred canonical vocabulary is CF Standard Names. External identifiers such as WMO GRIB2 parameter triples and SeaDataNet/NERC identifiers are crosswalks. Units, CRS/datum and provenance remain explicit because a standard variable name alone cannot fully define every geospatial measurement. + +A useful interoperability test is: + +```text +producer-local name -> controlled semantic name -> units/datum/CRS -> DataTiles coordinate +``` + +A consumer should be able to discover the semantic name without knowing the producer-local name. + +## 2. NetCDF convention mapping + +Consider a NetCDF variable: + +```text +float depth(time, lat, lon) + standard_name = "sea_floor_depth_below_geoid" + units = "m" +``` + +The importer maps it conceptually to: + +```text +variable coordinate token: depth +standard vocabulary: CF +standard name: sea_floor_depth_below_geoid +unit: m +other dimension: time +horizontal target CRS: EPSG:3857 +payload: DNT1 numeric matrix +``` + +If `standard_name` is missing, `netcdf2datatiles` does **not** invent a CF name. The token is registered with vocabulary `source` and can be curated later. + +## 3. GRIB2 convention mapping + +GRIB2 identifies parameters primarily using WMO tables. `grib2datatiles` decodes the file with cfgrib/ecCodes and preserves both semantic views when available: + +```text +CF standard_name canonical semantic discovery +WMO discipline.category.parameter GRIB2 crosswalk +GRIB shortName source/ecCodes crosswalk +``` + +This is why DataTiles should not choose either NetCDF variable names or GRIB codes as its only universal naming system. + +## 4. Install the import profile + +The DataTiles core remains dependency-free. Import utilities are an optional profile: + +```bash +python -m pip install -e '.[utils]' +``` + +This installs xarray/NetCDF support, the cfgrib/ecCodes stack required for GRIB, and the Zarr/fsspec stack required for chunked local or remote stores. + +## 5. Import a local NetCDF file + +```bash +python utils/netcdf2datatiles.py samples/ocean.nc ocean.datatiles \ + --variable depth \ + --zoom 7 \ + --bbox 12.8 39.9 15.8 41.3 +``` + +The source must expose a rectilinear latitude/longitude grid. The converter creates typed DataTiles dimensions for every non-spatial dimension, registers the semantic variable, resamples the selected grid to Web Mercator tile pixel centers with nearest-neighbour sampling, encodes the matrices as DNT1, and records source SHA-256 provenance. + +Inspect the result: + +```bash +datatiles variables ocean.datatiles +datatiles validate ocean.datatiles --require-variable-semantics +``` + +## 6. Import from a URL + +The source identifier can be HTTP(S): + +```bash +python utils/netcdf2datatiles.py \ + https://example.org/archive/ocean.nc \ + ocean.datatiles --variable depth --zoom 7 +``` + +The URL is downloaded to a temporary seekable file, hashed while streaming, converted, then deleted. The DataTiles provenance stores the original URL plus SHA-256 checksum. Reproducible publication should prefer immutable or versioned source URLs and independently published checksums where available. + +## 7. Import GRIB2 + +```bash +python utils/grib2datatiles.py forecast.grib2 forecast.datatiles \ + --variable t2m --zoom 6 +``` + +A GRIB file may contain messages that cannot be represented as one xarray hypercube. Narrow the cfgrib view with repeatable keys: + +```bash +python utils/grib2datatiles.py forecast.grib2 pressure.datatiles \ + --filter-by-keys typeOfLevel=isobaricInhPa \ + --variable t --zoom 6 +``` + +After import, query by semantic identity rather than by source encoding whenever possible: + +```bash +datatiles find-variable forecast.datatiles air_temperature +``` + +## 8. Import Zarr + +A local Zarr directory store is imported directly: + +```bash +python utils/zarr2datatiles.py samples/ocean.zarr ocean-zarr.datatiles \ + --variable depth --zoom 7 +``` + +A Zarr store is not one file. For a local store, DataTiles computes a deterministic digest over the sorted store keys, byte lengths, and exact object bytes (`zarr-tree-sha256-v1`). For a remote immutable store, supply the archival checksum published for that snapshot: + +```bash +python utils/zarr2datatiles.py \ + https://example.org/archive/ocean.zarr ocean-zarr.datatiles \ + --source-sha256 <64-hex-authoritative-sha256> \ + --group analysis --consolidated true --variable depth --zoom 7 +``` + +Object-store URLs such as `s3://` and `gcs://` can be used when the corresponding fsspec backend is installed. Runtime credentials belong in backend configuration or storage options; secret values must never enter DataTiles provenance. The converter records only storage-option names. A signed or credential-bearing access URL requires `--provenance-uri` with a stable credential-free source/PID URI. + +Zarr v2 and v3 differ in metadata and fill-value behavior. Use `--zarr-format 2` or `--zarr-format 3` when the publication workflow needs to require a specific format. Consolidated metadata improves store discovery but is not evidence of scientific validity or stronger provenance. + +## 9. Why the utility does not create image tiles + +The output is `application/vnd.datatiles.numeric` using DNT1. A color ramp, contour layer, shaded relief or nautical-style display is a reproducible *derivation* from the numeric evidence. Converting scientific source grids directly into PNG tiles would destroy machine-readable values and undermine the interoperability objective of this chapter. + +## 10. Limits are part of the convention + +The first utility profile intentionally rejects unsupported geometry rather than approximating it silently. Curvilinear coordinates, rotated grids, projected source grids, antimeridian-spanning extents and conservative cell-area remapping need explicit algorithms and provenance. `--max-tiles` is also part of the safety model: a mistaken high zoom must not expand an import without a deliberate operator decision. + +## 11. Laboratory checks + +Run an import and confirm all of the following: the source URI/checksum are present in metadata/provenance; `datatiles variables` exposes a CF name or explicitly says `source`; the DNT1 unit matches the imported variable unit; time/pressure/depth dimensions are typed rather than embedded in filenames; the horizontal CRS is explicit; and no raster portrayal has been substituted for the numeric payload. + +### Reflection + +1. Why is `depth` insufficient as a universal semantic identifier? +2. When should a WMO GRIB2 parameter be a crosswalk rather than the canonical name? +3. Why must an importer refuse to manufacture a CF `standard_name`? +4. Which provenance fields make a URL import reproducible? +5. Why is nearest-neighbour resampling acceptable only when declared explicitly? +6. Why must a Zarr directory store not be described as having the SHA-256 of a single file? +7. Why should credentials for S3/GCS/HTTP backends never be copied into provenance metadata? diff --git a/docs/tutorial/lesson-7-fair-provenance-and-licensing.md b/docs/tutorial/lesson-7-fair-provenance-and-licensing.md new file mode 100644 index 0000000..af1bffb --- /dev/null +++ b/docs/tutorial/lesson-7-fair-provenance-and-licensing.md @@ -0,0 +1,89 @@ +# Lesson 7 — FAIR evidence, provenance, and licensing + +## Objectives + +You will turn a technically valid DataTiles file into a defensible scholarly research object by separating persistent identity, scientific provenance, legal rights, citation metadata, and repository-level FAIR evidence. + +## 1. FAIR is not an adjective + +The FAIR principles are individually testable obligations. A file can be richly structured but still fail F1/F4/A2 because no PID, catalogue deposit, or metadata-retention policy exists. Conversely, restricted data can be FAIR when metadata is persistent, access conditions are explicit, and authorized reuse is well described. + +Run both local and publication-level checks: + +```python +from datatiles import DataTiles +with DataTiles("study.datatiles") as s: + print(s.fair_report()) + print(s.fair_report(strict_publication=True)) +``` + +Explain every failed principle instead of reporting a percentage. + +## 2. Record identity and scholarly relationships + +A released revision receives a PID. Do not reuse one DOI for materially different immutable versions unless the repository's versioning model explicitly supports it. + +```python +s.add_identifier("DOI", "10.xxxx/example.v1", uri="https://doi.org/10.xxxx/example.v1", primary=True) +s.add_related_identifier("DOI", "10.xxxx/source", "IsDerivedFrom", uri="https://doi.org/10.xxxx/source", resource_type="Dataset") +``` + +## 3. Separate rights layers + +```python +s.add_rights("dataset", "CC-BY-4.0", + license_uri="https://creativecommons.org/licenses/by/4.0/", + rights_holder="Example University", + attribution_text="Cite the dataset DOI and named creators") +s.add_rights("metadata", "CC0-1.0", + license_uri="https://creativecommons.org/publicdomain/zero/1.0/") +``` + +Every source entity receives its own `source` rights record. Never infer source rights from the output licence. + +## 4. Build a PROV graph + +Create source and generated entities, the transformation activity, software and responsible agents. Connect source -> activity -> generated object with `used`, `wasGeneratedBy`, `wasDerivedFrom`, and activity-agent associations. Record exact parameters and software versions. + +Export the graph: + +```python +import json +print(json.dumps(s.prov_json(), indent=2)) +``` + +Audit the graph: can an independent researcher identify every input that materially influenced a tile and the transformation that produced it? + +## 5. Prepare DataCite metadata + +```python +print(json.dumps(s.datacite_metadata(), indent=2)) +``` + +This is a DataCite-4.7-shaped candidate record. It is **not** evidence that a DOI exists. Deposit it through the chosen repository, then record catalogue/landing-page/retention evidence in the container or release manifest. + +## 6. Use the converters lawfully + +```bash +python utils/netcdf2datatiles.py source.nc study.datatiles \ + --variable depth --zoom 7 \ + --source-license CC-BY-4.0 \ + --source-license-uri https://creativecommons.org/licenses/by/4.0/ \ + --source-attribution "Source provider citation/attribution" \ + --dataset-license CC-BY-4.0 \ + --dataset-license-uri https://creativecommons.org/licenses/by/4.0/ +``` + +The utility records the exact source identifier, SHA-256, rights record and conversion activity. It does not decide licence compatibility for you. + +## 7. Publication exercise + +Prepare an evidence package containing the DataTiles file, checksums, source lock, rights manifest, runtime lock, configuration, PROV export, DataCite export, FAIR report and scientific QA. For a source that forbids redistribution, omit its raw bytes and document lawful reacquisition while retaining its checksum and provenance identity. + +## Reflection + +1. Why does a SHA-256 digest not replace a citation or licence? +2. Why must source and output licences be separate records? +3. Which FAIR principles cannot be proven by the SQLite container alone? +4. What is the difference between `wasDerivedFrom` and `used` in the provenance graph? +5. Why is a DataCite JSON export not proof of DOI registration? diff --git a/docs/tutorial/lesson-8-cryptographic-integrity.md b/docs/tutorial/lesson-8-cryptographic-integrity.md new file mode 100644 index 0000000..269847a --- /dev/null +++ b/docs/tutorial/lesson-8-cryptographic-integrity.md @@ -0,0 +1,71 @@ +# Lesson 8 — Cryptographic integrity and signed releases + +This lesson turns a reproducible DataTiles build into an optionally signed research object. Complete the scientific build, citations, licences, provenance, and FAIR release checks **before** signing; every later scientific mutation intentionally invalidates the signature. + +## 1. Why a signature is different from a checksum + +A SHA-256 checksum identifies bytes or logical content. A digital signature binds a digest to possession of a private key. DataTiles signs a canonical logical manifest so harmless SQLite storage rewrites do not break authenticity while changes to scientific tables do. + +Trust remains external: an embedded public key proves only self-consistency. Obtain a trusted public key through an institutional repository, project website, certificate infrastructure, signed Git release, or another independently authenticated channel. + +## 2. Install optional signing support + +```bash +python -m pip install -e '.[integrity]' +``` + +The base DataTiles package remains dependency-free; signing support is intentionally optional. + +## 3. Inspect the unsigned manifest + +```bash +datatiles-integrity manifest gaeta-maratea.datatiles \ + --output gaeta-maratea.integrity.json +``` + +Inspect `root_sha256`, table row counts, per-table digests, and schema revision. This file is deterministic for the same logical signed domain. + +## 4. Generate a release key + +For this exercise only: + +```bash +datatiles-integrity generate-key \ + --private-key tutorial-release-key.pem \ + --public-key tutorial-release-key.pub.pem +``` + +Production institutions should use their established key-management policy. Never publish the private key. + +## 5. Sign the final object + +```bash +datatiles-integrity sign gaeta-maratea.datatiles \ + --private-key tutorial-release-key.pem \ + --signer 'Tutorial publisher' \ + --detached gaeta-maratea.datatiles.sig.json +``` + +The command stores a signature in the container and creates a detached envelope suitable for a repository release. + +## 6. Verify with an independently obtained key + +```bash +datatiles-integrity verify gaeta-maratea.datatiles \ + --detached gaeta-maratea.datatiles.sig.json \ + --public-key tutorial-release-key.pub.pem +``` + +The report must show `cryptographically_valid: true`, `trusted_key_supplied: true`, `content_matches_signed_manifest: true`, and `valid: true`. + +Now copy the DataTiles file, modify one metadata value in the copy, and verify again. The signature remains mathematically valid over its historical manifest, but `content_matches_signed_manifest` becomes false. This separation is essential for forensic clarity. + +## 7. Connect signer and provenance + +Create or reuse a PROV agent representing the responsible person or organization and record its persistent identifier. When signing inside a publication workflow, pass its identifier as `--signer-agent-id`. The agent identity and cryptographic key identity remain distinct. + +## 8. Release checklist + +A signed scholarly release should archive the DataTiles object, detached signature, trusted public key/certificate reference, source manifest, provenance graph, source-specific citations, rights records, DataCite metadata, strict FAIR report, and any optional Sigstore/timestamp evidence. + +A signature is not a navigation approval. The Gaeta-to-Maratea product remains **Not for navigation** regardless of signature status. diff --git a/docs/tutorial/lesson-9-commercial-drm.md b/docs/tutorial/lesson-9-commercial-drm.md new file mode 100644 index 0000000..399d3dd --- /dev/null +++ b/docs/tutorial/lesson-9-commercial-drm.md @@ -0,0 +1,39 @@ +# Lesson 9 — Commercial distribution and DRM + +This lesson shows how a lawful proprietary DataTiles release can add technical access control without weakening FAIR metadata, provenance, licensing, citation, or cryptographic integrity. + +## 1. Decide whether commercial distribution is legally permitted + +Before encryption, inspect every source rights record and the frozen contribution manifest. The ability to encrypt a derivative does not grant commercial redistribution rights. Resolve attribution, share-alike, database rights, non-commercial restrictions, and contractual terms first. + +## 2. Register the product and policy + +Give the inner DataTiles object a stable product ID, publisher identity, edition, terms URI, optional licence-service URI, and machine-readable ODRL policy. Keep these facts inside the signed logical object so that policy metadata cannot be silently changed without invalidating the DataTiles integrity signature. + +## 3. Freeze and optionally sign + +Run FAIR/provenance/licensing checks, freeze the product, and optionally sign it with `datatiles-integrity`. The signature authenticates the exact scientific release; DRM comes afterward as a distribution wrapper. + +## 4. Protect the product + +Use `datatiles-drm protect`. The command generates a fresh AES-256 content key, encrypts the entire DataTiles file with authenticated encryption, records the plaintext SHA-256 in the authenticated package header, and writes the publisher content key to a restricted file. Move that secret to institutional key management as soon as practical. + +## 5. Provision a recipient + +Each recipient/device has an X25519 keypair. Only the public key is supplied to the publisher. The private key stays with the customer/device. + +## 6. Issue a licence + +The publisher wraps the content key to that recipient and signs the licence with its Ed25519 issuer key. The licence can include validity dates, a recipient identifier, technical permissions, and an ODRL policy. The same encrypted `.dtpkg` can therefore be distributed once while issuing different recipient-specific grants. + +## 7. Verify and decrypt + +The client verifies the issuer signature using an independently authenticated publisher public key, checks time/product/hash/recipient binding, unwraps the content key, authenticates/decrypts the package, and verifies the recovered DataTiles SHA-256. + +## 8. Understand the limits + +Portable DRM cannot guarantee that an authorized client will never copy plaintext. It protects distribution and access, not post-decryption behavior. Never market DRM as scientific certification, FAIR certification, copyright ownership, or navigation safety. + +## Exercise + +Create a small DataTiles release, sign it, protect it once, issue two licences for two recipient public keys, verify that each recipient can decrypt using only its own private key, and demonstrate that a licence copied to the other recipient fails. Then expire one licence and verify that decryption is denied. diff --git a/docs/white-paper-data-sources-addendum.md b/docs/white-paper-data-sources-addendum.md new file mode 100644 index 0000000..020c456 --- /dev/null +++ b/docs/white-paper-data-sources-addendum.md @@ -0,0 +1,20 @@ +## Data stewardship, source-specific citation, and licence boundaries + +An edge evidence substrate is scientifically defensible only when the identity and legal status of its inputs survive transformation. DataTiles therefore treats **data acknowledgement as executable release evidence**, not as generic prose. A derived tile pyramid, MBTiles compatibility export, map, figure, service, model input, or paper MUST be traceable to the exact source objects that contributed cells or features. The frozen run manifest is the authority for contributor status. + +The From Gaeta to Maratea reference build illustrates this principle. Its primary finite bathymetry is the JammeGaia22/MGDS multi-resolution multibeam product (Foglini, Tonielli & Rovere, 2024; doi:10.60521/331667). EMODnet Digital Bathymetry DTM 2024 (doi:10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1) is a conditional fallback only where JammeGaia22 has no finite measurement. This priority rule is scientific provenance: a consumer must be able to distinguish measured/local primary cells from harmonised fallback cells. Source coverage is therefore retained as data rather than erased after fusion. + +Land/ocean topology is an independent evidence class. GSHHG 2.3.7, cited through Wessel & Smith (1996; doi:10.1029/96JB00104), provides hierarchical shoreline/topology information. When enabled, S2Coast-2023 (Duan et al.; doi:10.5281/zenodo.17092775) contributes a Sentinel-2-derived high-water-line/coastline fact. These products must not be collapsed into a single undocumented notion of “coastline”: their definitions, resolutions, epochs, licences, and error modes differ. OpenStreetMap context remains separately attributable as © OpenStreetMap contributors under ODbL 1.0. + +This separation is both FAIR and operational. Findability requires persistent identifiers and indexed metadata; accessibility requires resolvable, documented access conditions; interoperability requires controlled semantics, units, CRS/datums, and explicit source roles; reusability requires qualified provenance, licences, attribution, limitations, and transformation history. A downloadable object is not necessarily reusable, and an open-source software licence does not relicense source data. + +DataTiles release engineering consequently applies four invariants: + +1. **No anonymous contribution.** Every contributing source is represented by a PID/stable URI, immutable object identity, rights record, and provenance entity. +2. **No generic acknowledgement.** Visible credits are source-specific and must agree with the frozen manifest. Candidate sources that contributed nothing are not credited as data contributors. +3. **No licence laundering.** Software, source-data, generated-dataset, metadata, and portrayal rights remain distinguishable. Combining data does not silently erase upstream obligations. +4. **No provenance-free fusion.** Priority, masking, resampling, reprojection, fallback and feature-selection decisions are generating activities with parameters and software identity; conditional fusion retains a source-coverage field or equivalent lineage evidence. + +For the current reference production build, the complete bibliographic records, required short-form map credit, licence statements, coverage/resolution cautions, manifest requirements, and FAIR release checklist are normative project guidance in `docs/data_sources_and_citation.md`. The map credit is a projection of the actual run evidence, not immutable boilerplate. GMRT, GEBCO, EMODnet thematic products, ISPRA, and other candidate/reference datasets must not be represented as contributors unless the specific run manifest proves their use. + +This discipline also constrains AI and statistical consumers. A model must be able to retrieve not merely a depth value but the source identity and transformation lineage that produced it. Where primary and fallback datasets differ in resolution, survey age, datum, or uncertainty, that distinction is part of the feature contract and may affect applicability. FAIR stewardship therefore becomes part of uncertainty management and reproducibility rather than a post-publication metadata exercise. diff --git a/docs/white-paper.md b/docs/white-paper.md index bc4d287..5c19a15 100644 --- a/docs/white-paper.md +++ b/docs/white-paper.md @@ -35,6 +35,35 @@ The same model can tile road geometry, grade, curvature, friction estimates, sur DataTiles must remain outside the safety boundary unless the complete system is engineered and assessed for that role. A portrayal is not a drivable-world model. Stale roadworks, uncertain floods, coordinate errors, or distribution shift can be hazardous. Deployment therefore needs integrity levels, authenticated updates, freshness rules, fail-operational or fail-safe behavior, independent sensing, runtime monitors, and applicable functional-safety and safety-of-the-intended-functionality processes. +## Data stewardship, source-specific citation, and licence boundaries + +An edge evidence substrate is scientifically defensible only when the identity and legal status of its inputs survive transformation. DataTiles therefore treats **data acknowledgement as executable release evidence**, not as generic prose. A derived tile pyramid, MBTiles compatibility export, map, figure, service, model input, or paper MUST be traceable to the exact source objects that contributed cells or features. The frozen run manifest is the authority for contributor status. + +The From Gaeta to Maratea reference build illustrates this principle. Its primary finite bathymetry is the JammeGaia22/MGDS multi-resolution multibeam product (Foglini, Tonielli & Rovere, 2024; doi:10.60521/331667). EMODnet Digital Bathymetry DTM 2024 (doi:10.12770/cf51df64-56f9-4a99-b1aa-36b8d7b743a1) is a conditional fallback only where JammeGaia22 has no finite measurement. This priority rule is scientific provenance: a consumer must be able to distinguish measured/local primary cells from harmonised fallback cells. Source coverage is therefore retained as data rather than erased after fusion. + +Land/ocean topology is an independent evidence class. GSHHG 2.3.7, cited through Wessel & Smith (1996; doi:10.1029/96JB00104), provides hierarchical shoreline/topology information. When enabled, S2Coast-2023 (Duan et al.; doi:10.5281/zenodo.17092775) contributes a Sentinel-2-derived high-water-line/coastline fact. These products must not be collapsed into a single undocumented notion of “coastline”: their definitions, resolutions, epochs, licences, and error modes differ. OpenStreetMap context remains separately attributable as © OpenStreetMap contributors under ODbL 1.0. + +This separation is both FAIR and operational. Findability requires persistent identifiers and indexed metadata; accessibility requires resolvable, documented access conditions; interoperability requires controlled semantics, units, CRS/datums, and explicit source roles; reusability requires qualified provenance, licences, attribution, limitations, and transformation history. A downloadable object is not necessarily reusable, and an open-source software licence does not relicense source data. + +DataTiles release engineering consequently applies four invariants: + +1. **No anonymous contribution.** Every contributing source is represented by a PID/stable URI, immutable object identity, rights record, and provenance entity. +2. **No generic acknowledgement.** Visible credits are source-specific and must agree with the frozen manifest. Candidate sources that contributed nothing are not credited as data contributors. +3. **No licence laundering.** Software, source-data, generated-dataset, metadata, and portrayal rights remain distinguishable. Combining data does not silently erase upstream obligations. +4. **No provenance-free fusion.** Priority, masking, resampling, reprojection, fallback and feature-selection decisions are generating activities with parameters and software identity; conditional fusion retains a source-coverage field or equivalent lineage evidence. + +For the current reference production build, the complete bibliographic records, required short-form map credit, licence statements, coverage/resolution cautions, manifest requirements, and FAIR release checklist are normative project guidance in `docs/data_sources_and_citation.md`. The map credit is a projection of the actual run evidence, not immutable boilerplate. GMRT, GEBCO, EMODnet thematic products, ISPRA, and other candidate/reference datasets must not be represented as contributors unless the specific run manifest proves their use. + +This discipline also constrains AI and statistical consumers. A model must be able to retrieve not merely a depth value but the source identity and transformation lineage that produced it. Where primary and fallback datasets differ in resolution, survey age, datum, or uncertainty, that distinction is part of the feature contract and may affect applicability. FAIR stewardship therefore becomes part of uncertainty management and reproducibility rather than a post-publication metadata exercise. + +## Cryptographic integrity, authenticity, and scholarly chain of custody + +DataTiles optionally signs a canonical logical manifest of the complete scientific object. This is deliberately stronger than attaching a checksum to the SQLite file: the manifest is invariant to storage-layout operations such as VACUUM while remaining sensitive to changes in scientific cells, vectors, metadata, semantics, provenance, rights, selected-slice state, and signed schema definitions. The native profile uses SHA-256 and Ed25519; the signing tables are excluded from the signed domain so multiple independent signatures can attest the same immutable scientific state. + +The trust claim is intentionally narrow. A mathematically valid signature proves possession of the corresponding private key at signing time, not the institutional identity of that key holder. Publication workflows must authenticate public keys independently, bind signers to persistent scholarly agents, preserve historical keys through rotation, and archive detached signatures and verification material with the research object. Sigstore/in-toto evidence can supplement the offline profile with certificate identity, transparency inclusion, and timestamps, but is not required for basic DataTiles verification. + +Cryptographic signatures complement FAIR metadata, W3C PROV lineage, DataCite identity, source-specific citation, and SPDX rights; they do not replace any of them. Nor do they establish scientific truth, licence compatibility, hydrographic authority, or navigation fitness. An unsigned object can still be FAIR; an institution may separately impose a signed-release policy for chain-of-custody assurance. + ## AI evidence contract Every onboard inference should identify: @@ -79,3 +108,21 @@ Cybersecurity matters because a plausible malicious tile can alter an AI decisio Future work includes uncertainty profiles, signed tile manifests, incremental synchronization, content-addressed deduplication, safety-oriented freshness policies, model/provenance profiles, non-Web-Mercator matrices, efficient vector/numeric co-query, and evaluation on real edge hardware. DataTiles is a bridge between scientific stewardship and edge intelligence. Its claim is not that a SQLite file makes autonomy safe. Its contribution is to make the data boundary explicit, local, inspectable, reproducible, interoperable, and accountable—conditions that systems protecting life and the environment need before algorithms can be trusted. + +## Commercial distribution, rights expression, and DRM + +DataTiles does not equate FAIR with open or zero-cost access. Commercial and institutionally restricted research/cartographic products can remain machine-discoverable and semantically interoperable while the data payload is distributed under controlled access. Revision 7 therefore separates four layers: scholarly/data rights, machine-readable ODRL usage policy, cryptographic integrity/authenticity, and optional DRM enforcement. + +The portable DRM profile wraps the finalized DataTiles SQLite object rather than encrypting individual scientific rows. This preserves the canonical research object, allows the existing integrity signature to attest the exact plaintext release, and ensures authorized decryption recovers byte-identical DataTiles content. AES-256-GCM provides authenticated confidentiality; recipient-specific X25519/HKDF key wrapping avoids sharing a universal customer secret; Ed25519 issuer signatures authenticate licence grants. + +DRM cannot manufacture legal rights. A publisher may sell a derivative only when every contributing source and contractual term permits the intended commercial exploitation. Source-specific citation, attribution, provenance, licence compatibility, and database-right obligations remain enforceable independently of the encryption layer. Nor can portable DRM prevent an authorized endpoint from copying plaintext after decryption. It should therefore be described as distribution/access control, not as an absolute anti-copy guarantee. + +For FAIR publication, public metadata should disclose the existence of access restrictions, product identity, terms, issuer, and access mechanism even when the payload requires purchase or authorization. Conversely, secrets, customer identifiers not required for scholarly provenance, private keys, and payment information are intentionally excluded from the scientific object and FAIR graph. + +## Versioned dissemination, acquisition history, and payment abstraction + +A scientific data product must distinguish identity of the logical product from identity of an individual release. Revision 8 therefore introduces a stable product identifier and an explicit monotonic release sequence in addition to the human-facing version label. This design prevents ambiguous lexical version ordering and supports reproducible citation of the exact release used in analysis. Published versioned objects are immutable; an update is a successor object, not a silent byte replacement. + +The Store uses this release identity to maintain user-level acquisition evidence. Purchases and downloads are separate records: a purchase represents a commercial entitlement transaction, while a download records actual delivery of an exact checksum-identified release. A higher sequence for the same product can generate an update notification to prior users without implying that the successor is free, licensed identically, or scientifically interchangeable. + +Commercial payment processing is intentionally abstracted from the scientific format and Store entitlement schema. A payment provider implements a narrow checkout/capture contract; PayPal Orders v2 is the reference implementation. Provider credentials and financial payment details remain outside DataTiles provenance. Commercial entitlement, data licensing, FAIR evidence, provenance, cryptographic signatures, DRM, scientific validity, and navigation fitness remain independent claims that must not be conflated. diff --git a/docs/zarr-source-profile.md b/docs/zarr-source-profile.md new file mode 100644 index 0000000..29bc465 --- /dev/null +++ b/docs/zarr-source-profile.md @@ -0,0 +1,78 @@ +# Zarr source-ingestion profile + +This document defines the DataTiles utility profile for importing Zarr datasets. It is normative for `utils/zarr2datatiles.py` and informative for independent implementations unless they claim this profile. + +## Scope + +Zarr is a chunked, N-dimensional storage format. DataTiles treats Zarr as a **source encoding**, not as the semantic identity of a scientific quantity. An importer MUST preserve or explicitly map variable semantics, units, scientific dimensions, source identity, provenance, rights, and transformation parameters before writing DNT1 numeric tiles. + +The profile supports Zarr format 2 and 3 stores that xarray can interpret as a Dataset. The source data variables used by the profile MUST expose rectilinear one-dimensional latitude and longitude coordinates. Unsupported curvilinear, projected, rotated, or antimeridian-crossing source grids MUST be rejected rather than silently approximated. + +## Source identifiers + +A source MAY be: + +- a local directory store or `file:` URI; +- an HTTP(S) URL; +- another fsspec URL such as `s3://` or `gcs://`, provided the required backend is installed and the operator supplies any necessary storage options. + +Authentication material is runtime configuration and MUST NOT be copied into DataTiles metadata, provenance attributes, logs, manifests, or tutorial artifacts. The importer records storage-option **keys only**, never secret values. + +## Store identity and checksums + +A Zarr store is a set of named objects, not a single file. Implementations MUST NOT describe a digest of a directory serialization as the SHA-256 of a Zarr file. + +For a local directory store, `zarr2datatiles` computes `zarr-tree-sha256-v1` as follows: + +1. Initialize SHA-256 with the ASCII/UTF-8 byte string `DataTiles-Zarr-Tree-SHA256-v1` followed by NUL. +2. Recursively enumerate regular files beneath the store root. Reject symbolic links. +3. Sort objects by their relative POSIX key using Unicode code-point order on the UTF-8 path string. +4. For each object, update the digest with `key`, NUL, decimal byte length, NUL, exact object bytes, NUL. +5. Store the lowercase hexadecimal digest with checksum algorithm `zarr-tree-sha256-v1`. + +This digest identifies the complete local object set and its byte content at the time of hashing. For rigorous publication, the source store SHOULD be immutable or snapshot/version identified so hashing and conversion cannot observe different states. + +For a remote store, the utility requires `--source-sha256`. If the access URL contains credentials, user-info, or a signed query string, the operator MUST also supply a stable credential-free `--provenance-uri`; the access URL is used only at runtime and MUST NOT be persisted. That value MUST identify the immutable published store or snapshot according to the provider's archival record. The utility does not traverse an arbitrary remote object store a second time merely to synthesize a checksum: doing so may be unbounded, expensive, authorization-sensitive, and still race with concurrent mutation. Publication workflows SHOULD record an object-version ID, DOI/PID, ETag/version manifest, or repository snapshot identifier in addition to the checksum when available. + +## Metadata and semantic mapping + +For every selected xarray data variable: + +- the source variable name becomes the local DataTiles variable token; +- a valid CF `standard_name`, when present, is the preferred canonical semantic identity; +- source `units` and `long_name` are retained; +- non-spatial dimensions are represented as typed DataTiles dimensions; +- producer-local names without a CF standard name remain in vocabulary `source` and MUST NOT be promoted to CF by inference; +- the native Zarr/xarray variable name is retained as a `Zarr-variable` external identifier. + +Zarr chunk shape, compressor/codec configuration, fill-value representation, and consolidated-metadata status are source-encoding properties. They MAY be retained as provenance attributes but MUST NOT replace scientific semantics. + +## Zarr v2/v3 and fill values + +An importer MAY accept either Zarr format 2 or 3. If `--zarr-format` is supplied, failure to match that format MUST be an error. CF decoding is delegated to xarray. Because Zarr v2 and v3 have different fill-value semantics in the xarray ecosystem, producers SHOULD make `_FillValue`/masking intent explicit in the source and SHOULD test the resulting nodata behavior before publication. + +## Consolidated metadata + +`--consolidated auto` delegates discovery to xarray: consolidated metadata is attempted where supported with fallback according to the backend. `true` requires consolidated metadata; `false` explicitly disables it. The selected policy is recorded in import provenance. Consolidation is a performance/discovery property and MUST NOT be interpreted as stronger scientific provenance. + +## Transformation and output + +The initial profile uses the same declared spatial transformation as the NetCDF/GRIB utilities: Web Mercator target tiles, nearest-neighbour sampling at tile-pixel centers, explicit source/bbox clipping, DNT1 numeric payloads, and a maximum-tile resource guard. The content schema records `source_kind=Zarr` and the resampling/grid parameters. + +A portrayal (PNG/JPEG/WebP, shaded relief, contour image, color ramp) is a downstream reproducible derivation. The importer MUST NOT substitute portrayal pixels for scientific arrays. + +## FAIR, provenance, and rights + +Transport accessibility does not imply permission. The operator MUST provide the source license/terms URI, SPDX expression or explicit `LicenseRef-*`, required attribution, generated-dataset license, and generated-dataset license URI. These are recorded separately. + +The source provenance entity MUST include the original source identifier, checksum algorithm, checksum, and source kind. The conversion activity MUST record zoom, tile size, bbox, Zarr group, requested Zarr format, consolidated-metadata policy, and non-secret storage-option keys. Tile-level lineage links generated tiles to the source entity. + +A DataTiles file created from Zarr is FAIR only to the extent supported by the complete DataTiles FAIR publication profile; successful conversion alone is not a FAIR certification. + +## Standards references + +This profile is designed against the Zarr core specification, with Zarr v3 core specification 3.1 as the current reference at the time of this extension, and the xarray Zarr backend contract. The Zarr core specification defines stores as mappings from keys to bytes and explicitly allows filesystem- and object-store realizations; that store model is the reason DataTiles uses a store-aware digest rather than pretending a Zarr hierarchy is one file. Xarray provides the dataset-level bridge used here and requires dimension/coordinate metadata that it can interpret. + +- Zarr core specification: https://zarr-specs.readthedocs.io/en/latest/v3/core/ +- Xarray `open_zarr`: https://docs.xarray.dev/en/latest/generated/xarray.open_zarr.html +- Xarray Zarr I/O guidance: https://docs.xarray.dev/en/latest/user-guide/io.html#zarr diff --git a/pyproject.toml b/pyproject.toml index fffd9b3..785dbd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "datatiles" -version = "0.10.0" +version = "0.19.0" description = "MBTiles-compatible multidimensional raster-matrix and vector-feature storage" readme = "README.md" requires-python = ">=3.10" @@ -22,6 +22,17 @@ classifiers = [ dependencies = [] [project.optional-dependencies] +integrity = ["cryptography>=45"] +drm = ["cryptography>=45"] +utils = [ + "numpy>=2.0", + "xarray>=2025.1", + "netCDF4>=1.7", + "cfgrib>=0.9.15", + "eccodes>=2.40", + "zarr>=3.0", + "fsspec>=2025.1", +] demo = [ "numpy==2.2.6; python_version < '3.11'", "numpy==2.3.5; python_version >= '3.11'", @@ -34,6 +45,8 @@ test = [ ] [project.scripts] +datatiles-integrity = "datatiles.integrity_cli:main" +datatiles-drm = "datatiles.drm_cli:main" datatiles = "datatiles.cli:main" datatiles-serve = "datatiles.server:main" datatiles-demo = "datatiles.demo:main" diff --git a/src/datatiles/__init__.py b/src/datatiles/__init__.py index 682458f..feff08f 100644 --- a/src/datatiles/__init__.py +++ b/src/datatiles/__init__.py @@ -4,4 +4,4 @@ from .numeric import NumericTile, decode_numeric_tile, encode_numeric_tile __all__ = ["DataTiles", "DataTilesError", "NumericTile", "encode_numeric_tile", "decode_numeric_tile"] -__version__ = "0.10.0" +__version__ = "0.19.0" diff --git a/src/datatiles/cli.py b/src/datatiles/cli.py index 6941180..54a61d0 100644 --- a/src/datatiles/cli.py +++ b/src/datatiles/cli.py @@ -31,6 +31,17 @@ def parser() -> argparse.ArgumentParser: dim.add_argument("--axis", choices=["T","Z","E","C","O"]); dim.add_argument("--unit"); dim.add_argument("--description") dim.add_argument("--optional", action="store_true") dim.add_argument("--extent", choices=["point","interval","point_or_interval"], default="point") + var = sub.add_parser("add-variable", help="register a semantic variable") + var.add_argument("file"); var.add_argument("name"); var.add_argument("--standard-name", required=True) + var.add_argument("--vocabulary", default="CF"); var.add_argument("--vocabulary-version") + var.add_argument("--canonical-unit"); var.add_argument("--long-name"); var.add_argument("--description") + vid = sub.add_parser("add-variable-id", help="add GRIB/SeaDataNet/NVS/etc crosswalk identifier") + vid.add_argument("file"); vid.add_argument("variable"); vid.add_argument("scheme"); vid.add_argument("identifier") + vid.add_argument("--scheme-version"); vid.add_argument("--uri") + vars_cmd = sub.add_parser("variables", help="list semantic variables as JSON"); vars_cmd.add_argument("file") + find_var = sub.add_parser("find-variable", help="find coordinate sets by semantic standard name") + find_var.add_argument("file"); find_var.add_argument("standard_name"); find_var.add_argument("--vocabulary", default="CF") + policy = sub.add_parser("set-variable-policy"); policy.add_argument("file"); policy.add_argument("policy", choices=["required","recommended"]) crs = sub.add_parser("add-crs") crs.add_argument("file"); crs.add_argument("role", choices=["horizontal","vertical","temporal","compound","engineering"]) crs.add_argument("--authority"); crs.add_argument("--code"); crs.add_argument("--uri"); crs.add_argument("--wkt2"); crs.add_argument("--projjson") @@ -52,7 +63,7 @@ def parser() -> argparse.ArgumentParser: contents.add_argument("file") metadata = sub.add_parser("set-metadata", help="set one non-managed metadata value") metadata.add_argument("file"); metadata.add_argument("name"); metadata.add_argument("value") - validate = sub.add_parser("validate"); validate.add_argument("file") + validate = sub.add_parser("validate"); validate.add_argument("file"); validate.add_argument("--cf-table", type=Path); validate.add_argument("--require-variable-semantics", action="store_true") return p @@ -65,6 +76,21 @@ def main(argv: list[str] | None = None) -> int: with DataTiles(args.file) as store: store.add_dimension(args.name, args.type, axis=args.axis, unit=args.unit, description=args.description, required=not args.optional, extent_kind=args.extent) + elif args.command == "add-variable": + with DataTiles(args.file) as store: + store.add_variable(args.name,args.standard_name,vocabulary=args.vocabulary,vocabulary_version=args.vocabulary_version, + canonical_unit=args.canonical_unit,long_name=args.long_name,description=args.description) + elif args.command == "add-variable-id": + with DataTiles(args.file) as store: + store.add_variable_identifier(args.variable,args.scheme,args.identifier,scheme_version=args.scheme_version,uri=args.uri) + elif args.command == "variables": + with DataTiles(args.file) as store: print(json.dumps({"variables":store.variables()},indent=2,ensure_ascii=False)) + elif args.command == "find-variable": + with DataTiles(args.file) as store: + print(json.dumps({"standard_name":args.standard_name,"vocabulary":args.vocabulary, + "coordinate_set_ids":store.find_coordinate_sets_by_standard_name(args.standard_name,vocabulary=args.vocabulary)},indent=2)) + elif args.command == "set-variable-policy": + with DataTiles(args.file) as store: store.set_variable_semantics(args.policy) elif args.command == "add-crs": with DataTiles(args.file) as store: store.add_crs(args.role, authority=args.authority, code=args.code, uri=args.uri, @@ -90,7 +116,7 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "set-metadata": with DataTiles(args.file) as store: store.set_metadata(args.name,args.value) elif args.command == "validate": - with DataTiles(args.file) as store: errors = store.validate() + with DataTiles(args.file) as store: errors = store.validate(cf_table=args.cf_table, require_variable_semantics=args.require_variable_semantics) if errors: for error in errors: print(error) return 1 diff --git a/src/datatiles/drm.py b/src/datatiles/drm.py new file mode 100644 index 0000000..2f4d3a1 --- /dev/null +++ b/src/datatiles/drm.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import struct +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, BinaryIO + +PACKAGE_MAGIC = b"DTPKG1\x00" +PACKAGE_PROFILE = "DataTiles-Protected-Distribution-1" +LICENSE_PROFILE = "DataTiles-Commercial-License-1" +POLICY_PROFILE = "W3C-ODRL-2.2" +CONTENT_ALGORITHM = "AES-256-GCM" +KEY_WRAP_ALGORITHM = "X25519-HKDF-SHA256+A256GCM" +SIGNATURE_ALGORITHM = "Ed25519" +DEFAULT_CHUNK_SIZE = 1024 * 1024 + + +class DRMError(ValueError): + pass + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def _b64(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _unb64(text: str) -> bytes: + return base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)) + + +def _require_crypto(): + try: + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey + from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + except ImportError as exc: + raise DRMError("DRM support requires the optional 'drm' dependency: pip install 'datatiles[drm]'") from exc + return hashes, serialization, Ed25519PrivateKey, Ed25519PublicKey, X25519PrivateKey, X25519PublicKey, Cipher, algorithms, modes, AESGCM, HKDF + + +def _sha256_file(path: str | Path, chunk_size: int = DEFAULT_CHUNK_SIZE) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(chunk_size) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def _raw_public_key(public_key) -> bytes: + _, serialization, *_ = _require_crypto() + return public_key.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + + +def key_id(public_key) -> str: + return "sha256:" + hashlib.sha256(_raw_public_key(public_key)).hexdigest() + + +def generate_recipient_keypair(private_key_path: str | Path, public_key_path: str | Path, *, overwrite: bool = False) -> dict[str, str]: + _, serialization, _, _, X25519PrivateKey, _, *_ = _require_crypto() + private_path, public_path = Path(private_key_path), Path(public_key_path) + if not overwrite and (private_path.exists() or public_path.exists()): + raise DRMError("refusing to overwrite an existing recipient key file") + private = X25519PrivateKey.generate() + public = private.public_key() + private_path.write_bytes(private.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())) + try: + os.chmod(private_path, 0o600) + except OSError: + pass + public_path.write_bytes(public.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo)) + return {"private_key": str(private_path), "public_key": str(public_path), "key_id": key_id(public)} + + +def load_recipient_private_key(path: str | Path): + _, serialization, _, _, X25519PrivateKey, *_ = _require_crypto() + key = serialization.load_pem_private_key(Path(path).read_bytes(), password=None) + if not isinstance(key, X25519PrivateKey): + raise DRMError("recipient private key must be X25519") + return key + + +def load_recipient_public_key(path: str | Path): + _, serialization, _, _, _, X25519PublicKey, *_ = _require_crypto() + key = serialization.load_pem_public_key(Path(path).read_bytes()) + if not isinstance(key, X25519PublicKey): + raise DRMError("recipient public key must be X25519") + return key + + +def load_issuer_private_key(path: str | Path): + _, serialization, Ed25519PrivateKey, *_ = _require_crypto() + key = serialization.load_pem_private_key(Path(path).read_bytes(), password=None) + if not isinstance(key, Ed25519PrivateKey): + raise DRMError("issuer private key must be Ed25519") + return key + + +def load_issuer_public_key(path: str | Path): + _, serialization, _, Ed25519PublicKey, *_ = _require_crypto() + key = serialization.load_pem_public_key(Path(path).read_bytes()) + if not isinstance(key, Ed25519PublicKey): + raise DRMError("issuer public key must be Ed25519") + return key + + +def write_content_key(path: str | Path, key: bytes, *, product_id: str) -> None: + if len(key) != 32: + raise DRMError("content key must be 32 bytes") + p = Path(path) + p.write_text(json.dumps({"profile": PACKAGE_PROFILE, "product_id": product_id, "content_key": _b64(key)}, indent=2) + "\n", encoding="utf-8") + try: + os.chmod(p, 0o600) + except OSError: + pass + + +def read_content_key(path: str | Path, *, expected_product_id: str | None = None) -> tuple[str, bytes]: + data = json.loads(Path(path).read_text(encoding="utf-8")) + product_id = data.get("product_id") + key = _unb64(data.get("content_key", "")) + if len(key) != 32 or not product_id: + raise DRMError("invalid content-key file") + if expected_product_id and product_id != expected_product_id: + raise DRMError("content-key product_id does not match package") + return product_id, key + + +def protect_file(source: str | Path, output: str | Path, *, content_key_file: str | Path, product_id: str | None = None, + issuer: str, issuer_uri: str | None = None, terms_uri: str, license_service_uri: str | None = None, + edition: str | None = None, metadata: dict[str, Any] | None = None, chunk_size: int = DEFAULT_CHUNK_SIZE) -> dict[str, Any]: + *_, Cipher, algorithms, modes, _, _ = _require_crypto() + source, output = Path(source), Path(output) + if not source.is_file(): + raise DRMError("source must be a finalized DataTiles file") + product_id = product_id or f"urn:uuid:{uuid.uuid4()}" + content_key = os.urandom(32) + nonce = os.urandom(12) + plaintext_sha256 = _sha256_file(source, chunk_size) + header = { + "profile": PACKAGE_PROFILE, + "product_id": product_id, + "edition": edition, + "issuer": issuer, + "issuer_uri": issuer_uri, + "terms_uri": terms_uri, + "license_service_uri": license_service_uri, + "content_algorithm": CONTENT_ALGORITHM, + "nonce": _b64(nonce), + "plaintext_sha256": plaintext_sha256, + "plaintext_size": source.stat().st_size, + "created_at": _utc_now(), + "metadata": metadata or {}, + } + header_bytes = _canonical_json(header) + encryptor = Cipher(algorithms.AES(content_key), modes.GCM(nonce)).encryptor() + encryptor.authenticate_additional_data(header_bytes) + cipher_hash = hashlib.sha256() + with source.open("rb") as src, output.open("wb") as dst: + dst.write(PACKAGE_MAGIC); dst.write(struct.pack(">I", len(header_bytes))); dst.write(header_bytes) + while True: + chunk = src.read(chunk_size) + if not chunk: + break + encrypted = encryptor.update(chunk) + dst.write(encrypted); cipher_hash.update(encrypted) + tail = encryptor.finalize() + if tail: + dst.write(tail); cipher_hash.update(tail) + dst.write(encryptor.tag) + write_content_key(content_key_file, content_key, product_id=product_id) + return {**header, "ciphertext_sha256": cipher_hash.hexdigest(), "content_key_file": str(content_key_file), "package": str(output)} + + +def read_package_header(path: str | Path) -> dict[str, Any]: + with open(path, "rb") as f: + if f.read(len(PACKAGE_MAGIC)) != PACKAGE_MAGIC: + raise DRMError("not a DataTiles protected-distribution package") + raw = f.read(4) + if len(raw) != 4: + raise DRMError("truncated package header") + length = struct.unpack(">I", raw)[0] + if length < 2 or length > 1024 * 1024: + raise DRMError("invalid package header length") + header_bytes = f.read(length) + if len(header_bytes) != length: + raise DRMError("truncated package header") + header = json.loads(header_bytes) + if header.get("profile") != PACKAGE_PROFILE or header.get("content_algorithm") != CONTENT_ALGORITHM: + raise DRMError("unsupported protected-distribution profile") + return header + + +def _license_payload(*, package_header: dict[str, Any], recipient_public_key, recipient_id: str | None, content_key: bytes, + issuer: str, issuer_uri: str | None, valid_from: str | None, valid_until: str | None, + permissions: list[str], policy: dict[str, Any] | None) -> dict[str, Any]: + hashes, serialization, _, _, X25519PrivateKey, _, _, _, _, AESGCM, HKDF = _require_crypto() + ephemeral = X25519PrivateKey.generate() + shared = ephemeral.exchange(recipient_public_key) + recipient_key_id = key_id(recipient_public_key) + salt = os.urandom(16) + kek = HKDF(algorithm=hashes.SHA256(), length=32, salt=salt, info=("DataTiles DRM " + package_header["product_id"]).encode()).derive(shared) + wrap_nonce = os.urandom(12) + aad = _canonical_json({"profile": LICENSE_PROFILE, "product_id": package_header["product_id"], "recipient_key_id": recipient_key_id}) + wrapped = AESGCM(kek).encrypt(wrap_nonce, content_key, aad) + ephemeral_raw = ephemeral.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return { + "profile": LICENSE_PROFILE, + "product_id": package_header["product_id"], + "package_plaintext_sha256": package_header["plaintext_sha256"], + "issuer": issuer, + "issuer_uri": issuer_uri, + "recipient_id": recipient_id, + "recipient_key_id": recipient_key_id, + "permissions": sorted(set(permissions)), + "valid_from": valid_from, + "valid_until": valid_until, + "issued_at": _utc_now(), + "key_wrap": { + "algorithm": KEY_WRAP_ALGORITHM, + "ephemeral_public_key": _b64(ephemeral_raw), + "salt": _b64(salt), + "nonce": _b64(wrap_nonce), + "wrapped_content_key": _b64(wrapped), + }, + "odrl": policy or {"@context": "http://www.w3.org/ns/odrl.jsonld", "@type": "Agreement", "target": package_header["product_id"]}, + } + + +def issue_license(package: str | Path, content_key_file: str | Path, recipient_public_key_path: str | Path, + issuer_private_key_path: str | Path, output: str | Path, *, issuer: str, issuer_uri: str | None = None, + recipient_id: str | None = None, valid_from: str | None = None, valid_until: str | None = None, + permissions: list[str] | None = None, policy: dict[str, Any] | None = None) -> dict[str, Any]: + header = read_package_header(package) + _, content_key = read_content_key(content_key_file, expected_product_id=header["product_id"]) + recipient = load_recipient_public_key(recipient_public_key_path) + issuer_key = load_issuer_private_key(issuer_private_key_path) + payload = _license_payload(package_header=header, recipient_public_key=recipient, recipient_id=recipient_id, + content_key=content_key, issuer=issuer, issuer_uri=issuer_uri, valid_from=valid_from, + valid_until=valid_until, permissions=permissions or ["read"], policy=policy) + signature = issuer_key.sign(_canonical_json(payload)) + envelope = { + "payload": payload, + "signature": {"algorithm": SIGNATURE_ALGORITHM, "issuer_key_id": key_id(issuer_key.public_key()), "value": _b64(signature)}, + } + Path(output).write_text(json.dumps(envelope, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8") + return envelope + + +def _parse_time(value: str | None): + if value is None: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise DRMError(f"invalid ISO-8601 licence time: {value}") from exc + + +def verify_license(envelope: dict[str, Any], issuer_public_key, *, at_time: datetime | None = None) -> dict[str, Any]: + payload = envelope.get("payload") or {} + sig = envelope.get("signature") or {} + if payload.get("profile") != LICENSE_PROFILE or sig.get("algorithm") != SIGNATURE_ALGORITHM: + raise DRMError("unsupported licence envelope") + issuer_key_id = key_id(issuer_public_key) + key_matches = sig.get("issuer_key_id") == issuer_key_id + crypto_valid = False + if key_matches: + try: + issuer_public_key.verify(_unb64(sig.get("value", "")), _canonical_json(payload)); crypto_valid = True + except Exception: + crypto_valid = False + now = at_time or datetime.now(timezone.utc) + start, end = _parse_time(payload.get("valid_from")), _parse_time(payload.get("valid_until")) + time_valid = (start is None or now >= start) and (end is None or now <= end) + return {"cryptographically_valid": crypto_valid, "issuer_key_matches": key_matches, "time_valid": time_valid, + "valid": crypto_valid and key_matches and time_valid, "product_id": payload.get("product_id"), + "recipient_key_id": payload.get("recipient_key_id"), "permissions": payload.get("permissions", [])} + + +def unwrap_content_key(envelope: dict[str, Any], recipient_private_key, issuer_public_key) -> bytes: + hashes, serialization, _, _, _, X25519PublicKey, _, _, _, AESGCM, HKDF = _require_crypto() + status = verify_license(envelope, issuer_public_key) + if not status["valid"]: + raise DRMError("licence is not valid for the supplied issuer key and current time") + payload = envelope["payload"] + if key_id(recipient_private_key.public_key()) != payload.get("recipient_key_id"): + raise DRMError("licence was issued to a different recipient key") + wrap = payload["key_wrap"] + ephemeral = X25519PublicKey.from_public_bytes(_unb64(wrap["ephemeral_public_key"])) + shared = recipient_private_key.exchange(ephemeral) + kek = HKDF(algorithm=hashes.SHA256(), length=32, salt=_unb64(wrap["salt"]), info=("DataTiles DRM " + payload["product_id"]).encode()).derive(shared) + aad = _canonical_json({"profile": LICENSE_PROFILE, "product_id": payload["product_id"], "recipient_key_id": payload["recipient_key_id"]}) + key = AESGCM(kek).decrypt(_unb64(wrap["nonce"]), _unb64(wrap["wrapped_content_key"]), aad) + if len(key) != 32: + raise DRMError("invalid unwrapped content key") + return key + + +def decrypt_package(package: str | Path, output: str | Path, *, license_path: str | Path, recipient_private_key_path: str | Path, + issuer_public_key_path: str | Path, chunk_size: int = DEFAULT_CHUNK_SIZE) -> dict[str, Any]: + *_, Cipher, algorithms, modes, _, _ = _require_crypto() + envelope = json.loads(Path(license_path).read_text(encoding="utf-8")) + recipient_key = load_recipient_private_key(recipient_private_key_path) + issuer_key = load_issuer_public_key(issuer_public_key_path) + content_key = unwrap_content_key(envelope, recipient_key, issuer_key) + package = Path(package); output = Path(output) + with package.open("rb") as src: + if src.read(len(PACKAGE_MAGIC)) != PACKAGE_MAGIC: + raise DRMError("not a DataTiles protected-distribution package") + length = struct.unpack(">I", src.read(4))[0] + header_bytes = src.read(length); header = json.loads(header_bytes) + if envelope["payload"].get("product_id") != header.get("product_id") or envelope["payload"].get("package_plaintext_sha256") != header.get("plaintext_sha256"): + raise DRMError("licence is bound to a different protected package") + total = package.stat().st_size + cipher_len = total - len(PACKAGE_MAGIC) - 4 - length - 16 + if cipher_len < 0: + raise DRMError("truncated protected package") + src.seek(total - 16); tag = src.read(16) + src.seek(len(PACKAGE_MAGIC) + 4 + length) + decryptor = Cipher(algorithms.AES(content_key), modes.GCM(_unb64(header["nonce"]), tag)).decryptor() + decryptor.authenticate_additional_data(header_bytes) + h = hashlib.sha256(); remaining = cipher_len + try: + with output.open("wb") as dst: + while remaining: + chunk = src.read(min(chunk_size, remaining)) + if not chunk: + raise DRMError("truncated protected package") + remaining -= len(chunk) + plain = decryptor.update(chunk); dst.write(plain); h.update(plain) + tail = decryptor.finalize(); dst.write(tail); h.update(tail) + except Exception: + try: output.unlink() + except FileNotFoundError: pass + raise + digest = h.hexdigest() + if digest != header["plaintext_sha256"]: + try: output.unlink() + except FileNotFoundError: pass + raise DRMError("decrypted product hash does not match protected package identity") + return {"product_id": header["product_id"], "output": str(output), "plaintext_sha256": digest, "permissions": envelope["payload"].get("permissions", [])} diff --git a/src/datatiles/drm_cli.py b/src/datatiles/drm_cli.py new file mode 100644 index 0000000..9a4c6fb --- /dev/null +++ b/src/datatiles/drm_cli.py @@ -0,0 +1,32 @@ +from __future__ import annotations +import argparse, json, sys +from pathlib import Path +from .drm import DRMError, decrypt_package, generate_recipient_keypair, issue_license, load_issuer_public_key, protect_file, read_package_header, verify_license + + +def _dump(v): print(json.dumps(v, indent=2, sort_keys=True, ensure_ascii=False)) + +def main(argv=None): + p=argparse.ArgumentParser(prog="datatiles-drm",description="Optional commercial protected-distribution tooling for DataTiles") + s=p.add_subparsers(dest="command",required=True) + k=s.add_parser("generate-recipient-key",help="generate an X25519 customer/device keypair"); k.add_argument("--private-key",required=True); k.add_argument("--public-key",required=True); k.add_argument("--overwrite",action="store_true") + x=s.add_parser("protect",help="encrypt a finalized DataTiles object into a .dtpkg package"); x.add_argument("source"); x.add_argument("output"); x.add_argument("--content-key-file",required=True); x.add_argument("--product-id"); x.add_argument("--issuer",required=True); x.add_argument("--issuer-uri"); x.add_argument("--terms-uri",required=True); x.add_argument("--license-service-uri"); x.add_argument("--edition") + i=s.add_parser("inspect",help="inspect the public package header"); i.add_argument("package") + l=s.add_parser("issue-license",help="issue an issuer-signed recipient licence and wrapped content key"); l.add_argument("package"); l.add_argument("--content-key-file",required=True); l.add_argument("--recipient-public-key",required=True); l.add_argument("--issuer-private-key",required=True); l.add_argument("--output",required=True); l.add_argument("--issuer",required=True); l.add_argument("--issuer-uri"); l.add_argument("--recipient-id"); l.add_argument("--valid-from"); l.add_argument("--valid-until"); l.add_argument("--permission",action="append",default=[]); l.add_argument("--odrl-policy") + v=s.add_parser("verify-license",help="verify issuer signature and licence time bounds"); v.add_argument("license"); v.add_argument("--issuer-public-key",required=True) + d=s.add_parser("decrypt",help="decrypt an authorized protected package"); d.add_argument("package"); d.add_argument("output"); d.add_argument("--license",required=True); d.add_argument("--recipient-private-key",required=True); d.add_argument("--issuer-public-key",required=True) + a=p.parse_args(argv) + try: + if a.command=="generate-recipient-key": _dump(generate_recipient_keypair(a.private_key,a.public_key,overwrite=a.overwrite)); return 0 + if a.command=="protect": _dump(protect_file(a.source,a.output,content_key_file=a.content_key_file,product_id=a.product_id,issuer=a.issuer,issuer_uri=a.issuer_uri,terms_uri=a.terms_uri,license_service_uri=a.license_service_uri,edition=a.edition)); return 0 + if a.command=="inspect": _dump(read_package_header(a.package)); return 0 + if a.command=="issue-license": + policy=json.loads(Path(a.odrl_policy).read_text()) if a.odrl_policy else None + env=issue_license(a.package,a.content_key_file,a.recipient_public_key,a.issuer_private_key,a.output,issuer=a.issuer,issuer_uri=a.issuer_uri,recipient_id=a.recipient_id,valid_from=a.valid_from,valid_until=a.valid_until,permissions=a.permission or ["read"],policy=policy); _dump({"output":a.output,"product_id":env["payload"]["product_id"],"recipient_key_id":env["payload"]["recipient_key_id"],"issuer_key_id":env["signature"]["issuer_key_id"]}); return 0 + if a.command=="verify-license": + env=json.loads(Path(a.license).read_text()); _dump(verify_license(env,load_issuer_public_key(a.issuer_public_key))); return 0 + if a.command=="decrypt": _dump(decrypt_package(a.package,a.output,license_path=a.license,recipient_private_key_path=a.recipient_private_key,issuer_public_key_path=a.issuer_public_key)); return 0 + except (DRMError,OSError,ValueError,json.JSONDecodeError) as e: + print(f"datatiles-drm: {e}",file=sys.stderr); return 2 + return 2 +if __name__=="__main__": raise SystemExit(main()) diff --git a/src/datatiles/fair.py b/src/datatiles/fair.py new file mode 100644 index 0000000..ce72c8e --- /dev/null +++ b/src/datatiles/fair.py @@ -0,0 +1,89 @@ +"""FAIR, provenance, rights, and citation helpers for DataTiles. + +The module is deliberately dependency-free. It validates the conservative lexical +subset of SPDX expressions needed at the container boundary; authoritative SPDX +list membership remains a publication/CI check against a pinned SPDX License List. +""" +from __future__ import annotations + +import json +import re +from typing import Any, Mapping + +SPDX_TOKEN = re.compile(r"(?:DocumentRef-[A-Za-z0-9.-]+:)?LicenseRef-[A-Za-z0-9.-]+|[A-Za-z0-9.-]+\+?") +SPDX_OPERATORS = {"AND", "OR", "WITH"} +PID_SCHEMES = {"DOI", "Handle", "ARK", "URN", "SWHID", "RAiD", "URL"} +ACCESS_RIGHTS = {"open", "embargoed", "restricted", "closed"} +RIGHTS_SCOPES = {"dataset", "metadata", "source", "portrayal", "software", "other"} + +class FairValidationError(ValueError): + pass + + +def validate_spdx_expression(expression: str) -> str: + """Reject malformed/ambiguous SPDX-like expressions without network access.""" + if not isinstance(expression, str) or not expression.strip() or expression != expression.strip(): + raise FairValidationError("SPDX expression must be non-empty and trimmed") + tokens = re.findall(r"\(|\)|[A-Za-z0-9.+:-]+", expression) + if "".join(tokens).replace("(", "").replace(")", "") == "": + raise FairValidationError("empty SPDX expression") + # Ensure no unparsed punctuation/whitespace ambiguity. + rebuilt = " ".join(tokens).replace("( ", "(").replace(" )", ")") + normalized = re.sub(r"\s+", " ", expression).replace("( ", "(").replace(" )", ")") + if rebuilt != normalized: + raise FairValidationError("invalid SPDX expression syntax") + depth = 0 + expect_operand = True + for token in tokens: + if token == "(": + if not expect_operand: raise FairValidationError("missing SPDX operator") + depth += 1 + elif token == ")": + if expect_operand or depth == 0: raise FairValidationError("unbalanced SPDX parentheses") + depth -= 1; expect_operand = False + elif token in SPDX_OPERATORS: + if expect_operand: raise FairValidationError("misplaced SPDX operator") + expect_operand = True + else: + if not SPDX_TOKEN.fullmatch(token): raise FairValidationError(f"invalid SPDX token: {token}") + if not expect_operand: raise FairValidationError("missing SPDX operator") + expect_operand = False + if depth or expect_operand: raise FairValidationError("incomplete SPDX expression") + return expression + + +def datacite_metadata(store: Any) -> dict[str, Any]: + """Produce a DataCite-4.7-shaped publication record from explicit container facts. + + This is an export helper, not a claim that a DOI has been registered. + """ + meta = store.metadata() + primary = store.primary_identifier() + creators = store.fair_agents(role="creator") + rights = store.rights() + related = store.related_identifiers() + result: dict[str, Any] = { + "schemaVersion": "http://datacite.org/schema/kernel-4", + "titles": [{"title": meta.get("name", store.path.stem)}], + "publisher": meta.get("datatiles:publisher", "UNSPECIFIED"), + "publicationYear": meta.get("datatiles:publication_year", "UNSPECIFIED"), + "types": {"resourceTypeGeneral": "Dataset", "resourceType": "DataTiles multidimensional tiled dataset"}, + "creators": creators, + "rightsList": [{"rights": r["license_expression"], "rightsUri": r.get("license_uri")} for r in rights if r["scope"] in ("dataset","metadata")], + "relatedIdentifiers": related, + } + if primary: + result["identifier"] = {"identifier": primary["identifier"], "identifierType": primary["scheme"]} + return result + + +def prov_json(store: Any) -> dict[str, Any]: + """Export the internal graph with explicit W3C PROV term mappings.""" + entities = {r["entity_id"]: {"prov:type": r["entity_type"], "prov:label": r["label"], **({"prov:location": r["uri"]} if r["uri"] else {})} + for r in store.db.execute("SELECT * FROM datatiles_provenance_entities")} + activities = {r["activity_id"]: {"prov:type": r["activity_type"], "prov:label": r["label"], "prov:startedAtTime": r["started_at"], "prov:endedAtTime": r["ended_at"]} + for r in store.db.execute("SELECT * FROM datatiles_provenance_activities")} + agents = {r["agent_id"]: {"prov:type": r["agent_type"], "prov:label": r["label"], **({"prov:location": r["uri"]} if r["uri"] else {})} + for r in store.db.execute("SELECT * FROM datatiles_provenance_agents")} + relations = [dict(r) for r in store.db.execute("SELECT * FROM datatiles_provenance_relations ORDER BY subject_id,predicate,object_id")] + return {"prefix": {"prov": "http://www.w3.org/ns/prov#"}, "entity": entities, "activity": activities, "agent": agents, "relations": relations} diff --git a/src/datatiles/integrity.py b/src/datatiles/integrity.py new file mode 100644 index 0000000..70453f3 --- /dev/null +++ b/src/datatiles/integrity.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import base64 +import hashlib +import heapq +import json +import os +import sqlite3 +import struct +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +INTEGRITY_PROFILE = "DataTiles-Integrity-Manifest-1" +CANONICALIZATION = "datatiles-sqlite-logical-v1" +HASH_ALGORITHM = "sha256" +PAYLOAD_TYPE = "application/vnd.openfairwind.datatiles.integrity-manifest.v1+json" +EXCLUDED_TABLES = {"datatiles_integrity_manifests", "datatiles_signatures"} + + +class IntegrityError(ValueError): + pass + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def _normalise_sql(sql: str | None) -> str: + return " ".join((sql or "").strip().split()) + + +def _typed_value(value: Any) -> list[str]: + if value is None: + return ["null", ""] + if isinstance(value, bool): + return ["integer", "1" if value else "0"] + if isinstance(value, int): + return ["integer", str(value)] + if isinstance(value, float): + return ["real64be", struct.pack(">d", value).hex()] + if isinstance(value, (bytes, bytearray, memoryview)): + return ["blob", base64.b64encode(bytes(value)).decode("ascii")] + if isinstance(value, str): + return ["text", value] + raise IntegrityError(f"unsupported SQLite value type: {type(value).__name__}") + + +def _row_digest(row: Iterable[Any]) -> bytes: + return hashlib.sha256(_canonical_json([_typed_value(v) for v in row])).digest() + + +def _digest_sorted_rows(rows: Iterable[Iterable[Any]], *, chunk_size: int = 100_000) -> tuple[int, str]: + """Digest a table as a deterministic multiset of row digests with bounded RAM. + + Row order in SQLite is intentionally ignored. Digests are externally sorted in + temporary files and then streamed into the table digest, preserving duplicate + multiplicity while avoiding an unbounded in-memory list for large tile tables. + """ + if chunk_size < 1: + raise IntegrityError("chunk_size must be positive") + count = 0 + chunk: list[bytes] = [] + paths: list[str] = [] + try: + for row in rows: + count += 1 + chunk.append(_row_digest(row)) + if len(chunk) >= chunk_size: + chunk.sort() + f = tempfile.NamedTemporaryFile(prefix="datatiles-integrity-", suffix=".digests", delete=False) + with f: + for digest in chunk: + f.write(digest.hex().encode("ascii") + b"\n") + paths.append(f.name) + chunk.clear() + + if not paths: + chunk.sort() + h = hashlib.sha256() + for digest in chunk: + h.update(digest) + return count, h.hexdigest() + + if chunk: + chunk.sort() + f = tempfile.NamedTemporaryFile(prefix="datatiles-integrity-", suffix=".digests", delete=False) + with f: + for digest in chunk: + f.write(digest.hex().encode("ascii") + b"\n") + paths.append(f.name) + + handles = [open(path, "rt", encoding="ascii") for path in paths] + try: + h = hashlib.sha256() + for line in heapq.merge(*handles): + h.update(bytes.fromhex(line.strip())) + return count, h.hexdigest() + finally: + for handle in handles: + handle.close() + finally: + for path in paths: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def _schema_objects(db: sqlite3.Connection) -> list[dict[str, str]]: + rows = db.execute( + "SELECT type,name,tbl_name,sql FROM sqlite_master " + "WHERE type IN ('table','view','trigger') AND name NOT LIKE 'sqlite_%' ORDER BY type,name" + ) + result = [] + for object_type, name, table_name, sql in rows: + if name in EXCLUDED_TABLES or table_name in EXCLUDED_TABLES: + continue + result.append({ + "type": object_type, + "name": name, + "table": table_name, + "sql": _normalise_sql(sql), + }) + return result + + +def build_manifest(db: sqlite3.Connection, *, chunk_size: int = 100_000) -> dict[str, Any]: + previous_row_factory = db.row_factory + try: + db.row_factory = None + schema_revision = int(db.execute("PRAGMA user_version").fetchone()[0]) + objects = _schema_objects(db) + tables = [] + for obj in objects: + if obj["type"] != "table": + continue + table = obj["name"] + quoted = '"' + table.replace('"', '""') + '"' + cursor = db.execute(f"SELECT * FROM {quoted}") + row_count, rows_sha256 = _digest_sorted_rows(cursor, chunk_size=chunk_size) + tables.append({ + "name": table, + "schema_sha256": hashlib.sha256(obj["sql"].encode("utf-8")).hexdigest(), + "row_count": row_count, + "rows_sha256": rows_sha256, + }) + + views_and_triggers = [ + { + "type": obj["type"], + "name": obj["name"], + "schema_sha256": hashlib.sha256(obj["sql"].encode("utf-8")).hexdigest(), + } + for obj in objects if obj["type"] != "table" + ] + body = { + "profile": INTEGRITY_PROFILE, + "payload_type": PAYLOAD_TYPE, + "canonicalization": CANONICALIZATION, + "hash_algorithm": HASH_ALGORITHM, + "schema_revision": schema_revision, + "tables": sorted(tables, key=lambda x: x["name"]), + "views_and_triggers": sorted(views_and_triggers, key=lambda x: (x["type"], x["name"])), + } + root = hashlib.sha256(_canonical_json(body)).hexdigest() + return {**body, "root_sha256": root} + finally: + db.row_factory = previous_row_factory + + +def manifest_bytes(manifest: dict[str, Any]) -> bytes: + required = {"profile", "payload_type", "canonicalization", "hash_algorithm", "schema_revision", "tables", "views_and_triggers", "root_sha256"} + missing = required - manifest.keys() + if missing: + raise IntegrityError("manifest missing fields: " + ", ".join(sorted(missing))) + body = {k: manifest[k] for k in manifest if k != "root_sha256"} + expected = hashlib.sha256(_canonical_json(body)).hexdigest() + if expected != manifest["root_sha256"]: + raise IntegrityError("manifest root_sha256 does not match canonical manifest body") + return _canonical_json(manifest) + + +def key_id_from_public_bytes(public_key_raw: bytes) -> str: + return "sha256:" + hashlib.sha256(public_key_raw).hexdigest() + + +def _require_crypto(): + try: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey + except ImportError as exc: + raise IntegrityError("digital signatures require the optional 'integrity' dependency: pip install 'datatiles[integrity]'") from exc + return serialization, Ed25519PrivateKey, Ed25519PublicKey + + +def generate_ed25519_keypair(private_key_path: str | Path, public_key_path: str | Path, *, overwrite: bool = False) -> tuple[str, str]: + serialization, Ed25519PrivateKey, _ = _require_crypto() + private_path, public_path = Path(private_key_path), Path(public_key_path) + if not overwrite and (private_path.exists() or public_path.exists()): + raise IntegrityError("refusing to overwrite an existing key file") + private = Ed25519PrivateKey.generate() + public = private.public_key() + private_pem = private.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + public_pem = public.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo) + private_path.write_bytes(private_pem) + try: + os.chmod(private_path, 0o600) + except OSError: + pass + public_path.write_bytes(public_pem) + raw_public = public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return str(private_path), key_id_from_public_bytes(raw_public) + + +def load_private_key(path: str | Path): + serialization, Ed25519PrivateKey, _ = _require_crypto() + key = serialization.load_pem_private_key(Path(path).read_bytes(), password=None) + if not isinstance(key, Ed25519PrivateKey): + raise IntegrityError("only Ed25519 private keys are supported by the native DataTiles signature profile") + return key + + +def load_public_key(path: str | Path): + serialization, _, Ed25519PublicKey = _require_crypto() + key = serialization.load_pem_public_key(Path(path).read_bytes()) + if not isinstance(key, Ed25519PublicKey): + raise IntegrityError("only Ed25519 public keys are supported by the native DataTiles signature profile") + return key + + +def create_signature_envelope(manifest: dict[str, Any], private_key, *, signer: str | None = None, signed_at: str | None = None) -> dict[str, Any]: + serialization, Ed25519PrivateKey, _ = _require_crypto() + if not isinstance(private_key, Ed25519PrivateKey): + raise IntegrityError("native signing requires an Ed25519 private key") + payload = manifest_bytes(manifest) + signature = private_key.sign(payload) + raw_public = private_key.public_key().public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return { + "profile": "DataTiles-Ed25519-Signature-1", + "payload_type": PAYLOAD_TYPE, + "manifest": manifest, + "signature": { + "scheme": "Ed25519", + "encoding": "base64", + "value": base64.b64encode(signature).decode("ascii"), + "key_id": key_id_from_public_bytes(raw_public), + "public_key_raw_base64": base64.b64encode(raw_public).decode("ascii"), + "signed_at": signed_at or _utc_now(), + **({"signer": signer} if signer else {}), + }, + } + + +def verify_signature_envelope(envelope: dict[str, Any], *, public_key=None, expected_key_id: str | None = None) -> dict[str, Any]: + serialization, _, Ed25519PublicKey = _require_crypto() + if envelope.get("profile") != "DataTiles-Ed25519-Signature-1": + raise IntegrityError("unsupported signature envelope profile") + sig = envelope.get("signature") or {} + if sig.get("scheme") != "Ed25519" or sig.get("encoding") != "base64": + raise IntegrityError("unsupported signature scheme/encoding") + manifest = envelope.get("manifest") + if not isinstance(manifest, dict): + raise IntegrityError("signature envelope has no manifest") + payload = manifest_bytes(manifest) + embedded_raw = base64.b64decode(sig.get("public_key_raw_base64", ""), validate=True) + embedded_key_id = key_id_from_public_bytes(embedded_raw) + if embedded_key_id != sig.get("key_id"): + raise IntegrityError("embedded public key does not match signature key_id") + if expected_key_id and embedded_key_id != expected_key_id: + raise IntegrityError("signature key_id does not match the required key") + trusted = public_key is not None + if public_key is None: + public_key = Ed25519PublicKey.from_public_bytes(embedded_raw) + else: + supplied_raw = public_key.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + if supplied_raw != embedded_raw: + raise IntegrityError("supplied trusted public key does not match the embedded signature key") + try: + public_key.verify(base64.b64decode(sig["value"], validate=True), payload) + except Exception as exc: + raise IntegrityError("digital signature verification failed") from exc + return { + "cryptographically_valid": True, + "trusted_key_supplied": trusted, + "key_id": embedded_key_id, + "root_sha256": manifest["root_sha256"], + "signed_at": sig.get("signed_at"), + "signer": sig.get("signer"), + } + + +def verify_database_against_envelope(db: sqlite3.Connection, envelope: dict[str, Any], *, public_key=None, expected_key_id: str | None = None, chunk_size: int = 100_000) -> dict[str, Any]: + signature_result = verify_signature_envelope(envelope, public_key=public_key, expected_key_id=expected_key_id) + current = build_manifest(db, chunk_size=chunk_size) + expected = envelope["manifest"] + content_match = current["root_sha256"] == expected["root_sha256"] + return { + **signature_result, + "content_matches_signed_manifest": content_match, + "current_root_sha256": current["root_sha256"], + "signed_root_sha256": expected["root_sha256"], + "valid": bool(signature_result["cryptographically_valid"] and content_match), + } + + +def store_envelope(db: sqlite3.Connection, envelope: dict[str, Any], *, signer_agent_id: str | None = None, verification_material: dict[str, Any] | None = None) -> str: + manifest = envelope["manifest"] + sig = envelope["signature"] + manifest_id = "urn:uuid:" + str(uuid.uuid4()) + signature_id = "urn:uuid:" + str(uuid.uuid4()) + manifest_json = _canonical_json(manifest).decode("utf-8") + created_at = sig.get("signed_at") or _utc_now() + with db: + row = db.execute("SELECT manifest_id FROM datatiles_integrity_manifests WHERE root_sha256=?", (manifest["root_sha256"],)).fetchone() + if row: + manifest_id = row[0] + else: + db.execute( + "INSERT INTO datatiles_integrity_manifests(manifest_id,profile,canonicalization,hash_algorithm,root_sha256,manifest_json,created_at) VALUES (?,?,?,?,?,?,?)", + (manifest_id, manifest["profile"], manifest["canonicalization"], manifest["hash_algorithm"], manifest["root_sha256"], manifest_json, created_at), + ) + db.execute( + "INSERT INTO datatiles_signatures(signature_id,manifest_id,signature_scheme,signature_encoding,signature,key_id,public_key,signer_agent_id,signed_at,verification_material_json) VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + signature_id, manifest_id, sig["scheme"], sig["encoding"], base64.b64decode(sig["value"]), sig["key_id"], + base64.b64decode(sig["public_key_raw_base64"]), signer_agent_id, sig.get("signed_at") or _utc_now(), + json.dumps(verification_material, sort_keys=True, separators=(",", ":")) if verification_material else None, + ), + ) + return signature_id + + +def list_stored_signatures(db: sqlite3.Connection) -> list[dict[str, Any]]: + rows = db.execute( + "SELECT s.signature_id,s.signature_scheme,s.signature_encoding,s.key_id,s.signer_agent_id,s.signed_at,m.manifest_id,m.profile,m.root_sha256,s.verification_material_json " + "FROM datatiles_signatures s JOIN datatiles_integrity_manifests m USING(manifest_id) ORDER BY s.signed_at,s.signature_id" + ) + result = [] + for row in rows: + item = { + "signature_id": row[0], "scheme": row[1], "encoding": row[2], "key_id": row[3], + "signer_agent_id": row[4], "signed_at": row[5], "manifest_id": row[6], + "manifest_profile": row[7], "root_sha256": row[8], + } + if row[9]: + item["verification_material"] = json.loads(row[9]) + result.append(item) + return result + + +def envelope_from_stored_signature(db: sqlite3.Connection, signature_id: str) -> dict[str, Any]: + row = db.execute( + "SELECT m.manifest_json,s.signature_scheme,s.signature_encoding,s.signature,s.key_id,s.public_key,s.signed_at,s.signer_agent_id " + "FROM datatiles_signatures s JOIN datatiles_integrity_manifests m USING(manifest_id) WHERE s.signature_id=?", + (signature_id,), + ).fetchone() + if row is None: + raise IntegrityError(f"unknown signature: {signature_id}") + manifest = json.loads(row[0]) + return { + "profile": "DataTiles-Ed25519-Signature-1", + "payload_type": PAYLOAD_TYPE, + "manifest": manifest, + "signature": { + "scheme": row[1], "encoding": row[2], "value": base64.b64encode(row[3]).decode("ascii"), + "key_id": row[4], "public_key_raw_base64": base64.b64encode(row[5]).decode("ascii"), + "signed_at": row[6], **({"signer_agent_id": row[7]} if row[7] else {}), + }, + } diff --git a/src/datatiles/integrity_cli.py b/src/datatiles/integrity_cli.py new file mode 100644 index 0000000..22e6f3b --- /dev/null +++ b/src/datatiles/integrity_cli.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +from .integrity import ( + IntegrityError, build_manifest, create_signature_envelope, envelope_from_stored_signature, + generate_ed25519_keypair, list_stored_signatures, load_private_key, load_public_key, + store_envelope, verify_database_against_envelope, +) + + +def _db(path: str, *, writable: bool = False) -> sqlite3.Connection: + if writable: + db = sqlite3.connect(path) + else: + db = sqlite3.connect(f"file:{Path(path).resolve()}?mode=ro", uri=True) + db.execute("PRAGMA foreign_keys=ON") + return db + + +def _dump(value, output: str | None = None): + text = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + if output: + Path(output).write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="datatiles-integrity", description="Create and verify DataTiles integrity manifests and optional Ed25519 signatures") + sub = p.add_subparsers(dest="command", required=True) + + g = sub.add_parser("generate-key", help="generate an Ed25519 signing keypair") + g.add_argument("--private-key", required=True); g.add_argument("--public-key", required=True); g.add_argument("--overwrite", action="store_true") + + m = sub.add_parser("manifest", help="compute the canonical logical integrity manifest") + m.add_argument("file"); m.add_argument("--output"); m.add_argument("--chunk-size", type=int, default=100000) + + s = sub.add_parser("sign", help="sign the current canonical manifest") + s.add_argument("file"); s.add_argument("--private-key", required=True); s.add_argument("--signer"); s.add_argument("--signer-agent-id") + s.add_argument("--detached", help="write a detached signature envelope JSON") + s.add_argument("--no-store", action="store_true", help="do not store the signature inside the DataTiles container") + s.add_argument("--chunk-size", type=int, default=100000) + + v = sub.add_parser("verify", help="verify a stored or detached signature and current DataTiles content") + v.add_argument("file"); v.add_argument("--signature-id"); v.add_argument("--detached"); v.add_argument("--public-key") + v.add_argument("--require-key-id"); v.add_argument("--chunk-size", type=int, default=100000) + + l = sub.add_parser("list", help="list signatures stored in a DataTiles container") + l.add_argument("file") + + args = p.parse_args(argv) + try: + if args.command == "generate-key": + private_path, key_id = generate_ed25519_keypair(args.private_key, args.public_key, overwrite=args.overwrite) + _dump({"private_key": private_path, "public_key": args.public_key, "key_id": key_id}); return 0 + if args.command == "manifest": + with _db(args.file) as db: manifest = build_manifest(db, chunk_size=args.chunk_size) + _dump(manifest, args.output); return 0 + if args.command == "sign": + key = load_private_key(args.private_key) + with _db(args.file, writable=not args.no_store) as db: + manifest = build_manifest(db, chunk_size=args.chunk_size) + envelope = create_signature_envelope(manifest, key, signer=args.signer) + signature_id = None + if not args.no_store: + signature_id = store_envelope(db, envelope, signer_agent_id=args.signer_agent_id) + if args.detached: _dump(envelope, args.detached) + _dump({"signature_id": signature_id, "root_sha256": manifest["root_sha256"], "key_id": envelope["signature"]["key_id"], "detached": args.detached}); return 0 + if args.command == "list": + with _db(args.file) as db: result = list_stored_signatures(db) + _dump({"signatures": result}); return 0 + if args.command == "verify": + key = load_public_key(args.public_key) if args.public_key else None + with _db(args.file) as db: + if args.detached: + envelope = json.loads(Path(args.detached).read_text(encoding="utf-8")) + else: + signature_id = args.signature_id + if not signature_id: + rows = list_stored_signatures(db) + if len(rows) != 1: + raise IntegrityError("specify --signature-id when the container does not contain exactly one signature") + signature_id = rows[0]["signature_id"] + envelope = envelope_from_stored_signature(db, signature_id) + result = verify_database_against_envelope(db, envelope, public_key=key, expected_key_id=args.require_key_id, chunk_size=args.chunk_size) + _dump(result); return 0 if result["valid"] else 2 + except (IntegrityError, sqlite3.Error, OSError, ValueError) as exc: + print(f"datatiles-integrity: {exc}", file=sys.stderr) + return 2 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/datatiles/migration_v5_to_v6.sql b/src/datatiles/migration_v5_to_v6.sql new file mode 100644 index 0000000..02850c8 --- /dev/null +++ b/src/datatiles/migration_v5_to_v6.sql @@ -0,0 +1,29 @@ +CREATE TABLE datatiles_integrity_manifests ( + manifest_id TEXT PRIMARY KEY, + profile TEXT NOT NULL, + canonicalization TEXT NOT NULL, + hash_algorithm TEXT NOT NULL CHECK(hash_algorithm = 'sha256'), + root_sha256 TEXT NOT NULL UNIQUE CHECK(length(root_sha256) = 64), + manifest_json TEXT NOT NULL, + created_at TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE datatiles_signatures ( + signature_id TEXT PRIMARY KEY, + manifest_id TEXT NOT NULL REFERENCES datatiles_integrity_manifests(manifest_id) ON DELETE CASCADE, + signature_scheme TEXT NOT NULL, + signature_encoding TEXT NOT NULL, + signature BLOB NOT NULL, + key_id TEXT NOT NULL, + public_key BLOB, + signer_agent_id TEXT REFERENCES datatiles_provenance_agents(agent_id) ON DELETE SET NULL, + signed_at TEXT NOT NULL, + verification_material_json TEXT, + UNIQUE(manifest_id, signature_scheme, key_id, signature) +) WITHOUT ROWID; +CREATE INDEX datatiles_signatures_manifest ON datatiles_signatures(manifest_id); +CREATE INDEX datatiles_signatures_key ON datatiles_signatures(key_id); + +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:integrity_profile','DataTiles-Integrity-Manifest-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:signature_profile','DataTiles-Ed25519-Signature-1'); +PRAGMA user_version = 6; diff --git a/src/datatiles/migration_v6_to_v7.sql b/src/datatiles/migration_v6_to_v7.sql new file mode 100644 index 0000000..5f24756 --- /dev/null +++ b/src/datatiles/migration_v6_to_v7.sql @@ -0,0 +1,24 @@ +CREATE TABLE datatiles_commercial_products ( + product_id TEXT PRIMARY KEY, + edition TEXT, + issuer TEXT NOT NULL, + issuer_uri TEXT, + terms_uri TEXT NOT NULL, + license_service_uri TEXT, + protection_profile TEXT NOT NULL DEFAULT 'DataTiles-Protected-Distribution-1', + created_at TEXT NOT NULL, + metadata_json TEXT +) WITHOUT ROWID; + +CREATE TABLE datatiles_drm_policies ( + product_id TEXT NOT NULL REFERENCES datatiles_commercial_products(product_id) ON DELETE CASCADE, + policy_id TEXT NOT NULL, + policy_profile TEXT NOT NULL DEFAULT 'W3C-ODRL-2.2', + policy_json TEXT NOT NULL, + PRIMARY KEY(product_id, policy_id) +) WITHOUT ROWID; +CREATE INDEX datatiles_drm_policy_product ON datatiles_drm_policies(product_id); + +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:drm_profile','DataTiles-Protected-Distribution-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:rights_policy_profile','W3C-ODRL-2.2'); +PRAGMA user_version = 7; diff --git a/src/datatiles/migration_v7_to_v8.sql b/src/datatiles/migration_v7_to_v8.sql new file mode 100644 index 0000000..2d8386a --- /dev/null +++ b/src/datatiles/migration_v7_to_v8.sql @@ -0,0 +1,16 @@ +CREATE TABLE datatiles_release ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), + product_id TEXT NOT NULL, + version TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK(sequence >= 1), + released_at TEXT NOT NULL, + previous_version TEXT, + previous_identifier TEXT, + release_notes_uri TEXT, + update_uri TEXT, + CHECK(product_id <> '' AND trim(product_id) = product_id), + CHECK(version <> '' AND trim(version) = version) +); +CREATE UNIQUE INDEX datatiles_release_product_version ON datatiles_release(product_id, version); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:versioning_profile','DataTiles-Release-Versioning-1'); +PRAGMA user_version = 8; diff --git a/src/datatiles/profile-demo.html b/src/datatiles/profile-demo.html index 91903f5..d7595df 100644 --- a/src/datatiles/profile-demo.html +++ b/src/datatiles/profile-demo.html @@ -2,6 +2,7 @@ DataTiles scientific playground

DataTiles · __TITLE__

DNT1 arrays · stored vectors · live scientific derivation
+ +async function loadIntegrityEvidence(){const r=await fetch(api+'/integrity');if(!r.ok)return;const x=await r.json(),n=(x.signatures||[]).length;document.getElementById('integrityEvidence').textContent=n+' signature record(s) · presence is not trust; verify with an independently authenticated public key';} + +async function loadCommercialEvidence(){const r=await fetch(api+'/commercial');if(!r.ok)return;const x=await r.json(),n=(x.products||[]).length;document.getElementById('commercialEvidence').textContent=n? n+' commercial product record(s) · access control does not alter source rights or safety status':'No commercial DRM product metadata recorded';} diff --git a/src/datatiles/schema.sql b/src/datatiles/schema.sql index 357a4f1..0842f9d 100644 --- a/src/datatiles/schema.sql +++ b/src/datatiles/schema.sql @@ -1,6 +1,6 @@ PRAGMA foreign_keys = ON; PRAGMA application_id = 0x44415441; -- "DATA" -PRAGMA user_version = 3; +PRAGMA user_version = 8; CREATE TABLE metadata ( name TEXT NOT NULL, @@ -20,6 +20,31 @@ CREATE TABLE datatiles_dimensions ( extent_kind TEXT NOT NULL DEFAULT 'point' CHECK(extent_kind IN ('point','interval','point_or_interval')) ); +CREATE TABLE datatiles_variables ( + variable_id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + standard_name TEXT NOT NULL, + standard_name_vocabulary TEXT NOT NULL DEFAULT 'CF', + standard_name_vocabulary_version TEXT, + canonical_unit TEXT, + long_name TEXT, + description TEXT, + CHECK(name <> '' AND trim(name) = name), + CHECK(standard_name <> '' AND trim(standard_name) = standard_name) +); +CREATE INDEX datatiles_variables_standard_name + ON datatiles_variables(standard_name_vocabulary, standard_name); +CREATE TABLE datatiles_variable_identifiers ( + variable_id INTEGER NOT NULL REFERENCES datatiles_variables(variable_id) ON DELETE CASCADE, + scheme TEXT NOT NULL, + identifier TEXT NOT NULL, + scheme_version TEXT, + uri TEXT, + PRIMARY KEY(variable_id, scheme, identifier), + UNIQUE(scheme, identifier), + CHECK(scheme <> '' AND trim(scheme) = scheme), + CHECK(identifier <> '' AND trim(identifier) = identifier) +) WITHOUT ROWID; CREATE TABLE datatiles_values ( value_id INTEGER PRIMARY KEY, dimension_id INTEGER NOT NULL REFERENCES datatiles_dimensions(dimension_id) ON DELETE CASCADE, @@ -141,6 +166,95 @@ CREATE TABLE datatiles_tile_provenance ( REFERENCES datatiles_tiles(zoom_level,tile_column,tile_row,coordinate_set_id) ON DELETE CASCADE ) WITHOUT ROWID; +CREATE TABLE datatiles_release ( + singleton INTEGER PRIMARY KEY CHECK(singleton = 1), + product_id TEXT NOT NULL, + version TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK(sequence >= 1), + released_at TEXT NOT NULL, + previous_version TEXT, + previous_identifier TEXT, + release_notes_uri TEXT, + update_uri TEXT, + CHECK(product_id <> '' AND trim(product_id) = product_id), + CHECK(version <> '' AND trim(version) = version) +); +CREATE UNIQUE INDEX datatiles_release_product_version ON datatiles_release(product_id, version); +CREATE TABLE datatiles_commercial_products ( + product_id TEXT PRIMARY KEY, + edition TEXT, + issuer TEXT NOT NULL, + issuer_uri TEXT, + terms_uri TEXT NOT NULL, + license_service_uri TEXT, + protection_profile TEXT NOT NULL DEFAULT 'DataTiles-Protected-Distribution-1', + created_at TEXT NOT NULL, + metadata_json TEXT +) WITHOUT ROWID; +CREATE TABLE datatiles_drm_policies ( + product_id TEXT NOT NULL REFERENCES datatiles_commercial_products(product_id) ON DELETE CASCADE, + policy_id TEXT NOT NULL, + policy_profile TEXT NOT NULL DEFAULT 'W3C-ODRL-2.2', + policy_json TEXT NOT NULL, + PRIMARY KEY(product_id, policy_id) +) WITHOUT ROWID; +CREATE INDEX datatiles_drm_policy_product ON datatiles_drm_policies(product_id); +CREATE TABLE datatiles_integrity_manifests ( + manifest_id TEXT PRIMARY KEY, + profile TEXT NOT NULL, + canonicalization TEXT NOT NULL, + hash_algorithm TEXT NOT NULL CHECK(hash_algorithm = 'sha256'), + root_sha256 TEXT NOT NULL UNIQUE CHECK(length(root_sha256) = 64), + manifest_json TEXT NOT NULL, + created_at TEXT NOT NULL +) WITHOUT ROWID; +CREATE TABLE datatiles_signatures ( + signature_id TEXT PRIMARY KEY, + manifest_id TEXT NOT NULL REFERENCES datatiles_integrity_manifests(manifest_id) ON DELETE CASCADE, + signature_scheme TEXT NOT NULL, + signature_encoding TEXT NOT NULL, + signature BLOB NOT NULL, + key_id TEXT NOT NULL, + public_key BLOB, + signer_agent_id TEXT REFERENCES datatiles_provenance_agents(agent_id) ON DELETE SET NULL, + signed_at TEXT NOT NULL, + verification_material_json TEXT, + UNIQUE(manifest_id, signature_scheme, key_id, signature) +) WITHOUT ROWID; +CREATE INDEX datatiles_signatures_manifest ON datatiles_signatures(manifest_id); +CREATE INDEX datatiles_signatures_key ON datatiles_signatures(key_id); +CREATE TABLE datatiles_identifiers ( + identifier_id INTEGER PRIMARY KEY, + scheme TEXT NOT NULL, + identifier TEXT NOT NULL, + uri TEXT, + is_primary INTEGER NOT NULL DEFAULT 0 CHECK(is_primary IN (0,1)), + UNIQUE(scheme,identifier) +); +CREATE UNIQUE INDEX datatiles_one_primary_identifier ON datatiles_identifiers(is_primary) WHERE is_primary=1; +CREATE TABLE datatiles_related_identifiers ( + scheme TEXT NOT NULL, identifier TEXT NOT NULL, relation_type TEXT NOT NULL, uri TEXT, + resource_type TEXT, relation_information TEXT, + PRIMARY KEY(scheme,identifier,relation_type) +) WITHOUT ROWID; +CREATE TABLE datatiles_rights ( + rights_id INTEGER PRIMARY KEY, + scope TEXT NOT NULL CHECK(scope IN ('dataset','metadata','source','portrayal','software','other')), + license_expression TEXT NOT NULL, license_uri TEXT, rights_holder TEXT, rights_holder_uri TEXT, + attribution_text TEXT, copyright_notice TEXT, + access_rights TEXT NOT NULL DEFAULT 'open' CHECK(access_rights IN ('open','embargoed','restricted','closed')), + source_entity_id TEXT REFERENCES datatiles_provenance_entities(entity_id) ON DELETE CASCADE, + applies_to TEXT, UNIQUE(scope,source_entity_id,license_expression,applies_to) +); +CREATE INDEX datatiles_rights_scope ON datatiles_rights(scope,access_rights); +CREATE TABLE datatiles_fair_agents ( + agent_id TEXT NOT NULL REFERENCES datatiles_provenance_agents(agent_id) ON DELETE CASCADE, + role TEXT NOT NULL, sequence INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(agent_id,role) +) WITHOUT ROWID; +CREATE TABLE datatiles_publication_evidence ( + evidence_type TEXT NOT NULL, uri TEXT NOT NULL, checked_at TEXT, checksum_algorithm TEXT, checksum TEXT, notes TEXT, + PRIMARY KEY(evidence_type,uri) +) WITHOUT ROWID; CREATE TABLE datatiles_selected_slice ( singleton INTEGER PRIMARY KEY CHECK(singleton = 1), coordinate_set_id INTEGER REFERENCES datatiles_coordinate_sets(coordinate_set_id) ON DELETE SET NULL diff --git a/src/datatiles/semantic.py b/src/datatiles/semantic.py new file mode 100644 index 0000000..6573ba9 --- /dev/null +++ b/src/datatiles/semantic.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import xml.etree.ElementTree as ET + + +class SemanticValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class CFStandardName: + name: str + canonical_units: str + description: str | None = None + + +def load_cf_standard_name_table(path: str | Path) -> dict[str, CFStandardName]: + """Load an official CF standard-name-table XML file using only stdlib. + + The caller supplies a local, pinned XML snapshot. DataTiles never downloads a + vocabulary implicitly, preserving offline/reproducible behavior. + """ + root = ET.parse(Path(path)).getroot() + result: dict[str, CFStandardName] = {} + for entry in root.findall("entry"): + name = entry.attrib.get("id", "").strip() + if not name: + continue + units_node = entry.find("canonical_units") + description_node = entry.find("description") + result[name] = CFStandardName( + name=name, + canonical_units=(units_node.text or "").strip() if units_node is not None else "", + description=(description_node.text or "").strip() if description_node is not None and description_node.text else None, + ) + if not result: + raise SemanticValidationError("CF table contains no standard-name entries") + return result + + +def validate_standard_name_syntax(name: str) -> None: + if not isinstance(name, str) or not name or name.strip() != name: + raise SemanticValidationError("standard name must be non-empty trimmed text") + if any(ch.isspace() for ch in name): + raise SemanticValidationError("standard name must not contain whitespace") + + +def validate_cf_standard_name( + standard_name: str, + *, + canonical_unit: str | None, + table: dict[str, CFStandardName], +) -> CFStandardName: + """Validate a CF standard name against a caller-pinned table. + + If canonical_unit is supplied, it must exactly match the canonical unit in + the CF table. This deliberately does not claim to perform UDUNITS physical + equivalence. Dataset/tile units may still differ when physically equivalent. + """ + validate_standard_name_syntax(standard_name) + try: + entry = table[standard_name] + except KeyError as exc: + raise SemanticValidationError(f"unknown CF standard name: {standard_name}") from exc + if canonical_unit is not None and canonical_unit != entry.canonical_units: + raise SemanticValidationError( + f"canonical unit mismatch for {standard_name}: expected {entry.canonical_units!r}, got {canonical_unit!r}" + ) + return entry diff --git a/src/datatiles/server.py b/src/datatiles/server.py index 23bd73c..e55dbe2 100644 --- a/src/datatiles/server.py +++ b/src/datatiles/server.py @@ -47,7 +47,7 @@ def _coordinates(query: dict[str, list[str]], dimensions: set[str]) -> dict[str, def handler_for(path: Path): class Handler(BaseHTTPRequestHandler): - server_version = "DataTiles/0.10" + server_version = "DataTiles/0.15" def do_GET(self) -> None: parsed = urlparse(self.path); parts = [unquote(p) for p in parsed.path.split("/") if p] @@ -80,6 +80,7 @@ def do_GET(self) -> None: "crs": [c["uri"] for c in crs if c["uri"]], "datatiles:dimensions": dims, "datatiles:contents": store.content_profiles(), + "datatiles:variables": store.variables(), "links": [{"rel":"tilesets-map","href":f"/collections/{collection_url}/tiles","type":"application/json"}, {"rel":"profile","href":f"/collections/{collection_url}/profile","type":"application/json"}, {"rel":"point","href":f"/collections/{collection_url}/point","type":"application/json"}, @@ -93,6 +94,16 @@ def do_GET(self) -> None: if parts == ["collections", collection]: return _json(self, 200, summary) if parts == ["collections", collection, "dimensions"]: return _json(self, 200, {"dimensions":dims}) if parts == ["collections", collection, "contents"]: return _json(self,200,{"contents":store.content_profiles()}) + if parts == ["collections", collection, "variables"]: return _json(self,200,{"variables":store.variables()}) + if parts == ["collections", collection, "commercial"]: return _json(self,200,store.drm_status()) + if parts == ["collections", collection, "release"]: return _json(self,200,{"release":store.release()}) + if parts == ["collections", collection, "commercial", "policies"]: return _json(self,200,{"policies":store.drm_policies()}) + if parts == ["collections", collection, "integrity"]: return _json(self,200,store.integrity_status(recompute=False)) + if parts == ["collections", collection, "integrity", "manifest"]: return _json(self,200,store.integrity_manifest()) + if parts == ["collections", collection, "fair"]: return _json(self,200,store.fair_report(strict_publication=False)) + if parts == ["collections", collection, "rights"]: return _json(self,200,{"rights":store.rights()}) + if parts == ["collections", collection, "provenance"]: return _json(self,200,store.prov_json()) + if parts == ["collections", collection, "datacite"]: return _json(self,200,store.datacite_metadata()) if parts == ["collections", collection, "crs"]: return _json(self, 200, {"crs":crs}) if parts == ["collections", collection, "fair"]: return _json(self,200,store.fair_report()) if parts == ["collections", collection, "provenance"]: @@ -187,7 +198,7 @@ def openapi(collection: str, dims: list[dict]) -> dict: {"name":"tileRow", "in":"path", "required":True, "schema":{"type":"integer","minimum":0}}] params += [{"name":d["name"], "in":"query", "required":bool(d["required"]), "schema":{"type":"string"}, "description":d["description"] or d["extent_kind"]} for d in dims] - return {"openapi":"3.0.3", "info":{"title":"DataTiles OGC API","version":"0.10.0"}, + return {"openapi":"3.0.3", "info":{"title":"DataTiles OGC API","version":"0.15.0"}, "paths":{f"/collections/{collection}/tiles/WebMercatorQuad/{{tileMatrix}}/{{tileCol}}/{{tileRow}}":{ "get":{"parameters":params, "responses":{"200":{"description":"tile"},"404":{"description":"not found"}}}}, f"/collections/{collection}/profile":{"get":{"parameters":[ diff --git a/src/datatiles/static_demo.py b/src/datatiles/static_demo.py index b52624d..b1d7699 100644 --- a/src/datatiles/static_demo.py +++ b/src/datatiles/static_demo.py @@ -29,7 +29,12 @@ def build_static_demo(source: Path, output: Path) -> None: manifest={"title":metadata.get("name",source.stem),"bounds":bbox, "warning":metadata.get("datatiles:warning","NOT FOR NAVIGATION"), "attribution":metadata.get("attribution",""), - "surface_sha256":surface["surface_sha256"],"source_container":source.name} + "surface_sha256":surface["surface_sha256"],"source_container":source.name, + "primary_identifier":store.primary_identifier(),"rights":store.rights(), + "fair":store.fair_report(strict_publication=False), + "provenance":store.prov_json(),"datacite":store.datacite_metadata(), + "integrity":store.integrity_status(recompute=False), + "commercial":store.drm_status()} for key in ("depth_m","seafloor_class","bathymetry_source","northwest_wind_shelter"): surface[key]=[value for row in surface[key] for value in row] (data_dir/"surface.json").write_text(json.dumps(surface,separators=(",",":"),ensure_ascii=False)+"\n") diff --git a/src/datatiles/store.py b/src/datatiles/store.py index b434933..1bdadd5 100644 --- a/src/datatiles/store.py +++ b/src/datatiles/store.py @@ -78,6 +78,15 @@ def __init__(self, path: str | Path, *, create: bool = False, name: str | None = ("datatiles:default_media_type", default_media), ("datatiles:version", "1.0-draft"), ("datatiles:dimensions", "[]"), + ("datatiles:variable_semantics", "recommended"), + ("datatiles:fair_profile", "FAIR-Guiding-Principles-2016"), + ("datatiles:provenance_profile", "W3C-PROV-2013"), + ("datatiles:metadata_profile", "DataCite-4.7"), + ("datatiles:license_profile", "SPDX-3.0.1-expression"), + ("datatiles:integrity_profile", "DataTiles-Integrity-Manifest-1"), + ("datatiles:signature_profile", "DataTiles-Ed25519-Signature-1"), + ("datatiles:drm_profile", "DataTiles-Protected-Distribution-1"), + ("datatiles:rights_policy_profile", "W3C-ODRL-2.2"), ]) self.db.commit() else: @@ -91,18 +100,43 @@ def _check_and_migrate(self, read_only: bool) -> None: if application_id != 0x44415441: raise DataTilesError("not a DataTiles container (application_id mismatch)") revision=int(self.db.execute("PRAGMA user_version").fetchone()[0]) - if revision not in (2,3): + if revision not in (2,3,4,5,6,7,8): raise DataTilesError(f"unsupported DataTiles schema revision: {revision}") objects={r[0] for r in self.db.execute("SELECT name FROM sqlite_master WHERE type IN ('table','view')")} required={"metadata","tiles","datatiles_dimensions","datatiles_values","datatiles_coordinate_sets", "datatiles_coordinates","datatiles_tiles","datatiles_selected_slice"} - if revision==3: required.add("datatiles_contents") + if revision>=3: required.add("datatiles_contents") + if revision>=4: required.update({"datatiles_variables","datatiles_variable_identifiers"}) + if revision>=5: required.update({"datatiles_identifiers","datatiles_related_identifiers","datatiles_rights","datatiles_fair_agents","datatiles_publication_evidence"}) + if revision>=6: required.update({"datatiles_integrity_manifests","datatiles_signatures"}) + if revision>=7: required.update({"datatiles_commercial_products","datatiles_drm_policies"}) + if revision==8: required.add("datatiles_release") missing=required-objects if missing: raise DataTilesError("incomplete DataTiles schema: "+", ".join(sorted(missing))) if revision == 2: if read_only: - raise DataTilesError("schema revision 2 requires writable migration to revision 3") - self._migrate_v2_to_v3() + raise DataTilesError("schema revision 2 requires writable migration to revision 4") + self._migrate_v2_to_v3(); revision = 3 + if revision == 3: + if read_only: + raise DataTilesError("schema revision 3 requires writable migration to revision 4") + self._migrate_v3_to_v4(); revision = 4 + if revision == 4: + if read_only: + raise DataTilesError("schema revision 4 requires writable migration to revision 5") + self._migrate_v4_to_v5(); revision = 5 + if revision == 5: + if read_only: + raise DataTilesError("schema revision 5 requires writable migration to revision 6") + self._migrate_v5_to_v6(); revision = 6 + if revision == 6: + if read_only: + raise DataTilesError("schema revision 6 requires writable migration to revision 7") + self._migrate_v6_to_v7(); revision = 7 + if revision == 7: + if read_only: + raise DataTilesError("schema revision 7 requires writable migration to revision 8") + self._migrate_v7_to_v8() def _migrate_v2_to_v3(self) -> None: metadata=dict(self.db.execute("SELECT name,value FROM metadata")) @@ -121,6 +155,90 @@ def _migrate_v2_to_v3(self) -> None: self.db.execute("UPDATE datatiles_coordinate_sets SET canonical_key=? WHERE coordinate_set_id=?",(key,set_row[0])) self.db.execute("PRAGMA user_version = 3") + def _migrate_v3_to_v4(self) -> None: + with self.db: + self.db.executescript(""" + CREATE TABLE datatiles_variables ( + variable_id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + standard_name TEXT NOT NULL, + standard_name_vocabulary TEXT NOT NULL DEFAULT 'CF', + standard_name_vocabulary_version TEXT, + canonical_unit TEXT, + long_name TEXT, + description TEXT, + CHECK(name <> '' AND trim(name) = name), + CHECK(standard_name <> '' AND trim(standard_name) = standard_name) + ); + CREATE INDEX datatiles_variables_standard_name + ON datatiles_variables(standard_name_vocabulary, standard_name); + CREATE TABLE datatiles_variable_identifiers ( + variable_id INTEGER NOT NULL REFERENCES datatiles_variables(variable_id) ON DELETE CASCADE, + scheme TEXT NOT NULL, + identifier TEXT NOT NULL, + scheme_version TEXT, + uri TEXT, + PRIMARY KEY(variable_id, scheme, identifier), + UNIQUE(scheme, identifier), + CHECK(scheme <> '' AND trim(scheme) = scheme), + CHECK(identifier <> '' AND trim(identifier) = identifier) + ) WITHOUT ROWID; + """) + self.db.execute("INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:variable_semantics','recommended')") + self.db.execute("PRAGMA user_version = 4") + + def _migrate_v4_to_v5(self) -> None: + sql = (Path(__file__).with_name("migration_v4_to_v5.sql").read_text() if Path(__file__).with_name("migration_v4_to_v5.sql").exists() else None) + if sql is None: + sql = """ +CREATE TABLE datatiles_identifiers (identifier_id INTEGER PRIMARY KEY,scheme TEXT NOT NULL,identifier TEXT NOT NULL,uri TEXT,is_primary INTEGER NOT NULL DEFAULT 0 CHECK(is_primary IN (0,1)),UNIQUE(scheme,identifier)); +CREATE UNIQUE INDEX datatiles_one_primary_identifier ON datatiles_identifiers(is_primary) WHERE is_primary=1; +CREATE TABLE datatiles_related_identifiers (scheme TEXT NOT NULL,identifier TEXT NOT NULL,relation_type TEXT NOT NULL,uri TEXT,resource_type TEXT,relation_information TEXT,PRIMARY KEY(scheme,identifier,relation_type)) WITHOUT ROWID; +CREATE TABLE datatiles_rights (rights_id INTEGER PRIMARY KEY,scope TEXT NOT NULL CHECK(scope IN ('dataset','metadata','source','portrayal','software','other')),license_expression TEXT NOT NULL,license_uri TEXT,rights_holder TEXT,rights_holder_uri TEXT,attribution_text TEXT,copyright_notice TEXT,access_rights TEXT NOT NULL DEFAULT 'open' CHECK(access_rights IN ('open','embargoed','restricted','closed')),source_entity_id TEXT REFERENCES datatiles_provenance_entities(entity_id) ON DELETE CASCADE,applies_to TEXT,UNIQUE(scope,source_entity_id,license_expression,applies_to)); +CREATE INDEX datatiles_rights_scope ON datatiles_rights(scope,access_rights); +CREATE TABLE datatiles_fair_agents (agent_id TEXT NOT NULL REFERENCES datatiles_provenance_agents(agent_id) ON DELETE CASCADE,role TEXT NOT NULL,sequence INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(agent_id,role)) WITHOUT ROWID; +CREATE TABLE datatiles_publication_evidence (evidence_type TEXT NOT NULL,uri TEXT NOT NULL,checked_at TEXT,checksum_algorithm TEXT,checksum TEXT,notes TEXT,PRIMARY KEY(evidence_type,uri)) WITHOUT ROWID; +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:fair_profile','FAIR-Guiding-Principles-2016'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:provenance_profile','W3C-PROV-2013'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:metadata_profile','DataCite-4.7'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:license_profile','SPDX-3.0.1-expression'); +PRAGMA user_version = 5; +""" + with self.db: self.db.executescript(sql) + + def _migrate_v5_to_v6(self) -> None: + sql = (Path(__file__).with_name("migration_v5_to_v6.sql").read_text() if Path(__file__).with_name("migration_v5_to_v6.sql").exists() else None) + if sql is None: + sql = """ +CREATE TABLE datatiles_integrity_manifests (manifest_id TEXT PRIMARY KEY,profile TEXT NOT NULL,canonicalization TEXT NOT NULL,hash_algorithm TEXT NOT NULL CHECK(hash_algorithm='sha256'),root_sha256 TEXT NOT NULL UNIQUE CHECK(length(root_sha256)=64),manifest_json TEXT NOT NULL,created_at TEXT NOT NULL) WITHOUT ROWID; +CREATE TABLE datatiles_signatures (signature_id TEXT PRIMARY KEY,manifest_id TEXT NOT NULL REFERENCES datatiles_integrity_manifests(manifest_id) ON DELETE CASCADE,signature_scheme TEXT NOT NULL,signature_encoding TEXT NOT NULL,signature BLOB NOT NULL,key_id TEXT NOT NULL,public_key BLOB,signer_agent_id TEXT REFERENCES datatiles_provenance_agents(agent_id) ON DELETE SET NULL,signed_at TEXT NOT NULL,verification_material_json TEXT,UNIQUE(manifest_id,signature_scheme,key_id,signature)) WITHOUT ROWID; +CREATE INDEX datatiles_signatures_manifest ON datatiles_signatures(manifest_id); +CREATE INDEX datatiles_signatures_key ON datatiles_signatures(key_id); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:integrity_profile','DataTiles-Integrity-Manifest-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:signature_profile','DataTiles-Ed25519-Signature-1'); +PRAGMA user_version = 6; +""" + with self.db: self.db.executescript(sql) + + def _migrate_v6_to_v7(self) -> None: + sql = (Path(__file__).with_name("migration_v6_to_v7.sql").read_text() if Path(__file__).with_name("migration_v6_to_v7.sql").exists() else None) + if sql is None: + sql = """ +CREATE TABLE datatiles_commercial_products (product_id TEXT PRIMARY KEY,edition TEXT,issuer TEXT NOT NULL,issuer_uri TEXT,terms_uri TEXT NOT NULL,license_service_uri TEXT,protection_profile TEXT NOT NULL DEFAULT 'DataTiles-Protected-Distribution-1',created_at TEXT NOT NULL,metadata_json TEXT) WITHOUT ROWID; +CREATE TABLE datatiles_drm_policies (product_id TEXT NOT NULL REFERENCES datatiles_commercial_products(product_id) ON DELETE CASCADE,policy_id TEXT NOT NULL,policy_profile TEXT NOT NULL DEFAULT 'W3C-ODRL-2.2',policy_json TEXT NOT NULL,PRIMARY KEY(product_id,policy_id)) WITHOUT ROWID; +CREATE INDEX datatiles_drm_policy_product ON datatiles_drm_policies(product_id); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:drm_profile','DataTiles-Protected-Distribution-1'); +INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:rights_policy_profile','W3C-ODRL-2.2'); +PRAGMA user_version = 7; +""" + with self.db: self.db.executescript(sql) + + def _migrate_v7_to_v8(self) -> None: + sql = (Path(__file__).with_name("migration_v7_to_v8.sql").read_text() if Path(__file__).with_name("migration_v7_to_v8.sql").exists() else None) + if sql is None: + sql = "CREATE TABLE datatiles_release (singleton INTEGER PRIMARY KEY CHECK(singleton=1),product_id TEXT NOT NULL,version TEXT NOT NULL,sequence INTEGER NOT NULL CHECK(sequence>=1),released_at TEXT NOT NULL,previous_version TEXT,previous_identifier TEXT,release_notes_uri TEXT,update_uri TEXT); INSERT OR IGNORE INTO metadata(name,value) VALUES ('datatiles:versioning_profile','DataTiles-Release-Versioning-1'); PRAGMA user_version=8;" + with self.db: self.db.executescript(sql) + def close(self) -> None: self.db.close() @@ -196,6 +314,10 @@ def _coordinate_set(self, coordinates: Mapping[str, Any], *, create: bool) -> in else: point = _canonical(row["value_type"], value) typed = (point[0], point, None, True, True, False) + if name == "variable" and not is_interval: + policy = dict(self.db.execute("SELECT name,value FROM metadata")).get("datatiles:variable_semantics", "recommended") + if policy == "required" and not self.db.execute("SELECT 1 FROM datatiles_variables WHERE name=?", (typed[0],)).fetchone(): + raise DataTilesError(f"unregistered variable coordinate: {typed[0]}") members.append((row["dimension_id"], name, typed)) members.sort(key=lambda item:item[1]) serial = json.dumps([[name, typed[0]] for _, name, typed in members], separators=(",", ":"), ensure_ascii=False) @@ -283,6 +405,183 @@ def content_profiles(self) -> list[dict[str,Any]]: result.append(item) return result + def set_variable_semantics(self, policy: str) -> None: + if policy not in ("required", "recommended"): + raise DataTilesError("variable semantics policy must be required or recommended") + with self.db: + self.db.execute("INSERT INTO metadata(name,value) VALUES ('datatiles:variable_semantics',?) ON CONFLICT(name) DO UPDATE SET value=excluded.value", (policy,)) + + def add_variable(self, name: str, standard_name: str, *, vocabulary: str = "CF", + vocabulary_version: str | None = None, canonical_unit: str | None = None, + long_name: str | None = None, description: str | None = None) -> int: + from .semantic import validate_standard_name_syntax, SemanticValidationError + if not name or name.strip() != name: + raise DataTilesError("variable name must be non-empty and trimmed") + if not vocabulary or vocabulary.strip() != vocabulary: + raise DataTilesError("variable vocabulary must be non-empty and trimmed") + try: + validate_standard_name_syntax(standard_name) + except SemanticValidationError as exc: + raise DataTilesError(str(exc)) from exc + with self.db: + cur = self.db.execute( + "INSERT INTO datatiles_variables(name,standard_name,standard_name_vocabulary,standard_name_vocabulary_version,canonical_unit,long_name,description) VALUES (?,?,?,?,?,?,?)", + (name, standard_name, vocabulary, vocabulary_version, canonical_unit, long_name, description), + ) + return int(cur.lastrowid) + + def add_variable_identifier(self, variable: str, scheme: str, identifier: str, *, + scheme_version: str | None = None, uri: str | None = None) -> None: + if not scheme or scheme.strip() != scheme or not identifier or identifier.strip() != identifier: + raise DataTilesError("identifier scheme and value must be non-empty and trimmed") + row = self.db.execute("SELECT variable_id FROM datatiles_variables WHERE name=?", (variable,)).fetchone() + if row is None: + raise DataTilesError(f"unknown registered variable: {variable}") + with self.db: + self.db.execute("INSERT INTO datatiles_variable_identifiers(variable_id,scheme,identifier,scheme_version,uri) VALUES (?,?,?,?,?)", + (row[0], scheme, identifier, scheme_version, uri)) + + def variables(self) -> list[dict[str, Any]]: + result = [] + for row in self.db.execute("SELECT * FROM datatiles_variables ORDER BY name"): + item = dict(row) + item["identifiers"] = [dict(r) for r in self.db.execute( + "SELECT scheme,identifier,scheme_version,uri FROM datatiles_variable_identifiers WHERE variable_id=? ORDER BY scheme,identifier", + (row["variable_id"],))] + item["coordinate_set_ids"] = self.find_coordinate_sets({"variable": row["name"]}) if self.db.execute( + "SELECT 1 FROM datatiles_dimensions WHERE name='variable'").fetchone() else [] + result.append(item) + return result + + def find_coordinate_sets_by_standard_name(self, standard_name: str, *, vocabulary: str = "CF") -> list[int]: + rows = self.db.execute("SELECT name FROM datatiles_variables WHERE standard_name_vocabulary=? AND standard_name=? ORDER BY name", + (vocabulary, standard_name)).fetchall() + found: set[int] = set() + for row in rows: + found.update(self.find_coordinate_sets({"variable": row["name"]})) + return sorted(found) + + def add_identifier(self, scheme: str, identifier: str, *, uri: str | None=None, primary: bool=False) -> None: + if not scheme or not identifier: raise DataTilesError("identifier scheme and value are required") + with self.db: + if primary: self.db.execute("UPDATE datatiles_identifiers SET is_primary=0 WHERE is_primary=1") + self.db.execute("INSERT INTO datatiles_identifiers(scheme,identifier,uri,is_primary) VALUES (?,?,?,?)",(scheme,identifier,uri,int(primary))) + + def primary_identifier(self) -> dict[str,Any] | None: + row=self.db.execute("SELECT scheme,identifier,uri FROM datatiles_identifiers WHERE is_primary=1").fetchone() + return dict(row) if row else None + + def add_related_identifier(self, scheme: str, identifier: str, relation_type: str, *, uri: str | None=None, resource_type: str | None=None, relation_information: str | None=None) -> None: + with self.db: self.db.execute("INSERT INTO datatiles_related_identifiers VALUES (?,?,?,?,?,?)",(scheme,identifier,relation_type,uri,resource_type,relation_information)) + + def related_identifiers(self) -> list[dict[str,Any]]: + return [dict(r) for r in self.db.execute("SELECT scheme AS relatedIdentifierType,identifier AS relatedIdentifier,relation_type AS relationType,uri,resource_type AS resourceTypeGeneral,relation_information AS relationTypeInformation FROM datatiles_related_identifiers ORDER BY relation_type,scheme,identifier")] + + def add_rights(self, scope: str, license_expression: str, *, license_uri: str | None=None, rights_holder: str | None=None, rights_holder_uri: str | None=None, attribution_text: str | None=None, copyright_notice: str | None=None, access_rights: str="open", source_entity_id: str | None=None, applies_to: str | None=None) -> None: + from .fair import validate_spdx_expression, RIGHTS_SCOPES, ACCESS_RIGHTS, FairValidationError + if scope not in RIGHTS_SCOPES or access_rights not in ACCESS_RIGHTS: raise DataTilesError("invalid rights scope/access_rights") + try: validate_spdx_expression(license_expression) + except FairValidationError as exc: raise DataTilesError(str(exc)) from exc + with self.db: self.db.execute("INSERT INTO datatiles_rights(scope,license_expression,license_uri,rights_holder,rights_holder_uri,attribution_text,copyright_notice,access_rights,source_entity_id,applies_to) VALUES (?,?,?,?,?,?,?,?,?,?)",(scope,license_expression,license_uri,rights_holder,rights_holder_uri,attribution_text,copyright_notice,access_rights,source_entity_id,applies_to)) + + def rights(self) -> list[dict[str,Any]]: + return [dict(r) for r in self.db.execute("SELECT * FROM datatiles_rights ORDER BY scope,rights_id")] + + def assign_fair_agent_role(self, agent_id: str, role: str, *, sequence: int=0) -> None: + with self.db: self.db.execute("INSERT INTO datatiles_fair_agents VALUES (?,?,?)",(agent_id,role,sequence)) + + def fair_agents(self, *, role: str) -> list[dict[str,Any]]: + rows=self.db.execute("SELECT a.label,a.uri,a.attributes_json FROM datatiles_fair_agents f JOIN datatiles_provenance_agents a USING(agent_id) WHERE f.role=? ORDER BY f.sequence,a.agent_id",(role,)) + return [{"name":r["label"], **({"nameIdentifiers":[{"nameIdentifier":r["uri"],"nameIdentifierScheme":"URI"}]} if r["uri"] else {})} for r in rows] + + def add_publication_evidence(self, evidence_type: str, uri: str, *, checked_at: str | None=None, checksum_algorithm: str | None=None, checksum: str | None=None, notes: str | None=None) -> None: + with self.db: self.db.execute("INSERT INTO datatiles_publication_evidence VALUES (?,?,?,?,?,?)",(evidence_type,uri,checked_at,checksum_algorithm,checksum,notes)) + + def fair_report(self, *, strict_publication: bool=False) -> dict[str,Any]: + meta=self.metadata(); checks=[] + def check(code, ok, detail, external=False): checks.append({"principle":code,"status":"pass" if ok else "fail","detail":detail,"external":external}) + pid=self.primary_identifier(); check("F1/F3", bool(pid), "primary persistent identifier is explicitly recorded") + check("F2", bool(meta.get("name") and list(self.db.execute("SELECT 1 FROM datatiles_dimensions LIMIT 1"))), "descriptive and structural metadata are embedded") + evidence={r[0] for r in self.db.execute("SELECT evidence_type FROM datatiles_publication_evidence")} + check("F4", "catalogue-registration" in evidence, "catalogue registration requires external publication evidence", True) + check("A1", "landing-page" in evidence or bool(pid and pid.get("uri")), "identifier/landing page is recorded", True) + check("A2", "metadata-retention-policy" in evidence, "metadata persistence after withdrawal requires repository evidence", True) + check("I1/I2", bool(list(self.db.execute("SELECT 1 FROM datatiles_variables LIMIT 1"))) and bool(list(self.db.execute("SELECT 1 FROM datatiles_crs LIMIT 1"))), "semantic variables and CRS are machine-readable") + check("I3/R1.2", bool(list(self.db.execute("SELECT 1 FROM datatiles_provenance_entities LIMIT 1"))) and bool(list(self.db.execute("SELECT 1 FROM datatiles_provenance_activities LIMIT 1"))), "PROV-aligned entities and activities are present") + dataset_rights=list(self.db.execute("SELECT 1 FROM datatiles_rights WHERE scope='dataset' LIMIT 1")); metadata_rights=list(self.db.execute("SELECT 1 FROM datatiles_rights WHERE scope='metadata' LIMIT 1")) + check("R1.1", bool(dataset_rights and metadata_rights), "dataset and metadata rights are separately declared") + source_count=self.db.execute("SELECT count(*) FROM datatiles_provenance_entities WHERE entity_type='dataset'").fetchone()[0] + licensed_sources=self.db.execute("SELECT count(DISTINCT source_entity_id) FROM datatiles_rights WHERE scope='source' AND source_entity_id IS NOT NULL").fetchone()[0] + check("R1.1/R1.2", source_count==0 or licensed_sources>=source_count, "every source dataset entity has an explicit rights record") + check("R1.3", meta.get("datatiles:fair_profile") is not None and meta.get("datatiles:metadata_profile")=="DataCite-4.7", "community profiles are declared") + failures=[c for c in checks if c["status"]=="fail" and (strict_publication or not c["external"])] + publishable=not failures + return {"profile":"DataTiles FAIR publication profile","fair_principles":"Wilkinson et al. 2016","checks":checks, + "publishable":publishable,"passes":publishable,"strict_publication":strict_publication, + "integrity":{"optional":True,"signatures":len(self.signatures()),"note":"digital signatures strengthen integrity/provenance evidence but are not required for FAIRness"}} + + def datacite_metadata(self) -> dict[str,Any]: + from .fair import datacite_metadata; return datacite_metadata(self) + + def prov_json(self) -> dict[str,Any]: + from .fair import prov_json; return prov_json(self) + + def integrity_manifest(self, *, chunk_size: int = 100000) -> dict[str,Any]: + from .integrity import build_manifest + return build_manifest(self.db, chunk_size=chunk_size) + + def signatures(self) -> list[dict[str,Any]]: + from .integrity import list_stored_signatures + return list_stored_signatures(self.db) + + def integrity_status(self, *, recompute: bool=False, chunk_size: int=100000) -> dict[str,Any]: + meta=self.metadata(); result={"profile":meta.get("datatiles:integrity_profile","DataTiles-Integrity-Manifest-1"),"signature_profile":meta.get("datatiles:signature_profile","DataTiles-Ed25519-Signature-1"),"signatures":self.signatures(),"trust_note":"embedded public keys establish cryptographic self-consistency, not publisher trust"} + if recompute: + result["current_manifest"]=self.integrity_manifest(chunk_size=chunk_size) + return result + + def add_commercial_product(self, product_id: str, *, issuer: str, terms_uri: str, edition: str | None=None, issuer_uri: str | None=None, license_service_uri: str | None=None, metadata: dict[str,Any] | None=None) -> None: + if not product_id or not issuer or not terms_uri: raise DataTilesError("product_id, issuer, and terms_uri are required") + import json as _json + from datetime import datetime as _dt, timezone as _tz + created=_dt.now(_tz.utc).replace(microsecond=0).isoformat().replace("+00:00","Z") + with self.db: self.db.execute("INSERT INTO datatiles_commercial_products(product_id,edition,issuer,issuer_uri,terms_uri,license_service_uri,protection_profile,created_at,metadata_json) VALUES (?,?,?,?,?,?,?,?,?)",(product_id,edition,issuer,issuer_uri,terms_uri,license_service_uri,"DataTiles-Protected-Distribution-1",created,_json.dumps(metadata or {},sort_keys=True,separators=(",",":")))) + + def add_drm_policy(self, product_id: str, policy_id: str, policy: dict[str,Any], *, profile: str="W3C-ODRL-2.2") -> None: + import json as _json + if not isinstance(policy,dict): raise DataTilesError("DRM/ODRL policy must be a JSON object") + with self.db: self.db.execute("INSERT INTO datatiles_drm_policies(product_id,policy_id,policy_profile,policy_json) VALUES (?,?,?,?)",(product_id,policy_id,profile,_json.dumps(policy,sort_keys=True,separators=(",",":"),ensure_ascii=False))) + + def commercial_products(self) -> list[dict[str,Any]]: + import json as _json + rows=[] + for r in self.db.execute("SELECT * FROM datatiles_commercial_products ORDER BY product_id"): + d=dict(r); d["metadata"]=_json.loads(d.pop("metadata_json") or "{}"); rows.append(d) + return rows + + def drm_policies(self, product_id: str | None=None) -> list[dict[str,Any]]: + import json as _json + q="SELECT * FROM datatiles_drm_policies"; args=() + if product_id is not None: q+=" WHERE product_id=?"; args=(product_id,) + q+=" ORDER BY product_id,policy_id" + out=[] + for r in self.db.execute(q,args): d=dict(r); d["policy"]=_json.loads(d.pop("policy_json")); out.append(d) + return out + + def drm_status(self) -> dict[str,Any]: + meta=self.metadata() + return {"optional":True,"profile":meta.get("datatiles:drm_profile","DataTiles-Protected-Distribution-1"),"rights_policy_profile":meta.get("datatiles:rights_policy_profile","W3C-ODRL-2.2"),"products":self.commercial_products(),"policies":self.drm_policies(),"warning":"DRM enforces distribution access; it does not create copyright, relicence sources, establish scientific validity, or imply navigation safety."} + + def set_release(self, product_id: str, version: str, sequence: int, *, released_at: str, previous_version: str | None=None, previous_identifier: str | None=None, release_notes_uri: str | None=None, update_uri: str | None=None) -> None: + if not product_id or not version or int(sequence) < 1: + raise DataTilesError("product_id, version, and positive sequence are required") + with self.db: + self.db.execute("INSERT INTO datatiles_release(singleton,product_id,version,sequence,released_at,previous_version,previous_identifier,release_notes_uri,update_uri) VALUES (1,?,?,?,?,?,?,?,?) ON CONFLICT(singleton) DO UPDATE SET product_id=excluded.product_id,version=excluded.version,sequence=excluded.sequence,released_at=excluded.released_at,previous_version=excluded.previous_version,previous_identifier=excluded.previous_identifier,release_notes_uri=excluded.release_notes_uri,update_uri=excluded.update_uri", (product_id,version,int(sequence),released_at,previous_version,previous_identifier,release_notes_uri,update_uri)) + + def release(self) -> dict[str,Any] | None: + row = self.db.execute("SELECT product_id,version,sequence,released_at,previous_version,previous_identifier,release_notes_uri,update_uri FROM datatiles_release WHERE singleton=1").fetchone() + return dict(row) if row else None + def add_crs(self, role: str, *, authority: str | None = None, code: str | None = None, uri: str | None = None, wkt2: str | None = None, projjson: str | None = None, coordinate_epoch: float | None = None) -> int: @@ -507,7 +806,7 @@ def export_mbtiles(self, target: str | Path, coordinates: Mapping[str, Any] | No temporary.unlink(missing_ok=True) raise - def validate(self) -> list[str]: + def validate(self, *, cf_table: str | Path | None = None, require_variable_semantics: bool = False) -> list[str]: errors: list[str] = [] metadata_columns=[row[1] for row in self.db.execute("PRAGMA table_info(metadata)")] tiles_columns=[row[1] for row in self.db.execute("PRAGMA table_info(tiles)")] @@ -523,7 +822,7 @@ def validate(self) -> list[str]: if fk: errors.append(f"foreign-key violations: {len(fk)}") if int(self.db.execute("PRAGMA application_id").fetchone()[0]) != 0x44415441: errors.append("invalid DataTiles application_id") - if int(self.db.execute("PRAGMA user_version").fetchone()[0]) != 3: errors.append("unsupported schema revision") + if int(self.db.execute("PRAGMA user_version").fetchone()[0]) != 8: errors.append("unsupported schema revision") missing_profiles=self.db.execute("SELECT count(*) FROM datatiles_coordinate_sets s WHERE EXISTS (SELECT 1 FROM datatiles_tiles t WHERE t.coordinate_set_id=s.coordinate_set_id) AND NOT EXISTS (SELECT 1 FROM datatiles_contents c WHERE c.coordinate_set_id=s.coordinate_set_id)").fetchone()[0] if missing_profiles: errors.append(f"coordinate sets missing content profiles: {missing_profiles}") for set_row in self.db.execute("SELECT coordinate_set_id,canonical_key FROM datatiles_coordinate_sets"): @@ -558,6 +857,26 @@ def validate(self) -> list[str]: value=json.loads(blob) if not isinstance(value,dict) or value.get("type") not in ("FeatureCollection","Feature"): raise ValueError except (UnicodeDecodeError,json.JSONDecodeError,ValueError): errors.append(f"invalid GeoJSON tile: {row['coordinate_set_id']}") + policy=dict(self.db.execute("SELECT name,value FROM metadata")).get("datatiles:variable_semantics", "recommended") + variable_dim=self.db.execute("SELECT dimension_id,value_type,extent_kind FROM datatiles_dimensions WHERE name='variable'").fetchone() + if variable_dim is not None: + if variable_dim["value_type"] != "text" or variable_dim["extent_kind"] != "point": + errors.append("semantic variable dimension must be text with point extent") + if policy == "required" or require_variable_semantics: + missing=self.db.execute( + "SELECT v.canonical_value FROM datatiles_values v WHERE v.dimension_id=? AND NOT EXISTS (SELECT 1 FROM datatiles_variables r WHERE r.name=v.canonical_value) ORDER BY v.canonical_value", + (variable_dim["dimension_id"],)).fetchall() + for row in missing: errors.append(f"unregistered variable coordinate: {row[0]}") + if cf_table is not None: + try: + from .semantic import load_cf_standard_name_table, validate_cf_standard_name, SemanticValidationError + table=load_cf_standard_name_table(cf_table) + for row in self.db.execute("SELECT name,standard_name,standard_name_vocabulary,canonical_unit FROM datatiles_variables"): + if row["standard_name_vocabulary"].upper() == "CF": + try: validate_cf_standard_name(row["standard_name"], canonical_unit=row["canonical_unit"], table=table) + except SemanticValidationError as exc: errors.append(f"variable {row['name']}: {exc}") + except (OSError, ValueError) as exc: + errors.append(f"CF standard-name table: {exc}") selected=self.db.execute("SELECT coordinate_set_id FROM datatiles_selected_slice WHERE singleton=1").fetchone() if selected is None: errors.append("selected-slice singleton is missing") elif selected[0] is not None: @@ -582,18 +901,3 @@ def validate(self) -> list[str]: if not self.db.execute("SELECT 1 FROM datatiles_provenance_activities LIMIT 1").fetchone(): errors.append("missing FAIR provenance activities") if not self.db.execute("SELECT 1 FROM datatiles_crs WHERE role='horizontal' LIMIT 1").fetchone(): errors.append("missing FAIR horizontal CRS") return errors - - def fair_report(self) -> dict[str, Any]: - metadata=self.metadata(); validation=self.validate() - checks={ - "persistent_identifier":bool(metadata.get("datatiles:identifier")), - "rich_metadata":bool(metadata.get("description") and metadata.get("datatiles:keywords")), - "standard_access":bool(metadata.get("datatiles:landing_page")), - "formal_semantics":bool(self.db.execute("SELECT 1 FROM datatiles_dimensions LIMIT 1").fetchone()), - "qualified_provenance":bool(self.db.execute("SELECT 1 FROM datatiles_provenance_relations LIMIT 1").fetchone()), - "explicit_license":bool(metadata.get("datatiles:license")), - "community_crs":bool(self.db.execute("SELECT 1 FROM datatiles_crs LIMIT 1").fetchone()), - } - return {"profile":"DataTiles FAIR-by-design 1.0-draft","declared":metadata.get("datatiles:fair_profile")=="1", - "checks":checks,"container_validation":validation,"passes":all(checks.values()) and not validation, - "caveat":"Catalogue registration, PID resolution, and metadata-retention policy require repository-level verification."} diff --git a/store/AGENTS.md b/store/AGENTS.md new file mode 100644 index 0000000..2415c4a --- /dev/null +++ b/store/AGENTS.md @@ -0,0 +1,19 @@ +# AGENTS.md — DataTiles Store + +## Scope +These instructions apply to `/store`. + +## Architecture rules +- Flask is the HTTP/PWA framework and SQLAlchemy 2.x is the only database interface. +- Every material browser interaction should have `/api/v1` parity. +- Authorization is always User → Group → Role → Permission for sessions and Bearer tokens. +- Never bypass licence/safety acceptance or paid entitlement gates. +- Payment providers implement `PaymentProvider`; PayPal is a reference adapter, not a schema dependency. +- Never store payment card credentials, OAuth/SMTP/provider secrets, DRM keys, or access tokens in DataTiles metadata/provenance. +- Published DataTiles releases are immutable; a new version is a new file/release. +- Update notifications do not imply entitlement to the newer release. +- Service-worker caches exclude authenticated APIs, data tiles, downloads, checkout/payment responses, and private library data. +- Preserve FAIR/provenance/citation/licensing/signature/DRM boundaries and the Not-for-navigation notice. + +## Security and QA +Use HTTPS and secure cookies in production; keep secrets in deployment/secret storage; verify/capture provider payments server-side; never log provider secrets. Update Store docs, API/OpenAPI descriptions, tests, and PWA Help with behavior changes. diff --git a/store/LICENSE.md b/store/LICENSE.md new file mode 100644 index 0000000..0a931df --- /dev/null +++ b/store/LICENSE.md @@ -0,0 +1,206 @@ +# Apache License 2.0 + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +SPDX-License-Identifier: Apache-2.0 diff --git a/store/README.md b/store/README.md new file mode 100644 index 0000000..c316bf3 --- /dev/null +++ b/store/README.md @@ -0,0 +1,19 @@ +# DataTiles Store + +The DataTiles Store is the optional Flask/SQLAlchemy PWA for institutional publication, discovery, preview, licensing, controlled download, optional commerce, and release-update workflows. + +See [`docs/index.md`](docs/index.md) for the complete manual, also rendered in the PWA Help section. + +## Quick start + +```bash +cd store +python -m pip install -e . +python run.py +``` + +Change the administrator password and `SECRET_KEY` before deployment. Payment is disabled by default. PayPal Orders v2 is the reference provider and starts in sandbox mode when enabled. + +## Licence + +Store software is Apache License 2.0. Distributed data retains independent source/product licences. See [`LICENSE.md`](LICENSE.md). diff --git a/store/config.py b/store/config.py new file mode 100644 index 0000000..d1dfec9 --- /dev/null +++ b/store/config.py @@ -0,0 +1,36 @@ +from __future__ import annotations +import os +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent +SECRET_KEY = os.environ.get("DATATILES_STORE_SECRET_KEY", "replace-this-development-secret") +DATABASE_URL = os.environ.get("DATATILES_STORE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'store.db'}") +CATALOG_DIR = Path(os.environ.get("DATATILES_STORE_CATALOG_DIR", BASE_DIR / "catalog")) +CATALOG_EXTENSIONS = (".datatiles", ".mbtiles", ".sqlite", ".db") +MAX_CONTENT_LENGTH = int(os.environ.get("DATATILES_STORE_MAX_UPLOAD", str(8 * 1024 * 1024 * 1024))) +ALLOW_PUBLIC_CATALOG = os.environ.get("DATATILES_STORE_PUBLIC", "0") == "1" +SESSION_COOKIE_HTTPONLY = True +SESSION_COOKIE_SAMESITE = "Lax" +SESSION_COOKIE_SECURE = os.environ.get("DATATILES_STORE_SESSION_COOKIE_SECURE", "0") == "1" + +# Bootstrap administrator. Production deployments should override the password +# with a secret manager/environment value and rotate after initial provisioning. +ADMIN_USERNAME = os.environ.get("DATATILES_STORE_ADMIN_USERNAME", "admin") +ADMIN_PASSWORD = os.environ.get("DATATILES_STORE_ADMIN_PASSWORD", "change-this-admin-password") +ADMIN_GROUP = os.environ.get("DATATILES_STORE_ADMIN_GROUP", "administrators") +ADMIN_ROLE = "admin" +MANAGERS_GROUP = os.environ.get("DATATILES_STORE_MANAGERS_GROUP", "managers") + +# Versioned mandatory safety/no-liability click-through. Changing either the +# version or text invalidates previous acceptance for protected interactions. +SAFETY_AGREEMENT_VERSION = os.environ.get("DATATILES_STORE_SAFETY_VERSION", "2026-08-29-v1") +SAFETY_AGREEMENT_TEXT = os.environ.get( + "DATATILES_STORE_SAFETY_TEXT", + "NOT SUITABLE FOR NAVIGATION. Data and portrayals are provided AS IS and AS AVAILABLE, without warranties of accuracy, completeness, fitness for a particular purpose, merchantability, non-infringement, continuity, or safety. They are not official nautical charts, ENCs, ECDIS products, collision-avoidance systems, or certified navigation aids. The user must independently verify information against authoritative sources and remains responsible for all operational and navigation decisions. To the maximum extent permitted by applicable law, data distributors, licensors, contributors, institutions, software providers, maintainers, and service operators disclaim liability for loss, damage, injury, grounding, collision, delay, business interruption, or other consequences arising from use of or reliance on the data, software, portrayals, APIs, or services. Nothing in this agreement overrides mandatory rights or liabilities that cannot lawfully be excluded." +) +RECORD_ACCEPTANCE_CLIENT_METADATA = os.environ.get("DATATILES_STORE_ACCEPTANCE_CLIENT_METADATA", "1") == "1" + +# Authentication and SMTP defaults are seeded into the SQLAlchemy settings +# table at first start and are thereafter editable by administrators in the +# PWA Configuration section. Secrets should be supplied/rotated there or by +# infrastructure automation using the authenticated configuration API. diff --git a/store/datatiles_store/__init__.py b/store/datatiles_store/__init__.py new file mode 100644 index 0000000..0bb8be4 --- /dev/null +++ b/store/datatiles_store/__init__.py @@ -0,0 +1,5 @@ +def create_app(config_object=None): + from .app import create_app as factory + return factory(config_object) + +__all__ = ["create_app"] diff --git a/store/datatiles_store/admin_ext.py b/store/datatiles_store/admin_ext.py new file mode 100644 index 0000000..3bfad20 --- /dev/null +++ b/store/datatiles_store/admin_ext.py @@ -0,0 +1,83 @@ +from __future__ import annotations +import json +from pathlib import Path +from flask import abort, current_app, jsonify, render_template, request +from markdown import markdown +from sqlalchemy import select +from .db import get_db +from .models import AppSetting +from .security import api_permission_required, permission_required +from .settings import SETTINGS, BY_KEY, ensure_settings, get_setting, set_setting +from .mailer import send_mail + +DOCS_DIR=Path(__file__).resolve().parents[1]/"docs" + +def _sections(): + out={} + for d in SETTINGS: out.setdefault(d.section,[]).append(d) + return out + +def register_admin(app,csrf): + with app.app_context(): + db=app.extensions["datatiles_store_sessionmaker"](); ensure_settings(db); db.commit(); db.close() + + @app.route("/admin/configuration",methods=["GET","POST"]) + @permission_required("users.manage") + def configuration(): + db=get_db(); ensure_settings(db) + if request.method=="POST": + for d in SETTINGS: + if d.secret and not request.form.get(d.key): continue + set_setting(db,d.key,request.form.get(d.key,"0" if d.default in {"0","1"} else "")) + db.commit() + vals={d.key:("" if d.secret else get_setting(db,d.key)) for d in SETTINGS} + deployment={"DATABASE_URL":current_app.config.get("DATABASE_URL"),"CATALOG_DIR":str(current_app.config.get("CATALOG_DIR")),"CATALOG_EXTENSIONS":", ".join(current_app.config.get("CATALOG_EXTENSIONS",())),"MAX_CONTENT_LENGTH":current_app.config.get("MAX_CONTENT_LENGTH"),"SESSION_COOKIE_SECURE":current_app.config.get("SESSION_COOKIE_SECURE"),"ADMIN_GROUP":current_app.config.get("ADMIN_GROUP"),"MANAGERS_GROUP":current_app.config.get("MANAGERS_GROUP")} + return render_template("configuration.html",sections=_sections(),values=vals,deployment=deployment) + + @app.post("/admin/configuration/test-smtp") + @permission_required("users.manage") + def test_smtp(): + to=request.form.get("email","").strip(); send_mail(get_db(),to,"DataTiles Store SMTP test","SMTP configuration is working.") + return jsonify({"ok":True}) + + @app.get("/help") + def help_index(): + docs=[] + for p in sorted(DOCS_DIR.glob("*.md")): + title=p.read_text(encoding="utf-8").splitlines()[0].lstrip("# ") + docs.append({"slug":p.stem,"title":title}) + return render_template("help.html",docs=docs,content=None,title="Help") + + @app.get("/help/") + def help_doc(slug): + if not slug.replace("-","").replace("_","").isalnum(): abort(404) + p=DOCS_DIR/f"{slug}.md" + if not p.exists(): abort(404) + text=p.read_text(encoding="utf-8"); title=text.splitlines()[0].lstrip("# ") + return render_template("help.html",docs=None,content=markdown(text,extensions=["fenced_code","tables","toc"]),title=title) + + @app.get("/api/v1/configuration") + @api_permission_required("users.manage") + def api_configuration(): + db=get_db(); ensure_settings(db) + return jsonify({"runtime":{d.key:{"value":None if d.secret else get_setting(db,d.key),"secret":d.secret,"section":d.section,"label":d.label,"restart_required":d.restart} for d in SETTINGS},"deployment":{"DATABASE_URL":current_app.config.get("DATABASE_URL"),"CATALOG_DIR":str(current_app.config.get("CATALOG_DIR")),"MAX_CONTENT_LENGTH":current_app.config.get("MAX_CONTENT_LENGTH"),"SESSION_COOKIE_SECURE":current_app.config.get("SESSION_COOKIE_SECURE"),"ADMIN_GROUP":current_app.config.get("ADMIN_GROUP"),"MANAGERS_GROUP":current_app.config.get("MANAGERS_GROUP")}}) + + @app.patch("/api/v1/configuration") + @csrf.exempt + @api_permission_required("users.manage") + def api_configuration_update(): + db=get_db(); data=request.get_json(silent=True) or {} + for k,v in data.items(): + if k not in BY_KEY: return jsonify({"error":"unknown_setting","key":k}),400 + set_setting(db,k,str(v)) + db.commit(); return jsonify({"updated":sorted(data)}) + + @app.get("/api/v1/help") + def api_help_index(): + return jsonify([{"slug":p.stem,"title":p.read_text(encoding="utf-8").splitlines()[0].lstrip("# "),"url":f"/api/v1/help/{p.stem}"} for p in sorted(DOCS_DIR.glob("*.md"))]) + + @app.get("/api/v1/help/") + def api_help_doc(slug): + p=DOCS_DIR/f"{slug}.md" + if not p.exists(): abort(404) + return current_app.response_class(p.read_text(encoding="utf-8"),mimetype="text/markdown") diff --git a/store/datatiles_store/agreements.py b/store/datatiles_store/agreements.py new file mode 100644 index 0000000..172614c --- /dev/null +++ b/store/datatiles_store/agreements.py @@ -0,0 +1,75 @@ +from __future__ import annotations +import hashlib +import json +from datetime import datetime, timezone + +from flask import current_app, request +from sqlalchemy import select + +from .models import AgreementAcceptance, CatalogItem +from .db import get_db +from .settings import get_bool, get_setting + + +def canonical_rights(item: CatalogItem) -> list[dict]: + try: + rights = json.loads(item.rights_json or "[]") + if not isinstance(rights, list): rights = [] + except Exception: + rights = [] + return rights + + +def license_fingerprint(item: CatalogItem) -> str: + payload = json.dumps(canonical_rights(item), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def agreement_document(item: CatalogItem) -> dict: + rights = canonical_rights(item) + return { + "catalog_item_id": item.id, + "filename": item.filename, + "file_sha256": item.sha256, + "license_fingerprint": license_fingerprint(item), + "rights": rights, + "license_notice": "Acceptance records acknowledgement of the licence/rights terms published with this exact DataTiles object. It does not grant rights beyond those terms and does not relicense upstream data.", + "safety_version": get_setting(get_db(), "agreement.safety.version"), + "safety_text": get_setting(get_db(), "agreement.safety.text"), + } + + +def current_acceptance(db, user, item: CatalogItem): + if user is None: + return None + doc = agreement_document(item) + return db.scalar(select(AgreementAcceptance).where( + AgreementAcceptance.user_id == user.id, + AgreementAcceptance.catalog_item_id == item.id, + AgreementAcceptance.file_sha256 == item.sha256, + AgreementAcceptance.license_fingerprint == doc["license_fingerprint"], + AgreementAcceptance.safety_version == doc["safety_version"], + ).order_by(AgreementAcceptance.accepted_at.desc())) + + +def accept_current(db, user, item: CatalogItem, *, source: str) -> AgreementAcceptance: + existing = current_acceptance(db, user, item) + if existing is not None: + return existing + doc = agreement_document(item) + record_client = get_bool(db, "agreement.record_client_metadata") + acc = AgreementAcceptance( + user_id=user.id, + catalog_item_id=item.id, + file_sha256=item.sha256, + license_fingerprint=doc["license_fingerprint"], + license_snapshot_json=json.dumps(doc["rights"], sort_keys=True, separators=(",", ":")), + safety_version=doc["safety_version"], + safety_text=doc["safety_text"], + source=source, + client_ip=(request.headers.get("X-Forwarded-For", request.remote_addr) or "")[:128] if record_client else None, + user_agent=(request.user_agent.string or "")[:512] if record_client else None, + accepted_at=datetime.now(timezone.utc), + ) + db.add(acc); db.flush() + return acc diff --git a/store/datatiles_store/app.py b/store/datatiles_store/app.py new file mode 100644 index 0000000..af0d014 --- /dev/null +++ b/store/datatiles_store/app.py @@ -0,0 +1,641 @@ +from __future__ import annotations +import json +import os +import secrets +import shutil +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from flask import Flask, Response, abort, flash, jsonify, redirect, render_template, request, send_file, url_for +from flask_login import LoginManager, current_user, login_required, login_user, logout_user +from flask_wtf.csrf import CSRFProtect +from sqlalchemy import or_, select +from werkzeug.security import check_password_hash, generate_password_hash +from werkzeug.utils import secure_filename + +from .agreements import accept_current, agreement_document, current_acceptance +from .catalog import extract_metadata, index_file, resolve_item_path, scan_catalog, tile_bytes +from .db import close_db, get_db, init_engine +from .models import AgreementAcceptance, ApiToken, AuditEvent, CatalogItem, Group, Role, User, PaymentTransaction, PurchaseRecord, DownloadRecord, UpdateNotification +from .security import api_permission_required, api_user, issue_token, permission_required, safe_next_url +from .settings import get_bool, get_setting +from .auth_ext import register_auth +from .admin_ext import register_admin +from .commerce import has_purchase,is_paid,provider_from_settings,new_transaction,complete_purchase,record_download,generate_update_notifications,serialize_purchase,serialize_download,serialize_notification,money +from .payments import CheckoutRequest + +login_manager = LoginManager() +csrf = CSRFProtect() + + +def create_app(config_object=None) -> Flask: + app = Flask(__name__, static_folder="static", template_folder="templates") + app.config.from_pyfile(str(Path(__file__).resolve().parents[1] / "config.py")) + if config_object: + app.config.from_mapping(config_object) + app.config["CATALOG_DIR"] = Path(app.config["CATALOG_DIR"]).resolve() + app.config["CATALOG_DIR"].mkdir(parents=True, exist_ok=True) + login_manager.init_app(app); login_manager.login_view = "login" + csrf.init_app(app); init_engine(app); app.teardown_appcontext(close_db) + register_auth(app, csrf) + register_admin(app, csrf) + + @login_manager.user_loader + def load_user(user_id: str): + try: return get_db().get(User, int(user_id)) + except (TypeError, ValueError): return None + + @app.context_processor + def inject_globals(): + return { + "can": lambda p: current_user.is_authenticated and current_user.can(p), + "managers_group": app.config["MANAGERS_GROUP"], + "registration_enabled": (lambda: get_bool(get_db(), "auth.registration.enabled")), + "google_enabled": (lambda: get_bool(get_db(), "auth.google.enabled")), + "microsoft_enabled": (lambda: get_bool(get_db(), "auth.microsoft.enabled")), + "oauth2_enabled": (lambda: get_bool(get_db(), "auth.oauth2.enabled")), + "oauth2_name": (lambda: get_setting(get_db(), "auth.oauth2.name")), + } + + def get_item(item_id, *, include_unavailable=False): + item = get_db().get(CatalogItem, item_id) + if item is None or (not include_unavailable and not item.available): abort(404) + return item + + def audit(action: str, object_type: str, object_id=None, detail=None, user=None): + u = user if user is not None else (api_user() if request.path.startswith("/api/") else (current_user if current_user.is_authenticated else None)) + get_db().add(AuditEvent(user_id=getattr(u, "id", None), action=action, object_type=object_type, + object_id=str(object_id) if object_id is not None else None, + detail_json=json.dumps(detail or {}, sort_keys=True))) + + def browser_agreement_gate(item): + if not current_user.is_authenticated: + return redirect(url_for("login", next=request.path)) + if current_acceptance(get_db(), current_user, item) is None: + return redirect(url_for("agreement", item_id=item.id, next=request.path)) + return None + + def browser_entitlement_gate(item): + if is_paid(item) and not has_purchase(get_db(),current_user,item): return redirect(url_for("checkout",item_id=item.id)) + return None + + def api_entitlement_gate(item): + u=api_user() + if is_paid(item) and not has_purchase(get_db(),u,item): return jsonify({"error":"purchase_required","checkout_url":f"/api/v1/catalog/{item.id}/checkout","amount":item.price_amount,"currency":item.price_currency}),402 + return None + + def api_agreement_gate(item): + user = api_user() + if user is None: + return jsonify({"error":"authentication_required"}), 401 + if current_acceptance(get_db(), user, item) is None: + return jsonify({ + "error":"agreement_required", + "message":"Current data licence and safety/no-liability agreement must be accepted before data access.", + "agreement_url":f"/api/v1/catalog/{item.id}/agreement", + }), 428 + return None + + def save_uploaded(upload, *, final_name=None, replacing: CatalogItem | None = None): + if not upload or not upload.filename: + abort(400, "file required") + name = secure_filename(final_name or upload.filename) + if not name or Path(name).suffix.lower() not in app.config["CATALOG_EXTENSIONS"]: + abort(400, "unsupported catalog file extension") + final = app.config["CATALOG_DIR"] / name + if app.config["CATALOG_DIR"] not in final.resolve().parents: + abort(400, "invalid filename") + if replacing is None and final.exists(): abort(409, "catalog file already exists") + if replacing is not None: + old = resolve_item_path(replacing, app.config["CATALOG_DIR"]) + if final.exists() and final != old: abort(409, "target filename already exists") + tmp = app.config["CATALOG_DIR"] / f".{uuid.uuid4().hex}.upload" + upload.save(tmp) + try: + extract_metadata(tmp) + if replacing is not None: + old = resolve_item_path(replacing, app.config["CATALOG_DIR"]) + if old != final: old.unlink(missing_ok=True) + os.replace(tmp, final) + if replacing is not None: + replacing.relative_path = final.relative_to(app.config["CATALOG_DIR"]).as_posix() + item = index_file(get_db(), app.config["CATALOG_DIR"], final) + get_db().flush() + return item + except Exception: + tmp.unlink(missing_ok=True) + raise + + # Browser authentication and pages ------------------------------------------------- + @app.get("/") + def root(): return redirect(url_for("catalog")) + + @app.route("/login", methods=["GET","POST"]) + def login(): + if current_user.is_authenticated: return redirect(url_for("catalog")) + if request.method == "POST": + if not get_bool(get_db(), "auth.local.enabled"): + abort(403, "managed password authentication is disabled") + username = request.form.get("username", "").strip().lower() + user = get_db().scalar(select(User).where(User.username == username)) + if user and user.active and user.password_hash and check_password_hash(user.password_hash, request.form.get("password", "")): + login_user(user, remember=bool(request.form.get("remember"))) + audit("login", "user", user.id, user=user); get_db().commit() + return redirect(safe_next_url(request.args.get("next")) or url_for("catalog")) + flash("Invalid username or password.", "error") + return render_template("login.html", local_enabled=get_bool(get_db(),"auth.local.enabled"), registration_enabled=get_bool(get_db(),"auth.registration.enabled"), google_enabled=get_bool(get_db(),"auth.google.enabled"), microsoft_enabled=get_bool(get_db(),"auth.microsoft.enabled"), oauth2_enabled=get_bool(get_db(),"auth.oauth2.enabled"), oauth2_name=get_setting(get_db(),"auth.oauth2.name")) + + @app.post("/logout") + @login_required + def logout(): + uid = current_user.id; audit("logout", "user", uid); get_db().commit(); logout_user() + return redirect(url_for("login")) + + @app.get("/catalog") + @permission_required("catalog.view") + def catalog(): + q = request.args.get("q", "").strip() + stmt = select(CatalogItem).where(CatalogItem.available.is_(True)) + if q: + for term in [t.casefold() for t in q.split() if t]: stmt = stmt.where(CatalogItem.search_text.like(f"%{term}%")) + items = list(get_db().scalars(stmt.order_by(CatalogItem.title))) + return render_template("catalog.html", items=items, q=q) + + @app.get("/catalog/") + @permission_required("catalog.view") + def detail(item_id): + item = get_item(item_id); path = resolve_item_path(item, app.config["CATALOG_DIR"]) + accepted = current_acceptance(get_db(), current_user, item) if current_user.is_authenticated else None + return render_template("detail.html", item=item, detail=extract_metadata(path), agreement=agreement_document(item), accepted=accepted, purchased=(has_purchase(get_db(),current_user,item) if current_user.is_authenticated else False), paid=is_paid(item)) + + @app.route("/catalog//agreement", methods=["GET", "POST"]) + @permission_required("agreements.accept") + def agreement(item_id): + item = get_item(item_id); doc = agreement_document(item) + if request.method == "POST": + if request.form.get("accept_license") != "yes" or request.form.get("accept_safety") != "yes": + abort(400, "both licence and safety/no-liability terms must be accepted") + acc = accept_current(get_db(), current_user, item, source="web") + audit("agreement.accept", "catalog_item", item.id, {"acceptance_id":acc.id,"file_sha256":item.sha256}) + get_db().commit(); flash("Agreement acceptance recorded for this exact DataTiles release.", "info") + return redirect(safe_next_url(request.args.get("next")) or url_for("detail", item_id=item.id)) + return render_template("agreement.html", item=item, agreement=doc, accepted=current_acceptance(get_db(), current_user, item)) + + @app.get("/catalog//tiles///") + @permission_required("catalog.preview") + def tile(item_id,z,x,y): + item=get_item(item_id); gate=browser_agreement_gate(item) + if gate: return gate + if z < 0 or z > 30 or x < 0 or y < 0 or x >= (1<= (1</download") + @permission_required("catalog.download") + def download(item_id): + item=get_item(item_id); gate=browser_agreement_gate(item) + if gate: return gate + gate=browser_entitlement_gate(item) + if gate: return gate + record_download(get_db(),current_user,item,"web"); audit("catalog.download", "catalog_item", item.id, {"sha256":item.sha256}); get_db().commit() + return send_file(resolve_item_path(item, app.config["CATALOG_DIR"]), as_attachment=True, download_name=item.filename, + mimetype="application/vnd.sqlite3", conditional=True) + + @app.post("/admin/catalog/scan") + @permission_required("catalog.manage") + def scan(): + result=scan_catalog(get_db(), app.config["CATALOG_DIR"], tuple(app.config["CATALOG_EXTENSIONS"])) + notices=sum(generate_update_notifications(get_db(),x) for x in get_db().scalars(select(CatalogItem).where(CatalogItem.available.is_(True)))) + result["update_notifications"]=notices; audit("catalog.scan", "catalog", detail=result); get_db().commit() + flash(f"Catalog scan complete: {result['indexed']} indexed, {result['failed']} failed.", "info") + return redirect(url_for("catalog")) + + @app.post("/admin/catalog/upload") + @permission_required("catalog.manage") + def upload(): + item=save_uploaded(request.files.get("file")); generate_update_notifications(get_db(),item); audit("catalog.create", "catalog_item", item.id, {"filename":item.filename,"sha256":item.sha256}) + get_db().commit(); flash(f"Uploaded and indexed {item.filename}.", "info"); return redirect(url_for("catalog")) + + @app.post("/admin/catalog//replace") + @permission_required("catalog.manage") + def replace_item(item_id): + old=get_item(item_id); old_sha=old.sha256 + if old.product_id and old.product_sequence is not None: abort(409,"versioned releases are immutable; upload a successor release instead") + item=save_uploaded(request.files.get("file"), final_name=request.form.get("filename") or old.filename, replacing=old) + audit("catalog.update", "catalog_item", item.id, {"old_sha256":old_sha,"new_sha256":item.sha256}) + get_db().commit(); flash("DataTiles file replaced and re-indexed; prior agreement acceptances no longer authorize data access.","info") + return redirect(url_for("detail", item_id=item.id)) + + @app.post("/admin/catalog//commerce") + @permission_required("catalog.manage") + def catalog_commerce(item_id): + item=get_item(item_id); required=request.form.get("purchase_required")=="yes"; amount=request.form.get("price_amount","").strip(); currency=request.form.get("price_currency",get_setting(get_db(),"commerce.default_currency")).strip().upper() + item.purchase_required=required; item.price_amount=money(amount) if amount else None; item.price_currency=(currency[:3] if currency else None) + audit("catalog.commerce.update","catalog_item",item.id,{"purchase_required":required,"price_amount":item.price_amount,"price_currency":item.price_currency}); get_db().commit(); flash("Commerce settings updated.","info"); return redirect(url_for("detail",item_id=item.id)) + + @app.post("/admin/catalog//delete") + @permission_required("catalog.manage") + def delete_item(item_id): + item=get_item(item_id); path=resolve_item_path(item, app.config["CATALOG_DIR"]); snapshot={"filename":item.filename,"sha256":item.sha256} + get_db().query(AgreementAcceptance).filter(AgreementAcceptance.catalog_item_id==item.id).delete(synchronize_session=False) + get_db().delete(item); path.unlink(); audit("catalog.delete","catalog_item",item_id,snapshot); get_db().commit() + flash(f"Deleted {snapshot['filename']}.","info"); return redirect(url_for("catalog")) + + @app.route("/admin/users", methods=["GET","POST"]) + @permission_required("users.manage") + def users(): + db=get_db() + if request.method == "POST": + username=request.form.get("username","").strip(); password=request.form.get("password",""); group_name=request.form.get("group","").strip() + if not username or len(password) < 12: abort(400, "username and password of at least 12 characters required") + if db.scalar(select(User).where(User.username==username)): abort(409, "username already exists") + user=User(username=username,password_hash=generate_password_hash(password)); db.add(user) + if group_name: + group=db.scalar(select(Group).where(Group.name==group_name)) + if group is None: abort(400,"unknown group") + user.groups.append(group) + db.flush(); audit("user.create","user",user.id,{"username":username}); db.commit(); flash(f"Created user {username}.","info") + return render_template("users.html", users=list(db.scalars(select(User).order_by(User.username))), groups=list(db.scalars(select(Group).order_by(Group.name)))) + + @app.route("/admin/groups", methods=["GET","POST"]) + @permission_required("users.manage") + def groups(): + db=get_db() + if request.method == "POST": + name=request.form.get("name","").strip(); roles=request.form.getlist("roles") + if not name: abort(400,"group name required") + if db.scalar(select(Group).where(Group.name==name)): abort(409,"group exists") + group=Group(name=name); group.roles=list(db.scalars(select(Role).where(Role.name.in_(roles)))) if roles else [] + db.add(group); db.flush(); audit("group.create","group",group.id,{"name":name}); db.commit(); flash(f"Created group {name}.","info") + return render_template("groups.html", groups=list(db.scalars(select(Group).order_by(Group.name))), roles=list(db.scalars(select(Role).order_by(Role.name)))) + + @app.get("/library") + @permission_required("catalog.view") + def library(): + db=get_db(); return render_template("library.html",purchases=list(db.scalars(select(PurchaseRecord).where(PurchaseRecord.user_id==current_user.id).order_by(PurchaseRecord.purchased_at.desc()))),downloads=list(db.scalars(select(DownloadRecord).where(DownloadRecord.user_id==current_user.id).order_by(DownloadRecord.downloaded_at.desc()))),notifications=list(db.scalars(select(UpdateNotification).where(UpdateNotification.user_id==current_user.id).order_by(UpdateNotification.created_at.desc())))) + + @app.route("/catalog//checkout",methods=["GET","POST"]) + @permission_required("catalog.download") + def checkout(item_id): + item=get_item(item_id) + if not is_paid(item) or has_purchase(get_db(),current_user,item): return redirect(url_for("detail",item_id=item.id)) + if request.method=="GET": return render_template("checkout.html",item=item) + provider=provider_from_settings(get_db()); tx=new_transaction(get_db(),current_user,item,provider.name); base=get_setting(get_db(),"store.public_base_url").rstrip("/") or request.url_root.rstrip("/") + s=provider.create_checkout(CheckoutRequest(tx.public_id,item.title,tx.amount,tx.currency,base+url_for("paypal_return"),base+url_for("paypal_cancel",txid=tx.public_id))); tx.provider_order_id=s.provider_order_id; tx.status=s.status.lower(); tx.approval_url=s.approval_url; tx.detail_json=json.dumps(s.raw,sort_keys=True); get_db().commit(); return redirect(s.approval_url) + + @app.get("/payments/paypal/return") + @login_required + def paypal_return(): + oid=request.args.get("token",""); tx=get_db().scalar(select(PaymentTransaction).where(PaymentTransaction.provider_order_id==oid,PaymentTransaction.user_id==current_user.id)) + if tx is None: abort(404) + r=provider_from_settings(get_db()).capture(oid); tx.status=r.status.lower(); tx.detail_json=json.dumps(r.raw,sort_keys=True) + if not r.completed: get_db().commit(); abort(402,"payment not completed") + complete_purchase(get_db(),tx,oid); get_db().commit(); flash("Purchase completed.","info"); return redirect(url_for("library")) + + @app.get("/payments/paypal/cancel") + @login_required + def paypal_cancel(): + tx=get_db().scalar(select(PaymentTransaction).where(PaymentTransaction.public_id==request.args.get("txid",""),PaymentTransaction.user_id==current_user.id)) + if tx: tx.status="cancelled"; get_db().commit() + return redirect(url_for("catalog")) + + # API authentication --------------------------------------------------------------- + @app.post("/api/v1/auth/token") + @csrf.exempt + def api_login_token(): + data=request.get_json(silent=True) or {}; username=str(data.get("username", "")); password=str(data.get("password", "")) + user=get_db().scalar(select(User).where(User.username==username)) + if not user or not user.active or not check_password_hash(user.password_hash,password): + return jsonify({"error":"invalid_credentials"}),401 + ttl=data.get("expires_in",86400*30) + try: ttl=max(300,min(int(ttl),86400*365)) + except Exception: return jsonify({"error":"invalid_expires_in"}),400 + raw,rec=issue_token(user,name=str(data.get("name","third-party client")),expires_at=datetime.now(timezone.utc)+timedelta(seconds=ttl)) + audit("token.create","api_token",rec.id,{"name":rec.name},user=user); get_db().commit() + return jsonify({"access_token":raw,"token_type":"Bearer","expires_at":rec.expires_at.isoformat(),"user":serialize_user(user)}),201 + + @app.get("/api/v1/auth/me") + @api_permission_required("catalog.view") + def api_me(): return jsonify(serialize_user(api_user())) + + @app.get("/api/v1/auth/tokens") + @api_permission_required("catalog.view") + def api_tokens(): + u=api_user(); rows=list(get_db().scalars(select(ApiToken).where(ApiToken.user_id==u.id).order_by(ApiToken.created_at.desc()))) + return jsonify([serialize_token(x) for x in rows]) + + @app.post("/api/v1/auth/tokens") + @csrf.exempt + @api_permission_required("catalog.view") + def api_create_token(): + u=api_user(); data=request.get_json(silent=True) or {}; raw,rec=issue_token(u,name=str(data.get("name","API token"))) + audit("token.create","api_token",rec.id,{"name":rec.name},user=u); get_db().commit() + return jsonify({"access_token":raw,"token":serialize_token(rec)}),201 + + @app.delete("/api/v1/auth/tokens/") + @csrf.exempt + @api_permission_required("catalog.view") + def api_revoke_token(token_id): + u=api_user(); rec=get_db().get(ApiToken,token_id) + if rec is None or (rec.user_id != u.id and not u.can("users.manage")): return jsonify({"error":"not_found"}),404 + rec.revoked_at=datetime.now(timezone.utc); audit("token.revoke","api_token",rec.id,user=u); get_db().commit(); return "",204 + + # Catalog API ---------------------------------------------------------------------- + @app.get("/api/v1/catalog") + @api_permission_required("catalog.view") + def api_catalog(): + q=request.args.get("q","").strip().casefold(); stmt=select(CatalogItem).where(CatalogItem.available.is_(True)) + if q: + for term in [t for t in q.split() if t]: stmt=stmt.where(CatalogItem.search_text.like(f"%{term}%")) + return jsonify([serialize_item(i) for i in get_db().scalars(stmt.order_by(CatalogItem.title))]) + + @app.get("/api/v1/catalog/") + @api_permission_required("catalog.view") + def api_detail(item_id): + item=get_item(item_id); path=resolve_item_path(item,app.config["CATALOG_DIR"]) + u=api_user(); return jsonify({**serialize_item(item),**extract_metadata(path),"agreement":agreement_status(get_db(),u,item)}) + + @app.get("/api/v1/catalog//agreement") + @api_permission_required("catalog.view") + def api_agreement(item_id): + item=get_item(item_id); return jsonify(agreement_status(get_db(),api_user(),item)) + + @app.post("/api/v1/catalog//agreement/accept") + @csrf.exempt + @api_permission_required("agreements.accept") + def api_accept_agreement(item_id): + item=get_item(item_id); data=request.get_json(silent=True) or {} + if data.get("accept_license") is not True or data.get("accept_safety") is not True: + return jsonify({"error":"explicit_acceptance_required","required":["accept_license","accept_safety"]}),400 + u=api_user(); acc=accept_current(get_db(),u,item,source="api") + audit("agreement.accept","catalog_item",item.id,{"acceptance_id":acc.id,"file_sha256":item.sha256},user=u); get_db().commit() + return jsonify(agreement_status(get_db(),u,item)),201 + + @app.get("/api/v1/catalog//tiles///") + @api_permission_required("catalog.preview") + def api_tile(item_id,z,x,y): + item=get_item(item_id); gate=api_agreement_gate(item) + if gate: return gate + if z < 0 or z > 30 or x < 0 or y < 0 or x >= (1<= (1</download") + @api_permission_required("catalog.download") + def api_download(item_id): + item=get_item(item_id); gate=api_agreement_gate(item) + if gate: return gate + gate=api_entitlement_gate(item) + if gate: return gate + u=api_user(); record_download(get_db(),u,item,"api"); audit("catalog.download","catalog_item",item.id,{"sha256":item.sha256},user=u); get_db().commit() + return send_file(resolve_item_path(item,app.config["CATALOG_DIR"]),as_attachment=True,download_name=item.filename,mimetype="application/vnd.sqlite3",conditional=True) + + @app.post("/api/v1/catalog") + @csrf.exempt + @api_permission_required("catalog.manage") + def api_create_catalog_item(): + item=save_uploaded(request.files.get("file"),final_name=request.form.get("filename") or None) + generate_update_notifications(get_db(),item) + audit("catalog.create","catalog_item",item.id,{"filename":item.filename,"sha256":item.sha256},user=api_user()); get_db().commit() + return jsonify(serialize_item(item)),201 + + @app.put("/api/v1/catalog//file") + @csrf.exempt + @api_permission_required("catalog.manage") + def api_replace_catalog_item(item_id): + old=get_item(item_id); old_sha=old.sha256 + if old.product_id and old.product_sequence is not None: return jsonify({"error":"immutable_versioned_release","message":"upload a successor release instead"}),409 + item=save_uploaded(request.files.get("file"),final_name=request.form.get("filename") or old.filename,replacing=old) + audit("catalog.update","catalog_item",item.id,{"old_sha256":old_sha,"new_sha256":item.sha256},user=api_user()); get_db().commit() + return jsonify({**serialize_item(item),"agreement_reacceptance_required":True}) + + @app.patch("/api/v1/catalog/") + @csrf.exempt + @api_permission_required("catalog.manage") + def api_rename_catalog_item(item_id): + item=get_item(item_id); data=request.get_json(silent=True) or {} + if "purchase_required" in data: item.purchase_required=bool(data["purchase_required"]) + if "price_amount" in data: item.price_amount=money(str(data["price_amount"])) if data["price_amount"] not in (None,"") else None + if "price_currency" in data: item.price_currency=str(data["price_currency"] or "").upper()[:3] or None + if "filename" not in data: get_db().commit(); return jsonify(serialize_item(item)) + name=secure_filename(str(data.get("filename", ""))) + if not name or Path(name).suffix.lower() not in app.config["CATALOG_EXTENSIONS"]: return jsonify({"error":"invalid_filename"}),400 + old=resolve_item_path(item,app.config["CATALOG_DIR"]); final=app.config["CATALOG_DIR"]/name + if final.exists() and final != old: return jsonify({"error":"target_exists"}),409 + old.rename(final); item.relative_path=final.relative_to(app.config["CATALOG_DIR"]).as_posix(); item.filename=name + audit("catalog.rename","catalog_item",item.id,{"filename":name},user=api_user()); get_db().commit(); return jsonify(serialize_item(item)) + + @app.delete("/api/v1/catalog/") + @csrf.exempt + @api_permission_required("catalog.manage") + def api_delete_catalog_item(item_id): + item=get_item(item_id); path=resolve_item_path(item,app.config["CATALOG_DIR"]); snapshot={"filename":item.filename,"sha256":item.sha256} + get_db().query(AgreementAcceptance).filter(AgreementAcceptance.catalog_item_id==item.id).delete(synchronize_session=False) + get_db().delete(item); path.unlink(); audit("catalog.delete","catalog_item",item_id,snapshot,user=api_user()); get_db().commit(); return "",204 + + @app.post("/api/v1/catalog/scan") + @csrf.exempt + @api_permission_required("catalog.manage") + def api_scan(): + result=scan_catalog(get_db(),app.config["CATALOG_DIR"],tuple(app.config["CATALOG_EXTENSIONS"])); result["update_notifications"]=sum(generate_update_notifications(get_db(),x) for x in get_db().scalars(select(CatalogItem).where(CatalogItem.available.is_(True)))); audit("catalog.scan","catalog",detail=result,user=api_user()); get_db().commit(); return jsonify(result) + + @app.get("/api/v1/payments/providers") + @api_permission_required("catalog.view") + def api_payment_providers(): + db=get_db(); return jsonify({"enabled":get_bool(db,"commerce.enabled"),"selected":get_setting(db,"commerce.provider"),"providers":{"paypal":{"enabled":get_bool(db,"payments.paypal.enabled"),"mode":get_setting(db,"payments.paypal.mode")}}}) + + @app.post("/api/v1/catalog//checkout") + @csrf.exempt + @api_permission_required("catalog.download") + def api_checkout(item_id): + item=get_item(item_id); u=api_user() + if not is_paid(item): return jsonify({"purchase_required":False}) + if has_purchase(get_db(),u,item): return jsonify({"purchased":True}) + provider=provider_from_settings(get_db()); tx=new_transaction(get_db(),u,item,provider.name); data=request.get_json(silent=True) or {}; base=get_setting(get_db(),"store.public_base_url").rstrip("/") or request.url_root.rstrip("/") + s=provider.create_checkout(CheckoutRequest(tx.public_id,item.title,tx.amount,tx.currency,str(data.get("return_url") or base+"/catalog"),str(data.get("cancel_url") or base+"/catalog"))); tx.provider_order_id=s.provider_order_id; tx.status=s.status.lower(); tx.approval_url=s.approval_url; tx.detail_json=json.dumps(s.raw,sort_keys=True); get_db().commit(); return jsonify({"transaction_id":tx.public_id,"provider":provider.name,"provider_order_id":s.provider_order_id,"approval_url":s.approval_url,"status":s.status}),201 + + @app.post("/api/v1/payments//capture") + @csrf.exempt + @api_permission_required("catalog.download") + def api_capture_payment(txid): + u=api_user(); tx=get_db().scalar(select(PaymentTransaction).where(PaymentTransaction.public_id==txid,PaymentTransaction.user_id==u.id)) + if tx is None: return jsonify({"error":"not_found"}),404 + r=provider_from_settings(get_db()).capture(tx.provider_order_id); tx.status=r.status.lower(); purchase=complete_purchase(get_db(),tx,tx.provider_order_id) if r.completed else None; get_db().commit(); return jsonify({"completed":r.completed,"status":r.status,"purchase":serialize_purchase(purchase) if purchase else None}) + + @app.get("/api/v1/library") + @api_permission_required("catalog.view") + def api_library(): + u=api_user(); db=get_db(); return jsonify({"purchases":[serialize_purchase(x) for x in db.scalars(select(PurchaseRecord).where(PurchaseRecord.user_id==u.id).order_by(PurchaseRecord.purchased_at.desc()))],"downloads":[serialize_download(x) for x in db.scalars(select(DownloadRecord).where(DownloadRecord.user_id==u.id).order_by(DownloadRecord.downloaded_at.desc()))],"notifications":[serialize_notification(x) for x in db.scalars(select(UpdateNotification).where(UpdateNotification.user_id==u.id).order_by(UpdateNotification.created_at.desc()))]}) + + @app.get("/api/v1/notifications") + @api_permission_required("catalog.view") + def api_notifications(): + u=api_user(); return jsonify([serialize_notification(x) for x in get_db().scalars(select(UpdateNotification).where(UpdateNotification.user_id==u.id).order_by(UpdateNotification.created_at.desc()))]) + + @app.patch("/api/v1/notifications/") + @csrf.exempt + @api_permission_required("catalog.view") + def api_notification_read(nid): + u=api_user(); n=get_db().get(UpdateNotification,nid) + if n is None or n.user_id!=u.id: return jsonify({"error":"not_found"}),404 + n.read_at=datetime.now(timezone.utc) if (request.get_json(silent=True) or {}).get("read",True) else None; get_db().commit(); return jsonify(serialize_notification(n)) + + # Administration API --------------------------------------------------------------- + @app.get("/api/v1/users") + @api_permission_required("users.manage") + def api_users(): return jsonify([serialize_user(x) for x in get_db().scalars(select(User).order_by(User.username))]) + + @app.post("/api/v1/users") + @csrf.exempt + @api_permission_required("users.manage") + def api_create_user(): + data=request.get_json(silent=True) or {}; username=str(data.get("username","")).strip(); password=str(data.get("password","")); groups=data.get("groups",[]) + if not username or len(password)<12: return jsonify({"error":"invalid_user"}),400 + if get_db().scalar(select(User).where(User.username==username)): return jsonify({"error":"username_exists"}),409 + user=User(username=username,password_hash=generate_password_hash(password)); get_db().add(user); get_db().flush() + if groups: user.groups=list(get_db().scalars(select(Group).where(Group.name.in_([str(x) for x in groups])))) + audit("user.create","user",user.id,{"username":username},user=api_user()); get_db().commit(); return jsonify(serialize_user(user)),201 + + @app.patch("/api/v1/users/") + @csrf.exempt + @api_permission_required("users.manage") + def api_update_user(user_id): + user=get_db().get(User,user_id) + if user is None: return jsonify({"error":"not_found"}),404 + data=request.get_json(silent=True) or {} + if "active" in data: user.active=bool(data["active"]) + if data.get("password"): + if len(str(data["password"]))<12: return jsonify({"error":"password_too_short"}),400 + user.password_hash=generate_password_hash(str(data["password"])) + if "groups" in data: user.groups=list(get_db().scalars(select(Group).where(Group.name.in_([str(x) for x in data["groups"]])))) + audit("user.update","user",user.id,user=api_user()); get_db().commit(); return jsonify(serialize_user(user)) + + @app.delete("/api/v1/users/") + @csrf.exempt + @api_permission_required("users.manage") + def api_delete_user(user_id): + actor=api_user(); user=get_db().get(User,user_id) + if user is None: return jsonify({"error":"not_found"}),404 + if user.id==actor.id: return jsonify({"error":"cannot_delete_current_user"}),409 + get_db().delete(user); audit("user.delete","user",user_id,user=actor); get_db().commit(); return "",204 + + @app.get("/api/v1/groups") + @api_permission_required("users.manage") + def api_groups(): return jsonify([serialize_group(x) for x in get_db().scalars(select(Group).order_by(Group.name))]) + + @app.post("/api/v1/groups") + @csrf.exempt + @api_permission_required("users.manage") + def api_create_group(): + data=request.get_json(silent=True) or {}; name=str(data.get("name","")).strip(); roles=[str(x) for x in data.get("roles",[])] + if not name: return jsonify({"error":"name_required"}),400 + if get_db().scalar(select(Group).where(Group.name==name)): return jsonify({"error":"group_exists"}),409 + group=Group(name=name); group.roles=list(get_db().scalars(select(Role).where(Role.name.in_(roles)))) if roles else [] + get_db().add(group); get_db().flush(); audit("group.create","group",group.id,{"name":name},user=api_user()); get_db().commit(); return jsonify(serialize_group(group)),201 + + @app.patch("/api/v1/groups/") + @csrf.exempt + @api_permission_required("users.manage") + def api_update_group(group_id): + group=get_db().get(Group,group_id) + if group is None: return jsonify({"error":"not_found"}),404 + data=request.get_json(silent=True) or {} + if "roles" in data: group.roles=list(get_db().scalars(select(Role).where(Role.name.in_([str(x) for x in data["roles"]])))) + audit("group.update","group",group.id,user=api_user()); get_db().commit(); return jsonify(serialize_group(group)) + + @app.delete("/api/v1/groups/") + @csrf.exempt + @api_permission_required("users.manage") + def api_delete_group(group_id): + group=get_db().get(Group,group_id) + if group is None: return jsonify({"error":"not_found"}),404 + if group.name in {app.config["ADMIN_GROUP"],app.config["MANAGERS_GROUP"]}: return jsonify({"error":"protected_group"}),409 + get_db().delete(group); audit("group.delete","group",group_id,user=api_user()); get_db().commit(); return "",204 + + @app.get("/api/v1/audit") + @api_permission_required("users.manage") + def api_audit(): + limit=min(max(int(request.args.get("limit",100)),1),1000) + rows=list(get_db().scalars(select(AuditEvent).order_by(AuditEvent.created_at.desc()).limit(limit))) + return jsonify([{"id":x.id,"user_id":x.user_id,"action":x.action,"object_type":x.object_type,"object_id":x.object_id,"detail":json.loads(x.detail_json),"created_at":x.created_at.isoformat()} for x in rows]) + + @app.get("/api/v1") + def api_root(): + return jsonify({"name":"DataTiles Store API","version":"v1","openapi":"/api/v1/openapi.json"}) + + @app.get("/api/v1/openapi.json") + def api_openapi(): return jsonify(openapi_document()) + + @app.get("/manifest.webmanifest") + def manifest(): return app.send_static_file("manifest.webmanifest") + + @app.get("/service-worker.js") + def service_worker(): + response=app.send_static_file("service-worker.js"); response.headers["Service-Worker-Allowed"]="/"; return response + + @app.get("/healthz") + def health(): return {"status":"ok"} + + return app + + +def agreement_status(db, user, item): + doc=agreement_document(item); acc=current_acceptance(db,user,item) if user is not None else None + return {**doc,"accepted":acc is not None,"acceptance":({"id":acc.id,"accepted_at":acc.accepted_at.isoformat(),"source":acc.source} if acc else None)} + + +def serialize_item(i: CatalogItem): + return {"id":i.id,"filename":i.filename,"title":i.title,"description":i.description,"format":i.format,"schema_revision":i.schema_revision,"size_bytes":i.size_bytes,"sha256":i.sha256,"bounds":[i.bounds_west,i.bounds_south,i.bounds_east,i.bounds_north],"minzoom":i.minzoom,"maxzoom":i.maxzoom,"indexed_at":i.indexed_at.isoformat(),"available":i.available,"product_id":i.product_id,"product_version":i.product_version,"product_sequence":i.product_sequence,"released_at":i.released_at,"previous_version":i.previous_version,"release_notes_uri":i.release_notes_uri,"update_uri":i.update_uri,"purchase_required":i.purchase_required,"price_amount":i.price_amount,"price_currency":i.price_currency} + + +def serialize_user(u: User): + return {"id":u.id,"username":u.username,"active":u.active,"groups":[g.name for g in u.groups],"permissions":sorted(u.permissions()),"created_at":u.created_at.isoformat()} + + +def serialize_group(g: Group): + return {"id":g.id,"name":g.name,"roles":[r.name for r in g.roles],"users":[u.username for u in g.users]} + + +def serialize_token(t: ApiToken): + return {"id":t.id,"name":t.name,"prefix":t.token_prefix,"created_at":t.created_at.isoformat(),"expires_at":t.expires_at.isoformat() if t.expires_at else None, + "last_used_at":t.last_used_at.isoformat() if t.last_used_at else None,"revoked_at":t.revoked_at.isoformat() if t.revoked_at else None} + + +def openapi_document(): + return { + "openapi":"3.1.0","info":{"title":"DataTiles Store API","version":"1.0.0"}, + "components":{"securitySchemes":{"bearerAuth":{"type":"http","scheme":"bearer"}}}, + "paths":{ + "/api/v1/auth/token":{"post":{"summary":"Exchange managed user credentials for a Bearer API token"}}, + "/api/v1/auth/providers":{"get":{"summary":"Discover enabled authentication and registration methods"}}, + "/api/v1/auth/external/{provider}":{"get":{"summary":"Start Google, Microsoft, or generic OIDC authorization"}}, + "/api/v1/register":{"post":{"summary":"Register managed account and send email verification"}}, + "/api/v1/verify-email":{"post":{"summary":"Verify registered email token"}}, + "/api/v1/configuration":{"get":{"summary":"Administrator configuration"},"patch":{"summary":"Update runtime configuration"}}, + "/api/v1/help":{"get":{"summary":"List Store help documents"}}, + "/api/v1/auth/me":{"get":{"summary":"Current API identity","security":[{"bearerAuth":[]}]}}, + "/api/v1/catalog":{"get":{"summary":"Search catalog"},"post":{"summary":"Upload/Create DataTiles asset","security":[{"bearerAuth":[]}]}}, + "/api/v1/catalog/{id}":{"get":{"summary":"Read catalog metadata"},"patch":{"summary":"Rename catalog asset"},"delete":{"summary":"Delete catalog asset"}}, + "/api/v1/catalog/{id}/file":{"put":{"summary":"Replace/Update DataTiles file"}}, + "/api/v1/catalog/{id}/agreement":{"get":{"summary":"Read current data licence and safety agreement/status"}}, + "/api/v1/catalog/{id}/agreement/accept":{"post":{"summary":"Explicitly accept current licence and safety/no-liability agreement"}}, + "/api/v1/catalog/{id}/download":{"get":{"summary":"Download exact DataTiles release after agreement acceptance and any required purchase entitlement"}}, + "/api/v1/payments/providers":{"get":{"summary":"Discover optional payment providers"}}, + "/api/v1/catalog/{id}/checkout":{"post":{"summary":"Create a provider-neutral checkout transaction"}}, + "/api/v1/payments/{transaction_id}/capture":{"post":{"summary":"Capture an approved payment"}}, + "/api/v1/library":{"get":{"summary":"List current user purchases, downloads, and update notifications"}}, + "/api/v1/notifications":{"get":{"summary":"List release update notifications"}}, + "/api/v1/catalog/{id}/tiles/{z}/{x}/{y}":{"get":{"summary":"Preview portrayal tile after acceptance"}}, + "/api/v1/catalog/scan":{"post":{"summary":"Rescan catalog directory"}}, + "/api/v1/users":{"get":{"summary":"List users"},"post":{"summary":"Create user"}}, + "/api/v1/groups":{"get":{"summary":"List groups"},"post":{"summary":"Create group"}}, + "/api/v1/audit":{"get":{"summary":"Read audit events"}}, + } + } diff --git a/store/datatiles_store/auth_ext.py b/store/datatiles_store/auth_ext.py new file mode 100644 index 0000000..54fe776 --- /dev/null +++ b/store/datatiles_store/auth_ext.py @@ -0,0 +1,125 @@ +from __future__ import annotations +import re +from urllib.parse import urljoin +from flask import abort, current_app, flash, jsonify, redirect, render_template, request, url_for +from flask_login import current_user, login_user +from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired +from sqlalchemy import select +from werkzeug.security import generate_password_hash +from authlib.integrations.flask_client import OAuth +from .db import get_db +from .mailer import send_mail +from .models import ExternalIdentity, Group, User +from .settings import get_bool,get_setting + +EMAIL_RE=re.compile(r"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$") + +def _serializer(): return URLSafeTimedSerializer(current_app.config["SECRET_KEY"],salt="datatiles-email-verification-v1") +def _base_url(): return get_setting(get_db(),"store.public_base_url").rstrip("/") or request.url_root.rstrip("/") +def _default_group(db): return db.scalar(select(Group).where(Group.name==get_setting(db,"auth.registration.default_group"))) + +def register_auth(app,csrf): + oauth=OAuth(app) + def provider_config(name): + db=get_db() + if name=="google" and get_bool(db,"auth.google.enabled"): + return dict(metadata="https://accounts.google.com/.well-known/openid-configuration",client_id=get_setting(db,"auth.google.client_id"),client_secret=get_setting(db,"auth.google.client_secret"),scope="openid email profile") + if name=="microsoft" and get_bool(db,"auth.microsoft.enabled"): + t=get_setting(db,"auth.microsoft.tenant") or "common" + return dict(metadata=f"https://login.microsoftonline.com/{t}/v2.0/.well-known/openid-configuration",client_id=get_setting(db,"auth.microsoft.client_id"),client_secret=get_setting(db,"auth.microsoft.client_secret"),scope="openid email profile") + if name=="oauth2" and get_bool(db,"auth.oauth2.enabled"): + return dict(metadata=get_setting(db,"auth.oauth2.metadata_url"),client_id=get_setting(db,"auth.oauth2.client_id"),client_secret=get_setting(db,"auth.oauth2.client_secret"),scope=get_setting(db,"auth.oauth2.scopes")) + return None + def client(name): + c=provider_config(name) + if not c: abort(404) + # Re-register to honor live admin configuration. + oauth._clients.pop(name,None) + return oauth.register(name,server_metadata_url=c["metadata"],client_id=c["client_id"],client_secret=c["client_secret"],client_kwargs={"scope":c["scope"]}) + + @app.get("/auth/providers") + def auth_providers(): + db=get_db(); return jsonify({"local":get_bool(db,"auth.local.enabled"),"registration":get_bool(db,"auth.registration.enabled"),"google":get_bool(db,"auth.google.enabled"),"microsoft":get_bool(db,"auth.microsoft.enabled"),"oauth2":get_bool(db,"auth.oauth2.enabled"),"oauth2_name":get_setting(db,"auth.oauth2.name")}) + + @app.get("/api/v1/auth/providers") + def api_auth_providers(): + db=get_db(); return jsonify({"local":get_bool(db,"auth.local.enabled"),"registration":get_bool(db,"auth.registration.enabled"),"providers":{"google":{"enabled":get_bool(db,"auth.google.enabled"),"authorize_url":"/api/v1/auth/external/google"},"microsoft":{"enabled":get_bool(db,"auth.microsoft.enabled"),"authorize_url":"/api/v1/auth/external/microsoft"},"oauth2":{"enabled":get_bool(db,"auth.oauth2.enabled"),"name":get_setting(db,"auth.oauth2.name"),"authorize_url":"/api/v1/auth/external/oauth2"}}}) + + @app.get("/api/v1/auth/external/") + def api_external_login(provider): + return client(provider).authorize_redirect(urljoin(_base_url()+"/",url_for("external_callback",provider=provider).lstrip("/"))) + + @app.route("/register",methods=["GET","POST"]) + def register(): + db=get_db() + if not get_bool(db,"auth.registration.enabled"): abort(404) + if request.method=="POST": + if not get_bool(db,"auth.local.enabled"): abort(403) + email=request.form.get("email","").strip().lower(); password=request.form.get("password","") + if not EMAIL_RE.match(email) or len(password)<12: abort(400,"valid email and password of at least 12 characters required") + if db.scalar(select(User).where(User.username==email)): abort(409,"account already exists") + user=User(username=email,email=email,email_verified=False,password_hash=generate_password_hash(password),active=False); db.add(user) + group=_default_group(db) + if group: user.groups.append(group) + db.commit(); token=_serializer().dumps({"uid":user.id,"email":email}) + verify=urljoin(_base_url()+"/",url_for("verify_email",token=token).lstrip("/")) + send_mail(db,email,"Verify your DataTiles Store account",f"Verify your email address by opening:\n\n{verify}\n\nIf you did not request this account, ignore this message.") + flash("Registration received. Check your email to verify the account.","info"); return redirect(url_for("login")) + return render_template("register.html") + + @app.get("/verify-email/") + def verify_email(token): + try: data=_serializer().loads(token,max_age=86400) + except SignatureExpired: abort(410,"verification link expired") + except BadSignature: abort(400,"invalid verification link") + db=get_db(); user=db.get(User,int(data["uid"])) + if not user or user.email!=data["email"]: abort(400) + user.email_verified=True; user.active=True; db.commit(); flash("Email verified. You can sign in.","info"); return redirect(url_for("login")) + + + @app.post("/api/v1/register") + @csrf.exempt + def api_register(): + db=get_db() + if not get_bool(db,"auth.registration.enabled") or not get_bool(db,"auth.local.enabled"): + return jsonify({"error":"registration_disabled"}),403 + data=request.get_json(silent=True) or {}; email=str(data.get("email","")).strip().lower(); password=str(data.get("password","")) + if not EMAIL_RE.match(email) or len(password)<12: return jsonify({"error":"invalid_registration"}),400 + if db.scalar(select(User).where(User.username==email)): return jsonify({"error":"account_exists"}),409 + user=User(username=email,email=email,email_verified=False,password_hash=generate_password_hash(password),active=False); db.add(user); group=_default_group(db) + if group: user.groups.append(group) + db.commit(); token=_serializer().dumps({"uid":user.id,"email":email}); verify=urljoin(_base_url()+"/",url_for("verify_email",token=token).lstrip("/")) + send_mail(db,email,"Verify your DataTiles Store account",f"Verify your email address by opening:\n\n{verify}\n") + return jsonify({"status":"verification_required","email":email}),201 + + @app.post("/api/v1/verify-email") + @csrf.exempt + def api_verify_email(): + token=str((request.get_json(silent=True) or {}).get("token","")) + try: data=_serializer().loads(token,max_age=86400) + except SignatureExpired: return jsonify({"error":"verification_expired"}),410 + except BadSignature: return jsonify({"error":"invalid_verification"}),400 + db=get_db(); user=db.get(User,int(data["uid"])) + if not user or user.email!=data["email"]: return jsonify({"error":"invalid_verification"}),400 + user.email_verified=True; user.active=True; db.commit(); return jsonify({"verified":True,"email":user.email}) + + @app.get("/auth/") + def external_login(provider): + return client(provider).authorize_redirect(urljoin(_base_url()+"/",url_for("external_callback",provider=provider).lstrip("/"))) + + @app.get("/auth//callback") + def external_callback(provider): + db=get_db(); c=client(provider); token=c.authorize_access_token(); info=token.get("userinfo") or c.userinfo(token=token) + sub=str(info.get("sub") or info.get("id") or ""); email=str(info.get("email") or info.get("preferred_username") or "").lower() + if not sub or not EMAIL_RE.match(email): abort(403,"identity provider did not supply a usable email identity") + if provider=="google": + allowed=[x.strip().lower() for x in get_setting(db,"auth.google.allowed_domains").split(",") if x.strip()] + if allowed and email.split("@")[-1] not in allowed: abort(403,"Google Workspace domain is not allowed") + identity=db.scalar(select(ExternalIdentity).where(ExternalIdentity.provider==provider,ExternalIdentity.subject==sub)) + user=identity.user if identity else db.scalar(select(User).where(User.username==email)) + if user is None: + if not get_bool(db,"auth.registration.enabled"): abort(403,"new-user registration is disabled") + user=User(username=email,email=email,email_verified=True,password_hash="!external",active=True); db.add(user); db.flush(); group=_default_group(db) + if group: user.groups.append(group) + if identity is None: db.add(ExternalIdentity(user_id=user.id,provider=provider,subject=sub,email=email)) + user.email=email; user.email_verified=True; user.active=True; db.commit(); login_user(user); return redirect(url_for("catalog")) diff --git a/store/datatiles_store/catalog.py b/store/datatiles_store/catalog.py new file mode 100644 index 0000000..528544e --- /dev/null +++ b/store/datatiles_store/catalog.py @@ -0,0 +1,146 @@ +from __future__ import annotations +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import quote + +from sqlalchemy import create_engine, select +from sqlalchemy.engine import Connection +from sqlalchemy.orm import Session +from sqlalchemy.pool import NullPool + +from .models import CatalogItem + + +def _ro_engine(path: Path): + # SQLite URI mode=ro keeps published DataTiles immutable. SQLAlchemy remains the + # sole database interface for both the Store DB and inspected DataTiles files. + uri = f"sqlite+pysqlite:///file:{quote(str(path.resolve()), safe='/')}?mode=ro&uri=true" + return create_engine(uri, future=True, poolclass=NullPool) + + +def _objects(con: Connection) -> set[str]: + return {r[0] for r in con.exec_driver_sql("SELECT name FROM sqlite_master WHERE type IN ('table','view')")} + + +def _safe_rows(con: Connection, sql: str, params=(), limit=10000) -> list[dict[str, Any]]: + try: + result = con.exec_driver_sql(sql, params) + return [dict(r._mapping) for r in result.fetchmany(limit)] + except Exception: + return [] + + +def sha256_file(path: Path, block=4 * 1024 * 1024) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + while chunk := fh.read(block): h.update(chunk) + return h.hexdigest() + + +def extract_metadata(path: Path) -> dict[str, Any]: + engine = _ro_engine(path) + try: + with engine.connect() as con: + con.exec_driver_sql("PRAGMA query_only=ON") + objects = _objects(con) + metadata = {r["name"]: r["value"] for r in _safe_rows(con, "SELECT name,value FROM metadata") } if "metadata" in objects else {} + revision = int(con.exec_driver_sql("PRAGMA user_version").scalar_one()) + variables = _safe_rows(con, "SELECT name,standard_name,standard_name_vocabulary,standard_name_vocabulary_version,canonical_unit,long_name,description FROM datatiles_variables ORDER BY name") if "datatiles_variables" in objects else [] + rights = _safe_rows(con, "SELECT scope,license_expression,license_uri,rights_holder,attribution_text,access_rights,applies_to FROM datatiles_rights ORDER BY scope,rights_id") if "datatiles_rights" in objects else [] + entities = _safe_rows(con, "SELECT entity_id,entity_type,label,uri,checksum_algorithm,checksum FROM datatiles_provenance_entities ORDER BY entity_id", limit=500) if "datatiles_provenance_entities" in objects else [] + activities = _safe_rows(con, "SELECT activity_id,activity_type,label,started_at,ended_at,software_name,software_version FROM datatiles_provenance_activities ORDER BY activity_id", limit=500) if "datatiles_provenance_activities" in objects else [] + contents = _safe_rows(con, "SELECT * FROM datatiles_contents ORDER BY content_id", limit=500) if "datatiles_contents" in objects else [] + dims = _safe_rows(con, "SELECT name,kind,unit,description FROM datatiles_dimensions ORDER BY dimension_id", limit=500) if "datatiles_dimensions" in objects else [] + integrity = _safe_rows(con, "SELECT signature_id,algorithm,key_id,signer,signed_at,manifest_root_sha256 FROM datatiles_signatures ORDER BY signed_at", limit=200) if "datatiles_signatures" in objects else [] + commercial = _safe_rows(con, "SELECT product_id,title,edition,issuer,terms_uri,license_service_uri FROM datatiles_commercial_products", limit=100) if "datatiles_commercial_products" in objects else [] + release_rows = _safe_rows(con, "SELECT product_id,version,sequence,released_at,previous_version,previous_identifier,release_notes_uri,update_uri FROM datatiles_release WHERE singleton=1", limit=1) if "datatiles_release" in objects else [] + finally: + engine.dispose() + bounds = metadata.get("bounds", "").split(",") + try: + parsed_bounds = [float(x) for x in bounds] if len(bounds) == 4 else None + except ValueError: + parsed_bounds = None + return { + "metadata": metadata, "revision": revision, "variables": variables, "rights": rights, + "provenance": {"entities": entities, "activities": activities}, "contents": contents, + "dimensions": dims, "integrity": integrity, "commercial": commercial, "release": (release_rows[0] if release_rows else {}), "bounds": parsed_bounds, + } + + +def index_file(db: Session, catalog_root: Path, path: Path) -> CatalogItem: + path = path.resolve(); root = catalog_root.resolve() + if root not in path.parents: raise ValueError("catalog file escapes configured catalog root") + rel = path.relative_to(root).as_posix(); st = path.stat() + existing = db.scalar(select(CatalogItem).where(CatalogItem.relative_path == rel)) + if existing and existing.file_mtime_ns == st.st_mtime_ns and existing.size_bytes == st.st_size: + existing.available = True; return existing + ext = extract_metadata(path); meta = ext["metadata"] + title = meta.get("name") or meta.get("title") or path.stem + desc = meta.get("description") or meta.get("abstract"); b = ext["bounds"] + search_parts = [title, desc or "", path.name] + search_parts.extend(f"{k} {v}" for k,v in meta.items()) + search_parts.extend(" ".join(str(v or "") for v in x.values()) for x in ext["variables"]) + search_parts.extend(" ".join(str(v or "") for v in x.values()) for x in ext["rights"]) + for e in ext["provenance"]["entities"]: + search_parts.extend(str(e.get(k) or "") for k in ("label","uri","entity_type")) + values = dict( + filename=path.name, title=title, description=desc, format=meta.get("format", "DataTiles"), + schema_revision=ext["revision"], size_bytes=st.st_size, file_mtime_ns=st.st_mtime_ns, sha256=sha256_file(path), + bounds_west=str(b[0]) if b else None, bounds_south=str(b[1]) if b else None, + bounds_east=str(b[2]) if b else None, bounds_north=str(b[3]) if b else None, + minzoom=_int_or_none(meta.get("minzoom")), maxzoom=_int_or_none(meta.get("maxzoom")), + metadata_json=json.dumps(meta, sort_keys=True), variables_json=json.dumps(ext["variables"], sort_keys=True), + rights_json=json.dumps(ext["rights"], sort_keys=True), provenance_json=json.dumps(ext["provenance"], sort_keys=True), + search_text="\n".join(search_parts).casefold(), indexed_at=datetime.now(timezone.utc), available=True, + product_id=(ext.get("release") or {}).get("product_id"), product_version=(ext.get("release") or {}).get("version"), product_sequence=_int_or_none((ext.get("release") or {}).get("sequence")), released_at=(ext.get("release") or {}).get("released_at"), previous_version=(ext.get("release") or {}).get("previous_version"), release_notes_uri=(ext.get("release") or {}).get("release_notes_uri"), update_uri=(ext.get("release") or {}).get("update_uri"), + ) + if existing is None: + existing = CatalogItem(relative_path=rel, **values); db.add(existing) + else: + for k,v in values.items(): setattr(existing, k, v) + return existing + + +def scan_catalog(db: Session, catalog_root: Path, extensions: tuple[str,...]) -> dict[str,int]: + catalog_root.mkdir(parents=True, exist_ok=True) + for item in db.scalars(select(CatalogItem)): item.available = False + indexed = failed = 0 + for path in sorted(catalog_root.rglob("*")): + if not path.is_file() or path.suffix.lower() not in extensions: continue + try: index_file(db, catalog_root, path); indexed += 1 + except Exception: failed += 1 + db.commit(); return {"indexed": indexed, "failed": failed} + + +def resolve_item_path(item: CatalogItem, root: Path) -> Path: + p = (root / item.relative_path).resolve() + if root.resolve() not in p.parents or not p.is_file(): raise FileNotFoundError(item.relative_path) + return p + + +def tile_bytes(path: Path, z: int, x: int, y_xyz: int): + y_tms = (1 << z) - 1 - y_xyz + engine = _ro_engine(path) + try: + with engine.connect() as con: + con.exec_driver_sql("PRAGMA query_only=ON") + if "tiles" not in _objects(con): return None + row = con.exec_driver_sql("SELECT tile_data FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", (z,x,y_tms)).first() + if row is None: return None + data = bytes(row[0]) + finally: + engine.dispose() + if data.startswith(b"\x89PNG\r\n\x1a\n"): return data, "image/png" + if data.startswith(b"\xff\xd8\xff"): return data, "image/jpeg" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": return data, "image/webp" + if data[:2] == b"\x1f\x8b": return data, "application/vnd.mapbox-vector-tile" + return data, "application/octet-stream" + + +def _int_or_none(value): + try: return int(value) if value is not None else None + except (TypeError, ValueError): return None diff --git a/store/datatiles_store/commerce.py b/store/datatiles_store/commerce.py new file mode 100644 index 0000000..26533b9 --- /dev/null +++ b/store/datatiles_store/commerce.py @@ -0,0 +1,47 @@ +from __future__ import annotations +import uuid +from decimal import Decimal, InvalidOperation +from sqlalchemy import select, func +from .models import DownloadRecord, PaymentTransaction, PurchaseRecord, UpdateNotification +from .payments import PayPalProvider +from .settings import get_bool,get_setting + +def money(value): + try: d=Decimal(str(value)).quantize(Decimal("0.01")) + except InvalidOperation as e: raise ValueError("invalid price") from e + if d<0: raise ValueError("price must not be negative") + return format(d,".2f") +def is_paid(item): return bool(item.purchase_required and item.price_amount and Decimal(item.price_amount)>0) +def has_purchase(db,user,item): + if not is_paid(item): return True + return db.scalar(select(PurchaseRecord.id).where(PurchaseRecord.user_id==user.id,PurchaseRecord.catalog_item_id==item.id)) is not None +def provider_from_settings(db): + if not get_bool(db,"commerce.enabled"): raise RuntimeError("commerce is disabled") + name=get_setting(db,"commerce.provider").strip().lower() + if name=="paypal": + if not get_bool(db,"payments.paypal.enabled"): raise RuntimeError("PayPal provider is disabled") + return PayPalProvider(get_setting(db,"payments.paypal.client_id"),get_setting(db,"payments.paypal.client_secret"),mode=get_setting(db,"payments.paypal.mode"),brand_name=get_setting(db,"payments.paypal.brand_name")) + raise RuntimeError(f"unknown payment provider: {name}") +def new_transaction(db,user,item,provider_name): + tx=PaymentTransaction(public_id=uuid.uuid4().hex,user_id=user.id,catalog_item_id=item.id,provider=provider_name,status="created",amount=money(item.price_amount or "0"),currency=(item.price_currency or "EUR").upper()); db.add(tx); db.flush(); return tx +def complete_purchase(db,tx,provider_reference=None): + existing=db.scalar(select(PurchaseRecord).where(PurchaseRecord.user_id==tx.user_id,PurchaseRecord.catalog_item_id==tx.catalog_item_id)) + if existing: return existing + i=tx.item; row=PurchaseRecord(user_id=tx.user_id,catalog_item_id=i.id,payment_transaction_id=tx.id,product_id=i.product_id,product_version=i.product_version,product_sequence=i.product_sequence,amount=tx.amount,currency=tx.currency,provider=tx.provider,provider_reference=provider_reference or tx.provider_order_id); db.add(row); db.flush(); return row +def record_download(db,user,item,source): + row=DownloadRecord(user_id=user.id,catalog_item_id=item.id,product_id=item.product_id,product_version=item.product_version,product_sequence=item.product_sequence,file_sha256=item.sha256,source=source); db.add(row); db.flush(); return row +def generate_update_notifications(db,new_item): + if not get_bool(db,"commerce.update_notifications"): return 0 + if not new_item.product_id or new_item.product_sequence is None: return 0 + users=set(db.scalars(select(DownloadRecord.user_id).where(DownloadRecord.product_id==new_item.product_id,DownloadRecord.product_sequence < new_item.product_sequence))) + users.update(db.scalars(select(PurchaseRecord.user_id).where(PurchaseRecord.product_id==new_item.product_id,PurchaseRecord.product_sequence < new_item.product_sequence))) + made=0 + for uid in users: + old=max(db.scalar(select(func.max(DownloadRecord.product_sequence)).where(DownloadRecord.user_id==uid,DownloadRecord.product_id==new_item.product_id)) or 0,db.scalar(select(func.max(PurchaseRecord.product_sequence)).where(PurchaseRecord.user_id==uid,PurchaseRecord.product_id==new_item.product_id)) or 0) + if old>=new_item.product_sequence: continue + if db.scalar(select(UpdateNotification.id).where(UpdateNotification.user_id==uid,UpdateNotification.catalog_item_id==new_item.id,UpdateNotification.kind=="update")): continue + db.add(UpdateNotification(user_id=uid,catalog_item_id=new_item.id,kind="update",product_id=new_item.product_id,previous_sequence=old,new_sequence=new_item.product_sequence,title=f"Update available: {new_item.title}",message=f"Version {new_item.product_version or new_item.product_sequence} is available; your newest accessed sequence is {old}.")); made+=1 + return made +def serialize_purchase(x): return {"id":x.id,"catalog_item_id":x.catalog_item_id,"product_id":x.product_id,"product_version":x.product_version,"product_sequence":x.product_sequence,"amount":x.amount,"currency":x.currency,"provider":x.provider,"provider_reference":x.provider_reference,"purchased_at":x.purchased_at.isoformat()} +def serialize_download(x): return {"id":x.id,"catalog_item_id":x.catalog_item_id,"product_id":x.product_id,"product_version":x.product_version,"product_sequence":x.product_sequence,"file_sha256":x.file_sha256,"downloaded_at":x.downloaded_at.isoformat(),"source":x.source} +def serialize_notification(x): return {"id":x.id,"catalog_item_id":x.catalog_item_id,"kind":x.kind,"product_id":x.product_id,"previous_sequence":x.previous_sequence,"new_sequence":x.new_sequence,"title":x.title,"message":x.message,"created_at":x.created_at.isoformat(),"read_at":x.read_at.isoformat() if x.read_at else None} diff --git a/store/datatiles_store/db.py b/store/datatiles_store/db.py new file mode 100644 index 0000000..9b38d0c --- /dev/null +++ b/store/datatiles_store/db.py @@ -0,0 +1,94 @@ +from __future__ import annotations +from flask import current_app, g +from sqlalchemy import create_engine, select, inspect, text +from sqlalchemy.orm import Session, sessionmaker +from werkzeug.security import generate_password_hash + +from .models import Base, Group, Role, User +from .settings import ensure_settings + +ROLE_PERMISSIONS = { + "viewer": "catalog.view", + "downloader": "catalog.view catalog.preview catalog.download agreements.accept", + "catalog_manager": "catalog.view catalog.preview catalog.download agreements.accept catalog.manage", + "admin": "*", +} + + +def init_engine(app): + engine = create_engine(app.config["DATABASE_URL"], future=True, pool_pre_ping=True) + app.extensions["datatiles_store_engine"] = engine + app.extensions["datatiles_store_sessionmaker"] = sessionmaker(engine, expire_on_commit=False, class_=Session) + Base.metadata.create_all(engine) + _upgrade_legacy_users(engine) + _upgrade_store_catalog(engine) + with app.extensions["datatiles_store_sessionmaker"]() as db: + bootstrap_security(db, app.config) + ensure_settings(db) + db.commit() + + + + +def _upgrade_store_catalog(engine): + cols={c["name"] for c in inspect(engine).get_columns("catalog_items")} + additions={"product_id":"VARCHAR(512)","product_version":"VARCHAR(128)","product_sequence":"INTEGER","released_at":"VARCHAR(64)","previous_version":"VARCHAR(128)","release_notes_uri":"VARCHAR(2048)","update_uri":"VARCHAR(2048)","purchase_required":"BOOLEAN NOT NULL DEFAULT 0","price_amount":"VARCHAR(32)","price_currency":"VARCHAR(3)"} + with engine.begin() as conn: + for name,sqltype in additions.items(): + if name not in cols: conn.execute(text(f"ALTER TABLE catalog_items ADD COLUMN {name} {sqltype}")) + +def _upgrade_legacy_users(engine): + """Small idempotent Store DB compatibility upgrade for pre-0.3 user tables.""" + cols={c["name"] for c in inspect(engine).get_columns("users")} + with engine.begin() as conn: + if "email" not in cols: + conn.execute(text("ALTER TABLE users ADD COLUMN email VARCHAR(320)")) + if "email_verified" not in cols: + conn.execute(text("ALTER TABLE users ADD COLUMN email_verified BOOLEAN NOT NULL DEFAULT 0")) + # Portable unique creation is intentionally left to migration tooling on existing DBs; + # application-level username uniqueness remains authoritative during transition. + + +def _ensure_group(db: Session, name: str, role: Role) -> Group: + group = db.scalar(select(Group).where(Group.name == name)) + if group is None: + group = Group(name=name) + db.add(group); db.flush() + if role not in group.roles: + group.roles.append(role) + return group + + +def bootstrap_security(db: Session, config): + roles: dict[str, Role] = {} + for name, permissions in ROLE_PERMISSIONS.items(): + role = db.scalar(select(Role).where(Role.name == name)) + if role is None: + role = Role(name=name, permissions=permissions) + db.add(role); db.flush() + elif role.permissions != permissions: + role.permissions = permissions + roles[name] = role + + admin_group = _ensure_group(db, config["ADMIN_GROUP"], roles[config["ADMIN_ROLE"]]) + _ensure_group(db, config["MANAGERS_GROUP"], roles["catalog_manager"]) + _ensure_group(db, "downloaders", roles["downloader"]) + + admin = db.scalar(select(User).where(User.username == config["ADMIN_USERNAME"])) + if admin is None: + admin = User(username=config["ADMIN_USERNAME"], password_hash=generate_password_hash(config["ADMIN_PASSWORD"])) + db.add(admin); db.flush() + if admin_group not in admin.groups: + admin.groups.append(admin_group) + + +def get_db() -> Session: + if "db" not in g: + g.db = current_app.extensions["datatiles_store_sessionmaker"]() + return g.db + + +def close_db(_exc=None): + db = g.pop("db", None) + if db is not None: + db.close() diff --git a/store/datatiles_store/mailer.py b/store/datatiles_store/mailer.py new file mode 100644 index 0000000..68f1c17 --- /dev/null +++ b/store/datatiles_store/mailer.py @@ -0,0 +1,15 @@ +from __future__ import annotations +import smtplib +from email.message import EmailMessage +from .settings import get_bool,get_setting + +def send_mail(db,to,subject,text): + if not get_bool(db,"smtp.enabled"): raise RuntimeError("SMTP is disabled") + host=get_setting(db,"smtp.host"); port=int(get_setting(db,"smtp.port") or 587) + ssl=get_bool(db,"smtp.ssl"); cls=smtplib.SMTP_SSL if ssl else smtplib.SMTP + msg=EmailMessage(); msg["To"]=to; msg["From"]=f'{get_setting(db,"smtp.from_name")} <{get_setting(db,"smtp.from_address")}>'; msg["Subject"]=subject; msg.set_content(text) + with cls(host,port,timeout=float(get_setting(db,"smtp.timeout") or 15)) as s: + if not ssl and get_bool(db,"smtp.starttls"): s.starttls() + u=get_setting(db,"smtp.username"); p=get_setting(db,"smtp.password") + if u: s.login(u,p) + s.send_message(msg) diff --git a/store/datatiles_store/models.py b/store/datatiles_store/models.py new file mode 100644 index 0000000..6a00180 --- /dev/null +++ b/store/datatiles_store/models.py @@ -0,0 +1,253 @@ +from __future__ import annotations +from datetime import datetime, timezone + +try: + from flask_login import UserMixin +except ImportError: + class UserMixin: + @property + def is_authenticated(self): return True + @property + def is_anonymous(self): return False + def get_id(self): return str(self.id) + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Table, Text, UniqueConstraint +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +class Base(DeclarativeBase): + pass + + +user_groups = Table( + "user_groups", Base.metadata, + Column("user_id", ForeignKey("users.id", ondelete="CASCADE"), primary_key=True), + Column("group_id", ForeignKey("groups.id", ondelete="CASCADE"), primary_key=True), +) + +group_roles = Table( + "group_roles", Base.metadata, + Column("group_id", ForeignKey("groups.id", ondelete="CASCADE"), primary_key=True), + Column("role_id", ForeignKey("roles.id", ondelete="CASCADE"), primary_key=True), +) + + +class Role(Base): + __tablename__ = "roles" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(64), unique=True, index=True) + permissions: Mapped[str] = mapped_column(Text, default="") + + def permission_set(self) -> set[str]: + return {p for p in self.permissions.split() if p} + + +class Group(Base): + __tablename__ = "groups" + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(128), unique=True, index=True) + roles: Mapped[list[Role]] = relationship(secondary=group_roles, lazy="selectin") + users: Mapped[list["User"]] = relationship(secondary=user_groups, back_populates="groups", lazy="selectin") + + +class User(UserMixin, Base): + __tablename__ = "users" + id: Mapped[int] = mapped_column(primary_key=True) + username: Mapped[str] = mapped_column(String(128), unique=True, index=True) + password_hash: Mapped[str | None] = mapped_column(String(512), nullable=True) + email: Mapped[str | None] = mapped_column(String(320), nullable=True, unique=True, index=True) + email_verified: Mapped[bool] = mapped_column(Boolean, default=False) + active: Mapped[bool] = mapped_column(Boolean, default=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + groups: Mapped[list[Group]] = relationship(secondary=user_groups, back_populates="users", lazy="selectin") + + @property + def is_active(self) -> bool: + return self.active + + def permissions(self) -> set[str]: + out: set[str] = set() + for group in self.groups: + for role in group.roles: + out.update(role.permission_set()) + return out + + def can(self, permission: str) -> bool: + perms = self.permissions() + return "*" in perms or permission in perms + + def in_group(self, name: str) -> bool: + return any(group.name == name for group in self.groups) + + +class ApiToken(Base): + __tablename__ = "api_tokens" + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + name: Mapped[str] = mapped_column(String(128), default="API token") + token_prefix: Mapped[str] = mapped_column(String(16), index=True) + token_hash: Mapped[str] = mapped_column(String(64), unique=True, index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + user: Mapped[User] = relationship(lazy="joined") + + +class CatalogItem(Base): + __tablename__ = "catalog_items" + __table_args__ = (UniqueConstraint("relative_path", name="uq_catalog_relative_path"),) + + id: Mapped[int] = mapped_column(primary_key=True) + relative_path: Mapped[str] = mapped_column(String(1024), index=True) + filename: Mapped[str] = mapped_column(String(512), index=True) + title: Mapped[str] = mapped_column(String(512), index=True) + description: Mapped[str | None] = mapped_column(Text) + format: Mapped[str] = mapped_column(String(64), default="DataTiles") + schema_revision: Mapped[int | None] = mapped_column(Integer) + size_bytes: Mapped[int] = mapped_column(Integer) + file_mtime_ns: Mapped[int] = mapped_column(Integer) + sha256: Mapped[str] = mapped_column(String(64), index=True) + bounds_west: Mapped[str | None] = mapped_column(String(64)) + bounds_south: Mapped[str | None] = mapped_column(String(64)) + bounds_east: Mapped[str | None] = mapped_column(String(64)) + bounds_north: Mapped[str | None] = mapped_column(String(64)) + minzoom: Mapped[int | None] = mapped_column(Integer) + maxzoom: Mapped[int | None] = mapped_column(Integer) + metadata_json: Mapped[str] = mapped_column(Text, default="{}") + variables_json: Mapped[str] = mapped_column(Text, default="[]") + rights_json: Mapped[str] = mapped_column(Text, default="[]") + provenance_json: Mapped[str] = mapped_column(Text, default="{}") + search_text: Mapped[str] = mapped_column(Text, default="", index=True) + indexed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + available: Mapped[bool] = mapped_column(Boolean, default=True, index=True) + product_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True) + product_version: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + product_sequence: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + released_at: Mapped[str | None] = mapped_column(String(64), nullable=True) + previous_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + release_notes_uri: Mapped[str | None] = mapped_column(String(2048), nullable=True) + update_uri: Mapped[str | None] = mapped_column(String(2048), nullable=True) + purchase_required: Mapped[bool] = mapped_column(Boolean, default=False, index=True) + price_amount: Mapped[str | None] = mapped_column(String(32), nullable=True) + price_currency: Mapped[str | None] = mapped_column(String(3), nullable=True) + + +class AgreementAcceptance(Base): + __tablename__ = "agreement_acceptances" + __table_args__ = (UniqueConstraint( + "user_id", "catalog_item_id", "file_sha256", "license_fingerprint", "safety_version", + name="uq_current_agreement_acceptance" + ),) + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + catalog_item_id: Mapped[int] = mapped_column(ForeignKey("catalog_items.id", ondelete="CASCADE"), index=True) + file_sha256: Mapped[str] = mapped_column(String(64), index=True) + license_fingerprint: Mapped[str] = mapped_column(String(64), index=True) + license_snapshot_json: Mapped[str] = mapped_column(Text) + safety_version: Mapped[str] = mapped_column(String(64), index=True) + safety_text: Mapped[str] = mapped_column(Text) + accepted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + source: Mapped[str] = mapped_column(String(32), default="web") + client_ip: Mapped[str | None] = mapped_column(String(128), nullable=True) + user_agent: Mapped[str | None] = mapped_column(String(512), nullable=True) + user: Mapped[User] = relationship(lazy="joined") + item: Mapped[CatalogItem] = relationship(lazy="joined") + + +class AuditEvent(Base): + __tablename__ = "audit_events" + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True) + action: Mapped[str] = mapped_column(String(128), index=True) + object_type: Mapped[str] = mapped_column(String(64), index=True) + object_id: Mapped[str | None] = mapped_column(String(256), nullable=True, index=True) + detail_json: Mapped[str] = mapped_column(Text, default="{}") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) + +class AppSetting(Base): + __tablename__ = "app_settings" + key: Mapped[str] = mapped_column(String(128), primary_key=True) + value: Mapped[str] = mapped_column(Text, default="") + secret: Mapped[bool] = mapped_column(Boolean, default=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + + +class ExternalIdentity(Base): + __tablename__ = "external_identities" + __table_args__ = (UniqueConstraint("provider", "subject", name="uq_external_identity"),) + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + provider: Mapped[str] = mapped_column(String(32), index=True) + subject: Mapped[str] = mapped_column(String(512), index=True) + email: Mapped[str] = mapped_column(String(320), index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + user: Mapped[User] = relationship(lazy="joined") + + +class PaymentTransaction(Base): + __tablename__ = "payment_transactions" + id: Mapped[int] = mapped_column(primary_key=True) + public_id: Mapped[str] = mapped_column(String(64), unique=True, index=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + catalog_item_id: Mapped[int] = mapped_column(ForeignKey("catalog_items.id", ondelete="CASCADE"), index=True) + provider: Mapped[str] = mapped_column(String(64), index=True) + provider_order_id: Mapped[str | None] = mapped_column(String(256), nullable=True, unique=True, index=True) + status: Mapped[str] = mapped_column(String(32), default="created", index=True) + amount: Mapped[str] = mapped_column(String(32)) + currency: Mapped[str] = mapped_column(String(3)) + approval_url: Mapped[str | None] = mapped_column(String(2048), nullable=True) + detail_json: Mapped[str] = mapped_column(Text, default="{}") + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc)) + user: Mapped[User] = relationship(lazy="joined") + item: Mapped[CatalogItem] = relationship(lazy="joined") + +class PurchaseRecord(Base): + __tablename__ = "purchase_records" + __table_args__ = (UniqueConstraint("user_id","catalog_item_id", name="uq_user_release_purchase"),) + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + catalog_item_id: Mapped[int] = mapped_column(ForeignKey("catalog_items.id", ondelete="CASCADE"), index=True) + payment_transaction_id: Mapped[int | None] = mapped_column(ForeignKey("payment_transactions.id", ondelete="SET NULL"), nullable=True) + product_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True) + product_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + product_sequence: Mapped[int | None] = mapped_column(Integer, nullable=True) + amount: Mapped[str | None] = mapped_column(String(32), nullable=True) + currency: Mapped[str | None] = mapped_column(String(3), nullable=True) + provider: Mapped[str | None] = mapped_column(String(64), nullable=True) + provider_reference: Mapped[str | None] = mapped_column(String(256), nullable=True) + purchased_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) + user: Mapped[User] = relationship(lazy="joined") + item: Mapped[CatalogItem] = relationship(lazy="joined") + +class DownloadRecord(Base): + __tablename__ = "download_records" + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + catalog_item_id: Mapped[int] = mapped_column(ForeignKey("catalog_items.id", ondelete="CASCADE"), index=True) + product_id: Mapped[str | None] = mapped_column(String(512), nullable=True, index=True) + product_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + product_sequence: Mapped[int | None] = mapped_column(Integer, nullable=True) + file_sha256: Mapped[str] = mapped_column(String(64), index=True) + downloaded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) + source: Mapped[str] = mapped_column(String(32), default="web") + user: Mapped[User] = relationship(lazy="joined") + item: Mapped[CatalogItem] = relationship(lazy="joined") + +class UpdateNotification(Base): + __tablename__ = "update_notifications" + __table_args__ = (UniqueConstraint("user_id","catalog_item_id","kind", name="uq_user_release_notification"),) + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True) + catalog_item_id: Mapped[int] = mapped_column(ForeignKey("catalog_items.id", ondelete="CASCADE"), index=True) + kind: Mapped[str] = mapped_column(String(32), default="update", index=True) + product_id: Mapped[str] = mapped_column(String(512), index=True) + previous_sequence: Mapped[int] = mapped_column(Integer) + new_sequence: Mapped[int] = mapped_column(Integer) + title: Mapped[str] = mapped_column(String(512)) + message: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) + read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + user: Mapped[User] = relationship(lazy="joined") + item: Mapped[CatalogItem] = relationship(lazy="joined") diff --git a/store/datatiles_store/payments.py b/store/datatiles_store/payments.py new file mode 100644 index 0000000..a2e38de --- /dev/null +++ b/store/datatiles_store/payments.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from dataclasses import dataclass +from abc import ABC, abstractmethod +import base64, json +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +@dataclass(frozen=True) +class CheckoutRequest: + reference: str; description: str; amount: str; currency: str; return_url: str; cancel_url: str +@dataclass(frozen=True) +class CheckoutSession: + provider_order_id: str; status: str; approval_url: str; raw: dict +@dataclass(frozen=True) +class CaptureResult: + provider_order_id: str; status: str; completed: bool; raw: dict + +class PaymentProvider(ABC): + name: str + @abstractmethod + def create_checkout(self, request: CheckoutRequest) -> CheckoutSession: ... + @abstractmethod + def capture(self, provider_order_id: str) -> CaptureResult: ... + +class PayPalProvider(PaymentProvider): + name="paypal" + def __init__(self, client_id: str, client_secret: str, *, mode: str="sandbox", brand_name: str="DataTiles Store", opener=urlopen): + if mode not in {"sandbox","live"}: raise ValueError("PayPal mode must be sandbox or live") + self.client_id=client_id; self.client_secret=client_secret; self.mode=mode; self.brand_name=brand_name; self._open=opener + self.base="https://api-m.sandbox.paypal.com" if mode=="sandbox" else "https://api-m.paypal.com" + def _json(self, req): + with self._open(req,timeout=30) as r: return json.loads(r.read().decode("utf-8")) + def _token(self): + basic=base64.b64encode(f"{self.client_id}:{self.client_secret}".encode()).decode() + return self._json(Request(self.base+"/v1/oauth2/token",data=urlencode({"grant_type":"client_credentials"}).encode(),method="POST",headers={"Authorization":"Basic "+basic,"Content-Type":"application/x-www-form-urlencoded","Accept":"application/json"}))["access_token"] + def _api(self,path,body,request_id=None): + h={"Authorization":"Bearer "+self._token(),"Content-Type":"application/json","Accept":"application/json"} + if request_id: h["PayPal-Request-Id"]=request_id[:25] + return self._json(Request(self.base+path,data=json.dumps(body).encode(),method="POST",headers=h)) + def create_checkout(self,r): + body={"intent":"CAPTURE","purchase_units":[{"reference_id":r.reference,"description":r.description[:127],"amount":{"currency_code":r.currency.upper(),"value":r.amount}}],"application_context":{"brand_name":self.brand_name[:127],"shipping_preference":"NO_SHIPPING","user_action":"PAY_NOW","return_url":r.return_url,"cancel_url":r.cancel_url}} + raw=self._api("/v2/checkout/orders",body,r.reference); approval=next((x.get("href") for x in raw.get("links",[]) if x.get("rel")=="approve"),None) + if not raw.get("id") or not approval: raise RuntimeError("PayPal create-order response lacks order id or approval URL") + return CheckoutSession(raw["id"],raw.get("status","CREATED"),approval,raw) + def capture(self,oid): + raw=self._api(f"/v2/checkout/orders/{oid}/capture",{}); status=str(raw.get("status","")); return CaptureResult(oid,status,status=="COMPLETED",raw) diff --git a/store/datatiles_store/security.py b/store/datatiles_store/security.py new file mode 100644 index 0000000..09a7ac8 --- /dev/null +++ b/store/datatiles_store/security.py @@ -0,0 +1,83 @@ +from __future__ import annotations +import hashlib +import secrets +from datetime import datetime, timezone +from functools import wraps + +from flask import abort, current_app, g, jsonify, request +from flask_login import current_user +from sqlalchemy import select + +from .db import get_db +from .models import ApiToken + + +def permission_required(permission: str): + def decorator(fn): + @wraps(fn) + def wrapped(*args, **kwargs): + if current_app.config.get("ALLOW_PUBLIC_CATALOG") and permission == "catalog.view": + return fn(*args, **kwargs) + if not current_user.is_authenticated: + return current_app.login_manager.unauthorized() + if not current_user.can(permission): + abort(403) + return fn(*args, **kwargs) + return wrapped + return decorator + + +def token_hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def issue_token(user, *, name="API token", expires_at=None): + raw = "dts_" + secrets.token_urlsafe(36) + rec = ApiToken(user_id=user.id, name=name[:128], token_prefix=raw[:12], token_hash=token_hash(raw), expires_at=expires_at) + db = get_db(); db.add(rec); db.flush() + return raw, rec + + +def api_user(): + cached = getattr(g, "api_user", None) + if cached is not None: + return cached + auth = request.headers.get("Authorization", "") + if auth.startswith("Bearer "): + raw = auth[7:].strip() + if raw: + rec = get_db().scalar(select(ApiToken).where(ApiToken.token_hash == token_hash(raw))) + now = datetime.now(timezone.utc) + expiry = rec.expires_at if rec else None + if expiry is not None and expiry.tzinfo is None: + expiry = expiry.replace(tzinfo=timezone.utc) + if rec and rec.revoked_at is None and (expiry is None or expiry > now) and rec.user.active: + rec.last_used_at = now + g.api_token = rec; g.api_user = rec.user + return rec.user + if current_user.is_authenticated: + g.api_user = current_user + return current_user + return None + + +def api_permission_required(permission: str): + def decorator(fn): + @wraps(fn) + def wrapped(*args, **kwargs): + user = api_user() + if current_app.config.get("ALLOW_PUBLIC_CATALOG") and permission == "catalog.view" and user is None: + return fn(*args, **kwargs) + if user is None: + return jsonify({"error":"authentication_required"}), 401 + if not user.can(permission): + return jsonify({"error":"forbidden","required_permission":permission}), 403 + return fn(*args, **kwargs) + return wrapped + return decorator + + +def safe_next_url(value: str | None) -> str | None: + if not value or not value.startswith("/") or value.startswith("//"): + return None + return value diff --git a/store/datatiles_store/settings.py b/store/datatiles_store/settings.py new file mode 100644 index 0000000..076faec --- /dev/null +++ b/store/datatiles_store/settings.py @@ -0,0 +1,66 @@ +from __future__ import annotations +from dataclasses import dataclass +from sqlalchemy import select +from .models import AppSetting + +@dataclass(frozen=True) +class SettingDef: + key: str; default: str; section: str; label: str; secret: bool=False; restart: bool=False; help: str="" + +SETTINGS = [ + SettingDef("auth.local.enabled","1","Authentication","Managed user/password"), + SettingDef("auth.registration.enabled","0","Authentication","Allow self-registration"), + SettingDef("auth.registration.default_group","downloaders","Authentication","Registration default group"), + SettingDef("auth.google.enabled","0","Google","Enable Google OpenID Connect"), + SettingDef("auth.google.client_id","","Google","Google client ID"), + SettingDef("auth.google.client_secret","","Google","Google client secret",True), + SettingDef("auth.google.allowed_domains","","Google","Allowed Google Workspace domains","",False,"Comma-separated; blank permits any verified Google email."), + SettingDef("auth.microsoft.enabled","0","Microsoft","Enable Microsoft Entra ID"), + SettingDef("auth.microsoft.tenant","common","Microsoft","Microsoft tenant ID/domain"), + SettingDef("auth.microsoft.client_id","","Microsoft","Microsoft client ID"), + SettingDef("auth.microsoft.client_secret","","Microsoft","Microsoft client secret",True), + SettingDef("auth.oauth2.enabled","0","OAuth2 / OIDC","Enable generic OpenID Connect"), + SettingDef("auth.oauth2.name","Institutional SSO","OAuth2 / OIDC","Provider display name"), + SettingDef("auth.oauth2.metadata_url","","OAuth2 / OIDC","OIDC discovery URL"), + SettingDef("auth.oauth2.client_id","","OAuth2 / OIDC","Client ID"), + SettingDef("auth.oauth2.client_secret","","OAuth2 / OIDC","Client secret",True), + SettingDef("auth.oauth2.scopes","openid email profile","OAuth2 / OIDC","Scopes"), + SettingDef("smtp.enabled","0","SMTP","Enable SMTP"), + SettingDef("smtp.host","localhost","SMTP","SMTP host"), SettingDef("smtp.port","587","SMTP","SMTP port"), + SettingDef("smtp.username","","SMTP","SMTP username"), SettingDef("smtp.password","","SMTP","SMTP password",True), + SettingDef("smtp.starttls","1","SMTP","Use STARTTLS"), SettingDef("smtp.ssl","0","SMTP","Use implicit TLS"), + SettingDef("smtp.from_address","datatiles@example.invalid","SMTP","From address"), + SettingDef("smtp.from_name","DataTiles Store","SMTP","From name"), + SettingDef("smtp.timeout","15","SMTP","Timeout seconds"), + SettingDef("store.public_base_url","","Store","Public base URL","",False,"Required for reliable email/OIDC callbacks behind proxies."), + SettingDef("store.help_title","DataTiles Store Help","Store","Help title"), + SettingDef("agreement.safety.version","2026-08-29-v1","Agreements","Safety agreement version"), + SettingDef("agreement.safety.text","NOT SUITABLE FOR NAVIGATION. Data and portrayals are provided AS IS and AS AVAILABLE, without warranties of accuracy, completeness, fitness for a particular purpose, merchantability, non-infringement, continuity, or safety. They are not official nautical charts, ENCs, ECDIS products, collision-avoidance systems, or certified navigation aids. The user must independently verify information against authoritative sources and remains responsible for all operational and navigation decisions. To the maximum extent permitted by applicable law, data distributors, licensors, contributors, institutions, software providers, maintainers, and service operators disclaim liability for loss, damage, injury, grounding, collision, delay, business interruption, or other consequences arising from use of or reliance on the data, software, portrayals, APIs, or services. Nothing in this agreement overrides mandatory rights or liabilities that cannot lawfully be excluded.","Agreements","Safety/no-liability text"), + SettingDef("agreement.record_client_metadata","1","Agreements","Record client IP/user-agent with acceptance"), + SettingDef("commerce.enabled","0","Commerce","Enable paid products"), + SettingDef("commerce.provider","paypal","Commerce","Payment provider","",False,"Provider adapter name; reference implementation supplies PayPal."), + SettingDef("commerce.default_currency","EUR","Commerce","Default currency"), + SettingDef("commerce.update_notifications","1","Commerce","Notify users about newer releases"), + SettingDef("payments.paypal.enabled","0","PayPal","Enable PayPal reference provider"), + SettingDef("payments.paypal.mode","sandbox","PayPal","PayPal mode","",False,"sandbox or live"), + SettingDef("payments.paypal.client_id","","PayPal","PayPal client ID"), + SettingDef("payments.paypal.client_secret","","PayPal","PayPal client secret",True), + SettingDef("payments.paypal.brand_name","DataTiles Store","PayPal","Checkout brand name"), +] +BY_KEY={x.key:x for x in SETTINGS} + +def ensure_settings(db): + existing={x.key for x in db.scalars(select(AppSetting))} + for d in SETTINGS: + if d.key not in existing: db.add(AppSetting(key=d.key,value=d.default,secret=d.secret)) + +def get_setting(db,key): + row=db.get(AppSetting,key); return row.value if row else BY_KEY.get(key,SettingDef(key,"","","" )).default + +def get_bool(db,key): return get_setting(db,key).strip().lower() in {"1","true","yes","on"} + +def set_setting(db,key,value): + if key not in BY_KEY: raise KeyError(key) + row=db.get(AppSetting,key) + if row is None: row=AppSetting(key=key,value=str(value),secret=BY_KEY[key].secret); db.add(row) + else: row.value=str(value) diff --git a/store/datatiles_store/static/app.css b/store/datatiles_store/static/app.css new file mode 100644 index 0000000..e478bd3 --- /dev/null +++ b/store/datatiles_store/static/app.css @@ -0,0 +1,2 @@ +:root{font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif;color:#e8f1f5;background:#06111b;--panel:#0b1a27;--line:#1f3848;--muted:#9eb2bf;--accent:#5bc7cf}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:linear-gradient(180deg,#06111b,#0a1721);color:#e8f1f5}a{color:inherit;text-decoration:none}.topbar{height:64px;display:flex;align-items:center;justify-content:space-between;padding:0 28px;border-bottom:1px solid var(--line);background:#07131ff0;position:sticky;top:0;z-index:1000}.brand{font-size:20px;font-weight:800;letter-spacing:.02em}.brand span{color:var(--accent)}nav{display:flex;gap:18px;align-items:center;color:var(--muted)}nav a:hover{color:white}.inline{display:inline}.linkbtn{background:none;border:0;color:var(--muted);cursor:pointer;font:inherit}.user{font-size:13px}.hero{padding:64px 28px 38px;max-width:1300px;margin:auto}.hero h1{font-size:clamp(38px,6vw,78px);line-height:.95;margin:.15em 0}.hero p{max-width:750px;color:var(--muted);font-size:18px}.eyebrow{text-transform:uppercase;letter-spacing:.18em;color:var(--accent)!important;font-size:12px!important}.catalog-shell{max-width:1300px;margin:auto;padding:0 28px 60px}.catalog-tools{display:flex;gap:18px;justify-content:space-between;align-items:center;margin-bottom:24px}.search{display:flex;flex:1;max-width:760px}.search input,input,select{background:#0e2231;border:1px solid var(--line);color:white;padding:12px;border-radius:8px}.search input{flex:1;border-radius:8px 0 0 8px}.search button,.primary,button{background:var(--accent);color:#031217;border:0;padding:12px 18px;border-radius:8px;font-weight:700;cursor:pointer}.search button{border-radius:0 8px 8px 0}.manager{display:flex;gap:10px}.manager form{display:flex;gap:7px}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:18px}.card{background:var(--panel);border:1px solid var(--line);border-radius:14px;overflow:hidden}.card-map{height:145px;background:radial-gradient(circle at 30% 40%,#174459 0 4%,transparent 5%),repeating-radial-gradient(ellipse at 55% 30%,#1b4b5f 0 2px,#0d2a3a 3px 10px);display:flex;align-items:flex-end;padding:12px}.card-map span{background:#06111bcf;padding:5px 8px;border-radius:5px;font-size:11px}.card-body{padding:18px}.card h2{margin:0 0 8px;font-size:20px}.card p{color:var(--muted);min-height:44px}.card dl{display:flex;gap:10px;font-size:12px}.card dt{color:var(--muted)}.card dd{margin:0}.actions{display:flex;gap:12px;align-items:center;margin-top:16px}.primary{display:inline-block}.explorer{height:calc(100vh - 64px);display:grid;grid-template-columns:1fr minmax(320px,390px)}#map{height:100%;background:#0b1a27}.panel{overflow:auto;padding:22px;background:#07131ff5;border-left:1px solid var(--line)}.panel h1{font-size:26px;margin:16px 0 8px}.panel>p{color:var(--muted)}.back{color:var(--accent);font-size:13px}.notice{background:#3b2615;color:#ffd19b;border:1px solid #77502a;padding:9px 12px;border-radius:7px;font-weight:800;text-align:center;margin:16px 0}details{border-top:1px solid var(--line);padding:14px 0}summary{font-weight:750;cursor:pointer;margin-bottom:12px}details label{display:block;color:var(--muted);margin:10px 0}.check{display:flex!important;gap:8px;align-items:center}.chips{display:flex;flex-wrap:wrap;gap:7px}.chips span{font-size:11px;border:1px solid var(--line);border-radius:999px;padding:6px 8px;color:#c7d6de}.fact{padding:8px 0;border-bottom:1px solid #142b39}.fact b,.fact span,.fact small{display:block}.fact small{color:var(--muted);overflow-wrap:anywhere}.metadata{font-size:11px;display:grid;grid-template-columns:120px 1fr;gap:5px}.metadata dt{color:var(--muted);overflow-wrap:anywhere}.metadata dd{margin:0;overflow-wrap:anywhere}.download{display:block;text-align:center;margin-top:14px}.login-card{max-width:420px;margin:10vh auto;padding:28px;background:var(--panel);border:1px solid var(--line);border-radius:14px}.login-card label{display:block;margin:14px 0;color:var(--muted)}.login-card input:not([type=checkbox]){width:100%;margin-top:6px}.admin-page{max-width:1100px;margin:40px auto;padding:0 24px}.admin-form{display:flex;gap:10px;align-items:center;flex-wrap:wrap;background:var(--panel);padding:16px;border-radius:10px}.admin-page table{width:100%;border-collapse:collapse;margin-top:20px}.admin-page th,.admin-page td{text-align:left;border-bottom:1px solid var(--line);padding:12px}.flash-wrap{position:fixed;right:20px;top:76px;z-index:2000}.flash{background:#173448;border:1px solid var(--line);padding:10px 14px;border-radius:8px;margin-bottom:8px}footer{padding:14px 28px;border-top:1px solid var(--line);display:flex;justify-content:space-between;color:var(--muted);font-size:12px}.explorer+footer{display:none}@media(max-width:820px){.explorer{grid-template-columns:1fr;grid-template-rows:56vh auto;height:auto}.panel{border-left:0;border-top:1px solid var(--line)}.catalog-tools{display:block}.manager{margin-top:10px;flex-wrap:wrap}.topbar{padding:0 16px}nav{gap:10px}.hero,.catalog-shell{padding-left:16px;padding-right:16px}} +.agreement-page{max-width:900px;margin:40px auto;padding:0 24px 60px}.agreement-page h1{font-size:38px}.agreement-lead,.api-note{color:var(--muted)}code{overflow-wrap:anywhere}.agreement-box{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:20px;margin:18px 0}.agreement-box.safety{border-color:#77502a;background:#1a1510}.right-record{display:grid;gap:5px;padding:12px 0;border-top:1px solid var(--line)}.right-record a{color:var(--accent);overflow-wrap:anywhere}.accept-form{display:grid;gap:14px;background:#0d2231;padding:20px;border-radius:12px}.accept-form label{display:flex;gap:10px;align-items:flex-start}.agreement-required{border:1px solid #77502a;background:#251b12;padding:14px;border-radius:9px;margin:14px 0}.accepted{border:1px solid #2f6855;background:#102a23;color:#bcebd8;padding:10px;border-radius:8px;margin:12px 0}.warning{padding:12px;background:#3b2615;color:#ffd19b;border-radius:8px}.manager-tools form{display:grid;gap:8px;margin:12px 0}.danger{background:#b74848;color:white}.manager-label{font-size:11px;color:var(--muted);align-self:center} diff --git a/store/datatiles_store/static/explorer.js b/store/datatiles_store/static/explorer.js new file mode 100644 index 0000000..2a718d1 --- /dev/null +++ b/store/datatiles_store/static/explorer.js @@ -0,0 +1 @@ +(()=>{const el=document.querySelector('.explorer');if(!el||typeof L==='undefined')return;const id=el.dataset.itemId;const accepted=el.dataset.accepted==='1';const map=L.map('map',{zoomControl:true,worldCopyJump:false}).setView([40.6,14.2],7);L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',{maxZoom:19,attribution:'© OpenStreetMap contributors'}).addTo(map);let overlay=null;if(accepted){overlay=L.tileLayer(`/catalog/${id}/tiles/{z}/{x}/{y}`,{maxZoom:Number(el.dataset.maxzoom||22),minZoom:Number(el.dataset.minzoom||0),opacity:.92,errorTileUrl:''}).addTo(map);}let bounds=null;try{const b=JSON.parse(el.dataset.bounds);if(b&&b.length===4){bounds=L.latLngBounds([b[1],b[0]],[b[3],b[2]]);map.fitBounds(bounds);L.rectangle(bounds,{weight:1,fill:false,dashArray:'5 6'}).addTo(map);}}catch(_e){}document.getElementById('overlayToggle')?.addEventListener('change',e=>{if(!overlay)return;e.target.checked?overlay.addTo(map):map.removeLayer(overlay)});document.getElementById('fit')?.addEventListener('click',()=>bounds&&map.fitBounds(bounds));map.on('click',e=>{L.popup().setLatLng(e.latlng).setContent(`${e.latlng.lat.toFixed(5)}, ${e.latlng.lng.toFixed(5)}`).openOn(map);});})(); diff --git a/store/datatiles_store/static/icons/icon-192.png b/store/datatiles_store/static/icons/icon-192.png new file mode 100644 index 0000000000000000000000000000000000000000..cd33678b27e59e1ec425638711cb7239ebb288cc GIT binary patch literal 1051 zcmeAS@N?(olHy`uVBq!ia0vp^2SAvE2}s`E_d9@rf%%cAi(^Q|oVRxk^Q7G+8Uigj zCNMu?uylX5g>%B8xw}l5g4~`5oMOnjv(h{9Kwj+cC_N62Mp+J({WEW>evNzC^k?hu z%rf7JbIk4k{J6|J=k@Nj><>0^cDOjGFewQNbT)XfxF{(IF;3(lB=qTT+xfrW9{=W@ z|M%3N->=`#ws}AMtGv#YvmxEyQBo61rbSM>xTs;t=1x}0wL3b)&WDM~{M@NMp`U;XC`Pdn^vtG!@)y~nRngE{`uPQn@nV*hZ|eU&z&O7=daumJ@MMM+Tg3sj+Ti$+$X-i-qd3i zFC4CLef{4V<$5uPhzzU=f)cjq$xBuO_E_vQ$pZG>T`@_p` z+o#+7(D(np=kJeyJ%4xF-90<=`Nunz@1CFeT=2=ISzQ5VWn*aa#^1bqRaqT{& zD0))q_f(Eh`a*T%$s-M7H7X6tXRXB#Gwr&gzp8ew+g$zW29tI&?(BMd@XBTbBZcsd z-i9B1b^@c7?}XBp`sDKyMLSY%)FuAk`srTf_TR;_`On*5rv`^DNEP4o`o{ch5U+b* z!Yl3B_qB5E4urd%-Tbd~vgY&+&B=8$_Rc-Qz4hV3c@h_T#f8d~@9Dp~)4-RqW#P^2 zKfri7eHxgWMYS&&dtWp@Zm!3_Wa literal 0 HcmV?d00001 diff --git a/store/datatiles_store/static/icons/icon-512.png b/store/datatiles_store/static/icons/icon-512.png new file mode 100644 index 0000000000000000000000000000000000000000..ae763992522987b7a146f46fa049b62994e6cc56 GIT binary patch literal 3412 zcmds4dr(tX8b3F|pa@2rA_UZEt*};CG*Ao*1i^~PYiq?|jRfQo1VJ#mpagRf6)le} zQbaU@f}*a-LlGMvBr6tB9>E2J5^O>PL`ZlOf(f~I19l((?##|?+nF=-JM-O{Z{~dS zdwloYpZq+{Osq@*0A^m>x9$W0W4L0#+!;u+^0;fLTYGKYvMZ4{*i{*8?TLZA)mvba z5HC|~zaoa{x4aX4;Wc)mYu`id*I5d8;q9l|Af=(BZzI_&yQ_OzN?Q$wh?-hjMEmMa zj8DGP&4CLzr7i%7R{-!e1}t3lyMv?mCxOio-BiWo!eGBbDx01(_nTugPk&G!r{+yn zn7Aaaf+*)bNM}C7c`k%&DVmVD9TimQO53w0~&pd{C6J9KQ?rd_901KS7E8KC8PcYhN{rIIr8Z#(x@nh(CmpYy1<_JB{ zWfl+JTfr(duHdyZ;X5frm+i=te(@~-2bTXj3wV6+TBkV0 zoS*GqWDe&Xy3`(&vVmXDU2Gyekl@8k7Y4+olCFIM6JoO(THxp&KkD|+K=ZIhz#NHj z(f@c4p@og4tB-Gt#>3u{u5Rp*s^L|ro>)K4!3(QTh$9=xorx_mRUfh(3%2+O*5()o zYn2)+`PvrBO8)iu2$z=aI0wzoP18$-@w6eT02yz`5*zIbkxoSPTKbiWUV=_L>ak7694fRVO54UECS|o1B8_iYJY<4l#p;q)L6o26XoIc%K#b_f zJKL^7YI0A`XjL_HQ>&KgSGO`^lQRA40W1(U_L&Mx`8YAT_cl2Mlsxeh4)kr_Yhwz4 z?&2T8_$n*Q`Y_w3gAfSzH&FJ=0AQ1}35#=RbFvQr;EmK7y12Sg*ihmW(>G7JC_-;s z0PAx_DMtW!X=5n(hxMwkk}Qf@U*dcRF90aJ9Dpl=m0V)FI{?tM5OxZxgF3=b<}l9y z5O-=nxMZh;^XB-U*$+UV5m>Gqz^?zBX1y@?(d4*d02_Bp7VR4B|6qKD1ptnQ?>hh} z`mMkKG&S8VZxadF+UTb#^!7=ghcww(&9|toNh`(kk%WA;z&cz{qv3^F?$~Vwq1T|5 zqRLkI*%FPWZ}71un%f-^OCl}Mszo@oRT^fM%AT%7l=l#c-{I@@`Tn5saBtp7q`c`k zFPr{%%Kc@`&-2=O>IKkLQML89%fpd!r-!flHXE3D*HJpm-;)HeZ&l~|Noi^Cv|`Ra zF?Bom^IMXkpW&yds^RqXbie3k;UN-fH)rur`%q~F1JzY_pqjTGXl$vPk!K5+`*5^& zp`!SI3C=q-OLxaOSSJ_#Q)$NCr_aZf5BO{~zibATM z|C8IgsH76fnz&<(yJ&KRemISeJ~pTz^B!`?Fi+24n4hf(jC2dh(j|S2kdZGQa`DD` zEi;w#{t!8Hiz_nAf*4z;on@zO;c)(#cVgA7o31;++rza#?;X{0JPokFmY5YcCsd8{ zB$4AwW1JPcFAr$@}z%Ar85d?P)FIXF5t>)v;n9 zd^o`cYrT(d%&=F+Z#(Su(I9rR#G9XBI^P+iI<<6-S&#b(?L>W>$S~nM)cM0@=%M@N zsQ!%-)v1)6jbd$iBSA8HAj74;-yglPp_O}8r=g4L!4pRSm1y3Wvv$kWje4tZQcn1$a z{UQM0+7F`2KfuXGAJeLDuJ8c|kEIvS^D7`?fN)U}G`B{D4Qh2J6dBU#}W>&QyP^xt2 z$}dF&EPKdTYJ2-gvZl^~aai}Y24K8?8un9=6|BjJ7m!r>k+16jF`k0;60mk~u*T5_BNSup5q?3@3VO|3$5rMR&u!N3DKPA##en{$CpzCKn6U%Up7RBGh>~F{ zlr7Qg2=L0)km-+njuAwGHoyUt35FN&^T*44cvClL%n@1kD@oA*d5r5xH+5p7BFxg) Q@V5lKzVh2z<{o_VA5qcTLjV8( literal 0 HcmV?d00001 diff --git a/store/datatiles_store/static/manifest.webmanifest b/store/datatiles_store/static/manifest.webmanifest new file mode 100644 index 0000000..b7a11af --- /dev/null +++ b/store/datatiles_store/static/manifest.webmanifest @@ -0,0 +1 @@ +{"name":"DataTiles Store","short_name":"DataTiles","start_url":"/catalog","display":"standalone","background_color":"#06111b","theme_color":"#07131f","description":"Institutional catalog and explorer for DataTiles scientific geospatial products","icons":[{"src":"/static/icons/icon-192.png","sizes":"192x192","type":"image/png"},{"src":"/static/icons/icon-512.png","sizes":"512x512","type":"image/png"}]} diff --git a/store/datatiles_store/static/service-worker.js b/store/datatiles_store/static/service-worker.js new file mode 100644 index 0000000..ccef944 --- /dev/null +++ b/store/datatiles_store/static/service-worker.js @@ -0,0 +1 @@ +const CACHE='datatiles-store-shell-v2';const SHELL=['/static/app.css','/static/explorer.js','/manifest.webmanifest'];self.addEventListener('install',e=>e.waitUntil(caches.open(CACHE).then(c=>c.addAll(SHELL))));self.addEventListener('activate',e=>e.waitUntil(caches.keys().then(keys=>Promise.all(keys.filter(k=>k!==CACHE).map(k=>caches.delete(k))))));self.addEventListener('fetch',e=>{const u=new URL(e.request.url);if(e.request.method!=='GET'||u.origin!==self.location.origin)return;if(u.pathname.startsWith('/catalog/')&&u.pathname.endsWith('/download'))return;if(u.pathname.startsWith('/api/'))return;e.respondWith(caches.match(e.request).then(r=>r||fetch(e.request).then(resp=>{if(resp.ok&&['style','script','manifest'].includes(e.request.destination)){const copy=resp.clone();caches.open(CACHE).then(c=>c.put(e.request,copy));}return resp;})));}); diff --git a/store/datatiles_store/templates/agreement.html b/store/datatiles_store/templates/agreement.html new file mode 100644 index 0000000..ff9349c --- /dev/null +++ b/store/datatiles_store/templates/agreement.html @@ -0,0 +1,24 @@ +{% extends 'base.html' %}{% block title %}Agreement · {{item.title}} · DataTiles Store{% endblock %} +{% block content %} +
+ ← {{item.title}} +

Data licence and safety agreement

+

Data access is bound to this exact release: {{agreement.file_sha256}}. A replacement or changed agreement requires new acceptance.

+ {% if accepted %}
Accepted {{accepted.accepted_at}} via {{accepted.source}}.
{% endif %} +
+

Data rights and licences

+

{{agreement.license_notice}}

+ {% for r in agreement.rights %} +
{{r.scope or 'dataset'}}{{r.license_expression or 'No SPDX expression supplied'}}{% if r.license_uri %}{{r.license_uri}}{% endif %}{% if r.attribution_text %}

{{r.attribution_text}}

{% endif %}
+ {% else %}
No structured rights record is embedded. Do not infer permission from file availability; contact the distributor before reuse.
{% endfor %} +
+

Not suitable for navigation · AS IS · no responsibility

{{agreement.safety_text}}

Agreement version: {{agreement.safety_version}}
+
+ + + + +
+

Third-party applications can retrieve and accept this agreement via /api/v1/catalog/{{item.id}}/agreement and /api/v1/catalog/{{item.id}}/agreement/accept.

+
+{% endblock %} diff --git a/store/datatiles_store/templates/base.html b/store/datatiles_store/templates/base.html new file mode 100644 index 0000000..0c6ef6a --- /dev/null +++ b/store/datatiles_store/templates/base.html @@ -0,0 +1,28 @@ + + + + + + + {% block title %}DataTiles Store{% endblock %} + + + {% block head %}{% endblock %} + + +
+ DataTiles Store + +
+{% with messages=get_flashed_messages(with_categories=true) %}{% if messages %}
{% for cat,msg in messages %}
{{msg}}
{% endfor %}
{% endif %}{% endwith %} +
{% block content %}{% endblock %}
+
DataTiles StoreNot for navigation
+ +{% block scripts %}{% endblock %} + diff --git a/store/datatiles_store/templates/catalog.html b/store/datatiles_store/templates/catalog.html new file mode 100644 index 0000000..410e000 --- /dev/null +++ b/store/datatiles_store/templates/catalog.html @@ -0,0 +1,12 @@ +{% extends 'base.html' %}{% block title %}Catalog · DataTiles Store{% endblock %} +{% block content %} +

Scientific geospatial catalog

Explore DataTiles

Search standardized metadata, variables, provenance and rights. Preview available portrayal slices and download authorized research products.

+
+
+
+ {% if can('catalog.manage') %}
{{managers_group}} CRUD
{% endif %} +
+
+ {% for item in items %}
{{ item.format }}

{{item.title}}

{{item.description or 'No description supplied.'}}

Revision
{{item.schema_revision}}
Size
{{'%.1f'|format(item.size_bytes/1048576)}} MiB
Explore{% if can('catalog.download') %}Licence / Download{% endif %}
{% else %}

No matching DataTiles

{% if q %}Try a broader metadata search.{% else %}An administrator can upload products or rescan the catalog directory.{% endif %}

{% endfor %} +
+
{% endblock %} diff --git a/store/datatiles_store/templates/checkout.html b/store/datatiles_store/templates/checkout.html new file mode 100644 index 0000000..55f7109 --- /dev/null +++ b/store/datatiles_store/templates/checkout.html @@ -0,0 +1 @@ +{% extends "base.html" %}{% block title %}Checkout · DataTiles Store{% endblock %}{% block content %}

Purchase {{item.title}}

Release {{item.product_version or 'unversioned'}}

{{item.price_amount}} {{item.price_currency}}

Payment is delegated to the configured provider; the Store does not store card/bank credentials.

{% endblock %} \ No newline at end of file diff --git a/store/datatiles_store/templates/configuration.html b/store/datatiles_store/templates/configuration.html new file mode 100644 index 0000000..f628360 --- /dev/null +++ b/store/datatiles_store/templates/configuration.html @@ -0,0 +1 @@ +{% extends "base.html" %}{% block title %}Configuration · DataTiles Store{% endblock %}{% block content %}

Configuration

Administrator-only application settings. Secret values are never rendered back into the browser; leave a secret field blank to keep its current value.

Deployment configuration

These startup settings are shown here for complete configuration visibility. Change them through the deployment configuration/environment and restart the service.

{% for k,v in deployment.items() %}
{{ k }}
{{ v }}
{% endfor %}

Runtime configuration

{% for section,defs in sections.items() %}
{{ section }}{% for d in defs %}{% endfor %}
{% endfor %}
{% endblock %} diff --git a/store/datatiles_store/templates/detail.html b/store/datatiles_store/templates/detail.html new file mode 100644 index 0000000..8f5f4b4 --- /dev/null +++ b/store/datatiles_store/templates/detail.html @@ -0,0 +1,22 @@ +{% extends 'base.html' %}{% block title %}{{item.title}} · DataTiles Store{% endblock %} +{% block head %}{% endblock %} +{% block content %} +
+
+ +
+{% endblock %} +{% block scripts %}{% endblock %} diff --git a/store/datatiles_store/templates/groups.html b/store/datatiles_store/templates/groups.html new file mode 100644 index 0000000..e335836 --- /dev/null +++ b/store/datatiles_store/templates/groups.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Groups · DataTiles Store{% endblock %}{% block content %}

Groups and roles

{% for r in roles %}{% endfor %}
{% for g in groups %}{% endfor %}
GroupRolesUsers
{{g.name}}{{g.roles|map(attribute='name')|join(', ')}}{{g.users|map(attribute='username')|join(', ')}}
{% endblock %} diff --git a/store/datatiles_store/templates/help.html b/store/datatiles_store/templates/help.html new file mode 100644 index 0000000..4cefb6b --- /dev/null +++ b/store/datatiles_store/templates/help.html @@ -0,0 +1 @@ +{% extends "base.html" %}{% block title %}{{ title }} · DataTiles Store{% endblock %}{% block content %}

{{ title }}

{% if docs %}{% else %}
{{ content|safe }}

Back to Help

{% endif %}
{% endblock %} diff --git a/store/datatiles_store/templates/library.html b/store/datatiles_store/templates/library.html new file mode 100644 index 0000000..c21e0bc --- /dev/null +++ b/store/datatiles_store/templates/library.html @@ -0,0 +1 @@ +{% extends "base.html" %}{% block title %}My Library · DataTiles Store{% endblock %}{% block content %}

My DataTiles

Updates

{% for n in notifications %}{% else %}

No updates.

{% endfor %}

Purchases

{% for x in purchases %}
{{x.item.title}}{{x.product_version or 'unversioned'}} · {{x.amount}} {{x.currency}}
{% else %}

No purchases.

{% endfor %}

Downloads

{% for x in downloads %}
{{x.item.title}}{{x.product_version or 'unversioned'}} · {{x.downloaded_at}}{{x.file_sha256}}
{% else %}

No downloads.

{% endfor %}
{% endblock %} \ No newline at end of file diff --git a/store/datatiles_store/templates/login.html b/store/datatiles_store/templates/login.html new file mode 100644 index 0000000..654d489 --- /dev/null +++ b/store/datatiles_store/templates/login.html @@ -0,0 +1,5 @@ +{% extends "base.html" %}{% block title %}Sign in · DataTiles Store{% endblock %} +{% block content %}

Sign in

+{% if local_enabled %}
{% endif %} +
{% if google_enabled %}Continue with Google{% endif %}{% if microsoft_enabled %}Continue with Microsoft{% endif %}{% if oauth2_enabled %}Continue with {{ oauth2_name }}{% endif %}
+{% if registration_enabled %}

New user? Create an account.

{% endif %}
{% endblock %} diff --git a/store/datatiles_store/templates/register.html b/store/datatiles_store/templates/register.html new file mode 100644 index 0000000..62b953c --- /dev/null +++ b/store/datatiles_store/templates/register.html @@ -0,0 +1 @@ +{% extends "base.html" %}{% block title %}Register · DataTiles Store{% endblock %}{% block content %}

Create account

Your email address becomes your username. The account remains disabled until email verification succeeds.

Alternatively use an enabled Google, Microsoft, or institutional identity provider from the sign-in page.

{% endblock %} diff --git a/store/datatiles_store/templates/users.html b/store/datatiles_store/templates/users.html new file mode 100644 index 0000000..fb2ae4f --- /dev/null +++ b/store/datatiles_store/templates/users.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Users · DataTiles Store{% endblock %}{% block content %}

Users

{% for u in users %}{% endfor %}
UserGroupsPermissions
{{u.username}}{{u.groups|map(attribute='name')|join(', ')}}{{u.permissions()|sort|join(', ')}}
{% endblock %} diff --git a/store/docs/README.md b/store/docs/README.md new file mode 100644 index 0000000..52d55ea --- /dev/null +++ b/store/docs/README.md @@ -0,0 +1,15 @@ +# DataTiles Store help library + +These Markdown files are the authoritative Store operations/help documentation. The Flask PWA renders them directly under `/help`, and third-party applications can retrieve them through `/api/v1/help`. + +- `installation.md` — creation, installation, deployment, persistence. +- `configuration.md` — administrator configuration model. +- `authentication.md` — managed passwords, registration, Google, Microsoft Entra ID, generic OIDC. +- `smtp.md` — SMTP and email verification. +- `management.md` — catalog/user/group management. +- `api.md` — third-party API integration. +- `agreements.md` — licence and safety acceptance. +- `security.md` — production security and operations. +- `troubleshooting.md` — common operational failures. + +See `payments.md` and `versioning-and-updates.md`. diff --git a/store/docs/agreements.md b/store/docs/agreements.md new file mode 100644 index 0000000..e94cc55 --- /dev/null +++ b/store/docs/agreements.md @@ -0,0 +1,5 @@ +# Data licences and safety agreement + +Before protected preview/download, users must accept both the current data licence snapshot and the versioned Store safety/no-liability statement. Acceptance is bound to user, DataTiles SHA-256, rights fingerprint, and safety-agreement version. + +The Store records acceptance as evidence of assent. It does not relicense upstream data, waive non-waivable law, certify accuracy, or make a product suitable for navigation. A changed file, rights statement, or safety version requires fresh acceptance. diff --git a/store/docs/api.md b/store/docs/api.md new file mode 100644 index 0000000..751dbf8 --- /dev/null +++ b/store/docs/api.md @@ -0,0 +1,13 @@ +# API integration + +All operational Store capabilities have `/api/v1` counterparts for third-party applications. Bearer tokens resolve to the same users, groups, roles, permissions, licence acceptance, and audit rules as browser sessions. + +Use `/api/v1/openapi.json` for machine-readable discovery. Configuration is available to administrators through `GET/PATCH /api/v1/configuration`; help content is available through `/api/v1/help` and `/api/v1/help/{slug}`. + +Data preview/download can return HTTP `428 Precondition Required` until the current licence and safety/no-liability agreement is accepted for the exact file hash and rights fingerprint. + +OAuth browser redirects remain interactive identity-provider flows; API clients should complete an approved interactive OIDC login or use Store-issued Bearer tokens according to institutional policy. Never embed administrator credentials in third-party applications. + + +## Commerce and library API +The API exposes provider discovery, checkout/capture, user library, and update notifications. diff --git a/store/docs/authentication.md b/store/docs/authentication.md new file mode 100644 index 0000000..635c372 --- /dev/null +++ b/store/docs/authentication.md @@ -0,0 +1,29 @@ +# Authentication, SSO, and registration + +The Store supports four optional authentication paths that all resolve to the same SQLAlchemy `User -> Group -> Role -> Permission` authorization model. + +## Managed accounts + +When local authentication is enabled, passwords are stored only as Werkzeug password hashes. Administrators can create managed users. For self-registration, the submitted email becomes both `username` and `email`; the account is inactive until verification succeeds. + +## Email registration + +When registration and local authentication are enabled, the user submits an email and a password of at least 12 characters. The Store sends a signed, time-limited verification link over the configured SMTP server. Successful verification activates the account and assigns the configured default group. Operators must use HTTPS and a correct public base URL. + +## Google + +Enable Google OIDC and configure client ID/client secret. The callback is `/auth/google/callback`. Optional allowed Workspace domains restrict new/existing Google sign-ins. Google must return a usable verified email identity. + +## Microsoft Entra ID + +Enable Microsoft authentication, configure the tenant (`common`, tenant UUID, or tenant domain), client ID, and client secret. The callback is `/auth/microsoft/callback`. For institutional deployment, use a specific tenant rather than `common` unless multi-tenant access is intended. + +## Generic OAuth2/OIDC + +The generic integration is OpenID Connect based: configure a discovery metadata URL, client ID, client secret, scopes, and display name. The callback is `/auth/oauth2/callback`. The provider must expose a stable subject identifier and email claim. + +## Registration policy + +If registration is disabled, an external identity may sign in only when it maps to an already provisioned Store user. If enabled, a first successful Google/Microsoft/OIDC login can create the account automatically and place it in the default group. Administrators remain responsible for assigning elevated groups. + +Never map external claims directly to `admin` or `managers` without an explicit institutional authorization policy. diff --git a/store/docs/configuration.md b/store/docs/configuration.md new file mode 100644 index 0000000..c3f72f0 --- /dev/null +++ b/store/docs/configuration.md @@ -0,0 +1,26 @@ +# Configuration + +Administrators open **Configuration** in the PWA. Runtime identity, registration, SMTP, and public-URL settings are stored through SQLAlchemy and can also be managed through the authenticated `/api/v1/configuration` API. + +Secret fields are write-only in the web/API representation: an existing client secret or SMTP password is not returned to browsers or API clients. Blank secret fields in the web form retain the existing value. + +## Authentication controls + +- `auth.local.enabled`: managed email/username and password authentication. +- `auth.registration.enabled`: permits new self-service accounts. +- `auth.registration.default_group`: group assigned to self-registered identities. +- Google, Microsoft, and generic OIDC settings enable infrastructure SSO. + +At least one usable authentication method must remain enabled before an administrator signs out. + +## Public base URL + +Set `store.public_base_url` to the canonical HTTPS origin when the Store is behind a proxy. Verification emails and OIDC redirect URIs use this value. Register the resulting callback URIs with each identity provider. + +## SMTP + +Configure host, port, STARTTLS or implicit TLS, username/password, sender address/name, and timeout. Email verification requires SMTP to be enabled and operational. Use the SMTP test operation after changes. + + +## Commerce and PayPal +Payment is disabled by default. Administrators configure the provider and PayPal sandbox/live credentials in Configuration. diff --git a/store/docs/index.md b/store/docs/index.md new file mode 100644 index 0000000..d001a7e --- /dev/null +++ b/store/docs/index.md @@ -0,0 +1,17 @@ +# DataTiles Store documentation + +This is the authoritative operational documentation for the DataTiles Store PWA. These Markdown files are rendered directly by the in-application **Help** section and are also available through `/api/v1/help` for third-party integrations. + +- [Installation and deployment](installation.md) +- [Configuration](configuration.md) +- [Authentication and registration](authentication.md) +- [SMTP and email verification](smtp.md) +- [Store and catalog management](management.md) +- [Payments and commerce](payments.md) +- [Versioning, user library, and updates](versioning-and-updates.md) +- [Licence and safety agreements](agreements.md) +- [REST API](api.md) +- [Security and operations](security.md) +- [Troubleshooting](troubleshooting.md) + +Production operators should read the configuration, security, agreements, and payment sections before exposing a Store to the Internet. DataTiles data licences remain independent of the Apache-2.0 Store software licence. diff --git a/store/docs/installation.md b/store/docs/installation.md new file mode 100644 index 0000000..d651dd8 --- /dev/null +++ b/store/docs/installation.md @@ -0,0 +1,27 @@ +# Installation and deployment + +## Purpose + +DataTiles Store is a Flask PWA and API service. It keeps its SQLAlchemy application database separate from the scientific DataTiles files. Catalog indexing is read-only with respect to published DataTiles objects; manager CRUD replaces whole files rather than editing scientific releases in place. + +## Requirements + +Python 3.10 or newer is required. From the repository root: + +```bash +cd store +python -m venv .venv +. .venv/bin/activate +python -m pip install -e . +python run.py +``` + +For production install `.[prod]` and run behind an HTTPS reverse proxy, for example Gunicorn. Set a strong `DATATILES_STORE_SECRET_KEY`, protect the application database and catalog filesystem, and set secure session cookies. + +## Initial administrator + +`store/config.py` defines bootstrap `ADMIN_USERNAME`, `ADMIN_PASSWORD`, `ADMIN_GROUP`, and the managers group. Environment variables should supply the initial secret in production. On first initialization the password is hashed into the SQLAlchemy database. Rotate the bootstrap credential immediately after provisioning. + +## Persistence + +Back up the SQLAlchemy database and catalog directory together. DataTiles files are independently checksum-indexed; acceptance evidence and audit events live in the application database. Do not restore one without considering consistency with the other. diff --git a/store/docs/management.md b/store/docs/management.md new file mode 100644 index 0000000..a94ab0e --- /dev/null +++ b/store/docs/management.md @@ -0,0 +1,28 @@ +# Store management + +## Roles + +Administrators manage configuration, users, groups, roles, audit evidence, and catalog operations. Members of the `managers` group carry the catalog-manager role and can create, read, replace/rename, and delete DataTiles files. + +## Catalog CRUD + +Create uploads a complete DataTiles/MBTiles-compatible SQLite file. Update validates a replacement before atomic publication. Delete removes the catalog item and file. Read includes catalog metadata, preview, agreement state, and download according to permissions. + +Replacing a file changes its SHA-256 and invalidates prior licence/safety acceptance for protected interactions. Never edit a signed scientific release in place merely to change Store metadata. + +## Indexing + +The Store automatically extracts searchable metadata, semantic variables, rights, provenance, bounds, zooms, signatures, and commercial metadata. A rescan reconciles the catalog database with the filesystem. + +## User administration + +Use groups for authorization. Keep ordinary users out of `administrators` and `managers` unless their duties require those capabilities. Disable accounts promptly when access is withdrawn. + + +## Pricing and releases +Managers can set purchase-required, price, and currency on a catalog release through the API. Publish new versioned DataTiles files rather than mutating released bytes. + + +### Versioned release immutability + +Versioned revision-8 releases are immutable in the Store. The replace operation returns HTTP 409 for a release that declares `product_id` and `sequence`; publish the successor as a new file/catalog item. This keeps historical purchase/download records and cryptographic citations meaningful. Unversioned staging assets may still use replacement CRUD. diff --git a/store/docs/payments.md b/store/docs/payments.md new file mode 100644 index 0000000..210d62f --- /dev/null +++ b/store/docs/payments.md @@ -0,0 +1,26 @@ +# Payments and commerce + +Payment is optional and provider-independent. `PaymentProvider` defines checkout creation and capture. Store records provider name, external order reference, amount/currency, state, and entitlement; it never stores card or bank credentials. + +## PayPal reference provider +The included implementation uses PayPal REST Orders v2 with server-side OAuth 2.0 client credentials. Configure Commerce and PayPal settings in the administrator Configuration page. Use sandbox first. The flow creates an order with `CAPTURE`, redirects to the PayPal approval URL, then captures server-side after approval. + +PayPal code is isolated in `payments.py`; other providers can implement the same interface without changing purchase/download tables. + +## API +- `GET /api/v1/payments/providers` +- `POST /api/v1/catalog/{id}/checkout` +- `POST /api/v1/payments/{transaction_id}/capture` +- `GET /api/v1/library` +- `GET /api/v1/notifications` + +Payment entitlement and data-licence/safety acceptance are independent gates. + +## PayPal implementation references + +The reference adapter follows the current PayPal REST guidance: + +- https://developer.paypal.com/api/rest/integration/orders-api +- https://developer.paypal.com/api/make-api-requests + +Use `https://api-m.sandbox.paypal.com` for sandbox testing and `https://api-m.paypal.com` only after production credentials and merchant readiness are established. diff --git a/store/docs/security.md b/store/docs/security.md new file mode 100644 index 0000000..95e1d1c --- /dev/null +++ b/store/docs/security.md @@ -0,0 +1,9 @@ +# Security and operations + +Use HTTPS, a strong secret key, secure cookies, least-privilege groups, protected database/catalog storage, and a production WSGI server. Put the service behind a maintained reverse proxy and apply infrastructure rate limiting and monitoring. + +Treat OAuth client secrets and SMTP credentials as secrets. Configuration APIs are administrator-only and secret values are not returned. Rotate provider credentials after suspected exposure. + +The PWA service worker caches only public application-shell assets, not authenticated API responses, map payloads, or downloadable DataTiles files. + +Back up audit/acceptance records. Review logs for authentication failures, token issuance/revocation, catalog CRUD, agreement acceptance, and administrative changes. Test restore procedures periodically. diff --git a/store/docs/smtp.md b/store/docs/smtp.md new file mode 100644 index 0000000..170a154 --- /dev/null +++ b/store/docs/smtp.md @@ -0,0 +1,7 @@ +# SMTP and email verification + +SMTP is configured from the administrator Configuration page. Use a dedicated service account where possible. Prefer STARTTLS on port 587 or implicit TLS where required by the provider. Do not use a personal mailbox password when an application credential is available. + +The verification message contains a signed, expiring URL. Verification proves control of the mailbox at that time; it is not proof of legal identity or organizational role. Group/role elevation remains an administrator action. + +Operational monitoring should alert on delivery failures. If SMTP is disabled, local self-registration requiring email verification cannot complete and should be disabled. diff --git a/store/docs/troubleshooting.md b/store/docs/troubleshooting.md new file mode 100644 index 0000000..fee7848 --- /dev/null +++ b/store/docs/troubleshooting.md @@ -0,0 +1,17 @@ +# Troubleshooting + +## OIDC redirect mismatch + +Set the Store public base URL to the exact externally visible HTTPS origin and register `/auth/google/callback`, `/auth/microsoft/callback`, or `/auth/oauth2/callback` with the provider. + +## Registration email does not arrive + +Confirm SMTP is enabled, sender policy permits the configured From address, TLS mode/port match the provider, and credentials are valid. Check spam/quarantine and application logs. + +## User can sign in but cannot download + +Check group/role permissions and whether the current DataTiles licence/safety agreement has been accepted. A recently replaced file requires a new acceptance. + +## External user is rejected + +When registration is disabled, provision the user first. For Google, check allowed domains. For Microsoft, check tenant configuration. For generic OIDC, confirm the provider returns `sub` and `email` claims. diff --git a/store/docs/versioning-and-updates.md b/store/docs/versioning-and-updates.md new file mode 100644 index 0000000..fc9457e --- /dev/null +++ b/store/docs/versioning-and-updates.md @@ -0,0 +1,7 @@ +# Versioning, user library, and update notifications + +Revision-8 DataTiles may declare stable `product_id`, human `version`, monotonic integer `sequence`, release timestamp, predecessor, release-notes URI, and update URI. The Store indexes these fields. + +The user library records purchases and actual downloads. Download history retains the exact file SHA-256. When a higher sequence for the same product enters the catalog, the Store can notify users who previously purchased/downloaded an older sequence. A notification does not grant entitlement to the new release. + +Never overwrite a published release's bytes. Publish a new object with a larger sequence, checksum, and new signature where used. diff --git a/store/pyproject.toml b/store/pyproject.toml new file mode 100644 index 0000000..5ca05a6 --- /dev/null +++ b/store/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "datatiles-store" +version = "0.4.0" +description = "Flask progressive web catalog and explorer for DataTiles" +requires-python = ">=3.10" +dependencies = [ + "Flask>=3.0,<4", + "SQLAlchemy>=2.0,<3", + "Flask-Login>=0.6,<1", + "Flask-WTF>=1.2,<2", + "Pillow>=10,<12", + "Authlib>=1.3,<2", + "Markdown>=3.6,<4", +] + +[project.optional-dependencies] +prod = ["gunicorn>=22,<24"] +test = ["pytest>=8,<9"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["datatiles_store*"] diff --git a/store/run.py b/store/run.py new file mode 100644 index 0000000..fc2723f --- /dev/null +++ b/store/run.py @@ -0,0 +1,5 @@ +from datatiles_store import create_app +app = create_app() + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080, debug=False) diff --git a/store/tests/test_catalog.py b/store/tests/test_catalog.py new file mode 100644 index 0000000..9967b27 --- /dev/null +++ b/store/tests/test_catalog.py @@ -0,0 +1,56 @@ +from __future__ import annotations +import sqlite3 +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from datatiles_store.catalog import extract_metadata, scan_catalog, tile_bytes +from datatiles_store.models import Base, CatalogItem + + +def make_tile(path: Path): + con=sqlite3.connect(path) + con.executescript(''' + PRAGMA user_version=7; + CREATE TABLE metadata(name TEXT PRIMARY KEY,value TEXT); + INSERT INTO metadata VALUES('name','Test Sea Depth'); + INSERT INTO metadata VALUES('description','Bathymetry test product'); + INSERT INTO metadata VALUES('bounds','12,39,16,42'); + INSERT INTO metadata VALUES('minzoom','4'); + INSERT INTO metadata VALUES('maxzoom','8'); + CREATE TABLE datatiles_variables(variable_id INTEGER PRIMARY KEY,name TEXT,standard_name TEXT,standard_name_vocabulary TEXT,standard_name_vocabulary_version TEXT,canonical_unit TEXT,long_name TEXT,description TEXT); + INSERT INTO datatiles_variables VALUES(1,'bathymetry','sea_floor_depth_below_geoid','CF','94','m','Depth',NULL); + CREATE TABLE datatiles_rights(rights_id INTEGER PRIMARY KEY,scope TEXT,license_expression TEXT,license_uri TEXT,rights_holder TEXT,attribution_text TEXT,access_rights TEXT,applies_to TEXT); + INSERT INTO datatiles_rights VALUES(1,'dataset','CC-BY-4.0','https://creativecommons.org/licenses/by/4.0/','Example','Credit Example','open',NULL); + CREATE TABLE tiles(zoom_level INTEGER,tile_column INTEGER,tile_row INTEGER,tile_data BLOB); + ''') + con.commit(); con.close() + + +def test_metadata_extraction(tmp_path): + p=tmp_path/'a.datatiles'; make_tile(p) + d=extract_metadata(p) + assert d['revision']==7 + assert d['variables'][0]['standard_name']=='sea_floor_depth_below_geoid' + assert d['rights'][0]['license_expression']=='CC-BY-4.0' + assert d['bounds']==[12.0,39.0,16.0,42.0] + + +def test_scan_indexes_searchable_metadata(tmp_path): + p=tmp_path/'a.datatiles'; make_tile(p) + engine=create_engine('sqlite:///:memory:'); Base.metadata.create_all(engine) + with Session(engine) as db: + result=scan_catalog(db,tmp_path,('.datatiles',)) + assert result=={'indexed':1,'failed':0} + item=db.query(CatalogItem).one() + assert 'sea_floor_depth_below_geoid' in item.search_text + assert item.sha256 and len(item.sha256)==64 + + +def test_tile_xyz_to_tms_and_png_detection(tmp_path): + p=tmp_path/'a.datatiles'; make_tile(p) + png=b'\x89PNG\r\n\x1a\nrest' + con=sqlite3.connect(p); con.execute('INSERT INTO tiles VALUES(2,1,2,?)',(png,)); con.commit(); con.close() + data,mime=tile_bytes(p,2,1,1) # XYZ y=1 -> TMS y=2 + assert data==png and mime=='image/png' diff --git a/store/tests/test_commerce.py b/store/tests/test_commerce.py new file mode 100644 index 0000000..aaf77bf --- /dev/null +++ b/store/tests/test_commerce.py @@ -0,0 +1,20 @@ +import json +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +from datatiles_store.models import Base,User,CatalogItem +from datatiles_store.commerce import money,record_download,generate_update_notifications +from datatiles_store.payments import PayPalProvider,CheckoutRequest + +def test_money(): assert money("12")=="12.00" +def test_update_notification_generation(): + e=create_engine("sqlite:///:memory:"); Base.metadata.create_all(e) + with Session(e) as db: + u=User(username="u",password_hash="x"); a=CatalogItem(relative_path="a",filename="a",title="A",size_bytes=1,file_mtime_ns=1,sha256="a"*64,product_id="urn:p",product_version="1",product_sequence=1); b=CatalogItem(relative_path="b",filename="b",title="B",size_bytes=1,file_mtime_ns=1,sha256="b"*64,product_id="urn:p",product_version="2",product_sequence=2); db.add_all([u,a,b]); db.flush(); record_download(db,u,a,"api"); db.flush(); assert generate_update_notifications(db,b)==1 +def test_paypal_adapter_without_network(): + replies=[{"access_token":"tok"},{"id":"ORDER","status":"CREATED","links":[{"rel":"approve","href":"https://example/approve"}]}] + class R: + def __init__(self,d): self.d=d + def __enter__(self): return self + def __exit__(self,*a): pass + def read(self): return json.dumps(self.d).encode() + p=PayPalProvider("id","secret",opener=lambda req,timeout=30:R(replies.pop(0))); s=p.create_checkout(CheckoutRequest("ref","Title","10.00","EUR","https://x/r","https://x/c")); assert s.provider_order_id=="ORDER" diff --git a/store/tests/test_policy_models.py b/store/tests/test_policy_models.py new file mode 100644 index 0000000..7394f8a --- /dev/null +++ b/store/tests/test_policy_models.py @@ -0,0 +1,40 @@ +from __future__ import annotations +from datetime import datetime, timezone + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +from datatiles_store.models import AgreementAcceptance, ApiToken, Base, CatalogItem, Group, Role, User + + +def test_policy_and_api_tables_are_sqlalchemy_managed(): + names=set(Base.metadata.tables) + assert {"api_tokens","agreement_acceptances","audit_events","catalog_items","users","groups","roles"} <= names + + +def test_acceptance_is_bound_to_user_file_rights_and_safety_version(): + engine=create_engine("sqlite:///:memory:") + Base.metadata.create_all(engine) + with Session(engine) as db: + role=Role(name="catalog_manager",permissions="catalog.view catalog.preview catalog.download agreements.accept catalog.manage") + group=Group(name="managers",roles=[role]) + user=User(username="manager",password_hash="not-used",groups=[group]) + item=CatalogItem(relative_path="a.datatiles",filename="a.datatiles",title="A",description=None,format="DataTiles",schema_revision=7,size_bytes=1,file_mtime_ns=1,sha256="a"*64,metadata_json="{}",variables_json="[]",rights_json='[{"license_expression":"CC-BY-4.0"}]',provenance_json="{}",search_text="a") + db.add_all([user,item]); db.flush() + acc=AgreementAcceptance(user_id=user.id,catalog_item_id=item.id,file_sha256=item.sha256,license_fingerprint="b"*64,license_snapshot_json=item.rights_json,safety_version="v1",safety_text="not for navigation",source="api",accepted_at=datetime.now(timezone.utc)) + db.add(acc); db.commit() + got=db.scalar(select(AgreementAcceptance)) + assert got.user_id==user.id and got.file_sha256=="a"*64 and got.safety_version=="v1" + assert user.in_group("managers") and user.can("catalog.manage") + +def test_store_auth_configuration_models_exist(): + from datatiles_store.models import AppSetting, ExternalIdentity + assert AppSetting.__tablename__ == "app_settings" + assert ExternalIdentity.__tablename__ == "external_identities" + + +def test_store_help_docs_present(): + from pathlib import Path + docs = Path(__file__).resolve().parents[1] / "docs" + required = {"installation.md","configuration.md","authentication.md","smtp.md","management.md","api.md","agreements.md","security.md","troubleshooting.md"} + assert required.issubset({p.name for p in docs.glob("*.md")}) diff --git a/tests/test_drm.py b/tests/test_drm.py new file mode 100644 index 0000000..fddd02e --- /dev/null +++ b/tests/test_drm.py @@ -0,0 +1,29 @@ +import json +from datetime import datetime, timezone +from pathlib import Path +import pytest +from datatiles.drm import DRMError, decrypt_package, generate_recipient_keypair, issue_license, load_issuer_public_key, protect_file, read_package_header, verify_license +from datatiles.integrity import generate_ed25519_keypair + + +def test_drm_roundtrip_and_tamper(tmp_path): + src=tmp_path/'product.datatiles'; src.write_bytes(b'SQLite-format-test\x00'+bytes(range(256))*20) + pkg=tmp_path/'product.dtpkg'; cek=tmp_path/'product.cek.json' + header=protect_file(src,pkg,content_key_file=cek,product_id='urn:test:product:1',issuer='Test Publisher',terms_uri='https://example.test/terms') + assert read_package_header(pkg)['plaintext_sha256']==header['plaintext_sha256'] + recip_priv=tmp_path/'customer.key'; recip_pub=tmp_path/'customer.pub.pem'; generate_recipient_keypair(recip_priv,recip_pub) + issuer_priv=tmp_path/'issuer.key'; issuer_pub=tmp_path/'issuer.pub.pem'; generate_ed25519_keypair(issuer_priv,issuer_pub) + lic=tmp_path/'license.json'; issue_license(pkg,cek,recip_pub,issuer_priv,lic,issuer='Test Publisher',recipient_id='customer-1',permissions=['read']) + status=verify_license(json.loads(lic.read_text()),load_issuer_public_key(issuer_pub)); assert status['valid'] + out=tmp_path/'out.datatiles'; decrypt_package(pkg,out,license_path=lic,recipient_private_key_path=recip_priv,issuer_public_key_path=issuer_pub); assert out.read_bytes()==src.read_bytes() + data=bytearray(pkg.read_bytes()); data[-20]^=1; pkg.write_bytes(data) + with pytest.raises(Exception): decrypt_package(pkg,tmp_path/'bad.datatiles',license_path=lic,recipient_private_key_path=recip_priv,issuer_public_key_path=issuer_pub) + + +def test_expired_license_rejected(tmp_path): + src=tmp_path/'p'; src.write_bytes(b'abc'); pkg=tmp_path/'p.dtpkg'; cek=tmp_path/'cek.json' + protect_file(src,pkg,content_key_file=cek,product_id='urn:p',issuer='I',terms_uri='https://example.test/t') + rpr=tmp_path/'r'; rpu=tmp_path/'r.pub'; generate_recipient_keypair(rpr,rpu) + ipr=tmp_path/'i'; ipu=tmp_path/'i.pub'; generate_ed25519_keypair(ipr,ipu) + lic=tmp_path/'l'; issue_license(pkg,cek,rpu,ipr,lic,issuer='I',valid_until='2000-01-01T00:00:00Z') + st=verify_license(json.loads(lic.read_text()),load_issuer_public_key(ipu),at_time=datetime.now(timezone.utc)); assert not st['time_valid'] and not st['valid'] diff --git a/tests/test_fair.py b/tests/test_fair.py new file mode 100644 index 0000000..707cb93 --- /dev/null +++ b/tests/test_fair.py @@ -0,0 +1,17 @@ +import pytest + +from datatiles.fair import FairValidationError, validate_spdx_expression + + +@pytest.mark.parametrize( + "expression", + ["CC-BY-4.0", "ODbL-1.0", "LicenseRef-Proprietary", "CC-BY-4.0 AND ODbL-1.0", "(CC-BY-4.0 OR CC0-1.0)"], +) +def test_spdx_expression_accepts_machine_actionable_forms(expression): + assert validate_spdx_expression(expression) == expression + + +@pytest.mark.parametrize("expression", ["", " CC-BY-4.0", "CC-BY-4.0 & MIT", "AND MIT", "MIT OR"]) +def test_spdx_expression_rejects_ambiguous_forms(expression): + with pytest.raises(FairValidationError): + validate_spdx_expression(expression) diff --git a/tests/test_import_utils.py b/tests/test_import_utils.py new file mode 100644 index 0000000..75c658d --- /dev/null +++ b/tests/test_import_utils.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import sys +from pathlib import Path + + +def load_common(): + path = Path(__file__).parents[1] / "utils" / "common.py" + spec = importlib.util.spec_from_file_location("datatiles_utils_common", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_safe_token_and_tile_ranges(): + common = load_common() + assert common.safe_token("Sea Floor Depth") == "sea_floor_depth" + xs, ys = common.tile_ranges((12.8, 39.9, 15.8, 41.3), 7) + assert len(xs) > 0 and len(ys) > 0 + + +def test_local_source_identity_and_checksum(tmp_path): + common = load_common() + source = tmp_path / "sample.nc" + source.write_bytes(b"example scientific source") + with common.resolve_source(str(source)) as resolved: + assert resolved.local_path == source.resolve() + assert resolved.uri == source.resolve().as_uri() + assert resolved.sha256 == hashlib.sha256(source.read_bytes()).hexdigest() + assert not resolved.temporary + + +def test_source_bbox_is_clipped_not_extrapolated(): + common = load_common() + class Axis: + def __init__(self, lo, hi): self.lo, self.hi = lo, hi + def min(self): return self.lo + def max(self): return self.hi + assert common.source_bbox(Axis(40, 41), Axis(13, 15), (12, 39, 14.5, 42)) == (13.0, 40.0, 14.5, 41.0) + + +def test_local_zarr_tree_digest_is_content_and_key_sensitive(tmp_path): + common = load_common() + store = tmp_path / "sample.zarr" + store.mkdir() + (store / "zarr.json").write_text('{"zarr_format":3}') + (store / "c").mkdir() + (store / "c" / "0").write_bytes(b"chunk") + with common.resolve_zarr_source(str(store)) as resolved: + first = resolved.checksum + assert resolved.checksum_algorithm == "zarr-tree-sha256-v1" + assert resolved.local_path == store.resolve() + (store / "c" / "0").write_bytes(b"changed") + with common.resolve_zarr_source(str(store)) as resolved: + assert resolved.checksum != first + + +def test_remote_zarr_requires_authoritative_sha256(): + common = load_common() + import pytest + with pytest.raises(common.ConversionError): + with common.resolve_zarr_source("https://example.org/data.zarr"): + pass + checksum = "a" * 64 + with common.resolve_zarr_source("s3://bucket/snapshot.zarr", source_sha256=checksum) as resolved: + assert resolved.local_path is None + assert resolved.checksum_algorithm == "sha256" + assert resolved.checksum == checksum + + +def test_remote_zarr_signed_url_requires_credential_free_provenance_uri(): + common = load_common() + import pytest + checksum = "b" * 64 + with pytest.raises(common.ConversionError): + with common.resolve_zarr_source("https://example.org/data.zarr?token=secret", source_sha256=checksum): + pass + with common.resolve_zarr_source( + "https://example.org/data.zarr?token=secret", + source_sha256=checksum, + provenance_uri="https://example.org/data.zarr", + ) as resolved: + assert resolved.identifier == "https://example.org/data.zarr" + assert "secret" not in resolved.identifier diff --git a/tests/test_integrity.py b/tests/test_integrity.py new file mode 100644 index 0000000..f211784 --- /dev/null +++ b/tests/test_integrity.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from datatiles.integrity import ( + IntegrityError, build_manifest, create_signature_envelope, generate_ed25519_keypair, + load_private_key, load_public_key, store_envelope, verify_database_against_envelope, + list_stored_signatures, +) + + +def make_db(path: Path): + db = sqlite3.connect(path) + db.executescript(''' + PRAGMA user_version=6; + CREATE TABLE metadata(name TEXT PRIMARY KEY,value TEXT); + CREATE TABLE tiles(z INTEGER,x INTEGER,y INTEGER,data BLOB,PRIMARY KEY(z,x,y)); + CREATE TABLE datatiles_integrity_manifests ( + manifest_id TEXT PRIMARY KEY, profile TEXT NOT NULL, canonicalization TEXT NOT NULL, + hash_algorithm TEXT NOT NULL, root_sha256 TEXT NOT NULL UNIQUE, manifest_json TEXT NOT NULL, + created_at TEXT NOT NULL + ) WITHOUT ROWID; + CREATE TABLE datatiles_signatures ( + signature_id TEXT PRIMARY KEY, manifest_id TEXT NOT NULL REFERENCES datatiles_integrity_manifests(manifest_id), + signature_scheme TEXT NOT NULL, signature_encoding TEXT NOT NULL, signature BLOB NOT NULL, + key_id TEXT NOT NULL, public_key BLOB, signer_agent_id TEXT, signed_at TEXT NOT NULL, + verification_material_json TEXT, UNIQUE(manifest_id,signature_scheme,key_id,signature) + ) WITHOUT ROWID; + INSERT INTO metadata VALUES ('name','demo'); + INSERT INTO tiles VALUES (1,2,3,x'010203'); + ''') + db.commit() + return db + + +def test_manifest_is_stable_and_excludes_signature_tables(tmp_path): + path = tmp_path / 'a.sqlite' + db = make_db(path) + m1 = build_manifest(db, chunk_size=1) + db.execute("INSERT INTO datatiles_integrity_manifests VALUES (?,?,?,?,?,?,?)", ('m','DataTiles-Integrity-Manifest-1','datatiles-sqlite-logical-v1','sha256','0'*64,'{}','2026-01-01T00:00:00Z')) + db.commit() + m2 = build_manifest(db, chunk_size=1) + assert m1 == m2 + db.close() + + +def test_manifest_changes_when_content_changes(tmp_path): + path = tmp_path / 'a.sqlite' + db = make_db(path) + m1 = build_manifest(db) + db.execute("UPDATE metadata SET value='changed' WHERE name='name'") + db.commit() + m2 = build_manifest(db) + assert m1['root_sha256'] != m2['root_sha256'] + db.close() + + +def test_ed25519_sign_store_and_verify(tmp_path): + path = tmp_path / 'a.sqlite' + db = make_db(path) + private_path = tmp_path / 'private.pem' + public_path = tmp_path / 'public.pem' + _, key_id = generate_ed25519_keypair(private_path, public_path) + envelope = create_signature_envelope(build_manifest(db), load_private_key(private_path), signer='test') + assert envelope['signature']['key_id'] == key_id + signature_id = store_envelope(db, envelope) + assert list_stored_signatures(db)[0]['signature_id'] == signature_id + result = verify_database_against_envelope(db, envelope, public_key=load_public_key(public_path), expected_key_id=key_id) + assert result['valid'] is True + assert result['trusted_key_supplied'] is True + db.execute("INSERT INTO tiles VALUES (4,5,6,x'ff')") + db.commit() + changed = verify_database_against_envelope(db, envelope, public_key=load_public_key(public_path)) + assert changed['cryptographically_valid'] is True + assert changed['content_matches_signed_manifest'] is False + assert changed['valid'] is False + db.close() + + +def test_wrong_trusted_key_fails(tmp_path): + path = tmp_path / 'a.sqlite'; db = make_db(path) + p1, q1 = tmp_path/'p1.pem', tmp_path/'q1.pem'; generate_ed25519_keypair(p1,q1) + p2, q2 = tmp_path/'p2.pem', tmp_path/'q2.pem'; generate_ed25519_keypair(p2,q2) + envelope = create_signature_envelope(build_manifest(db), load_private_key(p1)) + with pytest.raises(IntegrityError): + verify_database_against_envelope(db, envelope, public_key=load_public_key(q2)) + db.close() diff --git a/tests/test_semantic.py b/tests/test_semantic.py new file mode 100644 index 0000000..6651a75 --- /dev/null +++ b/tests/test_semantic.py @@ -0,0 +1,23 @@ +import pytest + +from datatiles.semantic import ( + SemanticValidationError, + load_cf_standard_name_table, + validate_cf_standard_name, +) + + +def test_cf_table_validation(tmp_path): + xml = tmp_path / "cf.xml" + xml.write_text( + '' + 'mDepth.' + '' + ) + table = load_cf_standard_name_table(xml) + result = validate_cf_standard_name("sea_floor_depth_below_geoid", canonical_unit="m", table=table) + assert result.canonical_units == "m" + with pytest.raises(SemanticValidationError, match="unknown CF"): + validate_cf_standard_name("depth", canonical_unit="m", table=table) + with pytest.raises(SemanticValidationError, match="canonical unit mismatch"): + validate_cf_standard_name("sea_floor_depth_below_geoid", canonical_unit="K", table=table) diff --git a/tests/test_store.py b/tests/test_store.py index a1cb6ec..e8becb8 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -126,10 +126,15 @@ def test_revision_two_is_migrated_with_content_profiles(tmp_path): with DataTiles(path,create=True,tile_format="png") as store: store.put(0,0,0,b"png",{}) store.db.execute("DROP TABLE datatiles_contents") + for table in ("datatiles_release", "datatiles_drm_policies", "datatiles_commercial_products", + "datatiles_signatures", "datatiles_integrity_manifests", "datatiles_publication_evidence", + "datatiles_fair_agents", "datatiles_rights", "datatiles_related_identifiers", + "datatiles_identifiers", "datatiles_variable_identifiers", "datatiles_variables"): + store.db.execute(f"DROP TABLE {table}") store.db.execute("DELETE FROM metadata WHERE name='datatiles:default_media_type'") store.db.execute("PRAGMA user_version=2"); store.db.commit() with DataTiles(path) as migrated: - assert migrated.db.execute("PRAGMA user_version").fetchone()[0]==3 + assert migrated.db.execute("PRAGMA user_version").fetchone()[0]==8 assert migrated.content_profiles()[0]["media_type"]=="image/png" assert migrated.get(0,0,0,{})==b"png" diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000..60ec5a5 --- /dev/null +++ b/utils/README.md @@ -0,0 +1,83 @@ +# DataTiles import utilities + +`netcdf2datatiles.py`, `grib2datatiles.py`, and `zarr2datatiles.py` convert scientific source grids into DNT1 numeric DataTiles. They deliberately do **not** pre-render PNG/JPEG/WebP map imagery. + +Both commands accept either a local path, a `file:` URI, or an HTTP(S) URL. HTTP(S) input is downloaded to a temporary seekable file, SHA-256 hashed, imported, and deleted. The original URL and checksum are retained as DataTiles source/provenance metadata. + +The first implementation targets rectilinear one-dimensional latitude/longitude grids. It resamples to Web Mercator tile pixel centers using nearest-neighbour sampling and stores rows through the DataTiles XYZ interface, which converts them to MBTiles/TMS storage. Curvilinear grids, rotated poles, projected source grids, conservative remapping, and antimeridian-spanning imports must use a future specialized resampling path rather than being silently approximated. + +Install optional dependencies from a repository checkout: + +```bash +python -m pip install -e '.[utils]' +``` + +NetCDF example: + +```bash +python utils/netcdf2datatiles.py ./ocean.nc ocean.datatiles \ + --variable depth --zoom 7 --bbox 12.8 39.9 15.8 41.3 +``` + +URL example: + +```bash +python utils/netcdf2datatiles.py \ + https://example.org/data/ocean.nc ocean.datatiles --variable depth +``` + +GRIB2 example: + +```bash +python utils/grib2datatiles.py ./forecast.grib2 weather.datatiles \ + --variable t2m --zoom 6 +``` + +When one GRIB file contains incompatible hypercubes, use repeatable cfgrib filters: + +```bash +python utils/grib2datatiles.py forecast.grib2 pressure.datatiles \ + --filter-by-keys typeOfLevel=isobaricInhPa --variable t +``` + +Important constraints: + +- The utilities are optional import tooling; `datatiles` core remains dependency-free. +- NetCDF semantic identity comes from `standard_name` when present; otherwise the local variable is registered under the non-CF `source` vocabulary. +- GRIB import preserves the decoded CF name where available and adds WMO GRIB2 discipline/category/parameter and GRIB short-name identifiers when exposed by cfgrib. +- Inputs without a CF name are never falsely labelled as CF. +- Output uses `application/vnd.datatiles.numeric` + DNT1; portrayal is a downstream reproducible derivation. +- `--max-tiles` is an intentional resource guard. + +## Zarr + +Local directory store: + +```bash +python utils/zarr2datatiles.py ./ocean.zarr ocean.datatiles \ + --variable depth --zoom 7 \ + --source-license CC-BY-4.0 --source-license-uri https://creativecommons.org/licenses/by/4.0/ \ + --source-attribution "Required source attribution" \ + --dataset-license CC-BY-4.0 --dataset-license-uri https://creativecommons.org/licenses/by/4.0/ +``` + +Remote store: + +```bash +python utils/zarr2datatiles.py https://example.org/ocean.zarr ocean.datatiles \ + --source-sha256 \ + --variable depth --zoom 7 \ + --source-license CC-BY-4.0 --source-license-uri https://creativecommons.org/licenses/by/4.0/ \ + --source-attribution "Required source attribution" \ + --dataset-license CC-BY-4.0 --dataset-license-uri https://creativecommons.org/licenses/by/4.0/ +``` + +Zarr stores are multi-object datasets. Local stores use the documented `zarr-tree-sha256-v1` canonical store digest. Remote stores require an externally established SHA-256 for the immutable store/snapshot instead of pretending that a directory URL has single-file bytes. `--group`, `--zarr-format {2,3}`, `--consolidated {auto,true,false}`, and repeatable `--storage-option KEY=VALUE` are supported. Secret storage-option values are never recorded in provenance. Signed or credential-bearing remote URLs require `--provenance-uri` with a stable credential-free identifier. See `docs/zarr-source-profile.md`. + +## Digital signatures + +Converters do not accept private signing keys and do not auto-sign outputs. Cryptographic signing is a release operation, not an ingestion operation. After the final DataTiles object has passed scientific QA, provenance, citation, rights, and FAIR checks, use `datatiles-integrity sign`. This separation reduces private-key exposure and ensures the signature covers the final immutable release state. + +## Commercial DRM + +Import utilities intentionally contain no DRM keys and no licence issuance logic. After a lawful final product is frozen and optionally signed, use `datatiles-drm protect` and `datatiles-drm issue-license`. This keeps source acquisition reproducible and keeps commercial secrets out of scientific ingestion workflows. diff --git a/utils/common.py b/utils/common.py new file mode 100644 index 0000000..701bed8 --- /dev/null +++ b/utils/common.py @@ -0,0 +1,472 @@ +"""Shared implementation for the DataTiles NetCDF, GRIB and Zarr import utilities. + +The repository core remains dependency-free. These utilities intentionally use +optional scientific-I/O dependencies and write only standard DataTiles DNT1 +numeric raster matrices. +""" +from __future__ import annotations + +import hashlib +import math +import os +import re +import tempfile +import urllib.parse +import urllib.request +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Iterator + +WEBMERCATOR_LAT_LIMIT = 85.0511287798066 +DEFAULT_NODATA = -3.4028234663852886e38 + + +class ConversionError(ValueError): + pass + + +@dataclass(frozen=True) +class ResolvedSource: + identifier: str + local_path: Path | None + uri: str + checksum_algorithm: str + checksum: str + temporary: bool + + @property + def sha256(self) -> str: + """Backward-compatible accessor for single-file sources only.""" + if self.checksum_algorithm != "sha256": + raise ConversionError(f"source checksum is {self.checksum_algorithm}, not sha256") + return self.checksum + + +@contextmanager +def resolve_source(source: str, *, timeout: float = 60.0) -> Iterator[ResolvedSource]: + """Resolve a local path or HTTP(S) URL to a seekable local source. + + URLs are downloaded to a temporary file while hashing the exact bytes. A + local source is hashed in place. The original identifier remains the + provenance URI so the resulting DataTiles file does not lose source identity. + """ + parsed = urllib.parse.urlparse(source) + if parsed.scheme in {"http", "https"}: + suffix = Path(parsed.path).suffix + fd, name = tempfile.mkstemp(prefix="datatiles-source-", suffix=suffix) + os.close(fd) + path = Path(name) + digest = hashlib.sha256() + try: + request = urllib.request.Request(source, headers={"User-Agent": "DataTiles-utils/1"}) + with urllib.request.urlopen(request, timeout=timeout) as response, path.open("wb") as output: + while True: + block = response.read(1024 * 1024) + if not block: + break + digest.update(block) + output.write(block) + yield ResolvedSource(source, path, source, "sha256", digest.hexdigest(), True) + finally: + path.unlink(missing_ok=True) + return + + if parsed.scheme not in {"", "file"}: + raise ConversionError("source must be a local path, file: URI, or HTTP(S) URL") + path = Path(urllib.request.url2pathname(parsed.path) if parsed.scheme == "file" else source).expanduser().resolve() + if not path.is_file(): + raise ConversionError(f"source file does not exist: {path}") + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + yield ResolvedSource(source, path, path.as_uri(), "sha256", digest.hexdigest(), False) + + + +def _zarr_tree_digest(path: Path) -> str: + """Hash a local Zarr directory using a canonical key+length+bytes stream. + + The algorithm is named ``zarr-tree-sha256-v1``. It is deliberately not + described as a SHA-256 of a file: a Zarr store is a set of named objects. + Symlinks are rejected so the digest cannot depend on filesystem traversal + semantics outside the declared store root. + """ + digest = hashlib.sha256(b"DataTiles-Zarr-Tree-SHA256-v1\0") + files = [] + for item in path.rglob("*"): + if item.is_symlink(): + raise ConversionError(f"Zarr provenance hashing refuses symlink: {item}") + if item.is_file(): + files.append(item) + for item in sorted(files, key=lambda x: x.relative_to(path).as_posix()): + rel = item.relative_to(path).as_posix().encode("utf-8") + size = item.stat().st_size + digest.update(rel); digest.update(b"\0") + digest.update(str(size).encode("ascii")); digest.update(b"\0") + with item.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + digest.update(b"\0") + return digest.hexdigest() + + +@contextmanager +def resolve_zarr_source(source: str, *, source_sha256: str | None = None, provenance_uri: str | None = None) -> Iterator[ResolvedSource]: + """Resolve a Zarr store without falsely reducing it to a single file. + + Local directory stores receive a deterministic ``zarr-tree-sha256-v1`` + digest over every key/object. Remote stores remain remote and require an + operator-supplied authoritative SHA-256 for the immutable store/snapshot. + This avoids an implicit, potentially unbounded second traversal of a cloud + store merely to manufacture a checksum. + """ + parsed = urllib.parse.urlparse(source) + if parsed.scheme in {"", "file"}: + path = Path(urllib.request.url2pathname(parsed.path) if parsed.scheme == "file" else source).expanduser().resolve() + if not path.is_dir(): + raise ConversionError(f"local Zarr source must be a directory store: {path}") + checksum = _zarr_tree_digest(path) + yield ResolvedSource(source, path, path.as_uri(), "zarr-tree-sha256-v1", checksum, False) + return + if not re.fullmatch(r"[0-9a-fA-F]{64}", source_sha256 or ""): + raise ConversionError("remote Zarr stores require --source-sha256 with the authoritative 64-hex SHA-256 of the immutable store/snapshot") + if (parsed.username or parsed.password or parsed.query) and not provenance_uri: + raise ConversionError("remote Zarr source contains user-info or query parameters; provide --provenance-uri with a stable credential-free identifier so secrets are not persisted") + identity = provenance_uri or source + identity_parsed = urllib.parse.urlparse(identity) + if identity_parsed.username or identity_parsed.password or identity_parsed.query: + raise ConversionError("--provenance-uri must not contain credentials or query parameters") + yield ResolvedSource(identity, None, identity, "sha256", str(source_sha256).lower(), False) + +def require_scientific_stack(): + try: + import numpy as np + import xarray as xr + except ImportError as exc: + raise ConversionError( + "the import utilities require the optional 'utils' dependencies; " + "install with: python -m pip install -e '.[utils]'" + ) from exc + return np, xr + + +def require_zarr_stack(): + np, xr = require_scientific_stack() + try: + import zarr # noqa: F401 + import fsspec # noqa: F401 + except ImportError as exc: + raise ConversionError( + "Zarr import requires zarr and fsspec from the optional 'utils' dependencies; " + "install with: python -m pip install -e '.[utils]'" + ) from exc + return np, xr + + +def safe_token(value: str) -> str: + token = re.sub(r"[^a-zA-Z0-9_]+", "_", value.strip()).strip("_").lower() + if not token: + raise ConversionError(f"cannot derive a variable token from {value!r}") + if token[0].isdigit(): + token = "v_" + token + return token + + +def cf_or_source_semantics(name: str, attrs: dict[str, Any]) -> tuple[str, str, str | None, str | None]: + standard = str(attrs.get("standard_name") or "").strip() + unit = str(attrs.get("units") or "").strip() or None + long_name = str(attrs.get("long_name") or "").strip() or None + if standard: + return standard, "CF", unit, long_name + # Never claim a producer-local name is a CF Standard Name. + return safe_token(name), "source", unit, long_name + + +def find_lat_lon(da: Any, *, lat_name: str | None = None, lon_name: str | None = None) -> tuple[str, str]: + candidates = set(da.coords) | set(da.dims) + if lat_name is None: + lat_name = next((n for n in candidates if n.lower() in {"lat", "latitude", "nav_lat"}), None) + if lon_name is None: + lon_name = next((n for n in candidates if n.lower() in {"lon", "longitude", "nav_lon"}), None) + if not lat_name or not lon_name or lat_name not in da.coords or lon_name not in da.coords: + raise ConversionError(f"{da.name}: unable to identify latitude/longitude coordinates; use --lat/--lon") + if da.coords[lat_name].ndim != 1 or da.coords[lon_name].ndim != 1: + raise ConversionError(f"{da.name}: only rectilinear 1-D latitude/longitude grids are supported") + if lat_name not in da.dims or lon_name not in da.dims: + raise ConversionError(f"{da.name}: latitude and longitude must be array dimensions") + return lat_name, lon_name + + +def normalized_axes(da: Any, lat_name: str, lon_name: str): + np, _ = require_scientific_stack() + lat = np.asarray(da.coords[lat_name].values, dtype="float64") + lon = np.asarray(da.coords[lon_name].values, dtype="float64") + if lat.ndim != 1 or lon.ndim != 1 or lat.size < 1 or lon.size < 1: + raise ConversionError("latitude/longitude axes must be non-empty 1-D arrays") + if not np.all(np.isfinite(lat)) or not np.all(np.isfinite(lon)): + raise ConversionError("latitude/longitude coordinates must be finite") + lon = ((lon + 180.0) % 360.0) - 180.0 + lat_order = np.argsort(lat) + lon_order = np.argsort(lon) + return lat[lat_order], lon[lon_order], lat_order, lon_order + + +def lon_to_tile_x(lon: float, z: int) -> int: + n = 1 << z + return min(n - 1, max(0, int(math.floor((lon + 180.0) / 360.0 * n)))) + + +def lat_to_tile_y(lat: float, z: int) -> int: + lat = min(WEBMERCATOR_LAT_LIMIT, max(-WEBMERCATOR_LAT_LIMIT, lat)) + rad = math.radians(lat) + n = 1 << z + y = (1.0 - math.asinh(math.tan(rad)) / math.pi) / 2.0 * n + return min(n - 1, max(0, int(math.floor(y)))) + + +def tile_ranges(bbox: tuple[float, float, float, float], z: int) -> tuple[range, range]: + west, south, east, north = bbox + if west > east: + raise ConversionError("antimeridian-crossing bboxes are not yet supported; split the import into two bboxes") + x0, x1 = lon_to_tile_x(west, z), lon_to_tile_x(east, z) + y0, y1 = lat_to_tile_y(north, z), lat_to_tile_y(south, z) + return range(min(x0, x1), max(x0, x1) + 1), range(min(y0, y1), max(y0, y1) + 1) + + +def tile_pixel_lon_lat(z: int, x: int, y: int, size: int): + np, _ = require_scientific_stack() + n = float(1 << z) + px = x + (np.arange(size, dtype="float64") + 0.5) / size + py = y + (np.arange(size, dtype="float64") + 0.5) / size + lon = px / n * 360.0 - 180.0 + merc = math.pi * (1.0 - 2.0 * py / n) + lat = np.degrees(np.arctan(np.sinh(merc))) + return lon, lat + + +def nearest_indices(axis, values): + np, _ = require_scientific_stack() + if len(axis) == 1: + return np.zeros(values.shape, dtype="int64") + idx = np.searchsorted(axis, values, side="left") + idx = np.clip(idx, 1, len(axis) - 1) + left = axis[idx - 1] + right = axis[idx] + return idx - (values - left <= right - values) + + +def infer_dimension_type(value: Any) -> str: + np, _ = require_scientific_stack() + if np.issubdtype(np.asarray(value).dtype, np.datetime64): + return "datetime" + if isinstance(value, (bool, np.bool_)): + return "boolean" + if isinstance(value, (int, np.integer)): + return "integer" + if isinstance(value, (float, np.floating)): + return "float" + return "text" + + +def coordinate_value(value: Any) -> Any: + np, _ = require_scientific_stack() + arr = np.asarray(value) + if np.issubdtype(arr.dtype, np.datetime64): + # xarray/numpy time values are normalized to UTC-looking ISO strings. + text = np.datetime_as_string(arr.astype("datetime64[us]"), unit="us") + return str(text) + "Z" if not str(text).endswith("Z") else str(text) + if isinstance(value, np.generic): + value = value.item() + return value if isinstance(value, (str, int, float, bool)) else str(value) + + +def ensure_dimensions(store: Any, dataset: Any, variables: list[Any], *, lat_names: set[str], lon_names: set[str]) -> None: + existing = {row[0] for row in store.db.execute("SELECT name FROM datatiles_dimensions")} + if "variable" not in existing: + store.add_dimension("variable", "text", description="Semantic variable token") + existing.add("variable") + for da in variables: + for dim in da.dims: + if dim in lat_names or dim in lon_names or dim in existing: + continue + coord = dataset.coords.get(dim) + sample = coord.values[0] if coord is not None and coord.size else "0" + axis = str((coord.attrs.get("axis") if coord is not None else "") or "").strip().upper() or None + unit = str((coord.attrs.get("units") if coord is not None else "") or "").strip() or None + store.add_dimension(dim, infer_dimension_type(sample), axis=axis, unit=unit, + description=f"Imported source dimension {dim}", required=False) + existing.add(dim) + + +def iter_non_spatial_slices(da: Any, lat_name: str, lon_name: str) -> Iterable[tuple[dict[str, Any], Any]]: + np, _ = require_scientific_stack() + dims = [d for d in da.dims if d not in {lat_name, lon_name}] + if not dims: + yield {}, da + return + shape = [da.sizes[d] for d in dims] + for index in np.ndindex(*shape): + selectors = dict(zip(dims, index)) + coords = {d: coordinate_value(da.coords[d].values[i]) if d in da.coords else int(i) + for d, i in selectors.items()} + yield coords, da.isel(selectors) + + +def register_variable(store: Any, da: Any, *, source_kind: str) -> str: + token = safe_token(str(da.name)) + standard, vocabulary, unit, long_name = cf_or_source_semantics(str(da.name), dict(da.attrs)) + if not store.db.execute("SELECT 1 FROM datatiles_variables WHERE name=?", (token,)).fetchone(): + store.add_variable(token, standard, vocabulary=vocabulary, + vocabulary_version=str(da.attrs.get("standard_name_vocabulary_version") or "") or None, + canonical_unit=unit, long_name=long_name, + description=f"Imported from {source_kind} variable {da.name}") + # Keep the native variable identifier as a crosswalk. + try: + store.add_variable_identifier(token, f"{source_kind}-variable", str(da.name)) + except Exception: + pass + if source_kind == "GRIB2": + attrs = da.attrs + discipline = attrs.get("GRIB_discipline") + category = attrs.get("GRIB_parameterCategory") + number = attrs.get("GRIB_parameterNumber") + if discipline is not None and category is not None and number is not None: + identifier = f"{discipline}.{category}.{number}" + try: + store.add_variable_identifier(token, "WMO-GRIB2", identifier) + except Exception: + pass + short_name = attrs.get("GRIB_shortName") + if short_name: + try: + store.add_variable_identifier(token, "GRIB-shortName", str(short_name)) + except Exception: + pass + return token + + +def source_bbox(lat_axis, lon_axis, override: tuple[float, float, float, float] | None): + native = (float(lon_axis.min()), float(lat_axis.min()), float(lon_axis.max()), float(lat_axis.max())) + if override is None: + return native + west, south, east, north = override + if west > east or south > north: + raise ConversionError("invalid --bbox ordering") + clipped = (max(west, native[0]), max(south, native[1]), min(east, native[2]), min(north, native[3])) + if clipped[0] > clipped[2] or clipped[1] > clipped[3]: + raise ConversionError("--bbox does not intersect the source grid") + return clipped + + +def convert_dataset(dataset: Any, target: str | Path, *, source: ResolvedSource, source_kind: str, + variables: list[str] | None, zoom: int, tile_size: int, + bbox: tuple[float, float, float, float] | None, max_tiles: int, + source_license: str, source_license_uri: str, source_attribution: str, + dataset_license: str, dataset_license_uri: str, dataset_attribution: str | None, + access_rights: str = "open", provenance_parameters: dict[str, Any] | None = None) -> dict[str, int]: + np, _ = require_scientific_stack() + try: + from datatiles.numeric import encode_numeric_tile + from datatiles.store import DataTiles + except ImportError as exc: + raise ConversionError("install DataTiles before running this utility") from exc + + if not 0 <= zoom <= 22: + raise ConversionError("--zoom must be between 0 and 22 for these import utilities") + if tile_size < 8 or tile_size > 1024: + raise ConversionError("--tile-size must be between 8 and 1024") + + names = variables or [name for name, da in dataset.data_vars.items() if da.ndim >= 2] + if not names: + raise ConversionError("no data variables selected") + arrays = [] + lat_names: set[str] = set() + lon_names: set[str] = set() + spatial: dict[str, tuple[str, str]] = {} + for name in names: + if name not in dataset.data_vars: + raise ConversionError(f"variable not found: {name}") + da = dataset[name] + lat_name, lon_name = find_lat_lon(da) + arrays.append(da) + spatial[name] = (lat_name, lon_name) + lat_names.add(lat_name); lon_names.add(lon_name) + + target = Path(target) + if target.exists(): + raise ConversionError(f"target already exists: {target}") + + stats = {"variables": 0, "slices": 0, "tiles": 0} + with DataTiles(target, create=True, name=target.stem, tile_format="application/vnd.datatiles.numeric") as store: + ensure_dimensions(store, dataset, arrays, lat_names=lat_names, lon_names=lon_names) + store.add_crs("horizontal", authority="EPSG", code="3857", uri="http://www.opengis.net/def/crs/EPSG/0/3857") + store.set_metadata("datatiles:source_identifier", source.identifier) + store.set_metadata("datatiles:source_checksum_algorithm", source.checksum_algorithm) + store.set_metadata("datatiles:source_checksum", source.checksum) + if source.checksum_algorithm == "sha256": + store.set_metadata("datatiles:source_sha256", source.checksum) + entity_id = "source:" + source.checksum_algorithm + ":" + source.checksum + store.add_provenance_entity(entity_id, "dataset", f"{source_kind} source", uri=source.uri, + checksum_algorithm=source.checksum_algorithm, checksum=source.checksum, + attributes={"source_kind": source_kind, "source_identifier": source.identifier}) + # Rights are never inferred from transport accessibility or file metadata. + # The operator must provide an explicit source licence/terms record and a + # separately concluded licence for the generated DataTiles object. + store.add_rights("source", source_license, license_uri=source_license_uri, + attribution_text=source_attribution, access_rights=access_rights, + source_entity_id=entity_id, applies_to=source.identifier) + store.add_rights("dataset", dataset_license, license_uri=dataset_license_uri, + attribution_text=dataset_attribution, access_rights=access_rights) + store.add_rights("metadata", "CC0-1.0", license_uri="https://creativecommons.org/publicdomain/zero/1.0/", + attribution_text="DataTiles machine-readable metadata; source attribution remains binding where applicable") + activity_id = "activity:import" + store.add_provenance_activity(activity_id, "conversion", f"{source_kind} to DataTiles", + software="DataTiles utils", + parameters={"zoom": zoom, "tile_size": tile_size, "bbox": bbox, **(provenance_parameters or {})}) + store.add_provenance_relation(activity_id, "used", entity_id) + + for da in arrays: + token = register_variable(store, da, source_kind=source_kind) + lat_name, lon_name = spatial[str(da.name)] + lat_axis, lon_axis, lat_order, lon_order = normalized_axes(da, lat_name, lon_name) + actual_bbox = source_bbox(lat_axis, lon_axis, bbox) + xs, ys = tile_ranges(actual_bbox, zoom) + planned = len(xs) * len(ys) + if stats["tiles"] + planned > max_tiles: + raise ConversionError(f"conversion would exceed --max-tiles={max_tiles}; select variables/bbox or change zoom") + unit = str(da.attrs.get("units") or "").strip() or None + fill = da.attrs.get("_FillValue", da.attrs.get("missing_value", DEFAULT_NODATA)) + try: + nodata = float(fill) + if not math.isfinite(nodata): + nodata = DEFAULT_NODATA + except (TypeError, ValueError): + nodata = DEFAULT_NODATA + + for extra_coords, slice_da in iter_non_spatial_slices(da, lat_name, lon_name): + values = np.asarray(slice_da.transpose(lat_name, lon_name).values, dtype="float64") + values = values[np.asarray(lat_order), :][:, np.asarray(lon_order)] + coordinates = {"variable": token, **extra_coords} + stats["slices"] += 1 + for x in xs: + for y in ys: + pixel_lon, pixel_lat = tile_pixel_lon_lat(zoom, x, y, tile_size) + iy = nearest_indices(lat_axis, pixel_lat) + ix = nearest_indices(lon_axis, pixel_lon) + tile = values[np.ix_(iy, ix)] + west, south, east, north = actual_bbox + inside = ((pixel_lat[:, None] >= south) & (pixel_lat[:, None] <= north) & + (pixel_lon[None, :] >= west) & (pixel_lon[None, :] <= east)) + tile = np.where(inside & np.isfinite(tile), tile, nodata).astype("float32", copy=False) + blob = encode_numeric_tile(tile.ravel().tolist(), tile.shape, dtype="float32", + compression="zlib", nodata=nodata, unit=unit) + store.put(zoom, x, y, blob, coordinates, xyz=True, data_type="raster", + media_type="application/vnd.datatiles.numeric", encoding="DNT1", + schema={"resampling": "nearest", "grid": "WebMercatorQuad", + "tile_size": tile_size, "source_kind": source_kind}) + store.link_tile_provenance(zoom, x, y, coordinates, entity_id, xyz=True) + stats["tiles"] += 1 + stats["variables"] += 1 + return stats diff --git a/utils/grib2datatiles.py b/utils/grib2datatiles.py new file mode 100644 index 0000000..1673ee7 --- /dev/null +++ b/utils/grib2datatiles.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Convert a local or URL-identified GRIB/GRIB2 dataset to numeric DataTiles.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from common import ConversionError, convert_dataset, require_scientific_stack, resolve_source + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("source", help="local GRIB/GRIB2 path, file: URI, or HTTP(S) URL") + p.add_argument("target", type=Path, help="new .datatiles file") + p.add_argument("--variable", action="append", dest="variables", help="decoded xarray variable; repeatable") + p.add_argument("--zoom", type=int, default=6, help="Web Mercator zoom level (default: 6)") + p.add_argument("--tile-size", type=int, default=256) + p.add_argument("--bbox", nargs=4, type=float, metavar=("WEST", "SOUTH", "EAST", "NORTH")) + p.add_argument("--max-tiles", type=int, default=10000, help="safety bound (default: 10000)") + p.add_argument("--filter-by-keys", action="append", default=[], metavar="KEY=VALUE", + help="cfgrib filter_by_keys entry; repeatable") + p.add_argument("--timeout", type=float, default=60.0, help="URL download timeout in seconds") + p.add_argument("--source-license", required=True, help="SPDX license expression for the source, e.g. CC-BY-4.0 or LicenseRef-Proprietary") + p.add_argument("--source-license-uri", required=True, help="authoritative source licence/terms URI") + p.add_argument("--source-attribution", required=True, help="verbatim attribution required by the source") + p.add_argument("--dataset-license", required=True, help="SPDX expression concluded for the generated dataset") + p.add_argument("--dataset-license-uri", required=True, help="authoritative generated-dataset licence URI") + p.add_argument("--dataset-attribution", help="recommended citation/attribution for the generated dataset") + p.add_argument("--access-rights", choices=("open","embargoed","restricted","closed"), default="open") + return p + + +def parse_filters(items: list[str]) -> dict[str, object]: + result: dict[str, object] = {} + for item in items: + if "=" not in item: + raise ConversionError(f"invalid --filter-by-keys value: {item!r}") + key, value = item.split("=", 1) + key, value = key.strip(), value.strip() + if not key: + raise ConversionError("empty cfgrib filter key") + try: + result[key] = int(value) + except ValueError: + result[key] = value + return result + + +def main() -> int: + args = parser().parse_args() + _, xr = require_scientific_stack() + try: + filters = parse_filters(args.filter_by_keys) + with resolve_source(args.source, timeout=args.timeout) as source: + backend_kwargs = {"indexpath": ""} + if filters: + backend_kwargs["filter_by_keys"] = filters + try: + with xr.open_dataset(source.local_path, engine="cfgrib", backend_kwargs=backend_kwargs) as ds: + stats = convert_dataset(ds, args.target, source=source, source_kind="GRIB2", + variables=args.variables, zoom=args.zoom, tile_size=args.tile_size, + bbox=tuple(args.bbox) if args.bbox else None, max_tiles=args.max_tiles, + source_license=args.source_license, source_license_uri=args.source_license_uri, + source_attribution=args.source_attribution, dataset_license=args.dataset_license, + dataset_license_uri=args.dataset_license_uri, dataset_attribution=args.dataset_attribution, + access_rights=args.access_rights) + except ImportError as exc: + raise ConversionError("GRIB conversion requires cfgrib and ecCodes; install .[utils]") from exc + print(f"created {args.target}: {stats['variables']} variables, {stats['slices']} slices, {stats['tiles']} tiles") + return 0 + except ConversionError as exc: + parser().error(str(exc)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/netcdf2datatiles.py b/utils/netcdf2datatiles.py new file mode 100644 index 0000000..03ab30c --- /dev/null +++ b/utils/netcdf2datatiles.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Convert a local or URL-identified NetCDF dataset to numeric DataTiles.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from common import ConversionError, convert_dataset, require_scientific_stack, resolve_source + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("source", help="local NetCDF path, file: URI, or HTTP(S) URL") + p.add_argument("target", type=Path, help="new .datatiles file") + p.add_argument("--variable", action="append", dest="variables", help="variable to import; repeatable") + p.add_argument("--zoom", type=int, default=6, help="Web Mercator zoom level (default: 6)") + p.add_argument("--tile-size", type=int, default=256) + p.add_argument("--bbox", nargs=4, type=float, metavar=("WEST", "SOUTH", "EAST", "NORTH")) + p.add_argument("--max-tiles", type=int, default=10000, help="safety bound (default: 10000)") + p.add_argument("--engine", help="xarray NetCDF engine, e.g. netcdf4 or h5netcdf") + p.add_argument("--timeout", type=float, default=60.0, help="URL download timeout in seconds") + p.add_argument("--source-license", required=True, help="SPDX license expression for the source, e.g. CC-BY-4.0 or LicenseRef-Proprietary") + p.add_argument("--source-license-uri", required=True, help="authoritative source licence/terms URI") + p.add_argument("--source-attribution", required=True, help="verbatim attribution required by the source") + p.add_argument("--dataset-license", required=True, help="SPDX expression concluded for the generated dataset") + p.add_argument("--dataset-license-uri", required=True, help="authoritative generated-dataset licence URI") + p.add_argument("--dataset-attribution", help="recommended citation/attribution for the generated dataset") + p.add_argument("--access-rights", choices=("open","embargoed","restricted","closed"), default="open") + return p + + +def main() -> int: + args = parser().parse_args() + _, xr = require_scientific_stack() + try: + with resolve_source(args.source, timeout=args.timeout) as source: + kwargs = {"decode_cf": True} + if args.engine: + kwargs["engine"] = args.engine + with xr.open_dataset(source.local_path, **kwargs) as ds: + stats = convert_dataset(ds, args.target, source=source, source_kind="NetCDF", + variables=args.variables, zoom=args.zoom, tile_size=args.tile_size, + bbox=tuple(args.bbox) if args.bbox else None, max_tiles=args.max_tiles, + source_license=args.source_license, source_license_uri=args.source_license_uri, + source_attribution=args.source_attribution, dataset_license=args.dataset_license, + dataset_license_uri=args.dataset_license_uri, dataset_attribution=args.dataset_attribution, + access_rights=args.access_rights) + print(f"created {args.target}: {stats['variables']} variables, {stats['slices']} slices, {stats['tiles']} tiles") + return 0 + except ConversionError as exc: + parser().error(str(exc)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/zarr2datatiles.py b/utils/zarr2datatiles.py new file mode 100644 index 0000000..61c05dc --- /dev/null +++ b/utils/zarr2datatiles.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Convert a local or URL-identified Zarr dataset to numeric DataTiles.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from common import ( + ConversionError, + convert_dataset, + require_zarr_stack, + resolve_zarr_source, +) + + +def _storage_options(values: list[str] | None) -> dict[str, object]: + """Parse repeatable KEY=JSON_VALUE storage options without guessing types.""" + result: dict[str, object] = {} + for item in values or []: + if "=" not in item: + raise ConversionError("--storage-option must be KEY=VALUE") + key, raw = item.split("=", 1) + key = key.strip() + if not key: + raise ConversionError("--storage-option key must not be empty") + raw = raw.strip() + try: + value: object = json.loads(raw) + except json.JSONDecodeError: + value = raw + result[key] = value + return result + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("source", help="local Zarr store path, file: URI, or fsspec URL such as https://, s3://, or gcs://") + p.add_argument("target", type=Path, help="new .datatiles file") + p.add_argument("--group", help="Zarr group/path within the store") + p.add_argument("--variable", action="append", dest="variables", help="variable to import; repeatable") + p.add_argument("--zoom", type=int, default=6, help="Web Mercator zoom level (default: 6)") + p.add_argument("--tile-size", type=int, default=256) + p.add_argument("--bbox", nargs=4, type=float, metavar=("WEST", "SOUTH", "EAST", "NORTH")) + p.add_argument("--max-tiles", type=int, default=10000, help="safety bound (default: 10000)") + p.add_argument("--consolidated", choices=("auto", "true", "false"), default="auto", + help="Zarr consolidated metadata policy (default: auto)") + p.add_argument("--zarr-format", type=int, choices=(2, 3), help="require Zarr format 2 or 3") + p.add_argument("--storage-option", action="append", metavar="KEY=VALUE", + help="fsspec backend option; repeatable; VALUE may be JSON") + p.add_argument("--provenance-uri", help="stable credential-free source/PID URI recorded in provenance when the access URL is signed or temporary") + p.add_argument("--source-sha256", + help="authoritative SHA-256 for a remote immutable Zarr store/snapshot; required for non-local stores") + p.add_argument("--source-license", required=True, help="SPDX license expression for the source") + p.add_argument("--source-license-uri", required=True, help="authoritative source licence/terms URI") + p.add_argument("--source-attribution", required=True, help="verbatim attribution required by the source") + p.add_argument("--dataset-license", required=True, help="SPDX expression concluded for the generated dataset") + p.add_argument("--dataset-license-uri", required=True, help="authoritative generated-dataset licence URI") + p.add_argument("--dataset-attribution", help="recommended citation/attribution for the generated dataset") + p.add_argument("--access-rights", choices=("open", "embargoed", "restricted", "closed"), default="open") + return p + + +def main() -> int: + args = parser().parse_args() + try: + _, xr = require_zarr_stack() + storage_options = _storage_options(args.storage_option) + with resolve_zarr_source(args.source, source_sha256=args.source_sha256, provenance_uri=args.provenance_uri) as source: + consolidated = None if args.consolidated == "auto" else args.consolidated == "true" + kwargs = { + "group": args.group, + "decode_cf": True, + "consolidated": consolidated, + "chunks": None, + } + if args.zarr_format is not None: + kwargs["zarr_format"] = args.zarr_format + if source.local_path is None and storage_options: + kwargs["storage_options"] = storage_options + store_arg = str(source.local_path) if source.local_path is not None else args.source + with xr.open_zarr(store_arg, **kwargs) as ds: + stats = convert_dataset( + ds, + args.target, + source=source, + source_kind="Zarr", + variables=args.variables, + zoom=args.zoom, + tile_size=args.tile_size, + bbox=tuple(args.bbox) if args.bbox else None, + max_tiles=args.max_tiles, + source_license=args.source_license, + source_license_uri=args.source_license_uri, + source_attribution=args.source_attribution, + dataset_license=args.dataset_license, + dataset_license_uri=args.dataset_license_uri, + dataset_attribution=args.dataset_attribution, + access_rights=args.access_rights, + provenance_parameters={ + "zarr_group": args.group, + "consolidated": args.consolidated, + "zarr_format": args.zarr_format, + "storage_option_keys": sorted(storage_options), + }, + ) + print(f"created {args.target}: {stats['variables']} variables, {stats['slices']} slices, {stats['tiles']} tiles") + return 0 + except ConversionError as exc: + parser().error(str(exc)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())