From 699c381dbc6adc7979542444a1b9a0588e7f7ae3 Mon Sep 17 00:00:00 2001 From: Alexander Belikov Date: Thu, 30 Jul 2026 09:24:00 +0200 Subject: [PATCH 1/2] implemented data sampling / profiling --- CHANGELOG.md | 18 + docs/concepts/index.md | 4 + .../concepts/schema/sampling_and_profiling.md | 163 +++++++ docs/guides/identity_inference.md | 23 + docs/guides/index.md | 3 +- .../architecture/contract/runtime.md | 11 +- .../contract/runtime/edge_derivation.md | 5 +- .../architecture/contract/runtime/resource.md | 5 +- .../architecture/database_features.md | 5 +- .../reference/architecture/edge_derivation.md | 5 +- .../pipeline/runtime/actor/config.md | 7 +- .../pipeline/runtime/actor/config/models.md | 5 +- .../runtime/actor/config/normalize.md | 5 +- .../pipeline/runtime/actor/config/parse.md | 5 +- docs/reference/connection_models.md | 5 +- docs/reference/db/connection.md | 7 +- .../reference/db/connection/config_mapping.md | 5 +- docs/reference/db/connection/onto.md | 5 +- docs/reference/db/connection/wsgi.md | 5 +- docs/reference/db/graflo_backend/config.md | 5 +- docs/reference/hq/connection_provider.md | 5 +- docs/reference/hq/fuzzy_matcher.md | 5 +- docs/reference/util/chunker.md | 5 +- docs/reference/util/merge.md | 5 +- docs/reference/util/onto.md | 5 +- graflo/architecture/graph_types/container.py | 31 +- graflo/architecture/onto_sample.py | 331 ++++++++++++++ .../architecture/pipeline/runtime/resource.py | 23 +- graflo/connections/onto.py | 49 +++ graflo/data_source/chunker.py | 23 +- graflo/db/arango/conn.py | 2 +- graflo/hq/graph_engine.py | 46 ++ graflo/hq/sampler.py | 407 ++++++++++++++++++ graflo/util/transform.py | 58 +-- mkdocs.yml | 1 + pyproject.toml | 2 +- test/architecture/test_layering.py | 1 + .../test_manifest_canonical_contract.py | 46 ++ test/data/sample-source/NOTES.txt | 1 + test/data/sample-source/api_orders.json | 37 ++ test/data/sample-source/customers.csv | 4 + test/data/sample-source/orders.csv | 4 + test/db/postgres/test_postgres_config_dsn.py | 74 ++++ test/hq/test_sampler.py | 312 ++++++++++++++ uv.lock | 62 +-- 45 files changed, 1694 insertions(+), 141 deletions(-) create mode 100644 docs/concepts/schema/sampling_and_profiling.md create mode 100644 graflo/architecture/onto_sample.py create mode 100644 graflo/hq/sampler.py create mode 100644 test/data/sample-source/NOTES.txt create mode 100644 test/data/sample-source/api_orders.json create mode 100644 test/data/sample-source/customers.csv create mode 100644 test/data/sample-source/orders.csv create mode 100644 test/db/postgres/test_postgres_config_dsn.py create mode 100644 test/hq/test_sampler.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 95e18093..0f42e4ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.10.1] + +### Added + +**Resource sampling** — a connector-driven sampling primitive, the shared input stage for schema inference (algorithmic or agentic). Previously `infer_manifest` performed this privately for PostgreSQL and nothing else could reach it. + +- **`architecture.onto_sample`** (new, L2 leaf models) — `ResourceSample` holds documents **verbatim as JSON**: flat rows for tables, arbitrarily nested for APIs, nothing flattened at the boundary. `connector` records where the documents came from, so the relation that later becomes a resource plus its `resource_connector` binding survives the round trip. `SourceSample.samples_by_resource` returns `dict[str, list[dict]]` — the shape cross-resource identity inference consumes, so no adapter is needed. `ForeignKeyHint` carries *declared* references, which are ground truth for edge inference rather than a name-suffix guess. +- **Sampling and profiling are separate operations.** `ResourceSampler` (`hq/sampler.py`) pulls documents; `profile_sample` derives the description. `ResourceProfile` is **path-keyed** (`address.city`, `items[].sku`) with a `FieldType`, null rate, cardinality and nesting depth, so hierarchical and tabular sources take one code path — a flat column list cannot represent a nested response at all. `ResourceProfile.flat_docs` projects nested documents into flat records, which is what makes `IdentityInferencer` (flat-records only) usable on an API source. +- **`GraphEngine.sample_resources(...)`** — accepts a `PostgresConfig`, a `Bindings` block, or a file/directory path. Bindings-driven sampling reuses `RegistryBuilder.discover_files` and the `resource_connector` mapping, so provenance comes for free instead of being reconstructed. PostgreSQL sampling carries `primary_key` and `foreign_keys` through from introspection. Documents are normalized to JSON-safe values (`datetime`, `Decimal`, `memoryview`, numpy scalars), capped by `max_docs` and `max_cell_chars`, with `truncated` set when anything was dropped or clipped. +- **`PostgresConfig.from_dsn(dsn)`** — a bare `uri=` derives only host and port, leaving `username`/`password`/`database` unset, so `to_sqlalchemy_connection_string()` raised. Parses credentials, database and a `search_path` from either `?schema=` or libpq `?options=-csearch_path=`. +- Concept page: [Sampling and profiling](docs/concepts/schema/sampling_and_profiling.md), cross-linked from the identity inference guide. + +### Fixed + +- **`strict_references=True` now rejects a pipeline `vertex:` step naming an undeclared vertex.** `filter_vertex_config_for_resource` intersects a resource's vertex names with the schema's and silently drops unknowns, so a resource that ingested nothing validated clean — a name mismatch between the vertex definition and the step was invisible. **Behaviour change:** manifests that previously passed under `strict_references=True` may now fail, which is the point; lenient validation is unchanged. +- `ChunkerFactory._guess_chunker_type` raises the documented `ValueError` for a file with no extension instead of `IndexError`, so callers scanning a directory can skip it like any other unknown type. +- The CSV/TSV chunker no longer raises `RuntimeError: generator raised StopIteration` (PEP 479) on an empty file; it yields no rows. + ## [1.10.0] diff --git a/docs/concepts/index.md b/docs/concepts/index.md index 5f665846..d1d18b1b 100644 --- a/docs/concepts/index.md +++ b/docs/concepts/index.md @@ -67,6 +67,10 @@ flowchart LR Entry points: **`GraphEngine.migrate_graph()`** (schema + data), **`infer_schema_from_graph()`** (schema only), **`export_graph()`** (in-memory `GraFloOutput`). See [Graph export and migration](operations/graph_export_migration.md) and the [Graph DB migration guide](../guides/graph_db_migration.md). +- **SourceSample** — a bounded, verbatim-JSON sample of one or more resources, retaining the + connector each document came from. The shared input stage for schema inference, algorithmic or + agentic; produced by `GraphEngine.sample_resources()`. See + [Sampling and profiling](schema/sampling_and_profiling.md). - **Source Instance** — a concrete data artifact (a file, a table, a SPARQL endpoint, an API, a Kafka topic), wrapped by an `AbstractDataSource` with a `DataSourceType` (`FILE`, `SQL`, `SPARQL`, `API`, `KAFKA`, `IN_MEMORY`). - **Resource** — a reusable transformation pipeline (actor steps: descend, transform, vertex, edge) that maps raw records to graph elements. Data sources bind to Resources by name via the `DataSourceRegistry`. - **GraphManifest** — the canonical top-level contract that composes `schema`, `ingestion_model`, and `bindings`. High-level **contract evolution** (remove/merge vertex types and keep ingestion aligned) is described in [Manifest evolution](schema/manifest_evolution.md). diff --git a/docs/concepts/schema/sampling_and_profiling.md b/docs/concepts/schema/sampling_and_profiling.md new file mode 100644 index 00000000..feb89ba2 --- /dev/null +++ b/docs/concepts/schema/sampling_and_profiling.md @@ -0,0 +1,163 @@ +# Sampling and profiling + +Before a schema can be inferred, something has to look at the data. GraFlo splits that into two +operations that are deliberately kept apart: + +| | What it does | Produced by | Model | +|---|---|---|---| +| **Sampling** | Pulls a bounded set of documents from a connector, **verbatim** | `ResourceSampler` | `SourceSample` / `ResourceSample` | +| **Profiling** | Derives paths, types, null rates and cardinality from those documents | `profile_sample` | `ResourceProfile` / `FieldProfile` | + +The split is what lets one code path serve both a CSV table and a paginated JSON API. A sample is +**pure JSON** — a list of flat rows for a table, an arbitrarily nested object for an API — and +nothing is flattened at the boundary. The flat, typed view is a *derived projection*, computed on +demand. Collapsing the two into a single "here are the columns and their types" model cannot +represent a hierarchical response at all. + +`GraphEngine.infer_manifest()` performed this privately for PostgreSQL and nothing else could reach +it. Sampling is the same input stage, exposed, so that **any** inferencer — the algorithmic +identity inferencers, an LLM agent, a studio preview — consumes the same substrate. + +## The sample + +```python +from graflo.hq.graph_engine import GraphEngine + +source = GraphEngine().sample_resources("data/", max_docs=100) +``` + +`sample_resources` dispatches on what it is given: a `PostgresConfig`, a `Bindings` block, or a +file/directory path (or list of paths). The result: + +```text +source_name: sample-source + resource='api_orders' connector='api_orders' docs=2 truncated=False + resource='customers' connector='customers' docs=3 truncated=False + resource='orders' connector='orders' docs=3 truncated=False +``` + +Three fields on `ResourceSample` carry more weight than the documents themselves: + +- **`connector`** — the connector the documents came from. This is the relation that later becomes + a [`resource_connector` binding](../index.md), so provenance survives the round trip + instead of being reconstructed downstream. `sample_bindings` treats the existing + `resource_connector` mapping as the authority, so what is sampled is exactly what will be + ingested. +- **`primary_key`** / **`foreign_keys`** — what the source *declared*. A `ForeignKeyHint` is ground + truth for edge inference; a `*_id` name-suffix guess is not, and must never be recorded here. + PostgreSQL sampling fills both from introspection; file sampling leaves them empty. +- **`truncated`** — set when documents were dropped or string values clipped. Sampling reads one + document past `max_docs` precisely so that a source holding exactly `max_docs` documents is + distinguishable from one that was cut short. + +`SourceSample.samples_by_resource` returns `dict[str, list[dict]]` — the input shape cross-resource +identity inference consumes, so no adapter sits between sampling and inference. + +### Guards + +Sampled documents leave the trust boundary: they land in prompts, previews and logs. Three caps +apply, all on `ResourceSampler`: + +- `max_docs` (default 100) — documents per resource +- `max_cell_chars` — length of any single string value +- JSON normalization — `datetime`, `Decimal`, `memoryview`, `UUID` and numpy scalars are coerced to + JSON-safe values, because `dict[str, Any]` accepts them but only best-effort serializes them + +Files that yield no documents (an empty CSV, a `NOTES.txt`) are skipped with a warning rather than +producing an empty resource. + +## The profile + +```python +from graflo.architecture.onto_sample import profile_sample + +profile = profile_sample(source.get("api_orders")) +``` + +Profiles are **path-keyed**. Nested objects extend the path with `.`; lists of objects extend it +with `[]`: + +```text +max_depth: 1 nested: True + +order_id STRING depth=0 null_ratio=0.00 +customer.id STRING depth=1 null_ratio=0.00 +customer.city STRING depth=1 null_ratio=0.50 +items[].sku STRING depth=1 null_ratio=0.00 +items[].qty INT depth=1 null_ratio=0.00 +tags LIST depth=0 null_ratio=0.00 +``` + +A list of *scalars* (`tags`) is typed whole as `LIST` with an `item_type`; a list of *objects* +(`items`) is descended into. `max_depth > 0` is the signal that ingestion needs +[`descend` steps](../ingestion/transforms.md) — a flat table is simply the `depth=0` case of the +same code path. + +Type inference checks `bool` before `int` deliberately: `bool` is an `int` subclass in Python, so +the naive order mistypes every boolean column as `INT`. + +!!! note "Types come from values, not from a declaration" + A CSV reader yields strings, so `orders.csv` profiles `total` and `paid` as `STRING`. A + PostgreSQL source carries real column types through introspection. Profiling describes what was + *observed*; it does not invent a declaration the source never made. + +### `flat_docs` — the bridge to identity inference + +`IdentityInferencer` operates on flat records. `ResourceProfile.flat_docs` projects nested documents +onto the profile's paths, which is how an API source becomes eligible for it at all: + +```python +profile.flat_docs(sample.docs)[0] +# {'order_id': 'o1', 'customer.id': 'c1', 'customer.city': 'Berlin', +# 'items[].sku': 'A-1', 'items[].qty': 2, 'tags': ['priority', 'gift']} +``` + +!!! warning "`unique` is a property of the sample, not of the source" + `FieldProfile.unique` means every non-null value observed was distinct — over as few as two + documents. Treat it as a candidate signal to be confirmed against a larger sample or a declared + `primary_key`, never as a uniqueness constraint. `min_sample_size` in identity inference exists + for this reason. + +## Where it fits + +```mermaid +flowchart LR + C["Connectors
File · Table · SPARQL · API"] + S["ResourceSampler
bounded, verbatim JSON"] + SS["SourceSample
docs + connector + declared keys"] + P["profile_sample
paths · types · null rate"] + II["IdentityInferencer"] + AG["Agentic inference
(ScheWea)"] + M["GraphManifest"] + + C --> S --> SS + SS --> P --> II --> M + SS -- samples_by_resource --> II + SS --> AG --> M +``` + +Sampling deliberately stops short of proposing anything. What consumes it: + +- **[Identity inference](../../guides/identity_inference.md)** — vertex `identity` and + `hash_identity_properties` from flat samples. +- **Cross-resource vertex discovery** — aligns fields across resources to find a shared key; + consumes `samples_by_resource` directly. +- **Agentic inference** — an external service receives a serialized `SourceSample` over the wire. + Because the model is defined once here, the producer and the consumer cannot drift into + disagreement. + +Note the asymmetry a `SourceSample` deliberately preserves: it names connectors but does not carry +their definitions, which hold paths, DSNs and credentials. A consumer can therefore propose +resources but cannot, on its own, emit a `bindings` block — the caller that did the sampling holds +the connectors and assembles it. This is the secret-free manifest doctrine falling out of the type +system rather than being enforced by convention. + +## API + +| Symbol | Module | +|---|---| +| `SourceSample`, `ResourceSample`, `ForeignKeyHint` | `graflo.architecture.onto_sample` | +| `ResourceProfile`, `FieldProfile` | `graflo.architecture.onto_sample` | +| `profile_sample`, `profile_source`, `iter_paths`, `infer_field_type` | `graflo.architecture.onto_sample` | +| `ResourceSampler` (`sample_file`, `sample_files`, `sample_postgres`, `sample_connector`, `sample_bindings`) | `graflo.hq.sampler` | +| `GraphEngine.sample_resources` | `graflo.hq.graph_engine` | diff --git a/docs/guides/identity_inference.md b/docs/guides/identity_inference.md index 61366821..ac103371 100644 --- a/docs/guides/identity_inference.md +++ b/docs/guides/identity_inference.md @@ -43,6 +43,28 @@ uv run python ingest.py Writes a chunked GraFlo file backend under `artifacts/csv-backend/`. +## Samples from a non-CSV source + +`IdentityInferencer.infer()` takes **flat records**, which is why this guide starts from CSV. To +reach a PostgreSQL table, a directory of mixed files, or a nested API response, sample the source +first and flatten through its profile: + +```python +from graflo.architecture.onto_sample import profile_sample +from graflo.hq.graph_engine import GraphEngine + +source = GraphEngine().sample_resources(pg_config, schema_name="public", max_docs=500) +sample = source.get("customers") +profile = profile_sample(sample) + +records = profile.flat_docs(sample.docs) # nested paths become 'customer.id', 'items[].sku' +``` + +A sampled source also carries the **declared** `primary_key` and `foreign_keys` when it has them — +prefer those over an inferred identity, and see +[Sampling and profiling](../concepts/schema/sampling_and_profiling.md) for the caps and the caveat +on `unique`. + ## Identity modes After inference, each vertex has a derived **`identity_mode`**: @@ -63,6 +85,7 @@ For attaching edges by a business key that is not the upsert identity, see ## Related documentation - [Vertex identity modes](../concepts/schema/vertex_identity.md) +- [Sampling and profiling](../concepts/schema/sampling_and_profiling.md) — where samples come from - [Example 16 — Secondary identities](../examples/example-16.md) - [Core components — Vertex](../concepts/architecture/core_components.md) - [Graph export and replay](graph_export_and_replay.md) — file backend ingest pattern diff --git a/docs/guides/index.md b/docs/guides/index.md index be128fac..df3dce5e 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -10,7 +10,8 @@ Task-oriented walkthroughs for common GraFlo workflows. Each guide links to the | Export a graph to disk and replay it | [Graph export and replay](graph_export_and_replay.md) | [Example 13](../examples/example-13.md) | [Graph export and migration](../concepts/operations/graph_export_migration.md) | | Wire REST API credentials from environment variables | [API env wiring](api_env_wiring.md) | [Example 14](../examples/example-14.md) | [API connector](../concepts/connectors/api_connector.md) | | Ingest JSON messages from Kafka topics | — | — | [Kafka connector](../concepts/connectors/kafka_connector.md) | -| Infer vertex identities from CSV samples | [Identity inference](identity_inference.md) | [Example 15](../examples/example-15.md) | [Vertex identity](../concepts/schema/vertex_identity.md) | +| Look at a source before writing a manifest | — | — | [Sampling and profiling](../concepts/schema/sampling_and_profiling.md) | +| Infer vertex identities from CSV samples | [Identity inference](identity_inference.md) | [Example 15](../examples/example-15.md) | [Vertex identity](../concepts/schema/vertex_identity.md), [Sampling and profiling](../concepts/schema/sampling_and_profiling.md) | | Attach edges by a business key (ISIN, LEI, …) without owning the vertex | — | [Example 16](../examples/example-16.md) | [Secondary identities](../concepts/schema/vertex_identity.md#secondary-identities-edge-endpoint-lookup) | | Bulk-load TigerGraph via CSV and S3 staging | [TigerGraph bulk load](tigergraph_bulk_load.md) | [Example 10](../examples/example-10.md) | [Object storage](../concepts/operations/object_storage.md) | | Use a pre-provisioned graph namespace (least privilege) | [Graph namespace and schema](graph_namespace_and_schema.md) | — | [Capabilities](../concepts/architecture/capabilities.md) | diff --git a/docs/reference/architecture/contract/runtime.md b/docs/reference/architecture/contract/runtime.md index 9ccd5e93..e5014ccd 100644 --- a/docs/reference/architecture/contract/runtime.md +++ b/docs/reference/architecture/contract/runtime.md @@ -1,5 +1,8 @@ -# `graflo.architecture.contract.runtime` +# Runtime (moved) -::: graflo.architecture.contract.runtime - options: - show_submodules: false +`graflo.architecture.contract.runtime` was removed in the contract ↔ pipeline +inversion. Schema-bound execution now lives under +[`graflo.architecture.pipeline.runtime`](../pipeline/runtime.md). + +- [`ResourceRuntime`](runtime/resource.md) → `graflo.architecture.pipeline.runtime.resource` +- [`EdgeDerivation`](runtime/edge_derivation.md) → `graflo.architecture.graph_types.edge_derivation` diff --git a/docs/reference/architecture/contract/runtime/edge_derivation.md b/docs/reference/architecture/contract/runtime/edge_derivation.md index 12ea09ee..bcbf7d94 100644 --- a/docs/reference/architecture/contract/runtime/edge_derivation.md +++ b/docs/reference/architecture/contract/runtime/edge_derivation.md @@ -1,3 +1,4 @@ -# `graflo.architecture.graph_types.edge_derivation` +# `graflo.architecture.contract.runtime.edge_derivation` (moved) -::: graflo.architecture.graph_types.edge_derivation +`graflo.architecture.contract.runtime.edge_derivation` moved to [`graflo.architecture.graph_types.edge_derivation`](../../../architecture/graph_types/edge_derivation.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../guides/importing.md). diff --git a/docs/reference/architecture/contract/runtime/resource.md b/docs/reference/architecture/contract/runtime/resource.md index 52252238..fb8cf0b4 100644 --- a/docs/reference/architecture/contract/runtime/resource.md +++ b/docs/reference/architecture/contract/runtime/resource.md @@ -1,3 +1,4 @@ -# `graflo.architecture.pipeline.runtime.resource` +# `graflo.architecture.contract.runtime.resource` (moved) -::: graflo.architecture.pipeline.runtime.resource +`graflo.architecture.contract.runtime.resource` moved to [`graflo.architecture.pipeline.runtime.resource`](../../../architecture/pipeline/runtime/resource.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../guides/importing.md). diff --git a/docs/reference/architecture/database_features.md b/docs/reference/architecture/database_features.md index 38f48703..978a469b 100644 --- a/docs/reference/architecture/database_features.md +++ b/docs/reference/architecture/database_features.md @@ -1,3 +1,4 @@ -# `graflo.architecture.schema.database_features` +# `graflo.architecture.database_features` (moved) -::: graflo.architecture.schema.database_features +`graflo.architecture.database_features` moved to [`graflo.architecture.schema.database_features`](../architecture/schema/database_features.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/architecture/edge_derivation.md b/docs/reference/architecture/edge_derivation.md index 12ea09ee..5236aec1 100644 --- a/docs/reference/architecture/edge_derivation.md +++ b/docs/reference/architecture/edge_derivation.md @@ -1,3 +1,4 @@ -# `graflo.architecture.graph_types.edge_derivation` +# `graflo.architecture.edge_derivation` (moved) -::: graflo.architecture.graph_types.edge_derivation +`graflo.architecture.edge_derivation` moved to [`graflo.architecture.graph_types.edge_derivation`](../architecture/graph_types/edge_derivation.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/architecture/pipeline/runtime/actor/config.md b/docs/reference/architecture/pipeline/runtime/actor/config.md index f4370391..1c84007a 100644 --- a/docs/reference/architecture/pipeline/runtime/actor/config.md +++ b/docs/reference/architecture/pipeline/runtime/actor/config.md @@ -1,5 +1,4 @@ -# `graflo.architecture.contract.ingestion.steps` +# `graflo.architecture.pipeline.runtime.actor.config` (moved) -::: graflo.architecture.contract.ingestion.steps - options: - show_submodules: false +`graflo.architecture.pipeline.runtime.actor.config` moved to [`graflo.architecture.contract.ingestion.steps`](../../../../architecture/contract/ingestion/steps.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../../guides/importing.md). diff --git a/docs/reference/architecture/pipeline/runtime/actor/config/models.md b/docs/reference/architecture/pipeline/runtime/actor/config/models.md index 4bb2afc2..d0ec861d 100644 --- a/docs/reference/architecture/pipeline/runtime/actor/config/models.md +++ b/docs/reference/architecture/pipeline/runtime/actor/config/models.md @@ -1,3 +1,4 @@ -# `graflo.architecture.contract.ingestion.steps.models` +# `graflo.architecture.pipeline.runtime.actor.config.models` (moved) -::: graflo.architecture.contract.ingestion.steps.models +`graflo.architecture.pipeline.runtime.actor.config.models` moved to [`graflo.architecture.contract.ingestion.steps.models`](../../../../../architecture/contract/ingestion/steps/models.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../../../guides/importing.md). diff --git a/docs/reference/architecture/pipeline/runtime/actor/config/normalize.md b/docs/reference/architecture/pipeline/runtime/actor/config/normalize.md index af6d7494..26d48151 100644 --- a/docs/reference/architecture/pipeline/runtime/actor/config/normalize.md +++ b/docs/reference/architecture/pipeline/runtime/actor/config/normalize.md @@ -1,3 +1,4 @@ -# `graflo.architecture.contract.ingestion.steps.normalize` +# `graflo.architecture.pipeline.runtime.actor.config.normalize` (moved) -::: graflo.architecture.contract.ingestion.steps.normalize +`graflo.architecture.pipeline.runtime.actor.config.normalize` moved to [`graflo.architecture.contract.ingestion.steps.normalize`](../../../../../architecture/contract/ingestion/steps/normalize.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../../../guides/importing.md). diff --git a/docs/reference/architecture/pipeline/runtime/actor/config/parse.md b/docs/reference/architecture/pipeline/runtime/actor/config/parse.md index b32d63af..adbcbada 100644 --- a/docs/reference/architecture/pipeline/runtime/actor/config/parse.md +++ b/docs/reference/architecture/pipeline/runtime/actor/config/parse.md @@ -1,3 +1,4 @@ -# `graflo.architecture.contract.ingestion.steps.parse` +# `graflo.architecture.pipeline.runtime.actor.config.parse` (moved) -::: graflo.architecture.contract.ingestion.steps.parse +`graflo.architecture.pipeline.runtime.actor.config.parse` moved to [`graflo.architecture.contract.ingestion.steps.parse`](../../../../../architecture/contract/ingestion/steps/parse.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../../../../guides/importing.md). diff --git a/docs/reference/connection_models.md b/docs/reference/connection_models.md index 008a5413..5fd11021 100644 --- a/docs/reference/connection_models.md +++ b/docs/reference/connection_models.md @@ -1,3 +1,4 @@ -# `graflo.connections.sources` +# `graflo.connection_models` (moved) -::: graflo.connections.sources +`graflo.connection_models` moved to [`graflo.connections.sources`](connections/sources.md) in 1.10.0. +Full old→new path table: [Importing and layering](../guides/importing.md). diff --git a/docs/reference/db/connection.md b/docs/reference/db/connection.md index c1880bcb..23d9f77b 100644 --- a/docs/reference/db/connection.md +++ b/docs/reference/db/connection.md @@ -1,5 +1,4 @@ -# `graflo.connections.onto` +# `graflo.db.connection` (moved) -::: graflo.connections.onto - options: - show_submodules: false +`graflo.db.connection` moved to [`graflo.connections.onto`](../connections/onto.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/db/connection/config_mapping.md b/docs/reference/db/connection/config_mapping.md index d94a75e1..a9c2edb5 100644 --- a/docs/reference/db/connection/config_mapping.md +++ b/docs/reference/db/connection/config_mapping.md @@ -1,3 +1,4 @@ -# `graflo.connections.mapping` +# `graflo.db.connection.config_mapping` (moved) -::: graflo.connections.mapping +`graflo.db.connection.config_mapping` moved to [`graflo.connections.mapping`](../../connections/mapping.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../guides/importing.md). diff --git a/docs/reference/db/connection/onto.md b/docs/reference/db/connection/onto.md index 9127bfe7..a901b5ab 100644 --- a/docs/reference/db/connection/onto.md +++ b/docs/reference/db/connection/onto.md @@ -1,3 +1,4 @@ -# `graflo.connections.onto` +# `graflo.db.connection.onto` (moved) -::: graflo.connections.onto +`graflo.db.connection.onto` moved to [`graflo.connections.onto`](../../connections/onto.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../../guides/importing.md). diff --git a/docs/reference/db/connection/wsgi.md b/docs/reference/db/connection/wsgi.md index bd146fd3..51dd0579 100644 --- a/docs/reference/db/connection/wsgi.md +++ b/docs/reference/db/connection/wsgi.md @@ -1,3 +1,4 @@ -# `graflo.connections.onto.wsgi` +# WSGI connection config (removed) -::: graflo.connections.onto.wsgi +`WSGIConfig` / `graflo.connections.onto.wsgi` was deleted. Use the connection +configs under [`graflo.connections.onto`](onto.md). diff --git a/docs/reference/db/graflo_backend/config.md b/docs/reference/db/graflo_backend/config.md index 9e4daf30..cdc4bb23 100644 --- a/docs/reference/db/graflo_backend/config.md +++ b/docs/reference/db/graflo_backend/config.md @@ -1,3 +1,4 @@ -# `graflo.db.graflo_backend.config` +# `graflo.db.graflo_backend.config` (moved) -::: graflo.db.graflo_backend.config +Backend connection config now lives under +[`graflo.connections.graflo_backend`](../../connections/graflo_backend.md). diff --git a/docs/reference/hq/connection_provider.md b/docs/reference/hq/connection_provider.md index de3c4e1f..2eafe0e3 100644 --- a/docs/reference/hq/connection_provider.md +++ b/docs/reference/hq/connection_provider.md @@ -1,3 +1,4 @@ -# `graflo.connections.provider` +# `graflo.hq.connection_provider` (moved) -::: graflo.connections.provider +`graflo.hq.connection_provider` moved to [`graflo.connections.provider`](../connections/provider.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/hq/fuzzy_matcher.md b/docs/reference/hq/fuzzy_matcher.md index 5be21787..d4886ecd 100644 --- a/docs/reference/hq/fuzzy_matcher.md +++ b/docs/reference/hq/fuzzy_matcher.md @@ -1,3 +1,4 @@ -# `graflo.util.fuzzy_matcher` +# `graflo.hq.fuzzy_matcher` (moved) -::: graflo.util.fuzzy_matcher +`graflo.hq.fuzzy_matcher` moved to [`graflo.util.fuzzy_matcher`](../util/fuzzy_matcher.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/util/chunker.md b/docs/reference/util/chunker.md index 9add37c3..aef5caf0 100644 --- a/docs/reference/util/chunker.md +++ b/docs/reference/util/chunker.md @@ -1,3 +1,4 @@ -# `graflo.data_source.chunker` +# `graflo.util.chunker` (moved) -::: graflo.data_source.chunker +`graflo.util.chunker` moved to [`graflo.data_source.chunker`](../data_source/chunker.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/util/merge.md b/docs/reference/util/merge.md index 256bfca1..f4d28928 100644 --- a/docs/reference/util/merge.md +++ b/docs/reference/util/merge.md @@ -1,3 +1,4 @@ -# `graflo.architecture.graph_types.merge` +# `graflo.util.merge` (moved) -::: graflo.architecture.graph_types.merge +`graflo.util.merge` moved to [`graflo.architecture.graph_types.merge`](../architecture/graph_types/merge.md) in 1.10.0. +Full old→new path table: [Importing and layering](../../guides/importing.md). diff --git a/docs/reference/util/onto.md b/docs/reference/util/onto.md index a28895be..e2c84a65 100644 --- a/docs/reference/util/onto.md +++ b/docs/reference/util/onto.md @@ -1,3 +1,4 @@ -# `graflo.util.onto` +# `graflo.util.onto` (moved) -::: graflo.util.onto +`graflo.util.onto` was removed. Shared ontology constants now live under +[`graflo.onto`](../onto.md). diff --git a/graflo/architecture/graph_types/container.py b/graflo/architecture/graph_types/container.py index d316cdb1..9fa94664 100644 --- a/graflo/architecture/graph_types/container.py +++ b/graflo/architecture/graph_types/container.py @@ -3,8 +3,6 @@ from __future__ import annotations from collections import defaultdict -from datetime import date, datetime, time -from decimal import Decimal from typing import Any from pydantic import Field, field_serializer, field_validator @@ -18,30 +16,7 @@ serialize_edge_key, serialize_entity_key, ) - - -def _pick_unique_dict(docs: list) -> list: - """Deduplicate dicts by structure; preserves original objects (local copy of merge logic).""" - - def make_hashable(obj: object) -> Any: - if isinstance(obj, dict): - return tuple(sorted((k, make_hashable(v)) for k, v in obj.items())) - if isinstance(obj, (list, tuple)): - return tuple(make_hashable(item) for item in obj) - if isinstance(obj, (datetime, date, time)): - return ("__datetime__", obj.isoformat()) - if isinstance(obj, Decimal): - return ("__decimal__", str(obj)) - if isinstance(obj, set): - return tuple(sorted(make_hashable(item) for item in obj)) - return obj - - seen: dict[Any, object] = {} - for doc in docs: - key = make_hashable(doc) - if key not in seen: - seen[key] = doc - return list(seen.values()) +from graflo.util.transform import pick_unique_dict def _serialize_linear_item( @@ -133,9 +108,9 @@ def items(self): def pick_unique(self): """Remove duplicate entries from vertices and edges.""" for k, v in self.vertices.items(): - self.vertices[k] = _pick_unique_dict(v) + self.vertices[k] = pick_unique_dict(v) for k, v in self.edges.items(): - self.edges[k] = _pick_unique_dict(v) + self.edges[k] = pick_unique_dict(v) @classmethod def from_docs_list( diff --git a/graflo/architecture/onto_sample.py b/graflo/architecture/onto_sample.py new file mode 100644 index 00000000..507eddf7 --- /dev/null +++ b/graflo/architecture/onto_sample.py @@ -0,0 +1,331 @@ +"""Resource sampling contract — pure-JSON samples and their derived profiles. + +Samples are the raw material every schema inferencer consumes, whether it reasons +algorithmically (:mod:`graflo.db.identity_inference`) or with a language model +(ScheWea). Two ideas, deliberately kept apart: + +* **Sampling** pulls documents from a connector. :class:`ResourceSample` holds + them **verbatim as JSON** — tabular sources yield flat ``list[dict]`` rows, + API sources yield arbitrarily nested documents. Nothing is flattened at this + boundary, so a hierarchical response survives intact. +* **Profiling** describes those documents. :func:`profile_sample` derives the + flat, path-keyed, typed view (:class:`ResourceProfile`) used for prompting, + studio previews and identity inference. + +:attr:`ResourceSample.connector` records *where* the documents came from. That +relation is what later becomes a resource plus its ``resource_connector`` +binding, so it must survive the round trip — a sample that has lost its +provenance cannot be turned back into an ingestion model. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from datetime import date, datetime, time +from typing import Any + +from pydantic import Field as PydanticField + +from graflo.architecture.base import ConfigBaseModel +from graflo.architecture.schema.identity_uuid import UUID_PATTERN +from graflo.architecture.schema.vertex import FieldType + +#: Path segment appended when descending into a list of objects, e.g. ``items[].sku``. +LIST_MARKER = "[]" + +_ISO_DATETIME_PATTERN = re.compile( + r"^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?$" +) +_LONG_TEXT_THRESHOLD = 256 + +DEFAULT_MAX_DOCS = 100 +DEFAULT_MAX_PATHS = 200 +DEFAULT_MAX_EXAMPLES = 3 + + +class ForeignKeyHint(ConfigBaseModel): + """A declared reference from one resource to another. + + Populated only when the source *declares* it (a SQL foreign key, an RDF + range). This is ground truth for edge inference and must not be confused + with the name-suffix guessing an inferencer falls back to. + """ + + field: str + references_resource: str + references_field: str | None = None + + +class ResourceSample(ConfigBaseModel): + """Documents sampled from one resource, plus what the source declared about it.""" + + resource_name: str + """Logical resource name; becomes ``ResourceConfig.name``.""" + + connector: str | None = None + """Name of the connector the documents came from; becomes the + ``resource_connector`` binding. ``None`` when sampled without bindings.""" + + docs: list[dict[str, Any]] = PydanticField(default_factory=list) + """Sampled documents, verbatim JSON. Flat rows for tables, nested for APIs.""" + + description: str | None = None + primary_key: list[str] = PydanticField(default_factory=list) + """Declared primary key, when the source has one.""" + + foreign_keys: list[ForeignKeyHint] = PydanticField(default_factory=list) + """Declared outbound references, when the source has them.""" + + truncated: bool = False + """True when documents were dropped or values clipped to respect caps.""" + + total_estimate: int | None = None + """Approximate total document count at the source, when cheaply available.""" + + +class SourceSample(ConfigBaseModel): + """A set of resource samples drawn from one logical source.""" + + source_name: str + description: str | None = None + samples: list[ResourceSample] = PydanticField(min_length=1) + + @property + def samples_by_resource(self) -> dict[str, list[dict[str, Any]]]: + """Documents keyed by resource name. + + This is the input shape consumed by cross-resource identity inference, + so no adapter is needed between sampling and inference. + """ + return {sample.resource_name: sample.docs for sample in self.samples} + + def get(self, resource_name: str) -> ResourceSample | None: + """Return the sample for *resource_name*, or ``None``.""" + for sample in self.samples: + if sample.resource_name == resource_name: + return sample + return None + + +class FieldProfile(ConfigBaseModel): + """Derived description of one field path within a resource sample.""" + + path: str + """Dotted path to the value, e.g. ``address.city`` or ``items[].sku``.""" + + type: FieldType = FieldType.STRING + item_type: FieldType | None = None + """Element type; set only when ``type`` is ``LIST``.""" + + depth: int = 0 + """Nesting depth of the path. ``0`` for a top-level scalar.""" + + present: int = 0 + """Documents in which the path occurred.""" + + null_count: int = 0 + distinct: int = 0 + examples: list[str] = PydanticField(default_factory=list) + + @property + def null_ratio(self) -> float: + """Fraction of occurrences whose value was null.""" + return self.null_count / self.present if self.present else 0.0 + + @property + def unique(self) -> bool: + """True when every non-null occurrence was distinct.""" + non_null = self.present - self.null_count + return non_null > 0 and self.distinct == non_null + + +class ResourceProfile(ConfigBaseModel): + """Derived, flat, typed view of a :class:`ResourceSample`.""" + + resource_name: str + connector: str | None = None + doc_count: int = 0 + max_depth: int = 0 + """Deepest nesting observed. ``> 0`` means ingestion needs ``descend`` steps.""" + + fields: list[FieldProfile] = PydanticField(default_factory=list) + primary_key: list[str] = PydanticField(default_factory=list) + foreign_keys: list[ForeignKeyHint] = PydanticField(default_factory=list) + truncated: bool = False + + @property + def field_paths(self) -> list[str]: + return [field.path for field in self.fields] + + @property + def nested(self) -> bool: + return self.max_depth > 0 + + def flat_docs(self, docs: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Project *docs* onto this profile's paths as flat records. + + Identity inference operates on flat records; this is how a nested source + becomes eligible for it. + """ + paths = set(self.field_paths) + flattened: list[dict[str, Any]] = [] + for doc in docs: + record: dict[str, Any] = {} + for path, _depth, value in iter_paths(doc): + if path in paths and path not in record: + record[path] = value + flattened.append(record) + return flattened + + +def iter_paths( + doc: dict[str, Any], prefix: str = "", depth: int = 0 +) -> Iterator[tuple[str, int, Any]]: + """Yield ``(path, depth, value)`` for every leaf in a JSON document. + + Nested objects extend the path with ``.``; lists of objects extend it with + ``[]``. Lists of scalars are yielded whole so they can be typed as ``LIST``. + """ + for key, value in doc.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + yield from iter_paths(value, path, depth + 1) + elif isinstance(value, list) and any(isinstance(item, dict) for item in value): + nested_path = f"{path}{LIST_MARKER}" + for item in value: + if isinstance(item, dict): + yield from iter_paths(item, nested_path, depth + 1) + else: + yield path, depth, value + + +def infer_field_type(values: list[Any]) -> tuple[FieldType, FieldType | None]: + """Infer a ``FieldType`` (and ``item_type`` for lists) from observed values. + + Checks ``bool`` before ``int`` deliberately — ``bool`` is an ``int`` subclass + in Python, so the naive order mistypes every boolean column as ``INT``. + """ + non_null = [value for value in values if value is not None] + if not non_null: + return FieldType.STRING, None + + if all(isinstance(value, list) for value in non_null): + items = [item for value in non_null for item in value] + item_type, _ = infer_field_type(items) if items else (FieldType.STRING, None) + return FieldType.LIST, item_type + + if all(isinstance(value, bool) for value in non_null): + return FieldType.BOOL, None + if all( + isinstance(value, int) and not isinstance(value, bool) for value in non_null + ): + return FieldType.INT, None + if all( + isinstance(value, (int, float)) and not isinstance(value, bool) + for value in non_null + ): + return FieldType.FLOAT, None + if all(isinstance(value, (datetime, date, time)) for value in non_null): + return FieldType.DATETIME, None + + if all(isinstance(value, str) for value in non_null): + if all(UUID_PATTERN.match(value) for value in non_null): + return FieldType.UUID, None + if all(_ISO_DATETIME_PATTERN.match(value) for value in non_null): + return FieldType.DATETIME, None + + return FieldType.STRING, None + + +def _example(value: Any) -> str: + text = str(value) + if len(text) > _LONG_TEXT_THRESHOLD: + return text[:_LONG_TEXT_THRESHOLD] + "…" + return text + + +def profile_sample( + sample: ResourceSample, + *, + max_paths: int = DEFAULT_MAX_PATHS, + max_examples: int = DEFAULT_MAX_EXAMPLES, +) -> ResourceProfile: + """Derive a :class:`ResourceProfile` from a sample's documents. + + Handles tabular and hierarchical documents through one code path: a flat row + is simply the depth-0 case. + """ + observed: dict[str, list[Any]] = {} + depths: dict[str, int] = {} + + for doc in sample.docs: + for path, depth, value in iter_paths(doc): + if path not in observed and len(observed) >= max_paths: + continue + observed.setdefault(path, []).append(value) + depths[path] = max(depths.get(path, depth), depth) + + truncated = sample.truncated or len(observed) >= max_paths + + fields: list[FieldProfile] = [] + for path, values in observed.items(): + field_type, item_type = infer_field_type(values) + non_null = [value for value in values if value is not None] + hashable = { + value if isinstance(value, (str, int, float, bool)) else repr(value) + for value in non_null + } + fields.append( + FieldProfile( + path=path, + type=field_type, + item_type=item_type, + depth=depths.get(path, 0), + present=len(values), + null_count=len(values) - len(non_null), + distinct=len(hashable), + examples=[_example(value) for value in non_null[:max_examples]], + ) + ) + + return ResourceProfile( + resource_name=sample.resource_name, + connector=sample.connector, + doc_count=len(sample.docs), + max_depth=max(depths.values(), default=0), + fields=fields, + primary_key=list(sample.primary_key), + foreign_keys=list(sample.foreign_keys), + truncated=truncated, + ) + + +def profile_source( + source_sample: SourceSample, + *, + max_paths: int = DEFAULT_MAX_PATHS, + max_examples: int = DEFAULT_MAX_EXAMPLES, +) -> list[ResourceProfile]: + """Profile every resource in a :class:`SourceSample`.""" + return [ + profile_sample(sample, max_paths=max_paths, max_examples=max_examples) + for sample in source_sample.samples + ] + + +__all__ = [ + "DEFAULT_MAX_DOCS", + "DEFAULT_MAX_EXAMPLES", + "DEFAULT_MAX_PATHS", + "LIST_MARKER", + "FieldProfile", + "ForeignKeyHint", + "ResourceProfile", + "ResourceSample", + "SourceSample", + "infer_field_type", + "iter_paths", + "profile_sample", + "profile_source", +] diff --git a/graflo/architecture/pipeline/runtime/resource.py b/graflo/architecture/pipeline/runtime/resource.py index 672b06a1..c63e1fa9 100644 --- a/graflo/architecture/pipeline/runtime/resource.py +++ b/graflo/architecture/pipeline/runtime/resource.py @@ -104,7 +104,9 @@ def __init__( self._vertex_config = runtime_vertex_config self._edge_config = local_edge_config - self._validate_vertex_references(vertex_config) + self._validate_vertex_references( + vertex_config, strict_references=strict_references + ) self._validate_infer_edge_spec_targets(self._edge_config) edge_derivation_registry = EdgeDerivationRegistry() @@ -190,8 +192,25 @@ def _filter_vertex_edge_configs( ) return runtime_vertex_config, local_edge_config - def _validate_vertex_references(self, vertex_config: VertexConfig) -> None: + def _validate_vertex_references( + self, vertex_config: VertexConfig, *, strict_references: bool = False + ) -> None: known_vertices = set(vertex_config.vertex_set) + + if strict_references: + # Vertex *steps* naming an undeclared vertex are otherwise silently + # dropped by filter_vertex_config_for_resource, so a pipeline that + # writes nothing validates clean. Under strict references that is an + # error: the resource claims to produce a vertex the schema lacks. + undeclared = sorted(self.collect_vertex_names() - known_vertices) + if undeclared: + raise ValueError( + f"Resource '{self.config.name}' pipeline references undefined " + f"vertices: {undeclared}. Declare them in vertex_config, or fix " + "the step names (a common cause is a naming mismatch between " + "the vertex definition and the pipeline step)." + ) + referenced_vertices: set[str] = set() for spec in self.config.infer_edge_only: diff --git a/graflo/connections/onto.py b/graflo/connections/onto.py index 4fc120d0..032163e2 100644 --- a/graflo/connections/onto.py +++ b/graflo/connections/onto.py @@ -1,6 +1,7 @@ import abc import logging import os +import re import warnings from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Self, TypeVar, cast @@ -1275,6 +1276,54 @@ def to_sqlalchemy_connection_string(self) -> str: else: return f"postgresql://{user_encoded}@{host}:{port}/{database_encoded}" + @classmethod + def from_dsn(cls, dsn: str, **overrides: Any) -> "PostgresConfig": + """Build a config from a libpq/SQLAlchemy DSN. + + ``uri`` alone is not enough: ``hostname`` and ``port`` are derived from + it, but ``username`` / ``password`` / ``database`` are independent fields, + so a config built from a bare URI has no database name and + :meth:`to_sqlalchemy_connection_string` raises. This parses the DSN and + populates those fields. + + Args: + dsn: e.g. ``postgresql://user:pass@host:5432/dbname?options=-csearch_path=sales`` + **overrides: Explicit field values that win over the parsed DSN. + + Returns: + PostgresConfig: Config with credentials and database populated. + """ + from urllib.parse import parse_qs, unquote, urlsplit + + parts = urlsplit(dsn) + if not parts.hostname: + raise ValueError(f"DSN has no host: {dsn!r}") + + fields: dict[str, Any] = { + "uri": f"{parts.scheme or 'postgresql'}://{parts.netloc}" + } + if parts.username: + fields["username"] = unquote(parts.username) + if parts.password: + fields["password"] = unquote(parts.password) + database = unquote(parts.path).lstrip("/") + if database: + fields["database"] = database + + # Honour a search_path passed either as ?schema= or in libpq options. + query = parse_qs(parts.query) + schema = next(iter(query.get("schema", [])), None) + if schema is None: + options = next(iter(query.get("options", [])), "") + match = re.search(r"-c\s*search_path=([^\s,]+)", options) + if match: + schema = match.group(1) + if schema: + fields["schema_name"] = schema + + fields.update(overrides) + return cls(**fields) + @classmethod def from_docker_env(cls, docker_dir: str | Path | None = None) -> "PostgresConfig": """Load PostgreSQL config from docker/postgres/.env file.""" diff --git a/graflo/data_source/chunker.py b/graflo/data_source/chunker.py index ddc09a72..500a987d 100644 --- a/graflo/data_source/chunker.py +++ b/graflo/data_source/chunker.py @@ -266,7 +266,15 @@ def _prepare_iteration(self): # After super()._prepare_iteration(), file_obj is guaranteed to be open if self.file_obj is None: raise RuntimeError("File should be opened by parent _prepare_iteration()") - header_line = next(self.file_obj) + try: + header_line = next(self.file_obj) + except StopIteration: + # An empty file has no header row. Leave the header empty and let + # iteration yield nothing; raising StopIteration from here would + # surface as `RuntimeError: generator raised StopIteration` (PEP 479) + # to any caller iterating batches. + self.header = [] + return if isinstance(header_line, bytes): header_line = header_line.decode(self.encoding or "utf-8") self.header = header_line.rstrip("\n").split(self.sep) @@ -486,10 +494,15 @@ def _guess_chunker_type(cls, filename: Path) -> ChunkerType: ValueError: If file extension is not recognized """ # Get all suffixes and remove compression extensions - suffixes = filename.suffixes - base_suffix = [y for y in suffixes if y.lower() not in (".gz", ".zip")][ - -1 - ].lower() + suffixes = [y for y in filename.suffixes if y.lower() not in (".gz", ".zip")] + if not suffixes: + # A file with no extension (or only .gz/.zip) has nothing to go on. + # Raise the documented ValueError rather than an IndexError, so + # callers scanning a directory can skip it like any other unknown type. + raise ValueError( + f"Could not guess chunker type: '{filename.name}' has no file extension" + ) + base_suffix = suffixes[-1].lower() if base_suffix == ".json": return ChunkerType.JSON diff --git a/graflo/db/arango/conn.py b/graflo/db/arango/conn.py index 27fdf113..55302a59 100644 --- a/graflo/db/arango/conn.py +++ b/graflo/db/arango/conn.py @@ -899,7 +899,7 @@ def insert_edges_batch( docs_edges = docs_edges[:head] if filter_uniques: docs_edges = pick_unique_dict(docs_edges) - docs_edges_str = json.dumps(docs_edges) + docs_edges_str = json.dumps(docs_edges, default=json_serializer) result_from, source_filter = _arango_edge_endpoint_aql( source_class, match_keys_source, 0 diff --git a/graflo/hq/graph_engine.py b/graflo/hq/graph_engine.py index 4260a21e..46f8119e 100644 --- a/graflo/hq/graph_engine.py +++ b/graflo/hq/graph_engine.py @@ -14,6 +14,7 @@ from graflo.architecture.contract.ingestion import IngestionModel from graflo.architecture.contract.manifest import GraphManifest from graflo.architecture.graph_types import GraphContainer +from graflo.architecture.onto_sample import DEFAULT_MAX_DOCS, SourceSample from graflo.architecture.onto_sql import SchemaIntrospectionResult from graflo.architecture.schema import GraFloOutput, Schema from graflo.connections.onto import DBConfig, PostgresConfig, SparqlEndpointConfig @@ -134,6 +135,51 @@ def introspect( include_raw_tables=include_raw_tables, ) + def sample_resources( + self, + source: "PostgresConfig | Bindings | Path | str | list[Path | str]", + *, + schema_name: str | None = None, + resources: list[str] | None = None, + max_docs: int = DEFAULT_MAX_DOCS, + source_name: str | None = None, + ) -> "SourceSample": + """Sample documents from a source, keeping connector provenance. + + The returned :class:`~graflo.architecture.onto_sample.SourceSample` holds + pure JSON documents per resource. It is the shared input for schema + inference — algorithmic or agentic — and + ``SourceSample.samples_by_resource`` is the shape cross-resource identity + inference consumes directly. + + Args: + source: A ``PostgresConfig``, a ``Bindings`` block, or a file/directory path. + schema_name: PostgreSQL schema to sample (PostgreSQL sources only). + resources: Restrict sampling to these resource names. + max_docs: Cap on documents fetched per resource. + source_name: Override the logical source name. + + Returns: + SourceSample: Per-resource samples with connector provenance. + """ + from graflo.hq.sampler import ResourceSampler + + sampler = ResourceSampler(max_docs=max_docs) + if isinstance(source, PostgresConfig): + return sampler.sample_postgres( + source, + schema_name=schema_name, + tables=resources, + source_name=source_name, + ) + if isinstance(source, Bindings): + return sampler.sample_bindings( + source, + resources=resources, + source_name=source_name or "bindings", + ) + return sampler.sample_files(source, source_name=source_name) + def infer_manifest( self, postgres_config: PostgresConfig, diff --git a/graflo/hq/sampler.py b/graflo/hq/sampler.py new file mode 100644 index 00000000..fe2e441e --- /dev/null +++ b/graflo/hq/sampler.py @@ -0,0 +1,407 @@ +"""Connector-driven resource sampling. + +Pulls a bounded number of documents from each resource behind a set of +connectors and returns them as a :class:`~graflo.architecture.onto_sample.SourceSample` +— pure JSON, with the originating connector recorded on every sample. + +This is the input stage shared by every schema inferencer. ``infer_manifest`` +performs it privately for PostgreSQL; exposing it lets an agentic or algorithmic +inferencer work from the same material, for any connector kind. + +Sampling is deliberately *not* profiling: this module fetches documents and does +not describe them. See :func:`~graflo.architecture.onto_sample.profile_sample` +for the derived typed view. +""" + +from __future__ import annotations + +import logging +import re +from datetime import date, datetime, time +from decimal import Decimal +from pathlib import Path +from typing import Any +from uuid import UUID + +from graflo.architecture.contract.bindings.connectors import ( + FileConnector, + ResourceConnector, + TableConnector, +) +from graflo.architecture.contract.bindings.core import Bindings +from graflo.architecture.onto_sample import ( + DEFAULT_MAX_DOCS, + ForeignKeyHint, + ResourceSample, + SourceSample, +) +from graflo.connections.onto import PostgresConfig +from graflo.data_source.factory import DataSourceFactory +from graflo.hq.registry_builder import RegistryBuilder + +logger = logging.getLogger(__name__) + +#: Values longer than this are clipped and the sample marked ``truncated``. +DEFAULT_MAX_CELL_CHARS = 512 + + +def _jsonable(value: Any, *, max_cell_chars: int) -> tuple[Any, bool]: + """Coerce a fetched value to something JSON-serialisable. + + Database and columnar readers hand back ``datetime``, ``Decimal``, + ``memoryview`` and numpy scalars. ``dict[str, Any]`` accepts them but only + serialises them best-effort, so normalise at sample time rather than leaving + it to whatever writes the JSON. + + Returns the coerced value and whether it was clipped. + """ + if value is None or isinstance(value, (bool, int)): + return value, False + if isinstance(value, float): + return value, False + if isinstance(value, str): + if len(value) > max_cell_chars: + return value[:max_cell_chars], True + return value, False + if isinstance(value, (datetime, date, time)): + return value.isoformat(), False + if isinstance(value, Decimal): + return float(value), False + if isinstance(value, UUID): + return str(value), False + if isinstance(value, (bytes, bytearray, memoryview)): + return f"<{len(bytes(value))} bytes>", True + if isinstance(value, dict): + out: dict[str, Any] = {} + clipped = False + for key, item in value.items(): + out[str(key)], item_clipped = _jsonable(item, max_cell_chars=max_cell_chars) + clipped = clipped or item_clipped + return out, clipped + if isinstance(value, (list, tuple, set, frozenset)): + items: list[Any] = [] + clipped = False + for item in value: + coerced, item_clipped = _jsonable(item, max_cell_chars=max_cell_chars) + items.append(coerced) + clipped = clipped or item_clipped + return items, clipped + # numpy scalars and anything else with a scalar view + item_method = getattr(value, "item", None) + if callable(item_method): + try: + return _jsonable(item_method(), max_cell_chars=max_cell_chars) + except Exception: + pass + return str(value), False + + +class ResourceSampler: + """Fetch bounded document samples from connectors. + + Args: + max_docs: Cap on documents fetched per resource. + max_cell_chars: Cap on the length of any single string value. + """ + + def __init__( + self, + *, + max_docs: int = DEFAULT_MAX_DOCS, + max_cell_chars: int = DEFAULT_MAX_CELL_CHARS, + ) -> None: + if max_docs < 1: + raise ValueError("max_docs must be at least 1") + self.max_docs = max_docs + self.max_cell_chars = max_cell_chars + + # ------------------------------------------------------------------ + # Normalisation + # ------------------------------------------------------------------ + + def _normalize_docs( + self, docs: list[dict[str, Any]] + ) -> tuple[list[dict[str, Any]], bool]: + normalized: list[dict[str, Any]] = [] + truncated = False + for doc in docs[: self.max_docs]: + coerced, clipped = _jsonable(doc, max_cell_chars=self.max_cell_chars) + normalized.append(coerced) + truncated = truncated or clipped + if len(docs) > self.max_docs: + truncated = True + return normalized, truncated + + def _read_data_source(self, data_source: Any) -> tuple[list[dict[str, Any]], bool]: + # Read one document past the cap so ``_normalize_docs`` can tell a source + # that happened to hold exactly ``max_docs`` from one that was truncated. + probe = self.max_docs + 1 + docs: list[dict[str, Any]] = [] + for batch in data_source.iter_batches(batch_size=probe, limit=probe): + docs.extend(batch) + if len(docs) >= probe: + break + return self._normalize_docs(docs) + + # ------------------------------------------------------------------ + # Files + # ------------------------------------------------------------------ + + def sample_file( + self, + path: Path | str, + *, + resource_name: str | None = None, + connector_name: str | None = None, + ) -> ResourceSample: + """Sample a single file, recording the connector that would read it.""" + path = Path(path) + connector = FileConnector( + name=connector_name or path.stem, + regex=f"^{re.escape(path.name)}$", + sub_path=path.parent, + ) + data_source = DataSourceFactory.create_file_data_source(path=path) + docs, truncated = self._read_data_source(data_source) + return ResourceSample( + resource_name=resource_name or path.stem, + connector=connector.name, + docs=docs, + truncated=truncated, + description=f"Sampled from file {path.name}", + ) + + def sample_files( + self, + paths_or_dir: Path | str | list[Path | str], + *, + source_name: str | None = None, + ) -> SourceSample: + """Sample every readable file in a directory, or an explicit file list. + + Files that cannot be read, or that yield no documents, are skipped with a + warning rather than failing the whole sample — a source directory + routinely holds READMEs and notes beside its data. + """ + if isinstance(paths_or_dir, (str, Path)): + root = Path(paths_or_dir) + if root.is_dir(): + paths = sorted(p for p in root.iterdir() if p.is_file()) + default_name = root.name + else: + paths = [root] + default_name = root.stem + else: + paths = [Path(p) for p in paths_or_dir] + default_name = paths[0].parent.name if paths else "source" + + samples: list[ResourceSample] = [] + for path in paths: + try: + sample = self.sample_file(path) + except (ValueError, OSError) as exc: + logger.warning("Skipping unsampleable file '%s': %s", path, exc) + continue + if not sample.docs: + logger.warning("Skipping file '%s': yielded no documents", path) + continue + samples.append(sample) + + if not samples: + raise ValueError(f"No sampleable files found in {paths_or_dir}") + + return SourceSample(source_name=source_name or default_name, samples=samples) + + # ------------------------------------------------------------------ + # PostgreSQL + # ------------------------------------------------------------------ + + def sample_postgres( + self, + config: PostgresConfig, + *, + schema_name: str | None = None, + tables: list[str] | None = None, + source_name: str | None = None, + ) -> SourceSample: + """Sample PostgreSQL tables, carrying declared keys through. + + Primary and foreign keys come from schema introspection, not from column + naming — they are ground truth for edge inference, so an inferencer never + has to guess at ``*_id`` suffixes when a real constraint exists. + """ + from graflo.db.postgres.conn import PostgresConnection + from graflo.hq.sql_inferencer import SQLInferenceManager + + effective_schema = schema_name or config.schema_name or "public" + samples: list[ResourceSample] = [] + + with PostgresConnection(config) as conn: + inferencer = SQLInferenceManager( + conn=conn, target_db_flavor=self.postgres_target_flavor() + ) + introspection = inferencer.introspect( + schema_name=effective_schema, include_raw_tables=True + ) + for table in introspection.raw_tables: + if tables is not None and table.name not in tables: + continue + connector = TableConnector( + name=table.name, + table_name=table.name, + schema_name=table.schema_name, + ) + rows = conn.get_table_sample_rows( + table.name, schema_name=table.schema_name, limit=self.max_docs + ) + docs, truncated = self._normalize_docs(rows) + samples.append( + ResourceSample( + resource_name=table.name, + connector=connector.name, + docs=docs, + primary_key=list(table.primary_key), + foreign_keys=[ + ForeignKeyHint( + field=fk.column, + references_resource=fk.references_table, + references_field=fk.references_column, + ) + for fk in table.foreign_keys + ], + truncated=truncated, + total_estimate=table.row_count_estimate, + description=f"Sampled from table {table.schema_name}.{table.name}", + ) + ) + + if not samples: + raise ValueError( + f"No tables found to sample in schema '{effective_schema}'" + ) + + return SourceSample( + source_name=source_name or config.database or effective_schema, + samples=samples, + ) + + @staticmethod + def postgres_target_flavor() -> Any: + """Target flavour used for introspection type mapping.""" + from graflo.connections.onto import DBType + + return DBType.ARANGO + + # ------------------------------------------------------------------ + # Bindings + # ------------------------------------------------------------------ + + def sample_connector( + self, + connector: ResourceConnector, + *, + resource_name: str, + config: PostgresConfig | None = None, + ) -> ResourceSample: + """Sample one resource through *connector*.""" + if isinstance(connector, FileConnector): + files = RegistryBuilder.discover_files( + connector.sub_path.expanduser(), connector=connector, limit_files=1 + ) + if not files: + raise ValueError( + f"FileConnector for resource '{resource_name}' matched no files " + f"under '{connector.sub_path}'" + ) + sample = self.sample_file( + files[0], + resource_name=resource_name, + connector_name=connector.name or resource_name, + ) + return sample + + if isinstance(connector, TableConnector): + if config is None: + raise ValueError( + f"Sampling TableConnector for resource '{resource_name}' requires " + "a PostgresConfig" + ) + from graflo.db.postgres.conn import PostgresConnection + + with PostgresConnection(config) as conn: + rows = conn.get_table_sample_rows( + connector.table_name, + schema_name=connector.schema_name, + limit=self.max_docs, + ) + docs, truncated = self._normalize_docs(rows) + return ResourceSample( + resource_name=resource_name, + connector=connector.name or resource_name, + docs=docs, + truncated=truncated, + ) + + raise ValueError( + f"Sampling is not implemented for {type(connector).__name__} " + f"(resource '{resource_name}')" + ) + + def sample_bindings( + self, + bindings: Bindings, + *, + resources: list[str] | None = None, + config: PostgresConfig | None = None, + source_name: str = "bindings", + ) -> SourceSample: + """Sample every resource wired up in *bindings*. + + The ``resource_connector`` mapping is the authority on which connector + feeds which resource, so the provenance recorded on each sample is the + same relation ingestion will later use. + """ + connectors_by_ref: dict[str, ResourceConnector] = {} + for connector in bindings.connectors: + if connector.name: + connectors_by_ref[connector.name] = connector + connectors_by_ref[connector.hash] = connector + + samples: list[ResourceSample] = [] + for mapping in bindings.resource_connector: + resource = ( + mapping.get("resource") + if isinstance(mapping, dict) + else mapping.resource + ) + connector_ref = ( + mapping.get("connector") + if isinstance(mapping, dict) + else mapping.connector + ) + if resources is not None and resource not in resources: + continue + connector = connectors_by_ref.get(str(connector_ref)) + if connector is None: + logger.warning( + "Skipping resource '%s': connector '%s' not found in bindings", + resource, + connector_ref, + ) + continue + try: + samples.append( + self.sample_connector( + connector, resource_name=str(resource), config=config + ) + ) + except ValueError as exc: + logger.warning("Skipping resource '%s': %s", resource, exc) + + if not samples: + raise ValueError("No resources could be sampled from the given bindings") + + return SourceSample(source_name=source_name, samples=samples) + + +__all__ = ["DEFAULT_MAX_CELL_CHARS", "ResourceSampler"] diff --git a/graflo/util/transform.py b/graflo/util/transform.py index 96bf1d6c..0345ae4c 100644 --- a/graflo/util/transform.py +++ b/graflo/util/transform.py @@ -10,7 +10,7 @@ - cast_ibes_analyst: Parse and standardize analyst names - clear_first_level_nones: Clean dictionaries by removing None values - parse_multi_item: Parse complex multi-item strings - - pick_unique_dict: Remove duplicate dictionaries + - pick_unique_dict: Remove duplicate structures by content hash Example: >>> name = standardize("John. Doe, Smith") @@ -24,6 +24,7 @@ from collections import defaultdict from datetime import datetime from functools import lru_cache +from typing import TypeVar ORDINAL_SUFFIX = ["st", "nd", "rd", "th"] _CAMEL_TO_SNAKE_STEP1_RE = re.compile(r"(.)([A-Z][a-z]+)") @@ -33,8 +34,10 @@ logger = logging.getLogger(__name__) +T = TypeVar("T") -def standardize(k): + +def standardize(k: str) -> str: """Standardizes a string key by removing periods and splitting. Handles comma and space-separated strings, normalizing their format. @@ -51,17 +54,17 @@ def standardize(k): >>> standardize("John Doe Smith") 'John,Doe,Smith' """ - k = k.translate(str.maketrans({".": ""})) + cleaned = k.translate(str.maketrans({".": ""})) # try to split by ", " - k = k.split(", ") - if len(k) < 2: - k = k[0].split(" ") + parts = cleaned.split(", ") + if len(parts) < 2: + parts = parts[0].split(" ") else: - k[1] = k[1].translate(str.maketrans({" ": ""})) - return ",".join(k) + parts[1] = parts[1].translate(str.maketrans({" ": ""})) + return ",".join(parts) -def parse_date_standard(input_str): +def parse_date_standard(input_str: str) -> tuple[int, int, int]: """Parse a date string in YYYY-MM-DD format. Args: @@ -78,7 +81,7 @@ def parse_date_standard(input_str): return dt.year, dt.month, dt.day -def parse_date_conf(input_str): +def parse_date_conf(input_str: str) -> tuple[int, int, int]: """Parse a date string in YYYYMMDD format. Args: @@ -95,7 +98,7 @@ def parse_date_conf(input_str): return dt.year, dt.month, dt.day -def parse_date_ibes(date0, time0): +def parse_date_ibes(date0: str | int, time0: str) -> str: """Converts IBES date and time to ISO 8601 format datetime. Args: @@ -116,7 +119,7 @@ def parse_date_ibes(date0, time0): return full_datetime -def parse_date_yahoo(date0): +def parse_date_yahoo(date0: str) -> str: """Convert Yahoo Finance date to ISO 8601 format. Args: @@ -133,7 +136,7 @@ def parse_date_yahoo(date0): return full_datetime -def round_str(x, **kwargs): +def round_str(x: str, **kwargs) -> float: """Round a string number to specified precision. Args: @@ -150,7 +153,7 @@ def round_str(x, **kwargs): return round(float(x), **kwargs) -def parse_date_standard_to_epoch(input_str): +def parse_date_standard_to_epoch(input_str: str) -> float: """Convert standard date string to Unix epoch timestamp. Args: @@ -168,7 +171,7 @@ def parse_date_standard_to_epoch(input_str): return timestamp -def cast_ibes_analyst(s): +def cast_ibes_analyst(s: str) -> tuple[str, str]: """Splits and normalizes analyst name strings. Handles various name formats like 'ADKINS/NARRA' or 'ARFSTROM J'. @@ -203,7 +206,7 @@ def cast_ibes_analyst(s): return r[0], r[1][:1] -def parse_date_reference(input_str): +def parse_date_reference(input_str: str) -> int | str: """Extract year from a date reference string. Args: @@ -219,7 +222,7 @@ def parse_date_reference(input_str): return _parse_date_reference(input_str)["year"] -def _parse_date_reference(input_str): +def _parse_date_reference(input_str: str) -> dict[str, int | str]: """Parse complex, human-written date references. Handles various date formats like: @@ -296,7 +299,9 @@ def try_int(x): return x -def clear_first_level_nones(docs, keys_keep_nones: list | None = None): +def clear_first_level_nones( + docs: list[dict], keys_keep_nones: list | None = None +) -> list[dict]: """Removes None values from dictionaries, with optional key exceptions. Args: @@ -319,7 +324,7 @@ def clear_first_level_nones(docs, keys_keep_nones: list | None = None): return docs -def parse_multi_item(s, mapper: dict, direct: list): +def parse_multi_item(s: str, mapper: dict, direct: list) -> defaultdict[str, list]: """Parses complex multi-item strings into structured data. Supports parsing strings with quoted or bracketed items. @@ -371,17 +376,18 @@ def parse_multi_item(s, mapper: dict, direct: list): return r -def pick_unique_dict(docs): - """Removes duplicate dictionaries from a list. +def pick_unique_dict(docs: list[T]) -> list[T]: + """Remove duplicate structures from a list by content hash. - Uses a hash-based approach to identify unique dictionaries, which is more - efficient than JSON serialization and preserves original object types. + Uses a hash-based approach that handles nested dicts, lists/tuples, + datetime objects, and Decimal types. Preserves original objects and + insertion order. Works for vertex dicts and edge triples alike. Args: - docs (list): List of dictionaries. + docs: List of structures to deduplicate. Returns: - list: List of unique dictionaries (preserving original objects). + List of unique structures (preserving original objects). Example: >>> docs = [{"a": 1}, {"a": 1}, {"b": 2}] @@ -421,7 +427,7 @@ def make_hashable(obj): return obj # Use a dict to preserve insertion order and original objects - seen = {} + seen: dict = {} for doc in docs: # Create hashable representation hashable_repr = make_hashable(doc) diff --git a/mkdocs.yml b/mkdocs.yml index 88dc04ba..5ea71856 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -61,6 +61,7 @@ nav: - Core components: concepts/architecture/core_components.md - Capabilities: concepts/architecture/capabilities.md - Schema and manifest: + - Sampling and profiling: concepts/schema/sampling_and_profiling.md - Vertex identity: concepts/schema/vertex_identity.md - Backend indexes: concepts/schema/backend_indexes.md - Manifest evolution: concepts/schema/manifest_evolution.md diff --git a/pyproject.toml b/pyproject.toml index d61ff317..07e4b46f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ description = "Manifest-driven Graph Schema & Transformation Language (GSTL): de name = "graflo" readme = "README.md" requires-python = ">=3.11" -version = "1.10.0" +version = "1.10.1" [project.optional-dependencies] dev = [ diff --git a/test/architecture/test_layering.py b/test/architecture/test_layering.py index 559be28d..0ce601ad 100644 --- a/test/architecture/test_layering.py +++ b/test/architecture/test_layering.py @@ -51,6 +51,7 @@ "graflo.architecture.evolution": 4, "graflo.architecture.util": 1, # helpers over graph_types only "graflo.architecture.onto_sql": 1, # leaf pydantic models (SQL introspection) + "graflo.architecture.onto_sample": 2, # leaf sample models; needs FieldType (L2) "graflo.data_source": 4, "graflo.db": 5, "graflo.object_storage": 5, diff --git a/test/architecture/test_manifest_canonical_contract.py b/test/architecture/test_manifest_canonical_contract.py index 8f57fa8d..b25d3376 100644 --- a/test/architecture/test_manifest_canonical_contract.py +++ b/test/architecture/test_manifest_canonical_contract.py @@ -86,6 +86,52 @@ def test_ingestion_model_strict_transform_reference_fails_fast() -> None: assert "was not found in ingestion_model.transforms" in str(exc) +def test_strict_mode_rejects_pipeline_vertex_step_naming_unknown_vertex() -> None: + """A vertex step naming an undeclared vertex is otherwise silently dropped. + + ``filter_vertex_config_for_resource`` intersects the resource's vertex names + with the schema's, so a pipeline that writes nothing used to validate clean. + A name mismatch between the vertex definition and the step is the common + cause, and it produces a manifest that ingests no data at all. + """ + schema = _minimal_schema() + ingestion_model = IngestionModel.model_validate( + { + "resources": [{"name": "r1", "pipeline": [{"vertex": "A"}]}], + "transforms": [], + } + ) + + try: + ingestion_model.finish_init(schema.core_schema, strict_references=True) + assert False, "Expected strict vertex reference validation to fail" + except ValueError as exc: + assert "references undefined vertices: ['A']" in str(exc) + + +def test_lenient_mode_still_tolerates_unknown_pipeline_vertex_step() -> None: + """Non-strict behaviour is unchanged: unknown vertex steps are filtered out.""" + schema = _minimal_schema() + ingestion_model = IngestionModel.model_validate( + { + "resources": [{"name": "r1", "pipeline": [{"vertex": "A"}]}], + "transforms": [], + } + ) + ingestion_model.finish_init(schema.core_schema, strict_references=False) + + +def test_strict_mode_accepts_matching_pipeline_vertex_step() -> None: + schema = _minimal_schema() + ingestion_model = IngestionModel.model_validate( + { + "resources": [{"name": "r1", "pipeline": [{"vertex": "a"}]}], + "transforms": [], + } + ) + ingestion_model.finish_init(schema.core_schema, strict_references=True) + + def test_registry_builder_strict_mode_aggregates_missing_connectors() -> None: schema = _minimal_schema() ingestion_model = IngestionModel.model_validate( diff --git a/test/data/sample-source/NOTES.txt b/test/data/sample-source/NOTES.txt new file mode 100644 index 00000000..6e0a74d5 --- /dev/null +++ b/test/data/sample-source/NOTES.txt @@ -0,0 +1 @@ +These are notes, not data. diff --git a/test/data/sample-source/api_orders.json b/test/data/sample-source/api_orders.json new file mode 100644 index 00000000..324d145e --- /dev/null +++ b/test/data/sample-source/api_orders.json @@ -0,0 +1,37 @@ +[ + { + "order_id": "o1", + "customer": { + "id": "c1", + "city": "Berlin" + }, + "items": [ + { + "sku": "A-1", + "qty": 2 + }, + { + "sku": "B-2", + "qty": 1 + } + ], + "tags": [ + "priority", + "gift" + ] + }, + { + "order_id": "o2", + "customer": { + "id": "c2", + "city": null + }, + "items": [ + { + "sku": "C-3", + "qty": 5 + } + ], + "tags": [] + } +] diff --git a/test/data/sample-source/customers.csv b/test/data/sample-source/customers.csv new file mode 100644 index 00000000..23c3ed17 --- /dev/null +++ b/test/data/sample-source/customers.csv @@ -0,0 +1,4 @@ +id,email,name +c1,alice@example.com,Alice +c2,bob@example.com,Bob +c3,cara@example.com,Cara diff --git a/test/data/sample-source/orders.csv b/test/data/sample-source/orders.csv new file mode 100644 index 00000000..77d36e6c --- /dev/null +++ b/test/data/sample-source/orders.csv @@ -0,0 +1,4 @@ +id,customer_id,total,paid +o1,c1,9.50,true +o2,c2,3.00,false +o3,c1,4.25,true diff --git a/test/db/postgres/test_postgres_config_dsn.py b/test/db/postgres/test_postgres_config_dsn.py new file mode 100644 index 00000000..e8933a53 --- /dev/null +++ b/test/db/postgres/test_postgres_config_dsn.py @@ -0,0 +1,74 @@ +"""``PostgresConfig.from_dsn`` — DSN parsing into connectable config. + +``uri`` alone derives only host and port; ``username`` / ``password`` / +``database`` are independent fields, so a config built from a bare URI cannot +produce a connection string. These tests pin the parsing that closes that gap. +""" + +import pytest + +from graflo.connections.onto import PostgresConfig + + +def test_from_dsn_populates_credentials_and_database(): + config = PostgresConfig.from_dsn("postgresql://alice:secret@db.example:5433/shop") + assert config.hostname == "db.example" + assert str(config.port) == "5433" + assert config.username == "alice" + assert config.password == "secret" + assert config.database == "shop" + + +def test_from_dsn_yields_usable_sqlalchemy_connection_string(): + """The regression this guards: a bare uri= config raised here.""" + config = PostgresConfig.from_dsn("postgresql://alice:secret@db.example:5433/shop") + assert ( + config.to_sqlalchemy_connection_string() + == "postgresql://alice:secret@db.example:5433/shop" + ) + + +def test_from_dsn_percent_decodes_credentials(): + config = PostgresConfig.from_dsn("postgresql://user%40corp:p%40ss@h/db") + assert config.username == "user@corp" + assert config.password == "p@ss" + + +def test_from_dsn_applies_default_port(): + config = PostgresConfig.from_dsn("postgresql://user@localhost/db") + assert str(config.port) == "5432" + + +@pytest.mark.parametrize( + "dsn,expected", + [ + ("postgresql://u@h/db?schema=sales", "sales"), + ("postgresql://u@h/db?options=-csearch_path=sales", "sales"), + ("postgresql://u@h/db?options=-c%20search_path=sales", "sales"), + ("postgresql://u@h/db", None), + ], +) +def test_from_dsn_reads_schema_from_query_or_libpq_options(dsn, expected): + assert PostgresConfig.from_dsn(dsn).schema_name == expected + + +def test_from_dsn_overrides_win_over_parsed_values(): + config = PostgresConfig.from_dsn( + "postgresql://u@h/db", database="other", schema_name="analytics" + ) + assert config.database == "other" + assert config.schema_name == "analytics" + + +def test_from_dsn_rejects_dsn_without_host(): + with pytest.raises(ValueError, match="no host"): + PostgresConfig.from_dsn("not-a-dsn") + + +def test_from_dsn_tolerates_missing_database(): + """A DSN may legitimately omit the database; the error should surface later, + at connection-string construction, not at parse time.""" + config = PostgresConfig.from_dsn("postgresql://u@h") + assert config.database is None + with pytest.raises(ValueError, match="database name is required"): + config.to_sqlalchemy_connection_string() diff --git a/test/hq/test_sampler.py b/test/hq/test_sampler.py new file mode 100644 index 00000000..ec789477 --- /dev/null +++ b/test/hq/test_sampler.py @@ -0,0 +1,312 @@ +"""Resource sampling and profiling. + +Covers the substrate shared by algorithmic and agentic schema inference: samples +stay pure JSON, connector provenance survives, and nesting is described by path +rather than flattened away at the boundary. +""" + +from datetime import datetime +from decimal import Decimal +from pathlib import Path + +import pytest + +from graflo.architecture.onto_sample import ( + ForeignKeyHint, + ResourceSample, + SourceSample, + infer_field_type, + iter_paths, + profile_sample, + profile_source, +) +from graflo.architecture.schema.vertex import FieldType +from graflo.hq.sampler import ResourceSampler, _jsonable + +SOURCE_DIR = Path(__file__).parent.parent / "data" / "sample-source" + + +@pytest.fixture +def sampler() -> ResourceSampler: + return ResourceSampler(max_docs=10) + + +@pytest.fixture +def file_sample(sampler: ResourceSampler) -> SourceSample: + return sampler.sample_files(SOURCE_DIR) + + +# ---------------------------------------------------------------------- +# Sampling +# ---------------------------------------------------------------------- + + +def test_sample_files_records_connector_provenance(file_sample): + """Every sample must know which connector produced it -- that relation is + what later becomes a resource_connector binding.""" + assert file_sample.source_name == "sample-source" + for sample in file_sample.samples: + assert sample.connector, f"{sample.resource_name} lost its connector" + assert sample.connector == sample.resource_name + + +def test_sample_files_skips_non_data_files(file_sample): + """A source directory routinely holds notes beside its data.""" + names = {sample.resource_name for sample in file_sample.samples} + assert names == {"customers", "orders", "api_orders"} + + +def test_sample_files_keeps_documents_verbatim(file_sample): + """Tabular rows arrive as flat dicts, nested documents keep their structure.""" + customers = file_sample.get("customers") + assert customers is not None + assert customers.docs[0]["email"] == "alice@example.com" + + api = file_sample.get("api_orders") + assert api is not None + assert api.docs[0]["customer"] == {"id": "c1", "city": "Berlin"} + assert api.docs[0]["items"][0]["sku"] == "A-1" + assert api.docs[0]["tags"] == ["priority", "gift"] + + +def test_samples_by_resource_matches_inferencer_input_shape(file_sample): + """dict[str, list[dict]] is what cross-resource identity inference consumes.""" + by_resource = file_sample.samples_by_resource + assert set(by_resource) == {"customers", "orders", "api_orders"} + assert all( + isinstance(docs, list) and all(isinstance(doc, dict) for doc in docs) + for docs in by_resource.values() + ) + + +def test_max_docs_caps_and_marks_truncated(): + sampler = ResourceSampler(max_docs=2) + sample = sampler.sample_file(SOURCE_DIR / "orders.csv") + assert len(sample.docs) == 2 + assert sample.truncated is True + + +def test_max_docs_below_one_is_rejected(): + with pytest.raises(ValueError, match="at least 1"): + ResourceSampler(max_docs=0) + + +def test_sample_single_file_directly(sampler): + sample = sampler.sample_file(SOURCE_DIR / "customers.csv") + assert sample.resource_name == "customers" + assert len(sample.docs) == 3 + assert sample.truncated is False + + +def test_sample_files_rejects_directory_without_data(tmp_path, sampler): + (tmp_path / "readme.txt").write_text("nothing here") + with pytest.raises(ValueError, match="No sampleable files"): + sampler.sample_files(tmp_path) + + +def test_source_sample_requires_at_least_one_resource(): + with pytest.raises(ValueError): + SourceSample(source_name="empty", samples=[]) + + +# ---------------------------------------------------------------------- +# JSON coercion +# ---------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value,expected", + [ + (datetime(2026, 7, 30, 12, 0), "2026-07-30T12:00:00"), + (Decimal("9.50"), 9.5), + (None, None), + (True, True), + (7, 7), + ("plain", "plain"), + ], +) +def test_jsonable_coerces_database_types(value, expected): + """DB and columnar readers hand back types json.dumps cannot serialise.""" + coerced, _ = _jsonable(value, max_cell_chars=64) + assert coerced == expected + + +def test_jsonable_clips_long_values_and_reports_it(): + coerced, clipped = _jsonable("x" * 100, max_cell_chars=10) + assert coerced == "x" * 10 + assert clipped is True + + +def test_jsonable_recurses_into_containers(): + coerced, _ = _jsonable( + {"when": datetime(2026, 1, 1), "amounts": [Decimal("1.5")]}, max_cell_chars=64 + ) + assert coerced == {"when": "2026-01-01T00:00:00", "amounts": [1.5]} + + +def test_jsonable_summarises_binary(): + coerced, clipped = _jsonable(b"\x00\x01\x02", max_cell_chars=64) + assert coerced == "<3 bytes>" + assert clipped is True + + +# ---------------------------------------------------------------------- +# Path walking and type inference +# ---------------------------------------------------------------------- + + +def test_iter_paths_describes_nesting_by_path(): + doc = { + "id": 1, + "customer": {"id": "c1", "address": {"city": "Berlin"}}, + "items": [{"sku": "A"}, {"sku": "B"}], + "tags": ["x"], + } + paths = {path for path, _depth, _value in iter_paths(doc)} + assert paths == { + "id", + "customer.id", + "customer.address.city", + "items[].sku", + "tags", + } + + +def test_iter_paths_reports_depth(): + doc = {"a": 1, "b": {"c": {"d": 2}}} + depths = {path: depth for path, depth, _value in iter_paths(doc)} + assert depths == {"a": 0, "b.c.d": 2} + + +@pytest.mark.parametrize( + "values,expected", + [ + ([True, False], FieldType.BOOL), + ([1, 2, 3], FieldType.INT), + ([1.5, 2], FieldType.FLOAT), + (["a", "b"], FieldType.STRING), + ([datetime(2026, 1, 1)], FieldType.DATETIME), + (["2026-07-30", "2026-07-31"], FieldType.DATETIME), + (["4f8c1d2e-1111-4222-8333-abcdefabcdef"], FieldType.UUID), + ([None, None], FieldType.STRING), + ], +) +def test_infer_field_type(values, expected): + field_type, _item_type = infer_field_type(values) + assert field_type == expected + + +def test_infer_field_type_checks_bool_before_int(): + """bool is an int subclass, so the naive order mistypes booleans as INT.""" + assert infer_field_type([True, False])[0] == FieldType.BOOL + assert infer_field_type([0, 1])[0] == FieldType.INT + + +def test_infer_field_type_lists_carry_item_type(): + field_type, item_type = infer_field_type([["a", "b"], ["c"]]) + assert field_type == FieldType.LIST + assert item_type == FieldType.STRING + + +# ---------------------------------------------------------------------- +# Profiling +# ---------------------------------------------------------------------- + + +def test_profile_sample_types_tabular_columns(sampler): + profile = profile_sample(sampler.sample_file(SOURCE_DIR / "orders.csv")) + assert profile.max_depth == 0 + assert profile.nested is False + types = {field.path: field.type for field in profile.fields} + assert types["id"] == FieldType.STRING + assert types["customer_id"] == FieldType.STRING + + +def test_profile_sample_describes_nesting(sampler): + """Nesting depth is the signal an ingestion model needs descend steps.""" + profile = profile_sample(sampler.sample_file(SOURCE_DIR / "api_orders.json")) + assert profile.nested is True + assert profile.max_depth == 1 + assert "items[].sku" in profile.field_paths + assert "customer.city" in profile.field_paths + + +def test_profile_sample_tracks_nulls_and_cardinality(sampler): + profile = profile_sample(sampler.sample_file(SOURCE_DIR / "api_orders.json")) + fields = {field.path: field for field in profile.fields} + assert fields["customer.city"].null_count == 1 + assert fields["order_id"].unique is True + assert fields["items[].sku"].present == 3 + + +def test_profile_sample_types_scalar_lists(sampler): + profile = profile_sample(sampler.sample_file(SOURCE_DIR / "api_orders.json")) + tags = next(field for field in profile.fields if field.path == "tags") + assert tags.type == FieldType.LIST + assert tags.item_type == FieldType.STRING + + +def test_profile_carries_declared_keys_not_guesses(): + """Declared PK/FK are ground truth for edge inference.""" + sample = ResourceSample( + resource_name="orders", + connector="orders", + docs=[{"id": "o1", "customer_id": "c1"}], + primary_key=["id"], + foreign_keys=[ + ForeignKeyHint( + field="customer_id", + references_resource="customers", + references_field="id", + ) + ], + ) + profile = profile_sample(sample) + assert profile.primary_key == ["id"] + assert profile.foreign_keys[0].references_resource == "customers" + + +def test_profile_max_paths_caps_and_marks_truncated(): + sample = ResourceSample( + resource_name="wide", + docs=[{f"col_{i}": i for i in range(50)}], + ) + profile = profile_sample(sample, max_paths=10) + assert len(profile.fields) == 10 + assert profile.truncated is True + + +def test_flat_docs_makes_nested_sources_eligible_for_identity_inference(sampler): + """Identity inference operates on flat records; this is the bridge.""" + sample = sampler.sample_file(SOURCE_DIR / "api_orders.json") + profile = profile_sample(sample) + flat = profile.flat_docs(sample.docs) + assert flat[0]["customer.id"] == "c1" + assert flat[0]["items[].sku"] == "A-1" + assert all(not isinstance(value, dict) for value in flat[0].values()) + + +def test_profile_source_covers_every_resource(file_sample): + profiles = profile_source(file_sample) + assert {profile.resource_name for profile in profiles} == { + "customers", + "orders", + "api_orders", + } + assert all(profile.connector for profile in profiles) + + +# ---------------------------------------------------------------------- +# Serialization +# ---------------------------------------------------------------------- + + +def test_source_sample_round_trips_through_json(file_sample): + """The sample crosses an HTTP boundary, so it must survive model_dump/validate + with nested documents intact.""" + payload = file_sample.model_dump(mode="json") + restored = SourceSample.model_validate(payload) + assert restored.samples_by_resource == file_sample.samples_by_resource + api = restored.get("api_orders") + assert api is not None + assert api.docs[0]["items"][0]["sku"] == "A-1" diff --git a/uv.lock b/uv.lock index 3ddb3afb..e8cb278a 100644 --- a/uv.lock +++ b/uv.lock @@ -73,11 +73,11 @@ dependencies = [ {name = "s3transfer"} ] name = "boto3" -sdist = {url = "https://files.pythonhosted.org/packages/18/95/bd6276870084c9c1a94e17d1dc73b2de275e59bd7a0ad8bfff0a1598cec4/boto3-1.43.58.tar.gz", hash = "sha256:12871fb50c383f1b9aa4ed6dd386ba689062baef730552e79d5a9cd782b53058", size = 112685, upload-time = "2026-07-28T19:35:09.336Z"} +sdist = {url = "https://files.pythonhosted.org/packages/dc/c8/ef9de1d7413da3adcdb6363258ba6b5cc703593409d8c1957825b20a69d3/boto3-1.43.59.tar.gz", hash = "sha256:4e9b14f89adc1a533c89312e86d8e00455a6f15d398796d92f9191b06e56b401", size = 112653, upload-time = "2026-07-29T19:33:25.703Z"} source = {registry = "https://pypi.org/simple"} -version = "1.43.58" +version = "1.43.59" wheels = [ - {url = "https://files.pythonhosted.org/packages/1c/85/b0709066efb4ce7b86aba4092c739ad0e458be9d62cf32550fc4c130c93f/boto3-1.43.58-py3-none-any.whl", hash = "sha256:ce1a20cbcfaa1d0b3c8f568e2b6c7fbd34b842ea00e729d49a4b4de522828db9", size = 140026, upload-time = "2026-07-28T19:35:06.993Z"} + {url = "https://files.pythonhosted.org/packages/2b/10/c5999e72b020012f2e0ccccf2a15632329edd34cb95b02b1ccfb1712ec08/boto3-1.43.59-py3-none-any.whl", hash = "sha256:58b9635deebf075c1c3d76df78df08eb2979c2a74283194676783a0bff3b4557", size = 140024, upload-time = "2026-07-29T19:33:23.751Z"} ] [[package]] @@ -87,11 +87,11 @@ dependencies = [ {name = "urllib3"} ] name = "botocore" -sdist = {url = "https://files.pythonhosted.org/packages/5e/64/5cd46a7e72b0647e6d78fc8da016259ad66b9ae0818f4c5d629c75e7ca49/botocore-1.43.58.tar.gz", hash = "sha256:e110ca53f65c128fe98df4d6d36a459b1db17c6114671c8a18becf151bf20909", size = 15742412, upload-time = "2026-07-28T19:34:57.468Z"} +sdist = {url = "https://files.pythonhosted.org/packages/e5/37/3712a70796583570a5a2e426163e13762ba5ec615a73e966ad18e5933954/botocore-1.43.59.tar.gz", hash = "sha256:8016da69ecc1d705249a8ef13548d3c95eec87ac1cd26133a8bdfa73ca175be0", size = 15788291, upload-time = "2026-07-29T19:33:14.871Z"} source = {registry = "https://pypi.org/simple"} -version = "1.43.58" +version = "1.43.59" wheels = [ - {url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z"} + {url = "https://files.pythonhosted.org/packages/f6/cd/62d749f824b25c152144665f7c5eb8b5ca8be967a87e0c63577b7d4501ae/botocore-1.43.59-py3-none-any.whl", hash = "sha256:21393c35d23b19d7ba95cc4156b59f4013f80696d667997e1abd9d4e29651708", size = 15471171, upload-time = "2026-07-29T19:33:10.864Z"} ] [[package]] @@ -434,11 +434,11 @@ wheels = [ [[package]] name = "filelock" -sdist = {url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z"} +sdist = {url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z"} source = {registry = "https://pypi.org/simple"} -version = "3.32.0" +version = "3.32.2" wheels = [ - {url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z"} + {url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z"} ] [[package]] @@ -491,7 +491,7 @@ dependencies = [ ] name = "graflo" source = {editable = "."} -version = "1.10.0" +version = "1.10.1" [package.metadata] provides-extras = ["dev", "docs", "plot"] @@ -2265,27 +2265,27 @@ wheels = [ [[package]] name = "ty" -sdist = {url = "https://files.pythonhosted.org/packages/8e/aa/14c9965d3b173105692473897cc89c34cd91241368b2044e43167e1c17ff/ty-0.0.64.tar.gz", hash = "sha256:d12ddbb05f15158bb518af619378b385486450def95fb06f8ab98037febe9f2c", size = 6350966, upload-time = "2026-07-27T18:32:45.403Z"} -source = {registry = "https://pypi.org/simple"} -version = "0.0.64" -wheels = [ - {url = "https://files.pythonhosted.org/packages/ae/4c/c54937e4ff3fa7b34a99ea3387ec766bf0ad98dc8df8d792e89b388e658e/ty-0.0.64-py3-none-linux_armv6l.whl", hash = "sha256:3830a6675ab43635ced1c4c557f380ac4a49e9414e03975e4e4e8db644c64944", size = 12118357, upload-time = "2026-07-27T18:32:08.702Z"}, - {url = "https://files.pythonhosted.org/packages/ce/aa/a839ee2bc78e943d079e6abe199a97b4eeffb7e5c9a57326d69de452186a/ty-0.0.64-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3ff07d7bc32a2135f58afe57393789a32ea2fed1a66216129a8e535e76043903", size = 11790882, upload-time = "2026-07-27T18:32:10.997Z"}, - {url = "https://files.pythonhosted.org/packages/4c/de/19f14357888a7198438926303753cf749428e3d62e8980ff1e9a72a78402/ty-0.0.64-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4f6d1c7f897cca05d12bacbf1435150d5ffa496099515aa6ed303c8b29e1d0bb", size = 11317394, upload-time = "2026-07-27T18:32:13.162Z"}, - {url = "https://files.pythonhosted.org/packages/08/2f/f54462300535ab99b551eda733177be2eef5dbc2997d3fdb357c4ddd760a/ty-0.0.64-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:138d6c37ad4bf8583aa7a9b29d90954151d7856f9910a03eae3eb34b34c57215", size = 11863042, upload-time = "2026-07-27T18:32:15.307Z"}, - {url = "https://files.pythonhosted.org/packages/5e/95/dbecf745520ebe8bd7b02fc55eee6441c9be312ebf6addce605eb52740dd/ty-0.0.64-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ed9719d1b7b66fb8efe073d860208a44c40af1f6cd5c2364aa9b323a1e579b4", size = 11910730, upload-time = "2026-07-27T18:32:17.467Z"}, - {url = "https://files.pythonhosted.org/packages/3b/26/12cfd40028e51ceed7b3cb645281c61c02eb64ff9fb0c09231d65c30ff25/ty-0.0.64-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d68b23e5169e2137b5f1de7169ab0cebecab6c8eda374c34c1f6394308f58242", size = 12631936, upload-time = "2026-07-27T18:32:19.533Z"}, - {url = "https://files.pythonhosted.org/packages/a0/d0/65ffc2b0a686347193c6f98e9421a7fc2a96fc3cd0b1001cf7cf284baff9/ty-0.0.64-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f41cb07d89d32626fcaf3ed4d262778fcb28b2628b6ed1e172cd7b18820668d8", size = 13171049, upload-time = "2026-07-27T18:32:22.026Z"}, - {url = "https://files.pythonhosted.org/packages/f6/a4/975a5961842dcd6fa60f0770a102c0bba7509da909ab652919bfcdcd4fc7/ty-0.0.64-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5f24e1504ab9e212f92356b82fe088fdeb3a39f9a2f4ff25e505d2e1d0db9056", size = 12826438, upload-time = "2026-07-27T18:32:24.178Z"}, - {url = "https://files.pythonhosted.org/packages/af/ef/dfb9b7f9bcc032d3b540b0d1f55f532a336e2fb41b1bd539c05ae81a151a/ty-0.0.64-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86db830cb914bb33bb8247b66ccea58de4496c2391bd42658aba744b439f3290", size = 12440880, upload-time = "2026-07-27T18:32:26.341Z"}, - {url = "https://files.pythonhosted.org/packages/ee/d0/bedac20505e8a8f5501ad73d7d15d8e421a563fef59993909a23036929a4/ty-0.0.64-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:652ee3d6d03bea76cd2fe8949c78bb5970394ebd7c5fd270e9518c0dee1b931d", size = 12782439, upload-time = "2026-07-27T18:32:28.642Z"}, - {url = "https://files.pythonhosted.org/packages/79/d5/795733f13ceff1378f08b3de0c49d0f518df220ed856b0dfac869f3b7c81/ty-0.0.64-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4838768295774a86e95f9ec5633e739d016adbbb69dcbdc62f3549578a11f624", size = 11814821, upload-time = "2026-07-27T18:32:30.632Z"}, - {url = "https://files.pythonhosted.org/packages/52/3e/9d99cd1e1831003434f508ed9f258a56543194afc3bbe051eba2545fa676/ty-0.0.64-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5a3d700669868599edf39ce5125196682f15a8880cc8c669bc2a83b99984d99b", size = 11928678, upload-time = "2026-07-27T18:32:32.888Z"}, - {url = "https://files.pythonhosted.org/packages/b8/3d/448f49a3503fb119a34348a5714bb92001f252fadbbb12d645f2e744b557/ty-0.0.64-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b161f0a82a8e2f2432db3bf7702b4d3924fa9486ba0014f6710a160fc157df0d", size = 12202249, upload-time = "2026-07-27T18:32:34.905Z"}, - {url = "https://files.pythonhosted.org/packages/dd/9b/75768e562cec990d189dc05807ae72890b20ffbd1e1f43597bb98db63c60/ty-0.0.64-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:39b9dd42908df47c2dc57dda87e656fab97097ffd2618474bbdea986af0d6a9d", size = 12548817, upload-time = "2026-07-27T18:32:36.995Z"}, - {url = "https://files.pythonhosted.org/packages/34/89/44cc276ea6ca0245495014758ffeadde1798fb0b5840c9cf36d8d2ed3250/ty-0.0.64-py3-none-win32.whl", hash = "sha256:d0676ab0e0935795e5843baa28dd5e366dc343d7c0945a996df9eab8e0644885", size = 11545474, upload-time = "2026-07-27T18:32:39.389Z"}, - {url = "https://files.pythonhosted.org/packages/01/7e/d1c8a871a38d17c8f168b9a6975f6247f7660f8334e517656a5e4b4a4858/ty-0.0.64-py3-none-win_amd64.whl", hash = "sha256:dcb9bd31f54097e362b776c26ab4564d4564cdd1355cb883481167a26c03cc3f", size = 12542987, upload-time = "2026-07-27T18:32:41.412Z"}, - {url = "https://files.pythonhosted.org/packages/35/4d/6d18640d0204cacd69abbaca95ad6a34c6d7e9169e9051d0117b17b827ec/ty-0.0.64-py3-none-win_arm64.whl", hash = "sha256:82cc34c1ad9a8feb6059aef193bebcecc656e548f6fab3d518bcd8b57d198d39", size = 11899263, upload-time = "2026-07-27T18:32:43.366Z"} +sdist = {url = "https://files.pythonhosted.org/packages/d6/54/cf561927e8e9ab5c1892a833b664aa9cd6f051a75f6280c66d8047246bda/ty-0.0.65.tar.gz", hash = "sha256:b7134bffcc00b715fa8291e84d845782ced810a998dc1f7f11d71c85c4046325", size = 6460098, upload-time = "2026-07-29T18:31:03.27Z"} +source = {registry = "https://pypi.org/simple"} +version = "0.0.65" +wheels = [ + {url = "https://files.pythonhosted.org/packages/7b/4e/71e2d325d2b53a1afad81624ad076b2ede413213fc4a18cb05b78c568571/ty-0.0.65-py3-none-linux_armv6l.whl", hash = "sha256:dc556c9f05408bef4c4ef02b2cc382e4e5f797b4b20d64410289848f0d76705f", size = 12298466, upload-time = "2026-07-29T18:30:12.744Z"}, + {url = "https://files.pythonhosted.org/packages/57/77/fec8f29647c55794efa430a7f365e44f5ce7ffb6459d9445a87fac569bec/ty-0.0.65-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:29d2e0d34cc0a28a17ef0cf81135c5ebabc3562131f9079138ba5e7bae0f56bd", size = 11942421, upload-time = "2026-07-29T18:30:16.076Z"}, + {url = "https://files.pythonhosted.org/packages/13/09/7f3766aef9dc627e2698cf4e3e59cf53389dcae3812040d33c1aa931230f/ty-0.0.65-py3-none-macosx_11_0_arm64.whl", hash = "sha256:685f49a9312bbf69d5b65bbb66384fed1f927403ea030c217b9289092d7e46c4", size = 11451922, upload-time = "2026-07-29T18:30:19.155Z"}, + {url = "https://files.pythonhosted.org/packages/cb/7b/1a77cd50e0befb50f55b8bf9bd3ed3eddf184bf28c61b56727039e0774fc/ty-0.0.65-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f564b5ebe78e2f3a8e7b8eacb1292eb88b7c0f3c8630671cfca31abc0709cd9", size = 11994999, upload-time = "2026-07-29T18:30:22.315Z"}, + {url = "https://files.pythonhosted.org/packages/63/7b/feda16f3a4a0a99be27431e0c9598eeeec0db1eb2fec9a15976698209418/ty-0.0.65-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c983e156fe9e113fb56389e13d327b6b8549fe866de9b269684723a88e9b732d", size = 12090662, upload-time = "2026-07-29T18:30:24.93Z"}, + {url = "https://files.pythonhosted.org/packages/ed/3e/3f69bf9c9307dbdc0771719f65ce5b556e7bdeeaccbdd599d4f57866d801/ty-0.0.65-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3663b7396e8b1a9954e20e732de7ccb0192bf4118473069b4945920d6923921", size = 12822094, upload-time = "2026-07-29T18:30:28.012Z"}, + {url = "https://files.pythonhosted.org/packages/90/38/8fa791b3bb503ee2b46ad81690cd1bdd54519582df6d805cee57fe143e85/ty-0.0.65-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:306ed01f29d6e108e98feb233dbbf5878a027603b71bd3743b343977933a9f16", size = 13357833, upload-time = "2026-07-29T18:30:31.122Z"}, + {url = "https://files.pythonhosted.org/packages/c1/73/4dda396a201e1dd0ed3594a9b48e559cb41c4bc048c6cd4c4d1b39eb4313/ty-0.0.65-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28bcfc8898c94f079a9100e684bcf312b6a64ad3a7d4ebb35a4591546030a2cd", size = 12977303, upload-time = "2026-07-29T18:30:33.944Z"}, + {url = "https://files.pythonhosted.org/packages/a5/26/c250c2c569adc53a8591716641388397bcb2a442e4a30b952ae81b50c0e0/ty-0.0.65-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a75bd0c245c38802a8f488378e74f92feb7dd33db7d63fbdd6fdf82791ba730", size = 12579338, upload-time = "2026-07-29T18:30:37.199Z"}, + {url = "https://files.pythonhosted.org/packages/d3/94/4a5647d44753ca218fc930d7e4d9bf468d0ed4a0ad4b3d57588bc1bbacf7/ty-0.0.65-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9e5e1bdea9662d2b5312b4e99f319f4e6e2ea427511b5fbc546141b79ec53f76", size = 12957731, upload-time = "2026-07-29T18:30:39.937Z"}, + {url = "https://files.pythonhosted.org/packages/36/b6/1e22fa11a1e0dfb20b1c7f3cbfd8170273aada2a82f9ecd3055275370c44/ty-0.0.65-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:03a88493d4842889f65280ae241e06b399d57eb3c63571054cad21a4c33b3b69", size = 11938625, upload-time = "2026-07-29T18:30:42.603Z"}, + {url = "https://files.pythonhosted.org/packages/5c/0a/fe5f22ef62b193201bc5566762e22049762cd485bfafb5095a7050760054/ty-0.0.65-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:600b8bf6f4940cf7ffb2f43d3716faaf38dcb97cd8617c55771451bc0276408f", size = 12105592, upload-time = "2026-07-29T18:30:45.419Z"}, + {url = "https://files.pythonhosted.org/packages/76/fd/922b3a6e9d697452cdbb4b7e3f636868add5ec652154518a736e4364f3b7/ty-0.0.65-py3-none-musllinux_1_2_i686.whl", hash = "sha256:0c28007bc79d648c1ddaf1e65885d07baec48eb87240da442f608e4107c1b7d8", size = 12387335, upload-time = "2026-07-29T18:30:48.405Z"}, + {url = "https://files.pythonhosted.org/packages/77/22/a1a08ebc84c083db2fb55e3b5cd186db0c067692f4921146f601360231e2/ty-0.0.65-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c852da96091ad22361e6586b7c7ba98e1334dcd4d8ffb67e47f4fb673de33f77", size = 12682710, upload-time = "2026-07-29T18:30:51.364Z"}, + {url = "https://files.pythonhosted.org/packages/81/14/eaaa410a25bbdea19722109b5422380a0e211b3afcf3071d15953ddbd5db/ty-0.0.65-py3-none-win32.whl", hash = "sha256:cf529d538f1403b14b0511e6ec3cdb95d3d974adabf24cc76cedc533368c3edc", size = 11692341, upload-time = "2026-07-29T18:30:54.35Z"}, + {url = "https://files.pythonhosted.org/packages/bc/0f/6d48f206dce9d7e53fe3b5ea0f0ab5800dd9d2365b2b48f736783436c43f/ty-0.0.65-py3-none-win_amd64.whl", hash = "sha256:234a321e33c7cbbfbd67bfa0b01b685dd9c21f1841781a21e5ca1fa0b25f1d5d", size = 12729355, upload-time = "2026-07-29T18:30:57.275Z"}, + {url = "https://files.pythonhosted.org/packages/96/aa/7446f7725e303cf78e058c893af1f0552b9451895454908706f4c6c3494b/ty-0.0.65-py3-none-win_arm64.whl", hash = "sha256:b9424be1ec56d93ff18609fb1c0a0a2283fe1282cd6d1c7604f97d73b94d61f2", size = 12051375, upload-time = "2026-07-29T18:31:00.579Z"} ] [[package]] From f666bab4a16cae41609c965c055e8bc7cfd6dcde Mon Sep 17 00:00:00 2001 From: Alexander Belikov Date: Thu, 30 Jul 2026 10:52:51 +0200 Subject: [PATCH 2/2] fixed rdf roundtrip --- CHANGELOG.md | 6 +- docs/assets/graflo-ontology-viz/embed.html | 32 ++++++++- .../graflo-ontology-viz/graph-data.json | 30 +++++++- docs/assets/graflo-ontology-viz/index.html | 34 ++++++++- graflo/onto.py | 2 +- graflo/rdf/deserializer.py | 56 +++++++++++++-- graflo/rdf/namespace.py | 6 +- graflo/rdf/ontology/graflo-context.jsonld | 13 ++++ graflo/rdf/ontology/graflo.ttl | 29 ++++++-- graflo/rdf/serializer.py | 40 +++++++++-- mkdocs.yml | 1 + pyproject.toml | 3 +- test/rdf/test_manifest_rdf.py | 71 +++++++++++++++++++ uv.lock | 29 +++++++- 14 files changed, 325 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f42e4ef..d337804e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -**Resource sampling** — a connector-driven sampling primitive, the shared input stage for schema inference (algorithmic or agentic). Previously `infer_manifest` performed this privately for PostgreSQL and nothing else could reach it. - +- **Meta-ontology `1.1.0`** (`graflo/rdf/ontology/graflo.ttl`) — `gf:assigned` (datatype, boolean), `gf:hasHashIdentity` (object property reusing `gf:Identity`), `gf:SecondaryIdentity` (class) and `gf:hasSecondaryIdentity`. `gf:hasIdentity`'s domain widens to include `gf:SecondaryIdentity`, so a secondary identity's field list reuses the existing identity-chain vocabulary rather than adding a parallel one. Mirrored in `graflo-context.jsonld` and `rdf/namespace.py`; `docs/assets/graflo-ontology-viz/` regenerated. +- **Resource sampling** — a connector-driven sampling primitive, the shared input stage for schema inference (algorithmic or agentic). Previously `infer_manifest` performed this privately for PostgreSQL and nothing else could reach it. - **`architecture.onto_sample`** (new, L2 leaf models) — `ResourceSample` holds documents **verbatim as JSON**: flat rows for tables, arbitrarily nested for APIs, nothing flattened at the boundary. `connector` records where the documents came from, so the relation that later becomes a resource plus its `resource_connector` binding survives the round trip. `SourceSample.samples_by_resource` returns `dict[str, list[dict]]` — the shape cross-resource identity inference consumes, so no adapter is needed. `ForeignKeyHint` carries *declared* references, which are ground truth for edge inference rather than a name-suffix guess. - **Sampling and profiling are separate operations.** `ResourceSampler` (`hq/sampler.py`) pulls documents; `profile_sample` derives the description. `ResourceProfile` is **path-keyed** (`address.city`, `items[].sku`) with a `FieldType`, null rate, cardinality and nesting depth, so hierarchical and tabular sources take one code path — a flat column list cannot represent a nested response at all. `ResourceProfile.flat_docs` projects nested documents into flat records, which is what makes `IdentityInferencer` (flat-records only) usable on an API source. - **`GraphEngine.sample_resources(...)`** — accepts a `PostgresConfig`, a `Bindings` block, or a file/directory path. Bindings-driven sampling reuses `RegistryBuilder.discover_files` and the `resource_connector` mapping, so provenance comes for free instead of being reconstructed. PostgreSQL sampling carries `primary_key` and `foreign_keys` through from introspection. Documents are normalized to JSON-safe values (`datetime`, `Decimal`, `memoryview`, numpy scalars), capped by `max_docs` and `max_cell_chars`, with `truncated` set when anything was dropped or clipped. @@ -20,6 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **RDF round-trip no longer silently degrades a vertex's identity mode** (`CORE-RDF-001`). The serializer wrote only `gf:blank` and `gf:identityName`, so `assigned`, `hash_identity_properties` and `secondary_identities` were dropped and every vertex read back as `natural` — a wrong-but-valid schema, which is the worst failure shape for a documented round-trip format. All four identity modes (`natural`, `hash`, `blank`, `assigned`) now survive, and `examples/16-secondary-identities` round-trips to canonical equality. +- **Identity field order survives.** Identity nodes are `BNode`s and RDF triples are unordered, so a multi-field `identity` could come back permuted. They now carry `gf:artifactIndex` and are read through `_ordered_nodes`; graphs written by the previous serializer still parse (a missing index degrades to arbitrary order rather than failing). - **`strict_references=True` now rejects a pipeline `vertex:` step naming an undeclared vertex.** `filter_vertex_config_for_resource` intersects a resource's vertex names with the schema's and silently drops unknowns, so a resource that ingested nothing validated clean — a name mismatch between the vertex definition and the step was invisible. **Behaviour change:** manifests that previously passed under `strict_references=True` may now fail, which is the point; lenient validation is unchanged. - `ChunkerFactory._guess_chunker_type` raises the documented `ValueError` for a file with no extension instead of `IndexError`, so callers scanning a directory can skip it like any other unknown type. - The CSV/TSV chunker no longer raises `RuntimeError: generator raised StopIteration` (PEP 479) on an empty file; it yields no rows. diff --git a/docs/assets/graflo-ontology-viz/embed.html b/docs/assets/graflo-ontology-viz/embed.html index da83e1b4..7c27f656 100644 --- a/docs/assets/graflo-ontology-viz/embed.html +++ b/docs/assets/graflo-ontology-viz/embed.html @@ -2,7 +2,7 @@ - GraFlo Ontology (v1.0.0) + GraFlo Ontology (v1.1.0) @@ -213,6 +213,13 @@ "source": "https://ontology.growgraph.dev/graflo/Schema", "target": "https://ontology.growgraph.dev/graflo/GrafloArtifact" }, + { + "id": "sub:https://ontology.growgraph.dev/graflo/SecondaryIdentity->https://ontology.growgraph.dev/graflo/GrafloArtifact", + "kind": "subClassOf", + "label": "subClassOf", + "source": "https://ontology.growgraph.dev/graflo/SecondaryIdentity", + "target": "https://ontology.growgraph.dev/graflo/GrafloArtifact" + }, { "id": "sub:https://ontology.growgraph.dev/graflo/SparqlConnector->https://ontology.growgraph.dev/graflo/BoundConnector", "kind": "subClassOf", @@ -367,6 +374,20 @@ "source": "https://ontology.growgraph.dev/graflo/Vertex", "target": "https://ontology.growgraph.dev/graflo/Identity" }, + { + "id": "prop:https://ontology.growgraph.dev/graflo/hasHashIdentity", + "kind": "objectProperty", + "label": "hasHashIdentity", + "source": "https://ontology.growgraph.dev/graflo/Vertex", + "target": "https://ontology.growgraph.dev/graflo/Identity" + }, + { + "id": "prop:https://ontology.growgraph.dev/graflo/hasSecondaryIdentity", + "kind": "objectProperty", + "label": "hasSecondaryIdentity", + "source": "https://ontology.growgraph.dev/graflo/Vertex", + "target": "https://ontology.growgraph.dev/graflo/SecondaryIdentity" + }, { "id": "prop:https://ontology.growgraph.dev/graflo/edgeSource", "kind": "objectProperty", @@ -791,6 +812,13 @@ "label": "Schema", "local": "Schema" }, + { + "comment": "Alternate named field-set that identifies a vertex for lookup only. Edge endpoints may match on it; upserts always use the primary identity.", + "id": "https://ontology.growgraph.dev/graflo/SecondaryIdentity", + "kind": "gf", + "label": "SecondaryIdentity", + "local": "SecondaryIdentity" + }, { "comment": null, "id": "https://ontology.growgraph.dev/graflo/SparqlConnector", @@ -877,7 +905,7 @@ } ], "ontology": "https://ontology.growgraph.dev/graflo", - "version": "1.0.0" + "version": "1.1.0" }; diff --git a/docs/assets/graflo-ontology-viz/graph-data.json b/docs/assets/graflo-ontology-viz/graph-data.json index e75addef..9c8ad710 100644 --- a/docs/assets/graflo-ontology-viz/graph-data.json +++ b/docs/assets/graflo-ontology-viz/graph-data.json @@ -189,6 +189,13 @@ "source": "https://ontology.growgraph.dev/graflo/Schema", "target": "https://ontology.growgraph.dev/graflo/GrafloArtifact" }, + { + "id": "sub:https://ontology.growgraph.dev/graflo/SecondaryIdentity->https://ontology.growgraph.dev/graflo/GrafloArtifact", + "kind": "subClassOf", + "label": "subClassOf", + "source": "https://ontology.growgraph.dev/graflo/SecondaryIdentity", + "target": "https://ontology.growgraph.dev/graflo/GrafloArtifact" + }, { "id": "sub:https://ontology.growgraph.dev/graflo/SparqlConnector->https://ontology.growgraph.dev/graflo/BoundConnector", "kind": "subClassOf", @@ -343,6 +350,20 @@ "source": "https://ontology.growgraph.dev/graflo/Vertex", "target": "https://ontology.growgraph.dev/graflo/Identity" }, + { + "id": "prop:https://ontology.growgraph.dev/graflo/hasHashIdentity", + "kind": "objectProperty", + "label": "hasHashIdentity", + "source": "https://ontology.growgraph.dev/graflo/Vertex", + "target": "https://ontology.growgraph.dev/graflo/Identity" + }, + { + "id": "prop:https://ontology.growgraph.dev/graflo/hasSecondaryIdentity", + "kind": "objectProperty", + "label": "hasSecondaryIdentity", + "source": "https://ontology.growgraph.dev/graflo/Vertex", + "target": "https://ontology.growgraph.dev/graflo/SecondaryIdentity" + }, { "id": "prop:https://ontology.growgraph.dev/graflo/edgeSource", "kind": "objectProperty", @@ -767,6 +788,13 @@ "label": "Schema", "local": "Schema" }, + { + "comment": "Alternate named field-set that identifies a vertex for lookup only. Edge endpoints may match on it; upserts always use the primary identity.", + "id": "https://ontology.growgraph.dev/graflo/SecondaryIdentity", + "kind": "gf", + "label": "SecondaryIdentity", + "local": "SecondaryIdentity" + }, { "comment": null, "id": "https://ontology.growgraph.dev/graflo/SparqlConnector", @@ -853,5 +881,5 @@ } ], "ontology": "https://ontology.growgraph.dev/graflo", - "version": "1.0.0" + "version": "1.1.0" } diff --git a/docs/assets/graflo-ontology-viz/index.html b/docs/assets/graflo-ontology-viz/index.html index 3b1cc55d..1b8a911b 100644 --- a/docs/assets/graflo-ontology-viz/index.html +++ b/docs/assets/graflo-ontology-viz/index.html @@ -2,13 +2,13 @@ - GraFlo Ontology (v1.0.0) + GraFlo Ontology (v1.1.0)