From 31592cb2315647b95b395118e736d3f619313b27 Mon Sep 17 00:00:00 2001 From: willpugh Date: Fri, 17 Jul 2026 14:10:26 -0700 Subject: [PATCH 1/3] Create the initial compliane suite --- compliance/ADAPTER_INTERFACE.md | 85 + compliance/README.md | 15 +- compliance/foundation/.gitignore | 9 + compliance/foundation/DATA_TESTS.md | 2258 +++++++++++++++++ compliance/foundation/README.md | 166 ++ compliance/foundation/SPEC.md | 90 + .../foundation/adapters/osi_python_adapter.py | 64 + compliance/foundation/conformance.yaml | 74 + .../datasets/f_prelude/description.md | 29 + .../foundation/datasets/f_prelude/schema.sql | 51 + compliance/foundation/decisions.yaml | 225 ++ compliance/foundation/proposals.yaml | 231 ++ compliance/foundation/pyproject.toml | 51 + compliance/foundation/tests/README.md | 87 + .../moderate/t-005a-single-step-sum/gold.sql | 4 + .../t-005a-single-step-sum/metadata.yaml | 13 + .../t-005a-single-step-sum/model.yaml | 37 + .../t-005a-single-step-sum/query.json | 12 + .../moderate/t-005b-single-step-avg/gold.sql | 4 + .../t-005b-single-step-avg/metadata.yaml | 13 + .../t-005b-single-step-avg/model.yaml | 37 + .../t-005b-single-step-avg/query.json | 12 + .../t-005d-single-step-vs-nested-sum/gold.sql | 4 + .../metadata.yaml | 13 + .../model.yaml | 39 + .../query.json | 12 + .../gold.sql | 4 + .../metadata.yaml | 14 + .../model.yaml | 37 + .../query.json | 12 + compliance/harness/README.md | 54 + compliance/harness/pyproject.toml | 40 + compliance/harness/src/harness/__init__.py | 18 + compliance/harness/src/harness/__main__.py | 22 + compliance/harness/src/harness/db_manager.py | 82 + compliance/harness/src/harness/models.py | 118 + .../harness/src/harness/proposals_check.py | 154 ++ compliance/harness/src/harness/reporter.py | 234 ++ .../harness/src/harness/result_compare.py | 156 ++ compliance/harness/src/harness/runner.py | 530 ++++ .../harness/src/harness/tests/__init__.py | 18 + .../src/harness/tests/test_proposals_check.py | 156 ++ .../src/harness/tests/test_registry_yaml.py | 198 ++ .../harness/tests/test_reporter_proposals.py | 119 + .../harness/src/harness/tests/test_runner.py | 307 +++ 45 files changed, 5907 insertions(+), 1 deletion(-) create mode 100644 compliance/ADAPTER_INTERFACE.md create mode 100644 compliance/foundation/.gitignore create mode 100644 compliance/foundation/DATA_TESTS.md create mode 100644 compliance/foundation/README.md create mode 100644 compliance/foundation/SPEC.md create mode 100644 compliance/foundation/adapters/osi_python_adapter.py create mode 100644 compliance/foundation/conformance.yaml create mode 100644 compliance/foundation/datasets/f_prelude/description.md create mode 100644 compliance/foundation/datasets/f_prelude/schema.sql create mode 100644 compliance/foundation/decisions.yaml create mode 100644 compliance/foundation/proposals.yaml create mode 100644 compliance/foundation/pyproject.toml create mode 100644 compliance/foundation/tests/README.md create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml create mode 100644 compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json create mode 100644 compliance/harness/README.md create mode 100644 compliance/harness/pyproject.toml create mode 100644 compliance/harness/src/harness/__init__.py create mode 100644 compliance/harness/src/harness/__main__.py create mode 100644 compliance/harness/src/harness/db_manager.py create mode 100644 compliance/harness/src/harness/models.py create mode 100644 compliance/harness/src/harness/proposals_check.py create mode 100644 compliance/harness/src/harness/reporter.py create mode 100644 compliance/harness/src/harness/result_compare.py create mode 100644 compliance/harness/src/harness/runner.py create mode 100644 compliance/harness/src/harness/tests/__init__.py create mode 100644 compliance/harness/src/harness/tests/test_proposals_check.py create mode 100644 compliance/harness/src/harness/tests/test_registry_yaml.py create mode 100644 compliance/harness/src/harness/tests/test_reporter_proposals.py create mode 100644 compliance/harness/src/harness/tests/test_runner.py diff --git a/compliance/ADAPTER_INTERFACE.md b/compliance/ADAPTER_INTERFACE.md new file mode 100644 index 00000000..d2f0a539 --- /dev/null +++ b/compliance/ADAPTER_INTERFACE.md @@ -0,0 +1,85 @@ + + +# Adapter Interface Contract + +**Version**: 1.0 + +--- + +## Purpose + +This document defines the boundary between the **test harness** and an **OSI implementation adapter**. It is the authoritative reference for what adapters may and must not do. + +## CLI Contract + +```bash + sql --model --query-file --dialect +``` + +| Stream | Content | +|--------|---------| +| **stdout** | Generated SQL string only — no headers, no decoration | +| **stderr** | Error messages on failure | +| **exit code** | 0 on success, non-zero on error | + +## Adapter Boundary Rules + +An adapter is a **thin translator** between the harness CLI contract and the implementation's programmatic API. It performs **format conversion only**. + +### An adapter MUST: + +1. Parse CLI arguments (`--model`, `--query-file`, `--dialect`) +2. Translate the query JSON into the implementation's query object (e.g., `LODQuery`) +3. Invoke the implementation's planner and transpiler +4. Print the resulting SQL to stdout +5. Print errors to stderr and exit non-zero on failure + +### An adapter MUST NOT: + +1. **Validate query semantics** — duplicate dimensions, ambiguous fields, ORDER BY validity, EXCLUDE dimension membership. These are planner responsibilities. If the implementation doesn't validate them, **file a bug on the implementation**. + +2. **Rewrite SQL expressions** — dialect function translation (IFF→CASE, ZEROIFNULL→COALESCE), expression normalization. These are transpiler responsibilities. If the transpiler mishandles a function, **fix the transpiler**. + +3. **Substitute parameters** — resolving `:param`/`$param` placeholders, type-aware quoting. These are parsing-layer responsibilities. If the parser doesn't handle parameters, **extend the parser**. + +4. **Handle special query shapes** — dimensions-only queries, empty-measure queries. These are planner responsibilities. If the planner can't handle a valid query shape, **fix the planner**. + +5. **Work around implementation bugs** — injecting synthetic measures, monkey-patching frozen models, generating SQL directly. If the implementation produces wrong SQL, **fix the implementation**. + +6. **Contain business logic** — aggregation classification, grain resolution, safety checks, filter routing. All BI semantics belong in the implementation. + +### An adapter MAY: + +1. Set up import paths to locate the implementation (`sys.path` manipulation) +2. Register dialect plugins (e.g., importing `osi.frontend.dialect` to register with sqlglot) +3. Translate between the JSON query format and the implementation's API types (this is format conversion, not business logic) + +## Size Guidance + +A compliant adapter should be about a page of code (or less). If an adapter exceeds this, it is almost certainly doing work that belongs in the implementation. The reference Python adapter is ~160 lines. + +## For AI Agents + +When working on adapters for the OSI compliance suite, you MUST follow these rules: + +1. **Never add business logic to an adapter.** If a test fails because the implementation doesn't handle something, fix the implementation — not the adapter. +2. **Never add more than 10 lines to an adapter without checking this document.** If you're adding validation, rewriting, parameter handling, or SQL generation, you're in the wrong file. +3. **The adapter line count is a hard signal.** If it exceeds 200 lines, something is wrong. +4. **When in doubt, the implementation is the right place.** The adapter exists only because different implementations have different API shapes. Everything else is implementation work. diff --git a/compliance/README.md b/compliance/README.md index 481db96f..23a3c6eb 100644 --- a/compliance/README.md +++ b/compliance/README.md @@ -17,4 +17,17 @@ under the License. --> -This is for compliance tests +# Ossie Compliance + +This directory holds the compliance test suites that validate an Ossie +implementation against a proposal's semantics, plus the shared runner +that executes them. + +- [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md) — the CLI contract an + implementation must satisfy to plug into a compliance suite. +- [`harness/`](harness/) — the engine-agnostic runner / reporter / DB + manager shared by every per-version suite below. +- [`foundation/`](foundation/) — the compliance suite for the + Foundation proposal (`osi_version: "0.1"`). See its + [README](foundation/README.md) for current status — this is + currently a bootstrap slice, not the full suite. diff --git a/compliance/foundation/.gitignore b/compliance/foundation/.gitignore new file mode 100644 index 00000000..f1c84633 --- /dev/null +++ b/compliance/foundation/.gitignore @@ -0,0 +1,9 @@ +# Everything in results/ is per-run runner output, written by +# `python -m harness.runner` into a subdirectory of results/ (the +# default is `results/latest/`; the Makefile under +# impl/python/ uses `results/osi_python/` and `results/osi_python_all/`). +# +# Only the curated baseline at results/REPORT.md is tracked. Repo-wide +# Python / cache / venv rules live in the root .gitignore. +results/* +!results/REPORT.md diff --git a/compliance/foundation/DATA_TESTS.md b/compliance/foundation/DATA_TESTS.md new file mode 100644 index 00000000..d4ce4b9e --- /dev/null +++ b/compliance/foundation/DATA_TESTS.md @@ -0,0 +1,2258 @@ + + +# DATA_TESTS.md — Concrete Test Vectors for the Foundation Compliance Suite + +This document is the **concrete realization** of the Foundation Conformance +Decisions catalogued in `Proposed_OSI_Semantics.md` Appendix B. Each +**T-NNN** entry below is a runnable test vector: a model, a dataset, a query, +and the row set or error code an implementation MUST produce. Appendix B's +**D-NNN** entries say *what* an implementation must do; this file says *what +inputs prove it*. + +Implementations are encouraged to translate each entry into a fixture under +the published Foundation compliance suite — for `osi_python` this is +[`compliance/foundation-v0.1/tests/`](/tests/) +(one folder per `T-NNN` containing `metadata.yaml + model.yaml + +query.json + gold_rows.json`). Implementations on other engines run the +same suite through the standard adapter contract documented in +[`compliance/foundation-v0.1/ADAPTER_INTERFACE.md`](../../compliance/ADAPTER_INTERFACE.md). +The expected outputs are normative and implementation-independent. + +> **Bootstrap-slice note.** The compliance suite currently ported into +> this repo only ships fixtures for the T-005 family +> (`compliance/foundation-v0.1/tests/cross_grain/moderate/`). Every +> other `T-NNN` entry below is normative documentation for tests that +> will land — with their own fixtures — in follow-up PRs. See +> `compliance/foundation-v0.1/README.md` for current status. + +--- + +## 1. Conventions + +### 1.1 Test entry shape + +Every test has the same five fields: + +| Field | Meaning | +|:---|:---| +| **Anchors** | The spec section(s) and `D-NNN` decision(s) the test pins. | +| **Fixture** | The named model fixture (§3) the test runs against. | +| **Data** | The concrete rows. For brevity, fixture-shared data lives in §3 and is referenced by name. Test-local data is shown inline. | +| **Query** | The semantic query — written in the Foundation surface used by tests (a thin YAML around §5's `Aggregation Query` / `Scalar Query` shapes). | +| **Expected** | Either a row set (presented as a markdown table, order-insensitive unless noted) or `error: ` with the diagnostic content the engine MUST surface. | + +A test passes iff the engine's observable output (rows or typed error) +matches the **Expected** field. Tests do **not** assert on SQL string, +plan shape, CTE layout, or any other internal artefact (§11.1 of the +semantics spec). + +### 1.2 Row-set comparison + +- **Order-insensitive** by default — sort both sides by all output columns before comparing. +- **NULL-aware** — `NULL == NULL` for the purposes of the test (the SQL three-valued logic does not apply to assertions). +- **Type-aware** — numeric values are compared by value, not by representation; `100`, `100.0`, and `1e2` are equal. String values must match exactly. + +### 1.3 Error comparison + +An error result is a triple `(code, anchor, must_contain)`: + +```yaml +error: + code: E_AGGREGATE_IN_WHERE + anchor: §5.3 # the spec section the engine MUST cite in its diagnostic + must_contain: # substrings the diagnostic MUST include (case-insensitive) + - "aggregate" + - "Where" +``` + +An engine MAY include additional diagnostic content. The code and the +`must_contain` substrings are the only assertions; wording is otherwise free. + +### 1.4 Determinism guarantee + +Every test that produces a row set MUST be deterministic: running it +twice on the same fixture MUST produce byte-identical SQL and the same +multiset of rows. This is also `D-014`'s test shape, applied uniformly. + +--- + +### 1.5 NULL handling, empty groups, and `0` (D-033) + +The Foundation follows **standard SQL** for aggregates over empty +input row sets (§6.11 / D-033): + +| Aggregate family | Empty-input result | +|:---|:---| +| `COUNT(*)`, `COUNT(x)`, `COUNT(DISTINCT x)` | `0` | +| `SUM`, `AVG`, `MIN`, `MAX`, `MEDIAN`, percentiles, `STDDEV`, `VARIANCE` | `NULL` | + +Tests in this catalog use these results literally — but **only for +dim values that actually appear in the result**. Per §1.6 (Plan A for +single-measure shapes), a dim value with no matching fact row is +dropped from the result entirely; there is no empty-aggregate cell +because there is no row. + +The empty-aggregate rule applies to: + +- **Multi-measure stitch cells** where one branch has no contribution + for a dim value present via the other branch (T-011, T-045) — + `SUM` → `NULL`, `COUNT` → `0`. +- **Dim values present via the `NULL`-key orphan bucket** when the + fact row has no matching dim row (T-047, T-053 etc.) — same rule + applies; the orphan-bucket row is part of the result, and its + cells follow standard SQL. + +Models that prefer `0` for a `SUM` cell MUST declare it explicitly: +`COALESCE(SUM(amount), 0) AS revenue`. The Foundation does NOT +auto-rewrite `SUM` to `COALESCE(SUM, 0)`. + +--- + +### 1.6 Resolved — single-measure shapes follow Plan A (preserve facts) + +Earlier drafts of this catalog had an unresolved tension between two +flagship tests: + +- **T-001** (`Dims: [customers.region]; Measures: [SUM(orders.amount)]`) + expected a `NORTH` row with `revenue = NULL` — implying the planner + preserves the dimension domain even when no fact rows match. +- **T-006** (`Dims: [customers.region]; Measures: [COUNT(*)]; Where + orders.status = 'completed'`) expected NORTH to *not* appear — + implying the planner anchors on the fact and drops dim values with + no matching fact. + +Both could not be correct under one planner. The Architect resolved +this in favour of **Plan A — preserve facts, drop unmatched dim values +— for single-measure aggregation queries**, with the **multi-measure +shape using FULL OUTER stitch (§6.6 row 3) to preserve both sides** as +a deliberate contrast. The spec was updated to match (§6.6, §6.2 step 7, +D-001 example), and existing tests were brought into alignment. + +#### Decision summary + +| Query shape | Default join | Which dim values appear in the result | +|:---|:---|:---| +| Single-measure aggregation: `Dims: [dim.X]; Measures: [SUM(fact.Y)]` | `fact LEFT JOIN dim` (§6.6 row 1) | Distinct `dim.X` reached by a surviving `fact` row, plus a `NULL`-key bucket for orphan facts. **A dim value with no matching fact does NOT appear.** | +| Multi-measure aggregation: `Dims: [dim.X]; Measures: [SUM(factA.Y), SUM(factB.Y)]` | Each measure independently resolved per row 1, then `FULL OUTER` stitch on shared dims (§6.6 row 3) | Union of dim values reachable from *either* branch. A dim value appears if either fact pulls it in. | + +This rule is now consistent across every test in the catalog: + +- **T-001, T-005a-e, T-026, T-028, T-047, T-053**: single-measure or + single-fact-source queries — NORTH does not appear. +- **T-006**: single-measure query with a fact-level WHERE — NORTH + does not appear. +- **T-011, T-045**: multi-measure stitch — NORTH appears via the + branch that has data for customer 4. + +#### How a user gets the "preserve all dim values" behaviour + +If a user wants every dim value to appear (BI-tool convention), they +make the query multi-measure by adding a measure sourced from the dim +dataset itself: + +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(customers.id) AS customer_count +``` + +This converts the shape into a multi-measure stitch — the `customers` +branch contributes every distinct region (because `COUNT(customers.id)` +is defined for every region in `customers`), and the FULL OUTER +preserves NORTH with `revenue = NULL`, `customer_count = 1`. + +This is the BI-tool "all regions appear" behaviour, surfaced +*deliberately* through the query shape rather than as a hidden +planner default. The user opts in by asking for it. + +#### Why Plan A (not preserve-dims by default) + +- **Matches raw SQL.** `SELECT dim.X, SUM(fact.Y) FROM fact LEFT JOIN + dim ... GROUP BY dim.X` produces exactly the single-measure Plan A + result. The Foundation does not introduce surprises relative to + the SQL the user could write by hand. +- **Matches Snowflake Semantic Views.** Snowflake's default plan + shape is also Plan A — porting Snowflake models to OSI is a no-op + for this dimension behaviour. +- **Composes cleanly with multi-measure stitch.** §6.6 row 3 already + mandates FULL OUTER for multi-measure shapes; row 1 says LEFT for + single-measure shapes. The contrast is the mental model: "the + multi-measure stitch is the *only* place both sides are preserved + — and that's because there is no other correct shape for merging + two independently-aggregated facts." +- **Surfaces orphan facts.** Plan A keeps the `NULL`-key bucket for + orphan fact rows (e.g. order 105 with `customer_id = 99`). This is + a data-quality signal — preserve-dims would lose it. + +#### Trade-off accepted + +A first-time BI user typing a single-measure query and *expecting* to +see every region might be surprised when a region with no fact data +is absent. The mitigation is the documented multi-measure pattern +above. The Foundation prefers "predictable, matches raw SQL" over +"BI-tool-magical". + +> **Note for test authors.** When writing a new test that groups by a +> dimension over a single-fact source, write the expected rows +> following Plan A: include only dim values reached by a fact row, +> plus the `NULL`-key bucket for orphans. If you want every dim value +> in the expected rows, the test must be multi-measure (with one of +> the measures sourced from the dim dataset). + +--- + +## 2. How to add a new test + +1. Pick the lowest-numbered free `T-NNN`. +2. Link it to a `D-NNN` in `Proposed_OSI_Semantics.md` Appendix B — if no `D-NNN` covers the behaviour, add one there first. +3. Write the test against an existing fixture (§3) if possible; create a new fixture only when the existing ones can't express the shape. +4. Cite the test from the relevant spec § so the doc and the test grow together. + +--- + +## 3. Common fixtures + +### 3.1 Fixture **F-PRELUDE** — single-fact star with multi-fact extension + +Mirrors the *Prelude model* in `Proposed_OSI_Semantics.md` Appendix A. + +```yaml +datasets: + - name: customers + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: region, type: string } + - { name: segment, type: string } + + - name: orders + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: customer_id, type: integer } + - { name: amount, type: decimal(10,2) } + - { name: status, type: string } + + - name: returns + primary_key: [id] + fields: + - { name: id, type: integer } + - { name: customer_id, type: integer } + - { name: amount, type: decimal(10,2) } + + - name: premium_customers + primary_key: [id] + fields: + - { name: id, type: integer } + +relationships: + - { name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id] } # N:1 + - { name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id] } # N:1 +``` + +**Data:** + +`customers`: + +| id | region | segment | +|:--:|:-------|:--------| +| 1 | EAST | retail | +| 2 | EAST | retail | +| 3 | WEST | wholesale | +| 4 | NORTH | retail | + +`orders` (note: orphan order 105 with `customer_id = 99` not in `customers`): + +| id | customer_id | amount | status | +|:---:|:-----------:|-------:|:-----------| +| 101 | 1 | 100.00 | completed | +| 102 | 1 | 50.00 | completed | +| 103 | 2 | 200.00 | pending | +| 104 | 3 | 75.00 | completed | +| 105 | 99 | 30.00 | completed | ← orphan: customer 99 not in customers + +`returns`: + +| id | customer_id | amount | +|:---:|:-----------:|-------:| +| 201 | 1 | 10.00 | +| 202 | 3 | 5.00 | +| 203 | 4 | 15.00 | ← customer 4 has a return but no order + +`premium_customers`: + +| id | +|:--:| +| 1 | +| 3 | + +### 3.2 Fixture **F-BRIDGE** — M:N through a bridge + +Mirrors *Mini-model M2* in Appendix A. This is the fixture for the +flagship `T-015` bridge-deduplication test. + +```yaml +datasets: + - name: actors + primary_key: [actor_id] + fields: + - { name: actor_id, type: integer } + - { name: name, type: string } + - { name: height, type: integer } + + - name: movies + primary_key: [movie_id] + fields: + - { name: movie_id, type: integer } + - { name: title, type: string } + - { name: gross, type: decimal(10,2) } + + - name: appearances + primary_key: [actor_id, movie_id] + role: bridge + fields: + - { name: actor_id, type: integer } + - { name: movie_id, type: integer } + +relationships: + - { name: app_to_actor, from: appearances, to: actors, from_columns: [actor_id], to_columns: [actor_id] } # N:1 + - { name: app_to_movie, from: appearances, to: movies, from_columns: [movie_id], to_columns: [movie_id] } # N:1 +``` + +**Data:** + +`actors`: + +| actor_id | name | height | +|:--------:|:------|-------:| +| 1 | Alice | 170 | +| 2 | Bob | 170 | +| 3 | Carol | 180 | + +`movies`: + +| movie_id | title | gross | +|:--------:|:-------|-------:| +| 10 | Action | 100.00 | +| 11 | Drama | 200.00 | +| 12 | Comedy | 50.00 | + +`appearances`: + +| actor_id | movie_id | +|:--------:|:--------:| +| 1 | 10 | +| 1 | 11 | +| 2 | 10 | ← M10 (Action) has two actors at height 170 +| 3 | 12 | + +### 3.3 Fixture **F-BRIDGE-NONE** — variant of F-BRIDGE without the bridge + +Same `actors` and `movies` data as F-BRIDGE, but no `appearances` dataset +and an undeclared M:N edge between `actors` and `movies`. Used to test +that the engine fails closed instead of guessing a join path. + +```yaml +datasets: + - name: actors + primary_key: [actor_id] + fields: [ { name: actor_id }, { name: name }, { name: height } ] + - name: movies + primary_key: [movie_id] + fields: [ { name: movie_id }, { name: title }, { name: gross } ] + +# Note: no `relationships:` block. The model declares no edge; the +# engine MUST NOT silently fabricate one. +``` + +### 3.4 Fixture **F-AMBIG** — two relationships between the same datasets + +Mirrors *Mini-model M3* in Appendix A. Used for `E_AMBIGUOUS_PATH`. + +```yaml +datasets: + - name: orders + primary_key: [id] + fields: + - { name: id } + - { name: placed_by_id } + - { name: fulfilled_by_id } + - { name: amount } + + - name: users + primary_key: [id] + fields: [ { name: id }, { name: region } ] + +relationships: + - { name: order_placed_by, from: orders, to: users, from_columns: [placed_by_id], to_columns: [id] } + - { name: order_fulfilled_by, from: orders, to: users, from_columns: [fulfilled_by_id], to_columns: [id] } +``` + +**Data:** + +`orders`: + +| id | placed_by_id | fulfilled_by_id | amount | +|:---:|:------------:|:---------------:|-------:| +| 301 | 1 | 2 | 100.00 | +| 302 | 2 | 2 | 50.00 | + +`users`: + +| id | region | +|:--:|:-------| +| 1 | EAST | +| 2 | WEST | + +### 3.5 Fixture **F-NOPATH** — two disconnected datasets + +Mirrors *Mini-model M1* in Appendix A. No relationships are declared. + +```yaml +datasets: + - name: orders + primary_key: [id] + fields: [ { name: id }, { name: customer_id }, { name: amount } ] + + - name: inventory_movements + primary_key: [movement_id] + fields: [ { name: movement_id }, { name: warehouse_id }, { name: quantity } ] +``` + +--- + +## 4. Tests + +### 4.A Query shape + +#### T-001 — Aggregation-query cardinality is `DISTINCT(Dimensions)` + +**Anchors:** §5.1.1, §5.2 · D-001 +**Fixture:** F-PRELUDE + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue +``` + +**Expected (single-measure ⇒ Plan A per §6.6 row 1 and §6.2 step 7):** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NULL | 30.00 | ← orphan order 105 (customer_id = 99 not in customers) + +Row count = `COUNT(DISTINCT region)` over the regions reached by an +`orders` row, plus one for the orphan-customer bucket. NORTH (customer +4's region) does **not** appear: customer 4 has no orders, and a +single-measure aggregation query follows the fact's natural LEFT join +shape (`orders LEFT JOIN customers`) — a dim value reachable only +from the dim domain is *not* in the result. + +Engines that "carry" `customer_id` into the output grain because the +measure is sourced from `orders` fail this test on row count (they +would group by `customer_id`, not `region`, and produce four data +rows + orphan). Engines that produce a `NORTH | NULL` row are +applying the multi-measure stitch shape to a single-measure query — +also wrong (see §1.6). + +If the user wants NORTH to appear with `revenue = NULL`, they must +make the query multi-measure (e.g. `Measures: [SUM(orders.amount) AS +revenue, COUNT(customers.id) AS customer_count]`) — the FULL OUTER +stitch in §6.6 row 3 then preserves every region present in +`customers`. See T-011 for the canonical multi-measure shape. + +#### T-002 — Mixing `Dimensions` and `Fields` is rejected + +**Anchors:** §5.1 · D-010 +**Fixture:** F-PRELUDE + +**Query:** +```yaml +Dimensions: [customers.region] +Fields: [customers.id] +Measures: [SUM(orders.amount)] +``` + +**Expected:** +```yaml +error: + code: E_MIXED_QUERY_SHAPE + anchor: §5.1 + must_contain: ["Dimensions", "Fields", "scalar"] +``` + +#### T-003 — Bare metric reference inside `Fields` is rejected + +**Anchors:** §5.1.2 · D-011 +**Fixture:** F-PRELUDE. Assume `orders.total_revenue = SUM(amount)` is declared. + +**Query:** +```yaml +Fields: [orders.id, orders.total_revenue] +``` + +**Expected:** +```yaml +error: + code: E_AGGREGATE_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["scalar", "total_revenue", "aggregate"] +``` + +A scalar query asks for row-level values; a metric is by definition an +aggregate at query grain, which has no defined value at the row grain +of `orders`. The engine MUST NOT silently aggregate, broadcast, or +"window" the metric. + +--- + +### 4.B Field & metric grain + +#### T-004 — Implicit home-grain aggregation in a field expression + +**Anchors:** §4.3 · D-003, D-015 +**Fixture:** F-PRELUDE. Add this field to the model: + +```yaml +fields: + - name: customers.lifetime_value + expression: "SUM(orders.amount)" +``` + +**Query (scalar):** +```yaml +Fields: [customers.region, customers.id, customers.lifetime_value] +``` + +**Expected:** + +| region | id | lifetime_value | +|:-------|:--:|---------------:| +| EAST | 1 | 150.00 | ← SUM(100, 50) +| EAST | 2 | 200.00 | +| WEST | 3 | 75.00 | +| NORTH | 4 | NULL | ← no orders (or 0, depending on engine's LEFT-aggregate convention — both are acceptable; test allows either) + +The inner `SUM` is evaluated at the **home grain of `customers`** (one +sum per customer row), regardless of any grouping the consuming query +applies. This is what makes `lifetime_value` reusable across queries. + +**Compilation-strategy equivalence (D-015):** an engine MAY compile the +inner `SUM` as (a) a correlated subquery, (b) a `LATERAL` / `CROSS APPLY`, +or (c) a pre-aggregated CTE joined back to `customers`. The test passes +under any of the three. + +#### T-005 — Cross-grain aggregation: single-step and nested forms + +This is the **flagship test family for D-020** — the most-debated +semantic in cross-vendor BI portability. The Foundation accepts **both** +single-step and explicit-nested forms over a `1 : N` edge; they give +different numerical answers for non-distributive aggregates (the +single-step form gives the "all rows at once" answer, the nested form +gives the "per-home-row first" answer). For distributive aggregates +they give identical answers, so either form is acceptable for style. + +**Anchors:** §4.5 form (1), §6.1 Semantic 2 · D-020 + +**Fixture:** F-PRELUDE. + +--- + +##### T-005a — Single-step cross-grain `SUM` over `1 : N` is accepted (distributive) + +**Metric declaration:** +```yaml +metrics: + - name: customers.total_order_amount + expression: "SUM(orders.amount)" # single-step; resolves at query grain +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.total_order_amount] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | total_order_amount | +|:-------|-------------------:| +| EAST | 350.00 | ← orders 101, 102, 103 +| WEST | 75.00 | ← order 104 +| NULL | 30.00 | ← orphan order 105 + +No error. For distributive aggregates the single-step form is +equivalent to the explicit-nested form `SUM(SUM(orders.amount))` — both +return identical numbers (cross-validated by T-005d below). + +--- + +##### T-005b — Single-step cross-grain `AVG` over `1 : N` is accepted (heavy-weighted) + +**Metric declaration:** +```yaml +metrics: + - name: customers.avg_order_amount + expression: "AVG(orders.amount)" # single-step; standard SQL semantics +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.avg_order_amount] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** the heavy-customer-weighted SQL average. + +| region | avg_order_amount | +|:-------|-----------------:| +| EAST | 116.67 | ← AVG(100, 50, 200) = 350/3 — three orders in EAST +| WEST | 75.00 | ← AVG(75) — one order in WEST +| NULL | 30.00 | ← orphan order 105 + +No error. Note that EAST's answer (`116.67`) is **not** the unweighted +average across customers — Customer 1 averages `75` over two orders, +Customer 2 averages `200` over one order, and the unweighted answer +would be `(75 + 200) / 2 = 137.5`. That second number is what the +**nested** form produces; see T-005c. + +--- + +##### T-005c — Explicit nested cross-grain `AVG` over `1 : N` is accepted (unweighted) + +The Snowflake-style explicit form is **still accepted**; it just gives a +different (and meaningfully different) answer for non-distributive +aggregates. The Foundation does not force this style — but neither does +it reject it. Users porting from Snowflake Semantic Views can leave their +nested expressions unchanged. + +**Metric declaration:** +```yaml +metrics: + - name: customers.avg_of_per_customer_avg + expression: "AVG(AVG(orders.amount))" # inner: per-customer avg; outer: avg across customers +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.avg_of_per_customer_avg] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** the unweighted average across customers in each region. + +| region | avg_of_per_customer_avg | +|:-------|------------------------:| +| EAST | 137.50 | ← AVG(Cust1=75, Cust2=200) = 275/2 — two customers in EAST +| WEST | 75.00 | ← AVG(Cust3=75) — one customer in WEST +| NULL | 30.00 | ← orphan order's customer is unknown; treated as one customer-of-one-order + +The user gets to pick: write `AVG(orders.amount)` for the +heavy-weighted answer (T-005b) or `AVG(AVG(orders.amount))` for the +unweighted answer (T-005c). Both are conformant. The two numbers differ +for non-distributive aggregates — this difference is the central +BI-portability decision in D-020. + +--- + +##### T-005d — Single-step and nested `SUM` are numerically equivalent (distributive) + +Run both forms against identical data and assert the row sets are +identical. + +**Metric A (single-step):** +```yaml +- name: customers.sum_single + expression: "SUM(orders.amount)" +``` + +**Metric B (nested):** +```yaml +- name: customers.sum_nested + expression: "SUM(SUM(orders.amount))" +``` + +**Query A:** +```yaml +Dimensions: [customers.region] +Measures: [customers.sum_single] +``` + +**Query B:** +```yaml +Dimensions: [customers.region] +Measures: [customers.sum_nested] +``` + +**Expected:** identical row sets (after column-name normalisation). +Both forms are single-measure aggregation queries ⇒ Plan A ⇒ NORTH +is absent and the orphan `NULL`-region row is present. This test is +what makes "port your Snowflake nested form unchanged" portable in +the distributive case — and is the canonical demonstration that +nested-vs-single is purely a stylistic choice when the aggregate is +distributive. + +--- + +##### T-005e — `COUNT(DISTINCT)` single-step over `1 : N` is accepted + +**Anchors:** §4.5, §6.1 · D-020 case (d), D-022 caveat + +**Metric declaration:** +```yaml +metrics: + - name: customers.distinct_order_statuses + expression: "COUNT(DISTINCT orders.status)" +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [customers.distinct_order_statuses] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent):** + +| region | distinct_order_statuses | +|:-------|------------------------:| +| EAST | 2 | ← `completed` (101, 102) + `pending` (103) +| WEST | 1 | ← `completed` (104) +| NULL | 1 | ← orphan 105 has `completed` + +No `E_UNSAFE_REAGGREGATION`. This pins D-020 (d) and the D-022 caveat +explicitly: a holistic aggregate over a plain `1 : N` edge is one SQL +aggregate over the joined rows — well-defined per D-020. The +`E_UNSAFE_REAGGREGATION` error only fires when the plan would require +*decomposing* a holistic aggregate across grains, which happens on M:N +(D-022) or fan-out shapes the engine cannot pre-aggregate safely. + +#### T-006 — `COUNT(*)` over a dataset is well-defined at the home grain + +**Anchors:** §7.2 · D-016 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - COUNT(*) AS order_count +Where: + - orders.status = 'completed' +``` + +**Expected:** + +| region | order_count | +|:-------|------------:| +| EAST | 2 | ← orders 101, 102 (102? wait 103 is pending). 101+102 completed. 103 pending excluded. +| WEST | 1 | ← order 104 +| NULL | 1 | ← orphan order 105 (status=completed) + +`COUNT(*)` is the count of rows of the **measure-bearing dataset** +that survive the predicates — here, `orders`. Engines that emit +`COUNT()` as a transparent rewrite (e.g. +Snowflake's `COUNT(*)`-rejection workaround) MUST produce the same +numbers. + +--- + +### 4.C Predicate routing + +#### T-007 — Aggregate in `Where` is rejected + +**Anchors:** §5.3 · D-005, D-012a +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount)] +Where: + - SUM(orders.amount) > 100 # aggregate inside Where +``` + +**Expected:** +```yaml +error: + code: E_AGGREGATE_IN_WHERE + anchor: §5.3 + must_contain: ["aggregate", "Where", "Having"] # diagnostic should hint at Having +``` + +The diagnostic MUST identify the offending aggregate by name and SHOULD +suggest moving the predicate to `Having`. + +#### T-008 — Pure row-level predicate in `Having` is rejected + +**Anchors:** §5.3 · D-012b +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount)] +Having: + - customers.region = 'EAST' # row-level dimension predicate +``` + +**Expected:** +```yaml +error: + code: E_NON_AGGREGATE_IN_HAVING + anchor: §5.3 + must_contain: ["Having", "Where"] +``` + +Note this test is about *routing*, not behaviour. An engine that +silently treats `Having` as `Where` produces the right numbers but +breaks the contract, and so fails the test on the error code, not +the rows. + +#### T-009 — Mixed-level boolean predicate is rejected + +**Anchors:** §5.3 · D-012c +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Having: + - customers.region = 'EAST' AND SUM(orders.amount) > 100 +``` + +**Expected:** +```yaml +error: + code: E_MIXED_PREDICATE_LEVEL + anchor: §5.3 + must_contain: ["row-level", "aggregate", "split"] +``` + +The diagnostic SHOULD suggest splitting the predicate (row-level half +→ `Where`, aggregate half → `Having`). + +--- + +### 4.D Default join behaviour + +#### T-010 — `N:1` enrichment defaults to `LEFT`; orphans appear under `NULL` + +**Anchors:** §6.4 · D-004a +**Fixture:** F-PRELUDE. + +Covered indirectly by T-001's expected rows — the `region = NULL` row +exists *only because* the default join from `orders → customers` is +`LEFT` and order 105's `customer_id = 99` does not match. An engine +that defaults to `INNER` returns one fewer row and fails the test on +row count. This is the single most common silent-correctness failure +in BI tools (Snowflake's "first matching row" rule is the worst case; +see Appendix B note B-1). + +#### T-011 — Multi-fact composition defaults to `FULL OUTER` on shared dims + +**Anchors:** §6.4 · D-004b +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - SUM(returns.amount) AS return_total +``` + +**Expected (multi-measure ⇒ FULL OUTER stitch per §6.6 row 3 / §6.2 +step 7; *both* sides preserved):** + +| region | revenue | return_total | +|:-------|--------:|-------------:| +| EAST | 350.00 | 10.00 | ← both facts have data +| WEST | 75.00 | 5.00 | ← both facts have data +| NORTH | NULL | 15.00 | ← only `returns` has data (customer 4); revenue is `SUM` over zero rows = `NULL` (§6.11 / D-033) +| NULL | 30.00 | NULL | ← only `orders` has data (orphan 105); return_total is `SUM` over zero rows = `NULL` + +This is the **multi-measure** counterpart to T-001's single-measure +Plan A: because there are two independently-aggregated facts +(`orders` and `returns`), §6.6 row 3 mandates a `FULL OUTER` between +the two pre-aggregated branches. The union of dim values appearing in +*either* branch is the result. NORTH appears here (via the `returns` +branch) because customer 4 has a return, even though customer 4 has +no orders. Without `returns` as a second measure, this would be a +single-measure query and Plan A would drop NORTH (see T-001). An +engine that defaults to `INNER` between the two pre-aggregated +branches loses NORTH and the orphan-bucket, failing on row count. + +**Companion `COUNT` test:** replacing either `SUM` with `COUNT(*)` or +`COUNT(orders.id)` would produce `0` in the missing-side cell, not +`NULL`, per standard SQL. See T-047 for the explicit +`SUM`-returns-`NULL` / `COUNT`-returns-`0` contrast. + +#### T-012 — Scalar grand total uses `CROSS JOIN` + +**Anchors:** §6.4 · D-004c +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [] # no dimensions ⇒ scalar grain +Measures: + - SUM(orders.amount) AS total_revenue + - SUM(returns.amount) AS total_returns +``` + +**Expected:** exactly one row. + +| total_revenue | total_returns | +|--------------:|--------------:| +| 455.00 | 30.00 | + +`455.00 = 100+50+200+75+30` (all orders including orphan); +`30.00 = 10+5+15` (all returns). The two scalars sit on a single row +joined via `CROSS JOIN` (`1 ⋈ 1`), not on separate rows. + +--- + +### 4.E M:N resolution — the flagship section + +#### T-013 — Two unrelated facts with no shared dimension raise `E3013` + +**Anchors:** §6.6 · D-007 +**Fixture:** F-NOPATH. + +**Query:** +```yaml +Measures: + - SUM(orders.amount) + - SUM(inventory_movements.quantity) +``` + +**Expected:** +```yaml +error: + code: E3013_NO_STITCHING_DIMENSION + anchor: §6.6 + must_contain: ["orders", "inventory_movements", "shared dimension"] +``` + +An engine that emits a Cartesian product here is producing a +plausible-but-wrong total and is the single failure mode Promise 4 is +designed to prevent. + +#### T-014 — `N:N` with no bridge and no shared dim raises `E3012` + +**Anchors:** §6.6 · D-007 +**Fixture:** F-BRIDGE-NONE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: [SUM(movies.gross)] +``` + +**Expected:** +```yaml +error: + code: E3012_MN_NO_SAFE_REWRITE + anchor: §6.6 + must_contain: ["actors", "movies", "bridge"] # diagnostic should suggest the remedy +``` + +#### T-015 — Bridge resolution de-duplicates per `(fact, group)` (FLAGSHIP) + +**Anchors:** §6.6.1, §6.3 Semantic 2 · D-026 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: + - SUM(movies.gross) AS total_gross +``` + +**Expected:** + +| height | total_gross | +|-------:|------------:| +| 170 | 300.00 | ← M10 (100) counted ONCE, plus M11 (200). Not 400. +| 180 | 50.00 | + +**Why this test exists.** The naive flat-join SQL +`actors ⋈ appearances ⋈ movies GROUP BY actors.height` produces +`(170 → 400, 180 → 50)` because M10's gross is replicated by Alice's +*and* Bob's appearances. The Foundation's bridge plan, per §6.6.1, +de-duplicates at the `(movie_id, height)` level before the final +`SUM`, and so M10 contributes 100 to height 170 exactly once. This +is **Semantic 2** in operation: no row of `movies` (the measure's +home dataset) contributes more than once to any output group. + +**Vendor cross-check.** + +| Tool | Same data, same query, expected output | +|:---|:---| +| Looker (symmetric aggregates on, declared PK on each view) | `170 → 300, 180 → 50` | +| Tableau (Relationships, 2020.2+) | `170 → 300, 180 → 50` | +| Power BI (bridge with single-direction filter + `SUMX(DISTINCT(...))`) | `170 → 300, 180 → 50` | +| Naive SQL join (no de-duplication) | `170 → 400, 180 → 50` — **wrong** | + +An OSI implementation that matches the naive-SQL number on this fixture +is non-conformant on D-026, regardless of how it generates the SQL. + +#### T-016 — Bare non-distributive aggregate over a bridge is rejected + +**Anchors:** §4.5, §6.1, §6.6.1 · D-027 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: [AVG(movies.gross)] +``` + +**Expected:** +```yaml +error: + code: E_UNSAFE_REAGGREGATION + anchor: §6.1 + must_contain: ["AVG", "non-distributive", "nested", "AVG(AVG"] +``` + +The diagnostic MUST suggest the nested form (e.g., `AVG(AVG(movies.gross))` +to mean "average across actors of their per-actor average gross"). + +#### T-017 — Nested aggregation over a bridge succeeds with per-endpoint plan + +**Anchors:** §4.5 form (1), §6.6.1 · D-020, D-027 +**Fixture:** F-BRIDGE. + +**Query:** +```yaml +Dimensions: [actors.height] +Measures: + - AVG(AVG(movies.gross)) AS avg_of_per_actor_avg +``` + +**Expected:** + +| height | avg_of_per_actor_avg | +|-------:|---------------------:| +| 170 | 125.00 | ← AVG(per-actor avg) = AVG(Alice=150, Bob=100) = 125 +| 180 | 50.00 | ← AVG(Carol=50) = 50 + +The inner `AVG` runs at actor grain (per actor: Alice averages M10/M11 += 150; Bob averages M10 = 100; Carol averages M12 = 50); the outer `AVG` +runs at query grain (per height). This is the explicit case where the +per-endpoint-first plan is correct and produces different numbers from +T-015's de-duplication plan — and that's the point of requiring nested +aggregation: user intent is now unambiguous. + +#### ~~T-018~~ — Deferred (was: `EXISTS_IN` compiles to `EXISTS`) + +The original T-018 exercised the semi-join filter form +(`EXISTS_IN` / `NOT EXISTS_IN`). That construct has been **removed +from the Foundation** and moved to a separate follow-up proposal that +will specify its surface syntax, NULL-safety guarantees, and +compilation contract in full. This test will be reinstated (and +expanded — positive case, `NOT`-form, NULL on both sides) once the +EXISTS_IN proposal lands. + +Until then: the Foundation has no semi-join construct, and there is +nothing for this slot to test. The slot is retained to keep T-NNN +numbering stable; future re-use of `T-018` for an unrelated test +would be misleading. + +--- + +### 4.F Path resolution & namespacing + +#### T-019 — Two relationships between the same datasets raise `E_AMBIGUOUS_PATH` + +**Anchors:** §6.7 · D-018a +**Fixture:** F-AMBIG. + +**Query:** +```yaml +Dimensions: [users.region] +Measures: [COUNT(orders.id)] +``` + +**Expected:** +```yaml +error: + code: E_AMBIGUOUS_PATH + anchor: §6.7 + must_contain: ["order_placed_by", "order_fulfilled_by"] # both candidate paths named +``` + +The diagnostic MUST list **every** candidate relationship by name so +the user can pick one (the planned remedy is to annotate the query +with the relationship to use; see §10). + +#### T-020 — Bare-name references resolve to the global namespace, not a dataset + +**Anchors:** §4.6 · D-006 +**Fixture:** F-PRELUDE. Declare `orders.total_revenue = SUM(amount)` as a +*dataset-scoped* metric. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [total_revenue] # bare name, not qualified +``` + +**Expected:** +```yaml +error: + code: E_NAME_NOT_FOUND + anchor: §4.6 + must_contain: ["total_revenue", "global", "orders.total_revenue"] +``` + +The bare name MUST NOT silently match `orders.total_revenue`. Using +the dataset-scoped metric requires the qualified reference: + +**Query (positive case):** +```yaml +Measures: [orders.total_revenue] +``` + +Resolves and returns the per-region revenue from T-001. + +--- + +### 4.G Grain error contracts + +#### T-021 — `COUNT(DISTINCT)` under fan-out raises `E_UNSAFE_REAGGREGATION` + +**Anchors:** §6.1 Starting Grain, §7.2 · D-022 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [orders.status] +Measures: [COUNT(DISTINCT customers.id) AS unique_customers] +``` + +**Expected:** +```yaml +error: + code: E_UNSAFE_REAGGREGATION + anchor: §6.1 + must_contain: ["COUNT(DISTINCT)", "holistic", "pre-aggregate"] +``` + +The aggregate is holistic and reads a column from the one-side +(`customers.id`) over a join that fans out to orders. The diagnostic +MUST identify the aggregate, classify it as holistic, and suggest a +pre-aggregation at the customer grain. + +For comparison, swapping in `SUM(orders.amount)` (distributive) on the +same shape succeeds with no error — this is the discriminator between +D-022 and the safe-by-pre-aggregation case. + +#### T-022 — Fan-out in a scalar query raises `E_FAN_OUT_IN_SCALAR_QUERY` + +**Anchors:** §6.1 Final Grain, §5.1.2 · D-023 +**Fixture:** F-PRELUDE. + +**Query (scalar):** +```yaml +Fields: [customers.region, orders.id] +``` + +**Expected:** +```yaml +error: + code: E_FAN_OUT_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["scalar", "fan-out", "customers", "orders"] +``` + +The scalar query has no aggregation step in which to absorb the +fan-out; either the user must aggregate one side or switch to an +aggregation query. The diagnostic MUST suggest both remedies. + +#### T-023 — Row-level reference to a finer grain raises `E_UNAGGREGATED_FINER_GRAIN_REFERENCE` + +**Anchors:** §6.1 Starting Grain · D-024 +**Fixture:** F-PRELUDE. Declare this *invalid* field on `customers`: + +```yaml +fields: + - name: customers.first_order_amount + expression: "orders.amount" # bare row-level reference to higher-grain table +``` + +**Expected (at model load or first query referencing the field):** +```yaml +error: + code: E_UNAGGREGATED_FINER_GRAIN_REFERENCE + anchor: §6.1 + must_contain: ["orders.amount", "aggregate", "customers"] +``` + +The remedy is to either wrap the reference in an aggregate +(`SUM(orders.amount)` — implicit home-grain aggregation per D-003), +or pull the value from a coarser-grain dataset where it is already +at the home grain. The diagnostic MUST surface both options. + +--- + +### 4.H Common-clause semantics + +This section pins behaviour that applies to **both** query shapes: +predicate-list interpretation, `Order By` NULL handling, `Limit` +without `Order By`, and home-grain scalar usability in `Where`. + +#### T-024 — Boolean home-grain scalar usable in `Where` + +**Anchors:** §4.3 routing rules, §6.1 Semantic 2 · D-005 (e) +**Fixture:** F-PRELUDE. + +**Model addition:** +```yaml +fields: + - dataset: customers + name: has_completed_orders + expression: "COUNT(orders.id WHERE orders.status = 'completed') > 0" +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [COUNT(customers.id) AS customer_count] +Where: ["customers.has_completed_orders"] +``` + +**Expected:** + +| region | customer_count | +|:-------|---------------:| +| EAST | 1 | ← Customer 1 (orders 101, 102 completed). Customer 2 has only pending order 103, excluded. +| WEST | 1 | ← Customer 3 (order 104 completed) + +No error. The field `has_completed_orders` contains `COUNT(orders.*) > 0` — +an aggregate over a finer-grain dataset — but it resolves at customer +grain via implicit home-grain aggregation (§4.3), so its **resolved +shape** is a boolean scalar at customer grain. By D-005 (e), it is +therefore valid in `Where` for any query at customer grain or coarser, +where it filters customers pre-aggregation. An engine that classifies +fields by surface syntax (i.e., "contains `COUNT`, therefore route to +`Having`") would raise `E_WRONG_PREDICATE_LOCATION` and fail this test. + +--- + +#### T-026 — Outer `Order By` defaults to `NULLS LAST` + +**Anchors:** §5.1 Common clause semantics, §6.10.2 · D-014, D-029 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Order By: ["customers.region ASC"] +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NULL | 30.00 | ← orphan-bucket sorts LAST (NULLS LAST default) + +The non-NULL `region` values sort by ASC; the orphan `NULL`-region +bucket sorts last per the `NULLS LAST` default. + +Compilation MUST emit explicit `NULLS LAST` into the compiled SQL even +though the query body did not say so: + +```yaml +sql_contract: + forbidden_substrings: + - "ORDER BY .* ASC$" # bare ASC without explicit NULLS LAST + required_substrings: + - "ORDER BY .* ASC NULLS LAST" +``` + +This MUST be byte-stable across compilations (D-014) and MUST agree +across dialects (Snowflake / Databricks / Postgres) even though each +dialect's *native* default differs. An engine that omits the explicit +`NULLS LAST` from compiled SQL fails on the SQL contract, even if the +runtime row order happens to match. + +--- + +#### T-028 — `Where: [P1, P2]` is `P1 AND P2` + +**Anchors:** §5.1 Common clause semantics · D-014 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Where: + - "orders.status = 'completed'" + - "orders.amount > 60" +``` + +**Expected (single-measure ⇒ Plan A; NORTH absent — see T-001):** + +| region | revenue | +|:-------|--------:| +| EAST | 100.00 | ← only order 101 (status=completed, amount=100). Order 102 (amount=50) excluded by amount filter. Order 103 (pending) excluded by status filter. +| WEST | 75.00 | ← order 104 (status=completed, amount=75) +| NULL | 200.00 | ← orphan 105 (status=completed, amount=200) + +The result MUST equal what a single equivalent `Where:` entry +`"orders.status = 'completed' AND orders.amount > 60"` produces — same +rows, same numbers, byte-identical compiled SQL (after alias +normalisation). An engine that interprets a predicate list as `OR` +would include order 102 (status=completed, amount=50) and inflate +EAST to `150` — a runtime test failure. + +--- + +#### T-052 — `Limit` without `Order By` compiles and runs + +**Anchors:** §5.1 Common clause semantics · D-014 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +Limit: 2 +``` + +**Expected:** + +(a) **No error.** The query compiles successfully — `Limit` without + `Order By` is a legitimate (if non-deterministic) BI query. + +(b) **No warning required.** The Foundation does NOT mandate that + engines emit a diagnostic. An engine that emits a "non-deterministic + Limit" advisory is conformant; an engine that emits no diagnostic at + all is also conformant. + +(c) **Compiled SQL is byte-stable (D-014).** Two compilations of the + same `(model, query, dialect)` MUST produce byte-identical SQL — + the *output rows* may differ between runs of that SQL (since the + engine is free to pick any two rows), but the SQL string itself is + deterministic. + +(d) **Result has exactly two rows**, both with valid `region` values + from the result set `{EAST, WEST, NORTH, NULL}`. Specific row + selection is engine-defined. + +--- + +### 4.I Window functions + +The Foundation supports a defined subset of SQL window functions +(§6.10). This section exercises both the supported shapes (with strong +result expectations) and the rejected shapes (with explicit error +codes). + +#### T-027 — `ORDER BY` inside `OVER (...)` defaults to `NULLS LAST` + +**Anchors:** §6.10.2 · D-029 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC) AS rn" +Where: + - "orders.status = 'completed'" +``` + +**Expected:** the compiled SQL MUST contain `ORDER BY ... DESC NULLS +LAST` inside the `OVER (...)` clause, even though the query body did +not specify it. Engines compiling to Snowflake (native `DESC NULLS +FIRST`) MUST emit the explicit clause to override the dialect default. + +```yaml +sql_contract: + required_substrings: + - "OVER (.*ORDER BY .* DESC NULLS LAST.*)" +``` + +The result row set: every `(customer_id, amount)` group's NULL `amount` +rows (if any existed) would sort after non-NULL rows. In F-PRELUDE all +orders have non-NULL `amount`, so this test exercises the SQL contract +above rather than data NULLs. + +--- + +#### T-029 — Window function in `Where` is rejected + +**Anchors:** §6.10.3 · D-030 (E_WINDOW_IN_WHERE) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [orders.customer_id] +Measures: [SUM(orders.amount) AS revenue] +Where: + - "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount) = 1" +``` + +**Expected:** the engine MUST reject: + +```yaml +error: + code: E_WINDOW_IN_WHERE + anchor: §6.10.3 + must_contain: ["window", "Where", "Qualify", "subquery"] +``` + +The diagnostic MUST mention both supported alternatives: (a) move the +predicate into a `Having` or `Qualify` clause, or (b) wrap the windowed +expression in a derived field and filter that field at a coarser grain. + +--- + +#### T-030 — Nested window functions rejected + +**Anchors:** §6.10.4 · D-031 +**Fixture:** F-PRELUDE. + +**Query (illegal — window over window):** +```yaml +Fields: + - orders.id + - "SUM(SUM(orders.amount) OVER (PARTITION BY orders.customer_id)) OVER ()" +``` + +**Expected:** rejected at compile time: + +```yaml +error: + code: E_NESTED_WINDOW + anchor: §6.10.4 + must_contain: ["nested window", "single window", "derived field"] +``` + +This matches every major SQL dialect (Snowflake, Databricks, Postgres, +BigQuery all reject nested windows). The diagnostic MUST suggest +factoring the inner window into a derived field. + +--- + +#### T-031 — Running-total window accepted (the canonical positive) + +**Anchors:** §6.10.1, §6.10.2 · D-030 (positive) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - orders.amount + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total" +Where: + - "orders.customer_id IS NOT NULL" +Order By: + - "orders.customer_id ASC" + - "orders.id ASC" +``` + +**Expected:** + +| id | customer_id | amount | running_total | +|----:|------------:|-------:|--------------:| +| 101 | 1 | 100.00 | 100.00 | +| 102 | 1 | 50.00 | 150.00 | +| 103 | 2 | 200.00 | 200.00 | +| 104 | 3 | 75.00 | 75.00 | + +The compiled SQL MUST contain explicit `ROWS BETWEEN UNBOUNDED +PRECEDING AND CURRENT ROW` (the literal frame; not a synonym), and the +`OVER (...)` `ORDER BY` MUST include the `NULLS LAST` default +resolution. + +--- + +#### T-032 — `ROW_NUMBER` with outer `Where` filter (qualify-style pattern) + +**Anchors:** §6.10.1, §6.10.3 · D-030 (positive) +**Fixture:** F-PRELUDE. + +**Strategy:** the legal way to "filter by `ROW_NUMBER = 1`" is to +expose the window as a derived field and filter at a coarser grain, +**OR** to use the engine's `Qualify`-style mechanism (deferred — +§10). This test exercises the derived-field strategy. + +**Model addition:** +```yaml +fields: + - dataset: orders + name: order_rank_in_customer + expression: "ROW_NUMBER() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC, orders.id ASC)" +``` + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - orders.amount + - orders.order_rank_in_customer +Where: + - "orders.order_rank_in_customer = 1" +``` + +**Expected:** the engine MUST compile the inner window in a CTE / inline +subquery and the outer `Where` MUST filter that subquery's result — +NEVER push the window inside the outer `WHERE`. The result is one row +per customer (their highest-amount order): + +| id | customer_id | amount | order_rank_in_customer | +|----:|------------:|-------:|-----------------------:| +| 101 | 1 | 100.00 | 1 | +| 103 | 2 | 200.00 | 1 | +| 104 | 3 | 75.00 | 1 | +| 105 | 99 | 200.00 | 1 | + +`Order By` was omitted; engines are not required to return rows in any +specific order, but the row *content* MUST match. + +--- + +#### T-033 — Composing onto a windowed metric is rejected + +**Anchors:** §6.10.5 · E_WINDOWED_METRIC_COMPOSITION +**Fixture:** F-PRELUDE. + +**Model addition:** +```yaml +metrics: + - name: orders.running_total_by_customer + expression: "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id)" + - name: orders.running_total_ratio + expression: "orders.running_total_by_customer / SUM(orders.amount)" # composition onto windowed metric +``` + +**Expected:** model load (or first reference to the second metric) MUST +fail: + +```yaml +error: + code: E_WINDOWED_METRIC_COMPOSITION + anchor: §6.10.5 + must_contain: ["window", "composition", "derived field", "§10"] +``` + +The diagnostic MUST acknowledge that the feature is deferred to §10 +(parameterised window composition) — not a permanent prohibition. + +--- + +#### T-034 — `GROUPS` frame mode rejected + +**Anchors:** §6.10.6 · D-032 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.id + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.amount GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS recent_sum" +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_FRAME_MODE + anchor: §6.10.6 + must_contain: ["GROUPS", "ROWS", "RANGE", "§10"] +``` + +Diagnostic MUST list the two supported alternatives (`ROWS` and `RANGE`) +and point at §10 as the future home of `GROUPS`. + +--- + +#### T-035 — Parameterised frame bound rejected + +**Anchors:** §6.10.6 · D-032 +**Fixture:** F-PRELUDE; query parameter `:n` declared. + +**Query:** +```yaml +Fields: + - orders.id + - "SUM(orders.amount) OVER (PARTITION BY orders.customer_id ORDER BY orders.id ROWS BETWEEN :n PRECEDING AND CURRENT ROW) AS recent_sum" +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_FRAME_BOUND + anchor: §6.10.6 + must_contain: ["parameter", "literal integer", "UNBOUNDED", "§10"] +``` + +Diagnostic MUST mention the three accepted bound shapes (integer +literal, `UNBOUNDED`, `CURRENT ROW`) and point at §10. + +--- + +#### T-036 — Window over a `1 : N` fan-out is accepted (no implicit aggregation) + +**Anchors:** §6.10.1, §6.7 · D-022, D-030 (positive) +**Fixture:** F-PRELUDE. + +The interesting question is what happens when a window-aggregate is +defined at the customer-row grain but its argument lives at the orders +grain. Windows do NOT trigger implicit home-grain aggregation — they +require the user to be explicit about which grain the window runs over. +This test exercises the *correct* shape: the window is defined inside a +scalar query at orders grain (its native home), so no fan-out: + +**Query:** +```yaml +Fields: + - orders.id + - orders.customer_id + - "RANK() OVER (PARTITION BY orders.customer_id ORDER BY orders.amount DESC) AS rank_by_customer" +``` + +**Expected:** the query compiles at orders grain (the natural home of +the window). The result is one row per order with the customer-internal +rank: + +| id | customer_id | rank_by_customer | +|----:|------------:|-----------------:| +| 101 | 1 | 1 | ← amount 100 > amount 50 +| 102 | 1 | 2 | +| 103 | 2 | 1 | +| 104 | 3 | 1 | +| 105 | 99 | 1 | + +**Companion negative — T-036b:** trying to use the same window +expression as a *field on customers* (i.e., declaring +`customers.rank_in_some_customer = RANK() OVER (...)`) MUST raise +`E_AMBIGUOUS_MEASURE_GRAIN` (D-025) — the field's home grain is +underspecified. + +--- + +### 4.J BI-shape coverage + +These are the four "shape archetypes" every real BI semantic layer +must handle. Without explicit tests, regressions can ship silently. + +#### T-043 — Multi-hop `N : 1` enrichment chain + +**Anchors:** §6.4 (cardinality inference), §6.6 (join defaults) +**Fixture:** F-CHAIN — a three-hop `N : 1` chain +(`order_lines → orders → customers → segments`): + +```yaml +datasets: + - { name: segments, primary_key: [id], fields: [name] } + - { name: customers, primary_key: [id], fields: [segment_id, region] } + - { name: orders, primary_key: [id], fields: [customer_id, amount, status] } + - { name: order_lines, primary_key: [id], fields: [order_id, sku, qty, price] } + +relationships: + - { from: customers, from_keys: [segment_id], to: segments, to_keys: [id], cardinality: 1:1 → many:1 } + - { from: orders, from_keys: [customer_id], to: customers, to_keys: [id], cardinality: N:1 } + - { from: order_lines, from_keys: [order_id], to: orders, to_keys: [id], cardinality: N:1 } +``` + +**Query:** +```yaml +Dimensions: [segments.name] +Measures: [SUM(order_lines.qty * order_lines.price) AS revenue] +``` + +**Expected:** the planner finds a path +`order_lines → orders → customers → segments` (three `N : 1` hops). +Cardinality inference proves no fan-out (every hop is many-to-one), so +the SQL is a chain of `LEFT JOIN`s with no de-duplication step. + +Compiled SQL contract: + +```yaml +sql_contract: + required_substrings: + - "FROM .*order_lines" + - "LEFT JOIN .*orders" + - "LEFT JOIN .*customers" + - "LEFT JOIN .*segments" + forbidden_substrings: + - "DISTINCT" # no de-duplication needed; chain is N:1 all the way +``` + +Wrong join direction (e.g., `customers LEFT JOIN order_lines`) would +fan out; the test fails if the row count differs from +`COUNT(DISTINCT segments.id) + 1` (for orphan segments). + +--- + +#### T-044 — Composite primary key / foreign key + +**Anchors:** D-009 (deferred-key rejection — composite keys ARE supported +in the foundation), §6.4 +**Fixture:** F-COMPOSITE — `inventory` keyed by `(store_id, sku)` with +a fact `sales` referencing the composite: + +```yaml +datasets: + - name: inventory + primary_key: [store_id, sku] + fields: [stock_level, reorder_point] + - name: sales + primary_key: [id] + fields: [store_id, sku, qty, sale_ts] + +relationships: + - from: sales + from_keys: [store_id, sku] + to: inventory + to_keys: [store_id, sku] + cardinality: N:1 +``` + +**Query:** +```yaml +Dimensions: [inventory.reorder_point] +Measures: [SUM(sales.qty) AS units_sold] +``` + +**Expected:** the engine generates a join `ON sales.store_id = +inventory.store_id AND sales.sku = inventory.sku`. The result is one +row per distinct `reorder_point` value. + +Compiled SQL contract: + +```yaml +sql_contract: + required_substrings: + - "ON .*store_id .* AND .*sku" # both key components in ON clause + forbidden_substrings: + - "ON .*store_id\\s+\\)" # only one key component + - "ON .*sku\\s+\\)" +``` + +An engine that uses only one key component fails this test (it would +produce row duplication if any `(store_id, sku)` collision exists, or +silent NULLs). + +--- + +#### T-045 — Two-measure stitch with bridge resolution + +**Anchors:** §6.8 (M:N), §6.8.2 (stitch) · D-026 +**Fixture:** F-BRIDGE — `customers ⇌ orders` with `customers ⇌ +returns` and a separate `customers ⇌ campaigns` many-to-many via +`customer_campaigns` bridge. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(DISTINCT campaigns.id) AS active_campaign_count +``` + +**Expected:** the planner must: + +1. Aggregate `orders` to customer grain (per-customer revenue). +2. Resolve the M:N path `customers → customer_campaigns → campaigns` + with bridge de-duplication per D-026 (count each campaign at most + once per customer). +3. Aggregate each branch to region grain. +4. `FULL OUTER` stitch on region (multi-measure ⇒ §6.6 row 3 / §6.2 + step 7). + +Result: every region present in *either* branch's join path appears +(the union of "regions reached by an order" and "regions reached by +a campaign-association"). A region with neither order nor campaign +activity does NOT appear — same Plan A logic as T-001, just applied +per-branch and then unioned by the FULL OUTER. Missing-side cells +follow standard SQL per §6.11: `revenue` (a `SUM`) returns `NULL` +when no orders contribute to the region; `active_campaign_count` (a +`COUNT DISTINCT`) returns `0` when no campaigns contribute. The +differing empty-set values across the two measure types is the +*point* of the test: an engine that returns the same value for both +(e.g., both `0` or both `NULL`) is doing a Foundation-level rewrite +the spec forbids. + +The CRITICAL invariant: `active_campaign_count` MUST NOT be inflated by +the M:N fan-out. An engine that joins everything first then groups +will inflate the count and fail; the engine MUST aggregate each branch +independently. + +```yaml +sql_contract: + required_substrings: + - "WITH .*orders_branch" # or two named CTEs + - "FULL OUTER JOIN" +``` + +--- + +#### T-046 — Reflexive (self-referential) relationship + +**Anchors:** §6.4 (cardinality), §6.6 (join defaults), §6.9 (path +resolution with roles) +**Fixture:** F-REFLEXIVE — `employees` references itself via +`manager_id`: + +```yaml +datasets: + - name: employees + primary_key: [id] + fields: [name, manager_id, region] + +relationships: + - name: reports_to + from: employees + from_keys: [manager_id] + to: employees + to_keys: [id] + cardinality: N:1 + role: manager # qualifier — the "to" side acts as `manager` +``` + +**Query:** +```yaml +Dimensions: + - "employees{role=manager}.region AS manager_region" +Measures: + - COUNT(employees.id) AS direct_report_count +``` + +**Expected:** the planner generates `employees AS employee LEFT JOIN +employees AS manager ON employee.manager_id = manager.id` and groups by +`manager.region`. Each employee is counted exactly once. + +```yaml +sql_contract: + required_substrings: + - "FROM .*employees" + - "LEFT JOIN .*employees" # self-join with distinct aliases +``` + +An engine that fails to disambiguate the two `employees` references +either errors with `E_AMBIGUOUS_PATH` (if the role qualifier is not +honoured) or silently produces wrong counts. + +--- + +### 4.K Boundary, empty, and NULL cases + +#### T-025 — Scalar query with two unrelated row-level facts is rejected + +**Anchors:** §5.1.2, Appendix B · D-023 (extended) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: + - orders.amount + - returns.amount + - customers.region +``` + +**Expected:** + +```yaml +error: + code: E_FAN_OUT_IN_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["orders", "returns", "aggregation", "single home"] +``` + +`orders` and `returns` are independent facts both on the many-side of +`customers`. A row-level scalar query referencing fields from both +would have to either fan out (replicate orders rows per return row, or +vice versa) or pick a single home — neither is unambiguous. The error +MUST suggest two resolutions: convert to an aggregation query, or pick +a single fact as the home. (A semi-join filter form is deferred to a +future proposal — see the §6.8 deferred-feature note.) + +--- + +#### T-040 — `E_NO_PATH` when no relationship connects two datasets + +**Anchors:** §6.9 · D-018b +**Fixture:** F-NOPATH — two datasets with no declared relationship +between them: + +```yaml +datasets: + - name: customers + primary_key: [id] + fields: [name, region] + - name: weather_observations # no relationship to customers + primary_key: [id] + fields: [station_id, observation_date, temperature] +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [AVG(weather_observations.temperature) AS avg_temp] +``` + +**Expected:** + +```yaml +error: + code: E_NO_PATH + anchor: §6.9 + must_contain: ["customers", "weather_observations", "no path", "relationship"] +``` + +Diagnostic MUST name both endpoint datasets and explicitly state that +no declared `relationships` entry connects them. + +--- + +#### T-041 — Reserved-name rejection + +**Anchors:** §3 reserved names · D-019 +**Fixture:** F-PRELUDE with a deliberately illegal addition. + +**Model addition (illegal):** +```yaml +fields: + - dataset: customers + name: select # reserved SQL keyword + expression: "'placeholder'" +``` + +**Expected:** model load fails: + +```yaml +error: + code: E_RESERVED_NAME + anchor: §3 + must_contain: ["select", "reserved", "rename"] +``` + +Diagnostic MUST quote the offending name verbatim. + +--- + +#### T-042 — Deferred-key rejection + +**Anchors:** §10 deferred keys · D-009 +**Fixture:** F-PRELUDE with a deliberately illegal addition. + +**Model addition (illegal — `grain: FIXED` is deferred to §10):** +```yaml +metrics: + - name: orders.revenue_at_orders_grain + expression: "SUM(orders.amount)" + grain: FIXED # deferred key +``` + +**Expected:** + +```yaml +error: + code: E_DEFERRED_KEY_REJECTED + anchor: §10 + must_contain: ["grain", "deferred", "§10"] +``` + +The diagnostic MUST identify which deferred key was used (`grain`) and +cite §10. This is the canonical instance; the full coverage of D-009 +requires one instance per deferred key — see §6 below. + +--- + +#### T-047 — Empty-set aggregate behaviour: `SUM` → `NULL`, `COUNT` → `0` (D-033) + +**Anchors:** §6.11 · D-033 +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - SUM(orders.amount) AS revenue + - COUNT(orders.id) AS order_count + - COUNT(*) AS row_count +``` + +**Expected (three measures over one fact — still a single fact source, +so Plan A applies; NORTH does not appear):** + +| region | revenue | order_count | row_count | +|:-------|--------:|------------:|----------:| +| EAST | 350.00 | 3 | 3 | +| WEST | 75.00 | 1 | 1 | +| NULL | 30.00 | 1 | 1 | + +> Why no NORTH? All three measures are sourced from `orders` — the +> measure dataset-set is the same for all of them, so step 6 of the +> §6.2 evaluation algorithm produces one combined row set per the +> single-measure Plan A rule. NORTH (customer 4 with no orders) has +> no `orders` rows pulling it into the result. If the user wanted +> NORTH to appear, they would add a measure sourced from a different +> dataset that *does* have data for customer 4 — e.g. +> `COUNT(customers.id) AS customer_count` — which would convert the +> shape into a true multi-measure-stitch query and the FULL OUTER +> rule of §6.6 row 3 would preserve NORTH via the customers branch. + +This pins the standard-SQL split codified by D-033: the `COUNT` family +(`COUNT(*)`, `COUNT(x)`, `COUNT(DISTINCT x)`) returns `0` on an empty +set because it counts rows; every other aggregate (`SUM`, `AVG`, +`MIN`, `MAX`, `MEDIAN`, percentiles, `STDDEV`, `VARIANCE`) returns +`NULL` because there is no defined result. + +**Companion T-053** below covers the `MIN`/`MAX`/`AVG` branch +(non-distributive, also `NULL` over empty). + +--- + +#### T-048 — NULL foreign key in `1 : N` enrichment + +**Anchors:** §6.6 (LEFT default), §6.11 +**Fixture:** F-PRELUDE-NULLFK — F-PRELUDE with one extra order row: + +```sql +-- order 106: amount 80, status completed, customer_id NULL +``` + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue] +``` + +**Expected:** + +| region | revenue | +|:-------|--------:| +| EAST | 350.00 | +| WEST | 75.00 | +| NORTH | 0 | +| NULL | 110.00 | ← orphan order 105 (30) + NULL-FK order 106 (80) — both bucketed under `region = NULL` + +The result demonstrates that a NULL foreign key behaves the same as an +unmatched key (orphan row 105 with `customer_id = 99` not in +`customers`): both produce a `NULL`-region bucket. An engine that +silently drops NULL-FK rows during the join would understate the +NULL-bucket by 80 and fail. + +--- + +#### T-049 — NULL in a dimension column + +**Anchors:** §5.1.1 (Result grain), §6.6 +**Fixture:** F-PRELUDE-NULLDIM — F-PRELUDE with customer 4's `region` +set to `NULL` (overriding NORTH). + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: [SUM(orders.amount) AS revenue, COUNT(customers.id) AS customer_count] +``` + +**Expected:** + +| region | revenue | customer_count | +|:-------|--------:|---------------:| +| EAST | 350.00 | 2 | +| WEST | 75.00 | 1 | +| NULL | 30.00 | 1 | ← customer 4 (NULL region) + orphan order 105's NULL bucket merge into one row + +The two distinct sources of NULL (the customer with NULL region + the +orphan order with unmatched customer) collapse into a *single* output +row. SQL `GROUP BY` treats all NULLs as one group; the Foundation +preserves this behaviour. An engine that emits two separate NULL rows +fails this test. + +--- + +#### T-050 — Empty aggregation query is rejected + +**Anchors:** §5.1.1 (At least one of `Dimensions` or `Measures` MUST be +non-empty), §6.2 normative evaluation step 1 · `E_EMPTY_AGGREGATION_QUERY` +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [] +Measures: [] +``` + +**Expected:** + +```yaml +error: + code: E_EMPTY_AGGREGATION_QUERY + anchor: §5.1.1 + must_contain: ["Dimensions", "Measures", "at least one"] +``` + +A query with neither dimensions nor measures has no shape at all — +this is distinct from "no dimensions" (which is legal and produces a +grand total) or "no measures" (which is the `Dimensions`-only listing +shape, also legal). The diagnostic MUST make this distinction explicit. + +--- + +#### T-051 — Empty scalar query is rejected + +**Anchors:** §5.1.2, §6.2 normative evaluation step 1 · `E_EMPTY_SCALAR_QUERY` +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Fields: [] +``` + +**Expected:** + +```yaml +error: + code: E_EMPTY_SCALAR_QUERY + anchor: §5.1.2 + must_contain: ["Fields", "at least one", "scalar"] +``` + +A scalar query with no fields has no projection; it cannot be compiled +to any SQL. The diagnostic MUST mention that `Fields` requires at least +one entry. + +--- + +#### T-053 — `AVG` / `MIN` / `MAX` over zero matching rows return `NULL` + +**Anchors:** §6.11 · D-033 (non-`COUNT` branch) +**Fixture:** F-PRELUDE. + +**Query:** +```yaml +Dimensions: [customers.region] +Measures: + - AVG(orders.amount) AS avg_amount + - MIN(orders.amount) AS min_amount + - MAX(orders.amount) AS max_amount +``` + +**Expected (three measures over one fact — Plan A; NORTH absent):** + +| region | avg_amount | min_amount | max_amount | +|:-------|-----------:|-----------:|-----------:| +| EAST | 116.67 | 50.00 | 200.00 | +| WEST | 75.00 | 75.00 | 75.00 | +| NULL | 30.00 | 30.00 | 30.00 | + +Combined with T-047 (`SUM` → `NULL` on empty, `COUNT` → `0`), this +fully pins D-033: only the `COUNT` family returns `0` over an empty +input. The NORTH row is absent for the same reason as in T-001 / T-047 +— a single-fact aggregation query follows Plan A. + +--- + +## 5. Coverage map + +This map is the authoritative cross-reference. Every `D-NNN` decision +in the spec's Appendix B that has observable runtime behaviour MUST +appear here with at least one covering `T-NNN`. + +| D-NNN | Decision | Covered by | +|:------|:---|:---| +| D-001 | Aggregation-query cardinality | T-001, T-050 (empty rejected) | +| D-003 | Implicit home-grain aggregation in fields | T-004, T-024 | +| D-004 | Default join types | T-001 (LEFT), T-010 (LEFT, indirect), T-011 (FULL OUTER), T-012 (CROSS), T-043 (multi-hop N:1 chain) | +| D-005 | Predicate routing by resolved expression shape | T-007, T-024 (e: boolean home-grain scalar in `Where`) | +| D-006 | Bare name → global namespace | T-020 | +| D-007 | M:N MUST be a safe rewrite | T-013, T-014 | +| D-009 | Deferred-key rejection | T-042 (canonical instance — `grain: FIXED`) | +| D-010 | Query shape determined by clauses | T-002 | +| D-011 | Bare metric in `Fields` rejected | T-003 | +| D-012 | Predicate-shape errors | T-007 (in WHERE), T-008 (in HAVING), T-009 (mixed) | +| D-014 | Determinism | §1.4 (applied to every test); explicit in T-026, T-052 (compiled SQL byte-stable) | +| D-015 | Three home-grain compilation strategies equivalent | T-004 | +| D-016 | `COUNT(*)` works | T-006, T-047 | +| ~~D-017~~ | **Deferred** — moved to a separate EXISTS_IN proposal | ~~T-018~~ (deferred slot) | +| D-018 | Path-resolution errors | T-019 (ambiguous), T-040 (E_NO_PATH) | +| D-019 | Reserved names rejected | T-041 | +| D-020 | Single-step AND nested cross-grain aggregation accepted | **T-005a (single-step SUM), T-005b (single-step AVG, heavy-weighted), T-005c (nested AVG, unweighted), T-005d (single-step SUM ≡ nested SUM), T-005e (single-step COUNT DISTINCT), T-017** | +| D-022 | Non-decomposable aggregate over fan-out | T-021, T-005e (caveat: D-022 fires on M:N decomposition, not plain 1:N) | +| D-023 | Fan-out in scalar query | T-022, T-025 (multiple unrelated row-level facts) | +| D-024 | Unaggregated finer-grain reference | T-023 | +| **D-026** | **Bridge resolution de-duplicates per `(fact, group)`** | **T-015 (flagship), T-045 (two-measure bridge)** | +| **D-027** | **Bare non-distributive aggregate over M:N rejected** | **T-016, T-017 (positive)** | +| D-029 | `Order By` defaults to `NULLS LAST` (outer + `OVER`) | T-026 (outer), T-027 (`OVER`) | +| D-030 | Windows: legal placements, `E_WINDOW_IN_WHERE` | T-029 (rejected), T-031 (running total positive), T-032 (qualify pattern), T-036 (window over `1:N`) | +| D-031 | Nested windows rejected | T-030 | +| D-032 | `ROWS` / `RANGE` accepted; `GROUPS` and parameterised bounds deferred | T-031 (positive `ROWS`), T-034 (`GROUPS` rejected), T-035 (param frame bound rejected) | +| **D-033** | **Empty-set aggregate behaviour follows standard SQL: `COUNT` family → `0`, everything else → `NULL`** | **T-011 (stitch missing-side `SUM` = `NULL`, the canonical test now that single-measure shapes drop missing-dim rows entirely per §1.6); T-045 (mixed `SUM`/`COUNT` branches); T-047, T-053 (one-fact multi-measure — `COUNT` = `0` for present dim values; missing dim values themselves are dropped per Plan A)** | +| E_WINDOWED_METRIC_COMPOSITION | Windowed metrics cannot be composed | T-033 | + +### 5.1 BI-shape coverage + +Independent of D-NNN coverage, these are the shapes every production +BI tool must handle. The catalog covers them as follows: + +| Shape | Covered by | +|:---|:---| +| Single-table fact | T-006, T-021, T-031, T-032, T-036 | +| `N : 1` enrichment (one hop) — single-measure (Plan A) | T-001, T-002, T-010 | +| `N : 1` enrichment (multi-hop chain) | T-043 | +| Composite primary/foreign key | T-044 | +| `1 : N` cross-grain aggregation (single-step) | T-005a, T-005b, T-005e, T-017 | +| `1 : N` cross-grain aggregation (nested) | T-005c, T-005d | +| `M : N` via bridge | T-013, T-014, T-015, T-016, T-017, T-045 | +| Multi-measure stitch (`FULL OUTER`, §6.6 row 3) | T-011, T-045 | +| Semi-join filter | ~~T-018~~ (deferred to future EXISTS_IN proposal) | +| Reflexive (self-referential) | T-046 | +| Windowed metrics | T-027, T-031, T-032, T-036 | +| Empty / boundary queries | T-050, T-051, T-052 | +| NULL semantics | T-011 (stitch missing-side), T-047, T-048, T-049, T-053 | + +## 6. Decisions awaiting coverage + +The following `D-NNN` entries do not yet have a `T-NNN` vector. Adding +one is the natural next sprint of this catalog. + +| D-NNN | Why deferred | +|:------|:---| +| D-002 | Multi-measure shared-grain composition — T-011 and T-045 cover the FULL-OUTER stitch path; D-002's "all measures resolve at the query's grain" claim is implicit in both. A direct test pinning the empty-dimensions / two-fact-no-dims case (Snowflake errata #4) is the natural next addition. | +| D-008 | No `UNSAFE`; no per-metric `joins.type` override. Test is a parser-level rejection — better placed in a parser-conformance suite than this row-set catalog. | +| D-009 | T-042 is the canonical instance; full coverage requires one test per deferred key (six total). Mechanically generated; out of scope for this seed but trivial follow-up. | +| D-013 | Same-named expressions reachable via qualification — needs a fixture with a deliberate name collision; planned for the next pass. | +| D-021 | Structured `expression` slot — parser/codegen-conformance. | +| D-025 | `E_AMBIGUOUS_MEASURE_GRAIN` catch-all — the D-025 enumeration lists the known cases; T-036b covers one (windowed field with no declared home grain). Remaining enumerated cases are mechanical follow-ups. | + +When this list shrinks to empty, every `D-NNN` in Appendix B has an +executable witness in this file and the Foundation Conformance Suite is +self-contained. diff --git a/compliance/foundation/README.md b/compliance/foundation/README.md new file mode 100644 index 00000000..c96966ea --- /dev/null +++ b/compliance/foundation/README.md @@ -0,0 +1,166 @@ + + +# OSI Compliance Test Suite — Foundation + +**Targets the +[Ossie Semantic Foundation](https://docs.google.com/document/d/1bEuIb2eUpwesi6cIZ5FOkXMQDh7KAPIjZsWchmLgOAk/edit?usp=sharing) proposal +[`SQL_EXPRESSION_SUBSET`](../core-spec/expression_language.md)** + +The Foundation is the deliberately narrow first-cut of Ossie semantics: +two query shapes, implicit home-grain aggregation, window functions, +and a small deferred-features list — pinned by numbered Conformance +Decisions (D-NNN) throughout the Foundation spec, and an error code +index in its appendix. + +The runner harness lives at [`../harness/`](../harness/) and is +shared with every future per-version suite under `compliance/`. + +## Status of this slice + +This is a **bootstrap slice**, not the full suite. It ports over just +enough of the compliance suite to make the mechanism (test discovery, +metadata schema, decision-coverage tracking, DuckDB-backed row +comparison, reporting) runnable and testable end to end: + +- **Tests present:** `tests/cross_grain/moderate/` only — 4 cases + (`t-005a`, `t-005b`, `t-005d`, `t-005e`) covering D-020 (single-step + cross-grain aggregation over a `1 : N` edge). Every other area listed + under `Layout` below (query shape, bridge/M:N, windows, namespace, + etc.) is the *target* design carried over from the source proposal + and will land — with its own tests — in follow-up PRs. +- **No adapter yet.** `adapters/osi_python_adapter.py` delegates to + `impl/python/conformance/adapter.py`, which does not exist in this + repo yet (`impl/python/` is still a placeholder). Until an + implementation lands, `python -m harness.runner --list` is the way + to exercise this suite — it discovers and validates the test corpus + without needing an engine. The 4 included tests carry + `status: planned` in their `metadata.yaml` (matching their status in + the source proposal — the semantics they pin are accepted but not + yet required-to-pass), so a full `harness.runner` run against a + partial adapter will correctly skip them unless invoked with + `--include-planned`. +- **`decisions.yaml`** keeps every `D-NNN` row from the source + proposal (so `harness`'s own registry-consistency tests still pass), + but only `D-020` has a populated `tests:` list — every other + decision's `tests:` is intentionally empty pending its witness + tests landing in a later PR. +- **Dropped `T-024`.** The source proposal's compliance suite also + shipped `tests/cross_grain/moderate/t-024-boolean-home-grain-scalar-in-where/`. + It was excluded from this slice: its `gold.sql` filters on + `customers.segment = 'PREMIUM'`, a value that does not exist in the + `f_prelude` fixture (segments are `retail` / `wholesale`), so the + query returns zero rows — and the test as implemented doesn't + exercise the aggregate-derived boolean-field case that + `DATA_TESTS.md` §4.G documents for T-024 (`has_completed_orders`, + an aggregate `COUNT(...) > 0` field routed through `Where`). See + the PR description for the full writeup; worth fixing at the + source before it's re-added here. + +## Layout + +``` +compliance/foundation-v0.1/ + README.md # this file + SPEC.md # the specs this suite targets (Foundation v0.1) + pyproject.toml # editable install; depends on ../harness (osi_compliance_harness) + conformance.yaml # test conformance levels; "foundation_v0_1" = required + proposals.yaml # mirrors §10 of the Foundation spec; every deferred is a proposal + decisions.yaml # D-NNN registry with anchor + status; tests: populated only for D-020 in this slice + adapters/ + osi_python_adapter.py # delegates to impl/python/conformance/adapter.py (not yet present) + datasets/ + f_prelude/ # mirrors DATA_TESTS.md §3.1 — the only fixture this slice needs + tests/ + cross_grain/ # T-005a/b/d/e (D-020) — the only populated area in this slice + results/ # created by the runner (gitignored); no committed baseline yet +``` + +The full target layout — `query_shape/`, `scalar_query/`, `bridge/`, +`windows/`, `namespace/`, etc. — is documented in `DATA_TESTS.md` and +`decisions.yaml`; those directories don't exist on disk yet. + +## Quick start + +```bash +# From the repo root +pip install -e compliance/harness +pip install -e compliance/foundation + +cd compliance/foundation + +# List discovered tests without running them (no adapter needed). +# The 4 cross_grain/moderate cases show status "planned" — they're +# skipped by a real run until an adapter lands, unless you pass +# --include-planned. +python -m harness.runner --list --tests tests/ --include-planned + +# Once an OSI implementation adapter exists (see ../ADAPTER_INTERFACE.md): +python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ \ + --include-planned +``` + +The harness ships under [`../harness/`](../harness/) and is installed +as the `osi_compliance_harness` package — every per-version suite under +`compliance/` shares this one runner / reporter / DB manager. The +harness's own mechanics (discovery, adapter invocation, row +comparison, reporting) are covered by its unit tests — see +[`../harness/README.md`](../harness/README.md). + +## Per-test layout + +Each test is a folder containing: + +| File | Purpose | +|:---|:---| +| `metadata.yaml` | `id: T-NNN`, `decision: D-NNN`, `spec_refs`, `required_features`, `expected_error_code`, `xfail_reason` (if applicable). | +| `model.yaml` | The semantic model (typically a thin wrapper around a fixture from `datasets/f_*`). | +| `query.json` | The semantic query, in the new two-shape format (`dimensions` + `measures` for aggregation queries; `fields` for scalar queries). | +| `gold.sql` | A hand-written reference SQL query the harness executes against the fixture data to produce the expected row multiset. Treated by the harness as a row oracle, not as a SQL-string comparison — D-014 is per-engine, not cross-engine. | + +Tests assert on observable behaviour only: + +- `expected_error_code: E_` ⇒ adapter must surface that code in + stderr (substring match — see `compliance/harness/src/harness/runner.py`). +- The harness runs both `gold.sql` and the adapter's emitted SQL against + the shared fixture and compares the resulting row multisets + (order-insensitive unless the query has an `Order By`). + +This means a `gold.sql` is a *witness* of the answer's shape and the +fixture data; the harness never compares SQL strings byte-for-byte +(per D-014, that's a per-engine concern). + +## Decision coverage + +Every `D-NNN` row in the Foundation spec should eventually have at +least one runnable case here. The mapping lives in `decisions.yaml`; +in this slice only `D-020` has one. + +## See also + +- `SPEC.md` — what the suite targets and why. +- `decisions.yaml` — the D-NNN status board. +- `proposals.yaml` — §10 deferred-features registry. +- `../../impl/python/` — where the upstream engine and its + `conformance/adapter.py` will land. +- `../ADAPTER_INTERFACE.md` — the CLI contract every + adapter satisfies. diff --git a/compliance/foundation/SPEC.md b/compliance/foundation/SPEC.md new file mode 100644 index 00000000..04bbf75a --- /dev/null +++ b/compliance/foundation/SPEC.md @@ -0,0 +1,90 @@ + + +# OSI Compliance Test Suite — Spec Coverage + +**Targets:** Foundation `osi_version: "0.1"`. Authoritative spec +documents: + +- [`../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md`](../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md) + — the semantic, query, join, M:N, and window contracts; Appendix B + (`D-001` … `D-033`) and Appendix C (error code index) are the + specific anchors this suite exercises. +- [`../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md`](../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md) + — the `OSI_SQL_2026` default expression dialect. +- [`DATA_TESTS.md`](DATA_TESTS.md) + — the normative test catalog (`T-001` … `T-NNN`) this suite encodes + as runnable cases. + +This suite is the runnable witness layer for the spec. Every +conformance decision in Appendix B has at least one `T-NNN` test here; +every error code in Appendix C has at least one negative test here. + +## What this suite does NOT cover + +- OSI features outside the Foundation surface (LOD grain modes, filter + context, named filters, semi-join filter form, per-metric + `joins.{type, using_relationships}`, non-equijoins, etc.). These + features are deferred per §10 of the Foundation proposal and are + exercised here only as negative cases that expect + `E_DEFERRED_KEY_REJECTED` (D-009). Future per-version compliance + suites under `compliance/` will cover them positively when their + proposals land. +- Engine-internal plan shape (CTE count, join order, alias names, + generated SQL string). The Foundation defines per-engine determinism + only (D-014); cross-engine SQL determinism is not required, so tests + assert on rows or `error.code` only. +- Performance characteristics. The suite is a correctness signal; any + performance regression checks live in the implementation's own + benchmark harness. + +## Compliance levels + +Defined in [`conformance.yaml`](conformance.yaml): + +| Level | Description | +|:---|:---| +| **`foundation_v0_1`** | Required for every Foundation-claiming engine. Every D-NNN in Appendix B must produce the expected outcome. | +| **`foundation_v0_1_strict`** | Adds determinism witnesses (D-014, D-029) — same `(model, query, dialect)` produces byte-identical SQL across runs. Optional. | + +Cross-engine portability is observable behaviour (rows / error codes), +not SQL text. `foundation_v0_1_strict` is per-engine determinism. + +## Decision coverage + +The mapping `D-NNN ↔ T-NNN` lives in [`decisions.yaml`](decisions.yaml). +The runner emits `results/decisions_coverage.md` after every run so +gaps are visible in PR review. + +## Adapter contract + +Adapters live under [`adapters/`](adapters/). The CLI contract is the +one published at +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md): + +``` + sql --model --query-file --dialect +``` + +stdout = generated SQL; stderr = `: ` on +failure; exit code = 0 on success, non-zero on error. The bundled +`osi_python_adapter.py` delegates to +[`../../impl/python/conformance/adapter.py`](../../impl/python/conformance/adapter.py) +so the existing CLI contract stays the source of truth — this suite +adds no new translation logic. diff --git a/compliance/foundation/adapters/osi_python_adapter.py b/compliance/foundation/adapters/osi_python_adapter.py new file mode 100644 index 00000000..76b26a22 --- /dev/null +++ b/compliance/foundation/adapters/osi_python_adapter.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""OSI Foundation compliance adapter for implementations. + +A thin delegator to ``impl/python/conformance/adapter.py``. The upstream +adapter implements the CLI contract published in +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md) (``sql --model M +--query-file Q --dialect D``); this file re-exposes it from inside the +compliance suite so the harness can resolve it from +``adapters/osi_python_adapter.py`` without reaching out of the suite. + +We deliberately do NOT re-implement any conversion logic here. If a suite +test case needs a YAML / JSON shape the upstream adapter does not +understand, fix it in the upstream adapter (a single source of truth) — +never fork the conversion logic. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_ADAPTER_DIR = Path(__file__).resolve().parent +# adapters/ is at compliance/foundation/adapters/. +# Walk up three levels (adapters → foundation → compliance → repo root), +# then descend into impl/python where the upstream adapter lives. +_REPO_ROOT = _ADAPTER_DIR.parent.parent.parent +_IMPL_PYTHON_ROOT = _REPO_ROOT / "impl" / "python" +_IMPL_PYTHON_CONFORMANCE = _IMPL_PYTHON_ROOT / "conformance" + +if not _IMPL_PYTHON_CONFORMANCE.exists(): + sys.stderr.write( + f"osi_python_adapter: cannot find upstream adapter at " + f"{_IMPL_PYTHON_CONFORMANCE}; check that impl/python/ is present " + f"at the OSI repo root.\n", + ) + sys.exit(2) + +# Make the upstream conformance package importable. The upstream adapter +# prepends impl/python/src to sys.path itself, so we only need to expose its +# parent directory here. +sys.path.insert(0, str(_IMPL_PYTHON_ROOT)) + +from conformance.adapter import main # noqa: E402 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compliance/foundation/conformance.yaml b/compliance/foundation/conformance.yaml new file mode 100644 index 00000000..5103d22b --- /dev/null +++ b/compliance/foundation/conformance.yaml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OSI Compliance Test Suite — Conformance Levels (updated Foundation) +# +# This file pins which subsets of the suite an engine MUST pass to claim +# a particular Foundation conformance level. The Foundation spec lives +# at ../../proposals/foundation/Proposed_OSI_Semantics.md (osi_version: 0.1). +# +# Each level lists the test areas that contribute to it; an +# implementation passes the level iff every must_pass case in those +# areas is green. +# +# Cross-engine SQL determinism is NOT required by Foundation v0.1 +# (per D-014); cross-engine portability is observable behaviour +# (rows or error code). The strict level adds per-engine SQL +# determinism witnesses on top of the basic foundation level. + +version: "0.1" + +levels: + foundation_v0_1: + description: > + Foundation v0.1 conformance. Required for any engine that claims + to implement the updated OSI Foundation. Every D-NNN row in + Appendix B of Proposed_OSI_Semantics.md must produce the + expected outcome (row set or error code). + pass_threshold: 100% + areas: + query_shape: all + scalar_query: all + field_metric_grain: all + cross_grain: all + nested_aggregates: all + bridge: all + joins_default: all + predicate_routing: all + namespace: all + windows: all + null_ordering: all + empty_inputs: all + deferred: all + error_taxonomy: all + + foundation_v0_1_strict: + description: > + Foundation v0.1 with per-engine SQL determinism witnesses + (D-014). Optional. Adds: same (model, query, dialect) compiled + twice on the same engine produces byte-identical SQL; outer + Order By and OVER ORDER BY emit explicit NULLS LAST per D-029. + pass_threshold: 100% + includes: [foundation_v0_1] + areas: + determinism: + all: all # D-014, D-029 SQL-text witnesses + +# Extension levels (e.g. foundation_v0_1_with_grain_modes) will be +# added when their feature proposals (§10 of the Foundation spec) +# are ratified. Each ratified proposal becomes a new level here that +# requires foundation_v0_1 plus the proposal's tests. diff --git a/compliance/foundation/datasets/f_prelude/description.md b/compliance/foundation/datasets/f_prelude/description.md new file mode 100644 index 00000000..2e6baa61 --- /dev/null +++ b/compliance/foundation/datasets/f_prelude/description.md @@ -0,0 +1,29 @@ +# F-PRELUDE + +The flagship Foundation fixture — a single-fact star (`customers ← +orders`, `customers ← returns`) with one orphan order (customer_id = +99) and one customer with no orders (id = 4). Every Foundation +semantic that exercises join defaults, NULL group keys, multi-fact +stitch, and basic aggregation runs against this fixture. + +Authoritative source: `../../DATA_TESTS.md §3.1`. + +The deliberate edge cases: + +- **Orphan order (105)**: tests Semantic 1 (`LEFT` enrichment surfaces + orphan orders under `region = NULL`). +- **Customer 4 with returns but no orders**: tests Semantic 3 (the + `NORTH` region appears in the multi-fact stitch even though it has + no orders). +- **Both `EAST` customers (1 and 2) have orders, customer 3 in `WEST` + has orders too**: gives the bridge / chasm tests a non-degenerate + cardinality. + +Aggregate witnesses: + +- `SUM(orders.amount)` total: 455.00 +- `SUM(orders.amount)` by region (LEFT enrichment): EAST=350.00, + WEST=75.00, NULL=30.00. +- `SUM(returns.amount)` by region: EAST=10.00, WEST=5.00, NORTH=15.00. +- `COUNT(orders.id)` total: 5; with `Where status='completed'`: 4. +- Distinct customers: 4. diff --git a/compliance/foundation/datasets/f_prelude/schema.sql b/compliance/foundation/datasets/f_prelude/schema.sql new file mode 100644 index 00000000..c4a8c3c8 --- /dev/null +++ b/compliance/foundation/datasets/f_prelude/schema.sql @@ -0,0 +1,51 @@ +-- F-PRELUDE — single-fact star with multi-fact extension. +-- Source: ../../DATA_TESTS.md §3.1 +-- +-- Used by: T-001, T-002, T-003, T-004, T-005x, T-006, T-011, T-016, +-- T-021, T-029, T-033, and most query-shape / predicate-routing +-- / namespace cases. + +CREATE TABLE customers ( + id INTEGER PRIMARY KEY, + region VARCHAR, + segment VARCHAR +); + +INSERT INTO customers VALUES + (1, 'EAST', 'retail'), + (2, 'EAST', 'retail'), + (3, 'WEST', 'wholesale'), + (4, 'NORTH', 'retail'); + +CREATE TABLE orders ( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + amount DECIMAL(10, 2), + status VARCHAR +); + +-- Note: order 105 is an orphan (customer_id = 99 is not in customers). +INSERT INTO orders VALUES + (101, 1, 100.00, 'completed'), + (102, 1, 50.00, 'completed'), + (103, 2, 200.00, 'pending'), + (104, 3, 75.00, 'completed'), + (105, 99, 30.00, 'completed'); + +CREATE TABLE returns ( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + amount DECIMAL(10, 2) +); + +-- Note: customer 4 has a return but no orders — Semantic 3 case. +INSERT INTO returns VALUES + (201, 1, 10.00), + (202, 3, 5.00), + (203, 4, 15.00); + +CREATE TABLE premium_customers ( + id INTEGER PRIMARY KEY +); + +INSERT INTO premium_customers VALUES (1), (3); diff --git a/compliance/foundation/decisions.yaml b/compliance/foundation/decisions.yaml new file mode 100644 index 00000000..c3c84fb7 --- /dev/null +++ b/compliance/foundation/decisions.yaml @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Foundation Conformance Decisions — runnable test mapping. +# +# Mirrors the D-NNN decisions called out throughout +# ../../proposals/foundation/Proposed_OSI_Semantics.md. Every D-NNN row +# from the source proposal is kept here (harness's registry-consistency +# tests require full D-NNN coverage), but this bootstrap slice only ships +# witness tests for D-020 — every other decision's tests: is intentionally +# empty pending its tests landing in a follow-up PR. The runner emits +# results/decisions_coverage.md after every run so gaps are visible. +# +# Status: +# * must_pass — the engine MUST pass every T-NNN test pinned to this +# decision; failure is a Foundation conformance failure. +# * xfail — the test is shipped but expected to fail until the +# named sprint flips it; xfail_pinned_to identifies the sprint. + +version: "0.1" + +decisions: + + - id: D-001 + title: Aggregation result cardinality + multi-measure stitch on incompatible roots + spec_ref: "§5.1.1, §6.2, §6.6" + tests: [] + status: must_pass + + - id: D-002 + title: All measures resolve at the query's grain (no per-aggregate native granularity leak) + spec_ref: §6.2 + tests: [] + status: must_pass + + - id: D-003 + title: Implicit home-grain aggregation in field expressions + spec_ref: §4.3.1 + tests: [] + status: must_pass + + - id: D-004 + title: "Default join shapes (LEFT for N:1; FULL OUTER stitch; CROSS JOIN of 1-row scalars)" + spec_ref: §6.6 + tests: [] + status: must_pass + + - id: D-005 + title: "Routing of predicates by resolved expression shape (no role: keyword)" + spec_ref: "§4.3, §6.3" + tests: [] + status: must_pass + + - id: D-006 + title: Bare-name resolves to global namespace; dataset-scoped via dataset.field + spec_ref: §4.6 + tests: [] + status: must_pass + + - id: D-007 + title: "M:N traversals must produce a result equivalent to bridge or stitch; otherwise E3012" + spec_ref: §6.8 + tests: [] + status: must_pass + + - id: D-008 + title: No per-metric joins.type override at Foundation level (deferred) + spec_ref: "§6.6, §6.9" + tests: [] + status: must_pass + + - id: D-009 + title: Deferred relationship-level keys rejected with E_DEFERRED_KEY_REJECTED in default mode + spec_ref: §11 + tests: [] + status: must_pass + + - id: D-010 + title: Aggregation vs Scalar query shape — mixing rejected with E_MIXED_QUERY_SHAPE + spec_ref: §5.1 + tests: [] + status: must_pass + + - id: D-011 + title: Bare metric reference inside Fields rejected with E_AGGREGATE_IN_SCALAR_QUERY + spec_ref: §5.1.2 + tests: [] + status: must_pass + + - id: D-012 + title: Predicate-shape errors (Where vs Having) + spec_ref: §6.3 + tests: [] + status: must_pass + + - id: D-014 + title: Per-engine determinism — same (model, query, dialect) → byte-identical SQL + spec_ref: §11 + tests: [] + status: must_pass + + - id: D-016 + title: COUNT(*) is required and counts rows of the home dataset / query grain + spec_ref: §7 + tests: [] + status: must_pass + + - id: D-018 + title: Path resolution — unique path used; multiple ⇒ E_AMBIGUOUS_PATH; none ⇒ E_NO_PATH + spec_ref: §6.9 + tests: [] + status: must_pass + + - id: D-019 + title: Reserved names (GRAIN, FILTER, QUERY_FILTER) cannot be used as user identifiers + spec_ref: §4.6.2 + tests: [] + status: must_pass + + - id: D-020 + title: "Single-step + nested cross-grain aggregates over 1:N (every aggregate category)" + spec_ref: "§4.5 form (1), §6.1 Semantic 2" + tests: + - tests/cross_grain/moderate/t-005a-single-step-sum/ + - tests/cross_grain/moderate/t-005b-single-step-avg/ + - tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/ + - tests/cross_grain/moderate/t-005e-count-distinct-single-step/ + status: must_pass + + - id: D-021 + title: "expression slot accepts string OR { dialects: [...] } per OSI_SQL_2026" + spec_ref: "§4.3 / §4.5" + tests: [] + status: must_pass + + - id: D-022 + title: Plan that forces decomposition the aggregate cannot survive ⇒ E_UNSAFE_REAGGREGATION (chasm pre-agg / stitch only — bridge plan is single-pass and not in scope) + spec_ref: "§6.2 Starting Grain, §6.7, §6.8.2" + tests: [] + status: must_pass + + - id: D-023 + title: Scalar query — fan-out / incompatible-home rejected with E_FAN_OUT_IN_SCALAR_QUERY + spec_ref: "§6.2 Final Grain, §5.1.2" + tests: [] + status: must_pass + + - id: D-024 + title: Row-level reference at finer grain ⇒ E_UNAGGREGATED_FINER_GRAIN_REFERENCE + spec_ref: §6.2 Starting Grain + tests: [] + status: must_pass + + - id: D-025 + title: Catch-all measure-grain ambiguity ⇒ E_AMBIGUOUS_MEASURE_GRAIN + spec_ref: §6.2 Starting Grain + tests: [] + status: must_pass + + - id: D-026 + title: Bridge resolution materializes distinct (fact, group-key) — Semantic 2 + spec_ref: "§6.1 Semantic 2, §6.8.1" + tests: [] + status: must_pass + + - id: D-027 + title: | + Bare cross-grain aggregate over an N:N edge resolves via the §6.8.1 + bridge-dedup construction in a single pass for every aggregate + category (distributive, algebraic, holistic). The "per-home-row-first" + interpretation requires the nested form AGG(AGG(...)), which is + deferred to §10 and raises E_NESTED_AGGREGATION_DEFERRED. + spec_ref: "§4.5 form (1), §6.8.1" + tests: [] + status: must_pass + + - id: D-028 + title: Standard SQL window functions in Measures / Fields / Order By / Having; Where ⇒ E_WINDOW_IN_WHERE + spec_ref: §6.10.1 + tests: [] + status: must_pass + + - id: D-029 + title: NULLS LAST default for outer Order By + OVER ORDER BY; emitted explicitly into compiled SQL + spec_ref: "§5.1, §6.10.2" + tests: [] + status: must_pass + + - id: D-030 + title: Window over fan-out ⇒ pre-fan-out materialization (or E_WINDOW_OVER_FANOUT_REWRITE) + spec_ref: §6.10.3 + tests: [] + status: must_pass + + - id: D-031 + title: Composing a metric on top of a windowed metric ⇒ E_WINDOWED_METRIC_COMPOSITION + spec_ref: §6.10.5 + tests: [] + status: must_pass + + - id: D-032 + title: "ROWS / RANGE with integer-literal bounds only; GROUPS or :param ⇒ E_DEFERRED_FRAME_MODE" + spec_ref: §6.10.6 + tests: [] + status: must_pass + + - id: D-033 + title: Empty / NULL aggregate behaviour follows standard SQL (COUNT⇒0, others⇒NULL) + spec_ref: "§6.11.1, §6.11.2" + tests: [] + status: must_pass diff --git a/compliance/foundation/proposals.yaml b/compliance/foundation/proposals.yaml new file mode 100644 index 00000000..59fc4a4a --- /dev/null +++ b/compliance/foundation/proposals.yaml @@ -0,0 +1,231 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OSI Foundation v0.1 — Proposal Registry +# ----------------------------------------------------------------------- +# Mirrors §10 of ../../proposals/foundation/Proposed_OSI_Semantics.md. Every +# entry is either: +# +# * status: foundation — IN scope for v0.1; engines MUST implement +# * status: deferred — OUT of scope; rejected at parse time with +# E_DEFERRED_KEY_REJECTED (D-009). +# +# Test metadata.yaml files reference these IDs via `required_features:`. +# A test that references an unknown ID is a CI failure. +# +# Adapters advertise the proposals they implement via +# `enabled_proposals.yaml` (or equivalent) per the harness contract. +# A Foundation-only adapter advertises every `status: foundation` ID +# and rejects every `status: deferred` ID with E_DEFERRED_KEY_REJECTED. + +version: "0.1" + +proposals: + # -------- Foundation (v0.1) — required for every conforming engine -- + - id: two_query_shapes + status: foundation + title: Aggregation vs Scalar query shapes (Fields clause) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + decisions: [D-001, D-010, D-011, D-023] + - id: routing_by_expression_shape + status: foundation + title: Predicate routing by resolved expression shape (no role:) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.3 + decisions: [D-005, D-012] + - id: implicit_home_grain_aggregation_rejection + status: foundation + title: Implicit home-grain aggregation in field expressions is rejected (E_AGGREGATE_IN_FIELD) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.3 + rejection_code: E_AGGREGATE_IN_FIELD + decisions: [D-003] + # D-003 and D-015 are both struck/deferred in Appendix B; the + # Foundation's active surface is the rejection rule (any aggregate + # in a field body MUST raise E_AGGREGATE_IN_FIELD per §4.3 / + # Appendix C). Per-strategy equivalence (D-015) returns when + # §10's grain-aware-functions proposal lands. + - id: cross_grain_aggregates + status: foundation + title: Single-step cross-grain aggregates over 1:N (nested form is deferred per D-020(c)/D-027(d)) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.5 + decisions: [D-020, D-024] + - id: default_join_shapes + status: foundation + title: Default LEFT for N:1, FULL OUTER stitch for incompatible-root multi-fact + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.6 + decisions: [D-001, D-004, D-008] + - id: bridge_dedup + status: foundation + title: Bridge resolution materializes distinct (fact, group-key) in a single-pass aggregate (every category) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8.1 + decisions: [D-026, D-027] + - id: decomposition_safety + status: foundation + title: E_UNSAFE_REAGGREGATION fires only when the chosen plan forces decomposition the aggregate cannot survive (chasm pre-agg / stitch — bridge plan is single-pass and not in scope) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.2 + decisions: [D-022] + - id: identifier_resolution + status: foundation + title: Global / dataset-scoped namespace, ambiguous-path / no-path errors + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.6 + decisions: [D-006, D-018, D-019] + - id: window_functions_standard + status: foundation + title: Standard SQL window functions (ranking / navigation / aggregate-windows; ROWS / RANGE; integer-literal bounds) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10 + decisions: [D-028, D-030, D-031, D-032] + - id: nulls_last_default + status: foundation + title: Default NULLS LAST emission for outer Order By + OVER ORDER BY + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + decisions: [D-014, D-029] + - id: empty_aggregate_standard_sql + status: foundation + title: Standard SQL empty / NULL aggregate behaviour + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.11 + decisions: [D-033] + - id: osi_sql_2026_dialect + status: foundation + title: OSI_SQL_2026 default dialect with per-dialect expression form + spec_ref: ../../proposals/foundation/SQL_EXPRESSION_SUBSET.md + decisions: [D-021, D-016] + - id: windowed_metric_composition + status: foundation + title: Composing a metric on top of a windowed metric is rejected with E_WINDOWED_METRIC_COMPOSITION + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10.5 + rejection_code: E_WINDOWED_METRIC_COMPOSITION + decisions: [D-031] + + # -------- Deferred (§10 of the Foundation spec) --------------------- + # Each deferred entry MUST be rejected at parse time with + # E_DEFERRED_KEY_REJECTED (D-009). The compliance suite ships exactly + - id: explicit_grain_overrides + status: deferred + title: Explicit grain overrides (FIXED / INCLUDE / EXCLUDE / TABLE) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: filter_context_propagation + status: deferred + title: Filter context propagation (reset, filter.expression on metrics) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: metric_composition_grain_filter + status: deferred + title: Metric composition with grain or filter inheritance + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: natural_grain_top_level + status: deferred + title: Model-level natural_grain declaration + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: path_disambiguation_using_relationships + status: deferred + title: Per-metric joins.using_relationships path disambiguation + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: per_metric_joins_type + status: deferred + title: Per-metric joins.type override (INNER / LEFT / FULL) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: non_equijoin_relationships + status: deferred + title: condition / cardinality on relationships (non-equijoin) + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: asof_range_relationships + status: deferred + title: ASOF and Range relationships + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: referential_integrity_keys + status: deferred + title: referential_integrity / from_all_rows_match / to_all_rows_match + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: semi_additive_measures + status: deferred + title: Semi-additive measures over snapshot facts + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: grouping_sets + status: deferred + title: GROUPING SETS / ROLLUP / CUBE + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: pivot_operator + status: deferred + title: PIVOT / UNPIVOT + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: parameterized_window_frame_bounds + status: deferred + title: Parameterized window frame bounds (e.g. ROWS BETWEEN :n PRECEDING ...) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_FRAME_MODE + - id: groups_frame_mode + status: deferred + title: GROUPS frame mode (non-portable) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_FRAME_MODE + - id: ordered_set_aggregates + status: deferred + title: WITHIN GROUP ordered-set aggregates (LISTAGG, PERCENTILE_CONT) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + # ----- moved up: this is a Foundation rejection rule, not deferral + # See the foundation block above for the actual entry. + - id: named_filters + status: deferred + title: Reusable named boolean filters referenced by name + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: dataset_scope_filters + status: deferred + title: Dataset-level filters with scope-based propagation + spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + rejection_code: E_DEFERRED_KEY_REJECTED + - id: semi_join_filter_form + status: deferred + title: EXISTS_IN / NOT EXISTS_IN semi-join filter (separate proposal pending) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: role_keyword + status: deferred + title: "role: keyword on fields/datasets (replaced by routing-by-shape D-005)" + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: legacy_attr_unsafe_agg_grainagg + status: deferred + title: Legacy ATTR / UNSAFE / AGG / GRAIN_AGG keyword set + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: multi_hop_bridge + status: deferred + title: Multi-hop bridge resolution (more than one bridge between the same endpoints) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: symmetric_aggregates + status: deferred + title: Looker-style symmetric aggregates (codegen-side, not a correctness mechanism) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + rejection_code: E_DEFERRED_KEY_REJECTED + - id: snowflake_partition_by_excluding + status: deferred + title: Snowflake ``PARTITION BY ... EXCLUDING`` window extension (vendor-specific, outside OSI_SQL_2026) + spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#12a + rejection_code: E_DEFERRED_KEY_REJECTED diff --git a/compliance/foundation/pyproject.toml b/compliance/foundation/pyproject.toml new file mode 100644 index 00000000..15aa728f --- /dev/null +++ b/compliance/foundation/pyproject.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "osi_compliance_foundation_v0_1" +version = "0.1.0" +description = "OSI Foundation v0.1 compliance test suite (osi_version 0.1)." +readme = "README.md" +requires-python = ">=3.11" +authors = [{ name = "OSI Foundation contributors" }] +license = { text = "Apache-2.0" } +dependencies = [ + "duckdb>=0.10", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", +] + +# The compliance runner harness is vendored at compliance/harness/ inside +# this OSI repo. To install everything needed to run the suite: +# +# pip install -e ../harness +# pip install -e . +# +# Both packages are editable so iterating on either is one-step. + +[tool.setuptools.packages.find] +where = ["."] +include = ["adapters*"] +exclude = ["tests*", "datasets*", "results*"] diff --git a/compliance/foundation/tests/README.md b/compliance/foundation/tests/README.md new file mode 100644 index 00000000..feb149b1 --- /dev/null +++ b/compliance/foundation/tests/README.md @@ -0,0 +1,87 @@ + + +# Foundation v0.1 compliance tests + +> **Bootstrap-slice note.** Only `cross_grain/moderate/` is physically +> present on disk right now (4 tests: `t-005a`, `t-005b`, `t-005d`, +> `t-005e`, covering D-020). Every other row in the Layout table below +> describes the target design and will land, with its own tests, in +> follow-up PRs — see `../README.md` for current status. + +Each test is a folder containing: + +| File | Purpose | +|:---|:---| +| `metadata.yaml` | `id: T-NNN`, `decision: D-NNN`, `dataset: f_*`, `spec_refs`, `required_features`, `expected_error_code` (negative cases), `xfail_reason` (if pinned to a sprint). | +| `model.yaml` | The semantic model the test runs against. | +| `query.json` | The semantic query, in the new two-shape format (`dimensions` + `measures` for aggregation queries; `fields` for scalar queries). | +| `gold.sql` | Hand-written reference SQL the harness executes against the fixture data to produce the expected row multiset. Positive cases only; negative cases keep a stub `gold.sql` so the harness can locate the directory. | + +Tests assert on observable behaviour only. The harness executes both +`gold.sql` and the adapter's emitted SQL against the shared fixture +and compares the resulting row multisets — order-insensitive unless +the query has an `Order By`. SQL strings are never compared +byte-for-byte (per D-014 that's a per-engine concern). + +## Layout + +| Area | What it covers | Anchor decisions | +|:---|:---|:---| +| `query_shape/` | Aggregation vs Scalar shape, mixing rejection, COUNT(*), expression-form dialects | D-001, D-010, D-002, D-016, D-021 | +| `scalar_query/` | Scalar query specifics: bare metric in Fields, fan-out rejection | D-011, D-023 | +| `field_metric_grain/` | Field expressions with implicit home-grain aggregation | D-003, D-015 | +| `cross_grain/` | Single-step cross-grain aggregates (1:N) | D-020, D-024 | +| `nested_aggregates/` | Nested aggregates over M:N (inner-grain inference) | D-022, D-027 | +| `bridge/` | M:N bridge resolution + de-duplication (Semantic 2) | D-026 (T-015 flagship) | +| `joins_default/` | LEFT for N:1, FULL OUTER stitch, CROSS JOIN of scalars | D-001, D-004, D-008 | +| `predicate_routing/` | Where vs Having shape errors | D-005, D-012 | +| `namespace/` | Global / dataset-scoped resolution, ambiguous / no path, reserved names | D-006, D-018, D-019 | +| `windows/` | Window placement, pre-fan-out, deferred frame modes, composition | D-028, D-030, D-031, D-032 | +| `null_ordering/` | NULLS LAST default emission, per-engine determinism witnesses | D-014, D-029 | +| `empty_inputs/` | Standard SQL empty / NULL aggregate behaviour | D-033 | +| `deferred/` | One negative test per `E_DEFERRED_KEY_REJECTED` family | D-009 | +| `validation_errors/` | Structural validation errors that are NOT deferred features (e.g. `E_FIELD_DEPENDENCY_CYCLE` — fields must form a DAG). | Appendix C | +| `error_taxonomy/` | Remaining Appendix C codes (E_PRIMARY_KEY_REQUIRED, E_AMBIGUOUS_MEASURE_GRAIN, etc.) | Appendix C | + +## Sprint timeline + +- **S-B**: this scaffold + `tests/README.md` (no test cases yet). +- **S-C**: lands `T-001 … T-033` plus the `E_DEFERRED_KEY_REJECTED` + negative tests, derived from `DATA_TESTS.md §4`. +- **S-E**: lands additional `T-NNN` cases from the differential / + edge-case audit before any S-1..S-17 sprint runs. + +## Negative tests + +A test is negative when its `metadata.yaml` carries +`expected_error_code: E_` (no `gold_rows.json`). The runner +asserts that the adapter exited non-zero AND that stderr contains the +named error code. Negative tests do NOT carry `required_features:` +unless the rejection itself depends on a Foundation-level feature; the +goal is for every adapter (Foundation or extended) to produce the same +error. + +## xfail policy + +A case marked `xfail_reason: ` is shipped but expected to fail +until the named sprint flips it to `must_pass`. Every `xfail_reason` +MUST cite the sprint ID (e.g., `xfail_reason: "Sprint S-7 — default +join shape rewrite (D-004)"`) so the rollout can find every red row +when the implementation lands. diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql new file mode 100644 index 00000000..72c1f429 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, SUM(o.amount) AS total_order_amount +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml new file mode 100644 index 00000000..c2e07555 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005a-single-step-sum +description: "Single-step cross-grain SUM over 1:N accepted; identical to nested SUM (distributive)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, distributive] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005a +status: planned +xfail_reason: "Sprint S-5 — single-step cross-grain aggregation (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml new file mode 100644 index 00000000..f40095f4 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: total_order_amount, expression: "SUM(orders.amount)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json new file mode 100644 index 00000000..0cdd1322 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "total_order_amount", + "metric": "total_order_amount" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql new file mode 100644 index 00000000..b02fe99a --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, AVG(o.amount) AS avg_order_amount +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml new file mode 100644 index 00000000..700f35a9 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005b-single-step-avg +description: "Single-step cross-grain AVG over 1:N accepted; heavy-customer-weighted (standard SQL AVG semantics)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, non-distributive] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005b +status: planned +xfail_reason: "Sprint S-5 — single-step cross-grain AVG (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml new file mode 100644 index 00000000..ddbd0a0d --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: avg_order_amount, expression: "AVG(orders.amount)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json new file mode 100644 index 00000000..4a39b104 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "avg_order_amount", + "metric": "avg_order_amount" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql new file mode 100644 index 00000000..45078103 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, SUM(o.amount) AS sum_single +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml new file mode 100644 index 00000000..a46fdd62 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml @@ -0,0 +1,13 @@ +name: t-005d-single-step-vs-nested-sum +description: "Single-step SUM and nested SUM(SUM(...)) over 1:N produce identical row sets (distributive)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" +tags: [cross-grain, 1:N, distributive, equivalence] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005d +status: planned +xfail_reason: "Sprint S-5 — single-step ≡ nested SUM equivalence (D-020)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml new file mode 100644 index 00000000..7dbe616f --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml @@ -0,0 +1,39 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: sum_single, expression: "SUM(orders.amount)"} + # sum_nested intentionally removed — nested aggregation in a metric + # body is deferred (D-027(d)); see Proposed_OSI_Semantics.md §10. diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json new file mode 100644 index 00000000..42ed57b7 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "sum_single", + "metric": "sum_single" + } + ] +} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql new file mode 100644 index 00000000..cf18293f --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/gold.sql @@ -0,0 +1,4 @@ +SELECT c.region AS region, COUNT(DISTINCT o.status) AS distinct_order_statuses +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml new file mode 100644 index 00000000..5979567a --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml @@ -0,0 +1,14 @@ +name: t-005e-count-distinct-single-step +description: "Single-step COUNT(DISTINCT) over 1:N accepted (holistic but well-defined on plain 1:N)" +area: cross_grain +difficulty: moderate +dataset: f_prelude +spec_refs: + - "Proposed_OSI_Semantics.md#D-020" + - "Proposed_OSI_Semantics.md#D-022" +tags: [cross-grain, 1:N, count-distinct, holistic] +conformance_level: foundation_v0_1 +decision: D-020 +test_id: T-005e +status: planned +xfail_reason: "Sprint S-5 — COUNT(DISTINCT) single-step over 1:N (D-020 (d), D-022 caveat)" diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml new file mode 100644 index 00000000..916557fc --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/model.yaml @@ -0,0 +1,37 @@ +name: f_prelude_model +datasets: + - name: customers + source: customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: region, expression: region, dimension: {}} + - {name: segment, expression: segment, dimension: {}} + - name: orders + source: orders + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: status, expression: status, dimension: {}} + - {name: amount, expression: amount} + - name: returns + source: returns + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} + - {name: customer_id, expression: customer_id, dimension: {}} + - {name: amount, expression: amount} + - name: premium_customers + source: premium_customers + primary_key: [id] + fields: + - {name: id, expression: id, dimension: {}} +relationships: + - {name: orders_to_customer, from: orders, to: customers, from_columns: [customer_id], to_columns: [id]} + - {name: returns_to_customer, from: returns, to: customers, from_columns: [customer_id], to_columns: [id]} +metrics: + - {name: total_revenue, expression: "SUM(orders.amount)"} + - {name: total_returns, expression: "SUM(returns.amount)"} + - {name: order_count, expression: "COUNT(orders.id)"} + - {name: distinct_order_statuses, expression: "COUNT(DISTINCT orders.status)"} diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json new file mode 100644 index 00000000..09b7c631 --- /dev/null +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/query.json @@ -0,0 +1,12 @@ +{ + "dataset": "customers", + "dimensions": [ + "customers.region" + ], + "measures": [ + { + "name": "distinct_order_statuses", + "metric": "distinct_order_statuses" + } + ] +} diff --git a/compliance/harness/README.md b/compliance/harness/README.md new file mode 100644 index 00000000..cee3a148 --- /dev/null +++ b/compliance/harness/README.md @@ -0,0 +1,54 @@ + + +# OSI Compliance Harness + +Shared runner / reporter / DB manager for the OSI compliance suite. + +This package is the engine behind every per-version compliance suite +under `compliance/`. It is **engine-agnostic** — it does not know about +any specific OSI implementation. Engines plug in via an *adapter* that +implements the CLI contract documented in +[`../ADAPTER_INTERFACE.md`](../ADAPTER_INTERFACE.md). + +## Install + +```bash +pip install -e . +``` + +This installs the `harness` package. The compliance suites then depend on +this package to run their tests. + +## Run + +The harness resolves ``--output`` relative to the current working +directory, so run it from the suite root (per-run artifacts then land +under ``/results/latest/`` by default): + +```bash +cd ../foundation +python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ +``` + +See [`../foundation-v0.1/README.md`](../foundation/README.md) for +the suite-level entry point and reporting layout. diff --git a/compliance/harness/pyproject.toml b/compliance/harness/pyproject.toml new file mode 100644 index 00000000..abc62adc --- /dev/null +++ b/compliance/harness/pyproject.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "osi_compliance_harness" +version = "0.1.0" +description = "Shared runner / reporter / DB manager for the OSI compliance suite." +readme = "README.md" +requires-python = ">=3.11" +authors = [{ name = "OSI Foundation contributors" }] +license = { text = "Apache-2.0" } +dependencies = [ + "duckdb>=0.10", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = ["pytest>=7.4"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["harness*"] diff --git a/compliance/harness/src/harness/__init__.py b/compliance/harness/src/harness/__init__.py new file mode 100644 index 00000000..9aa0a35e --- /dev/null +++ b/compliance/harness/src/harness/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""OSI Compliance Test Suite — Test Harness.""" diff --git a/compliance/harness/src/harness/__main__.py b/compliance/harness/src/harness/__main__.py new file mode 100644 index 00000000..69b68036 --- /dev/null +++ b/compliance/harness/src/harness/__main__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Allow running the harness as: python -m harness""" + +from .runner import main + +raise SystemExit(main()) diff --git a/compliance/harness/src/harness/db_manager.py b/compliance/harness/src/harness/db_manager.py new file mode 100644 index 00000000..b1f98c38 --- /dev/null +++ b/compliance/harness/src/harness/db_manager.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""DuckDB database management for the test harness.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import duckdb + + +class DBManager: + """Manages DuckDB connections and dataset loading.""" + + def __init__(self) -> None: + self._conn: duckdb.DuckDBPyConnection | None = None + self._loaded_datasets: set[str] = set() + + def connect(self) -> duckdb.DuckDBPyConnection: + """Create a fresh in-memory DuckDB connection.""" + self.close() + self._conn = duckdb.connect(":memory:") + self._loaded_datasets.clear() + return self._conn + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + self._loaded_datasets.clear() + + def load_dataset(self, dataset_name: str, datasets_dir: Path) -> None: + """Load a dataset's schema.sql into the current connection.""" + if self._conn is None: + raise RuntimeError("No active database connection") + + if dataset_name in self._loaded_datasets: + return + + schema_path = datasets_dir / dataset_name / "schema.sql" + if not schema_path.exists(): + raise FileNotFoundError(f"Dataset schema not found: {schema_path}") + + sql_text = schema_path.read_text() + for statement in sql_text.split(";"): + lines = [line for line in statement.splitlines() if not line.strip().startswith("--")] + stmt = "\n".join(lines).strip() + if stmt: + self._conn.execute(stmt) + + self._loaded_datasets.add(dataset_name) + + def execute_sql(self, sql: str) -> list[dict[str, Any]]: + """Execute SQL and return results as a list of dicts.""" + if self._conn is None: + raise RuntimeError("No active database connection") + + result = self._conn.execute(sql) + columns = [desc[0] for desc in result.description] + rows = result.fetchall() + + return [dict(zip(columns, row)) for row in rows] + + def reset(self) -> None: + """Drop all tables and reset the connection.""" + self.connect() diff --git a/compliance/harness/src/harness/models.py b/compliance/harness/src/harness/models.py new file mode 100644 index 00000000..76ae181c --- /dev/null +++ b/compliance/harness/src/harness/models.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Data models for the OSI compliance test harness.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from enum import Enum +from functools import cached_property +from pathlib import Path +from typing import Any + + +class Difficulty(str, Enum): + EASY = "easy" + MODERATE = "moderate" + HARD = "hard" + CONVERSION = "conversion" + + +class TestStatus(str, Enum): + PASS = "pass" + FAIL = "fail" + ERROR = "error" + SKIP = "skip" + + +@dataclass(frozen=True) +class TestCase: + """A single compliance test case loaded from disk.""" + + test_id: str + name: str + description: str + area: str + difficulty: str + dataset: str + spec_refs: list[str] + tags: list[str] + model_path: Path + query_path: Path + gold_sql_path: Path + test_dir: Path + expected_error: bool = False + expected_error_code: str = "" + conformance_level: str = "full" + status: str = "active" # "active" or "planned" — planned tests skipped unless --include-planned + required_features: list[str] = field(default_factory=list) # Feature IDs; skip if adapter doesn't support + + @cached_property + def has_order_by(self) -> bool: + """Check if the query specifies an order_by clause.""" + qdict = json.loads(self.query_path.read_text()) + return bool(qdict.get("order_by")) + + +@dataclass +class TestResult: + """Result of running a single test case.""" + + test_id: str + area: str + difficulty: str + status: TestStatus + spec_refs: list[str] = field(default_factory=list) + error_type: str = "" + error_detail: str = "" + generated_sql: str = "" + generated_rows: list[dict[str, Any]] = field(default_factory=list) + gold_rows: list[dict[str, Any]] = field(default_factory=list) + duration_ms: float = 0.0 + required_features: list[str] = field(default_factory=list) + + +@dataclass +class SuiteResult: + """Aggregate results for a test suite run.""" + + adapter: str + results: list[TestResult] = field(default_factory=list) + # None means "no filter applied" (all proposals implicitly enabled). + adapter_features: frozenset[str] | None = None + + @property + def total(self) -> int: + return len(self.results) + + @property + def passed(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.PASS) + + @property + def failed(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.FAIL) + + @property + def errors(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.ERROR) + + @property + def skipped(self) -> int: + return sum(1 for r in self.results if r.status == TestStatus.SKIP) diff --git a/compliance/harness/src/harness/proposals_check.py b/compliance/harness/src/harness/proposals_check.py new file mode 100644 index 00000000..32b525c2 --- /dev/null +++ b/compliance/harness/src/harness/proposals_check.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate ``required_features`` in test metadata against proposals.yaml. + +Run via ``python -m harness.proposals_check`` (see ``__main__`` guard). +CI calls this; a non-zero exit signals an unknown or misspelled proposal +ID, which would otherwise silently skip tests on every adapter. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Iterable + +import yaml + +PROPOSALS_FILE = "proposals.yaml" +TESTS_DIR = "tests" +METADATA_FILE = "metadata.yaml" +# ``foundation`` denotes a proposal that is in scope for Foundation v0.1 +# (the proposal at proposals/foundation/); ``deferred`` denotes a +# §10 deferred feature; ``proposed`` / ``thin_slice`` were the +# pre-Foundation rollout statuses. New rows should use ``foundation`` +# or ``deferred`` — the older two are retained so legacy proposal rows +# parse without churn. +VALID_STATUS = {"foundation", "thin_slice", "proposed", "deferred"} + + +class ProposalsError(Exception): + """Raised when the registry itself is malformed.""" + + +def load_proposal_ids(root: Path) -> set[str]: + """Return the set of declared proposal IDs. + + Raises :class:`ProposalsError` if the registry is missing keys, + has duplicates, or lists an unknown ``status``. + """ + path = root / PROPOSALS_FILE + data = yaml.safe_load(path.read_text()) + if not isinstance(data, dict) or "proposals" not in data: + raise ProposalsError(f"{path}: missing top-level 'proposals:' key") + + ids: set[str] = set() + for entry in data["proposals"]: + if not isinstance(entry, dict): + raise ProposalsError(f"{path}: every proposal must be a mapping") + try: + pid = entry["id"] + status = entry["status"] + except KeyError as exc: + raise ProposalsError( + f"{path}: proposal entry missing required key: {exc}" + ) from exc + if status not in VALID_STATUS: + raise ProposalsError( + f"{path}: proposal {pid!r} has invalid status {status!r}; " + f"expected one of {sorted(VALID_STATUS)}" + ) + if pid in ids: + raise ProposalsError(f"{path}: duplicate proposal id {pid!r}") + ids.add(pid) + return ids + + +def iter_metadata_files(root: Path) -> Iterable[Path]: + """Yield every ``metadata.yaml`` under the tests tree.""" + yield from sorted((root / TESTS_DIR).rglob(METADATA_FILE)) + + +def collect_unknown_references( + root: Path, valid_ids: set[str] +) -> list[tuple[Path, list[str]]]: + """Return [(metadata_path, unknown_ids)] for every offending file.""" + offenders: list[tuple[Path, list[str]]] = [] + for meta_path in iter_metadata_files(root): + meta = yaml.safe_load(meta_path.read_text()) or {} + features = meta.get("required_features", []) or [] + if not isinstance(features, list): + offenders.append((meta_path, [f": {features!r}"])) + continue + unknown = [f for f in features if f not in valid_ids] + if unknown: + offenders.append((meta_path, unknown)) + return offenders + + +def _format_report( + offenders: list[tuple[Path, list[str]]], + valid_ids: set[str], + root: Path, +) -> str: + lines = [ + "ERROR: unknown proposal IDs referenced in test metadata.", + f"Registry: {(root / PROPOSALS_FILE).as_posix()}", + "", + "Offending files:", + ] + for path, unknown in offenders: + rel = path.relative_to(root) + lines.append(f" {rel}") + for u in unknown: + lines.append(f" - {u}") + lines.append("") + lines.append( + "Either add the proposal to proposals.yaml (with a status), " + "or fix the spelling in the metadata file." + ) + lines.append(f"Known IDs ({len(valid_ids)}): {', '.join(sorted(valid_ids))}") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + """Entry point. Returns 0 on success, 1 on any violation.""" + argv = argv if argv is not None else sys.argv[1:] + root = Path(argv[0]).resolve() if argv else Path(__file__).resolve().parent.parent + + try: + valid_ids = load_proposal_ids(root) + except ProposalsError as exc: + print(f"FATAL: {exc}", file=sys.stderr) + return 2 + + offenders = collect_unknown_references(root, valid_ids) + if offenders: + print(_format_report(offenders, valid_ids, root), file=sys.stderr) + return 1 + + count = sum(1 for _ in iter_metadata_files(root)) + print( + f"OK: {count} metadata files validated against {len(valid_ids)} " + f"registered proposal IDs." + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/compliance/harness/src/harness/reporter.py b/compliance/harness/src/harness/reporter.py new file mode 100644 index 00000000..0966eefb --- /dev/null +++ b/compliance/harness/src/harness/reporter.py @@ -0,0 +1,234 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Report generation for the OSI compliance test suite.""" + +from __future__ import annotations + +import csv +import io +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +import yaml + +from .models import SuiteResult, TestResult, TestStatus + +PROPOSALS_FILE_NAME = "proposals.yaml" + + +def write_reports(suite: SuiteResult, output_dir: Path) -> tuple[Path, Path]: + """Write failure CSV and summary MD to output_dir. Returns (csv_path, md_path).""" + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = output_dir / "failures.csv" + md_path = output_dir / "summary.md" + + _write_failures_csv(suite, csv_path) + _write_summary_md(suite, md_path) + + return csv_path, md_path + + +def _write_failures_csv(suite: SuiteResult, path: Path) -> None: + failures = [r for r in suite.results if r.status in (TestStatus.FAIL, TestStatus.ERROR)] + + with open(path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow( + [ + "test_id", + "area", + "difficulty", + "spec_refs", + "error_type", + "details", + ] + ) + for r in failures: + writer.writerow( + [ + r.test_id, + r.area, + r.difficulty, + "; ".join(r.spec_refs), + r.error_type, + r.error_detail, + ] + ) + + +def _write_summary_md(suite: SuiteResult, path: Path) -> None: + buf = io.StringIO() + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + buf.write("# OSI Compliance Test Suite — Summary\n\n") + buf.write(f"**Adapter**: {suite.adapter} \n") + buf.write(f"**Date**: {now} \n\n") + + pct = (suite.passed / suite.total * 100) if suite.total > 0 else 0 + buf.write("## Overall\n\n") + buf.write("| Metric | Count |\n") + buf.write("|--------|-------|\n") + buf.write(f"| Total | {suite.total} |\n") + buf.write(f"| Passed | {suite.passed} |\n") + buf.write(f"| Failed | {suite.failed} |\n") + buf.write(f"| Errors | {suite.errors} |\n") + buf.write(f"| Skipped | {suite.skipped} |\n") + buf.write(f"| **Compliance** | **{pct:.1f}%** |\n\n") + + _write_breakdown(buf, "By Area", suite.results, key=lambda r: r.area) + _write_breakdown(buf, "By Difficulty", suite.results, key=lambda r: r.difficulty) + _write_proposals_status(buf, suite) + + failures = [r for r in suite.results if r.status in (TestStatus.FAIL, TestStatus.ERROR)] + if failures: + buf.write("## Failures\n\n") + buf.write("| Test | Area | Difficulty | Error |\n") + buf.write("|------|------|------------|-------|\n") + for r in failures: + detail = r.error_detail[:80].replace("|", "\\|") if r.error_detail else "" + buf.write(f"| {r.test_id} | {r.area} | {r.difficulty} | {detail} |\n") + buf.write("\n") + + path.write_text(buf.getvalue()) + + +def _write_breakdown( + buf: io.StringIO, + title: str, + results: list[TestResult], + key, +) -> None: + groups: dict[str, list[TestResult]] = defaultdict(list) + for r in results: + groups[key(r)].append(r) + + buf.write(f"## {title}\n\n") + buf.write("| Group | Total | Pass | Fail | Error | Skip | % |\n") + buf.write("|-------|-------|------|------|-------|------|---|\n") + + for group_name in sorted(groups.keys()): + group = groups[group_name] + total = len(group) + passed = sum(1 for r in group if r.status == TestStatus.PASS) + failed = sum(1 for r in group if r.status == TestStatus.FAIL) + errored = sum(1 for r in group if r.status == TestStatus.ERROR) + skipped = sum(1 for r in group if r.status == TestStatus.SKIP) + pct = (passed / total * 100) if total > 0 else 0 + buf.write(f"| {group_name} | {total} | {passed} | {failed} " f"| {errored} | {skipped} | {pct:.0f}% |\n") + buf.write("\n") + + +def _load_proposal_registry(*starts: Path) -> dict[str, str]: + """Return ``{proposal_id: status}``, searching upwards for ``proposals.yaml``. + + Walks up from every given start path. Empty dict when the registry + isn't findable; the caller falls back to the inferred set of IDs + seen across results. + """ + tried: set[Path] = set() + for start in starts: + cursor = start.resolve() + for _ in range(6): + if cursor in tried: + break + tried.add(cursor) + candidate = cursor / PROPOSALS_FILE_NAME + if candidate.exists(): + data = yaml.safe_load(candidate.read_text()) or {} + return {p["id"]: p.get("status", "") for p in data.get("proposals", [])} + if cursor.parent == cursor: + break + cursor = cursor.parent + return {} + + +def _write_proposals_status(buf: io.StringIO, suite: SuiteResult) -> None: + """Emit the Proposals-status section. + + For every proposal ID that is either advertised by the adapter or + referenced by at least one result, report: + + * whether the adapter advertised it (``enabled``), + * total tests that require it, + * tests that ran vs. were skipped, + * pass rate among the ones that ran. + """ + registry = _load_proposal_registry(Path.cwd(), Path(__file__).parent) + + referenced: set[str] = set() + for r in suite.results: + referenced.update(r.required_features) + advertised: set[str] = set(suite.adapter_features or ()) + all_ids = sorted(referenced | advertised | set(registry.keys())) + if not all_ids: + return + + counts_total: dict[str, int] = defaultdict(int) + counts_ran: dict[str, int] = defaultdict(int) + counts_passed: dict[str, int] = defaultdict(int) + counts_skipped: dict[str, int] = defaultdict(int) + + for r in suite.results: + for feat in r.required_features: + counts_total[feat] += 1 + if r.status == TestStatus.SKIP and r.error_type == "unsupported_proposal": + counts_skipped[feat] += 1 + else: + counts_ran[feat] += 1 + if r.status == TestStatus.PASS: + counts_passed[feat] += 1 + + buf.write("## Proposals Status\n\n") + if suite.adapter_features is None: + buf.write("_No `--proposals` filter applied; every proposal implicitly enabled._\n\n") + else: + buf.write(f"Adapter advertised {len(advertised)} proposal(s): " f"`{', '.join(sorted(advertised)) or '(none)'}`\n\n") + + buf.write("| Proposal | Status | Enabled | Tests | Ran | Passed | Skipped | Pass% |\n") + buf.write("|----------|--------|---------|-------|-----|--------|---------|-------|\n") + for pid in all_ids: + status = registry.get(pid, "unknown") + enabled = "yes" if (suite.adapter_features is None or pid in advertised) else "no" + total = counts_total.get(pid, 0) + ran = counts_ran.get(pid, 0) + passed = counts_passed.get(pid, 0) + skipped = counts_skipped.get(pid, 0) + pct = f"{(passed / ran * 100):.0f}%" if ran else "—" + buf.write(f"| `{pid}` | {status} | {enabled} | {total} | {ran} | {passed} | {skipped} | {pct} |\n") + buf.write("\n") + + +def format_summary_console(suite: SuiteResult) -> str: + """Format a brief console summary.""" + pct = (suite.passed / suite.total * 100) if suite.total > 0 else 0 + lines = [ + f"\n{'=' * 50}", + f"OSI Compliance Test Results — {suite.adapter}", + f"{'=' * 50}", + f" Total: {suite.total}", + f" Passed: {suite.passed}", + f" Failed: {suite.failed}", + f" Errors: {suite.errors}", + f" Skipped: {suite.skipped}", + f" Compliance: {pct:.1f}%", + ] + if suite.adapter_features is not None: + lines.append(f" Proposals: {', '.join(sorted(suite.adapter_features)) or '(none)'}") + lines.append(f"{'=' * 50}") + return "\n".join(lines) diff --git a/compliance/harness/src/harness/result_compare.py b/compliance/harness/src/harness/result_compare.py new file mode 100644 index 00000000..36421f89 --- /dev/null +++ b/compliance/harness/src/harness/result_compare.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Result comparison logic with order-insensitive and numeric-tolerant matching.""" + +from __future__ import annotations + +import math +from datetime import date, datetime, time +from decimal import Decimal +from typing import Any + +DEFAULT_EPSILON = 0.0001 + + +def compare_results( + generated: list[dict[str, Any]], + gold: list[dict[str, Any]], + *, + ordered: bool = False, + epsilon: float = DEFAULT_EPSILON, +) -> tuple[bool, str]: + """Compare two result sets. + + Returns (is_match, detail_message). + """ + if len(generated) != len(gold): + return False, (f"Row count mismatch: generated {len(generated)} rows, " f"gold {len(gold)} rows") + + if not generated: + return True, "Both result sets are empty" + + gen_cols = set(_normalize_key(k) for k in generated[0].keys()) + gold_cols = set(_normalize_key(k) for k in gold[0].keys()) + + if gen_cols != gold_cols: + missing = gold_cols - gen_cols + extra = gen_cols - gold_cols + parts = [] + if missing: + parts.append(f"missing columns: {sorted(missing)}") + if extra: + parts.append(f"extra columns: {sorted(extra)}") + return False, f"Column mismatch: {'; '.join(parts)}" + + gen_normalized = [_normalize_row(r) for r in generated] + gold_normalized = [_normalize_row(r) for r in gold] + + if not ordered: + gen_normalized = _sort_rows(gen_normalized) + gold_normalized = _sort_rows(gold_normalized) + + for i, (gen_row, gold_row) in enumerate(zip(gen_normalized, gold_normalized)): + match, detail = _compare_rows(gen_row, gold_row, epsilon) + if not match: + return False, f"Row {i} mismatch: {detail}" + + return True, f"All {len(generated)} rows match" + + +def _normalize_key(key: str) -> str: + return key.lower().strip() + + +def _normalize_row(row: dict[str, Any]) -> dict[str, Any]: + return {_normalize_key(k): v for k, v in row.items()} + + +def _sort_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Sort rows by all columns for order-insensitive comparison.""" + if not rows: + return rows + columns = sorted(rows[0].keys()) + + def sort_key(row: dict[str, Any]) -> tuple: + return tuple(_sortable_value(row.get(c)) for c in columns) + + return sorted(rows, key=sort_key) + + +def _sortable_value(val: Any) -> tuple: + """Convert a value to a sortable tuple (type_tag, comparable_value).""" + if val is None: + return (0, "") + if isinstance(val, bool): + return (1, int(val)) + if isinstance(val, (int, float, Decimal)): + f = float(val) + if math.isnan(f): + return (2, float("inf")) + return (2, f) + return (3, str(val)) + + +def _compare_rows( + gen: dict[str, Any], + gold: dict[str, Any], + epsilon: float, +) -> tuple[bool, str]: + for key in gold: + gen_val = gen.get(key) + gold_val = gold.get(key) + + if not _values_equal(gen_val, gold_val, epsilon): + return False, (f"column '{key}': generated={gen_val!r}, gold={gold_val!r}") + return True, "" + + +def _values_equal(a: Any, b: Any, epsilon: float) -> bool: + if a is None and b is None: + return True + if a is None or b is None: + return False + + if isinstance(a, (int, float, Decimal)) and isinstance(b, (int, float, Decimal)): + fa, fb = float(a), float(b) + if fa == fb: + return True + if math.isnan(fa) and math.isnan(fb): + return True + if math.isnan(fa) or math.isnan(fb): + return False + if abs(fa - fb) < epsilon: + return True + denom = max(abs(fa), abs(fb)) + if denom == 0: + return True + return abs(fa - fb) / denom < epsilon + + if isinstance(a, datetime) and isinstance(b, datetime): + return a == b + if isinstance(a, datetime) and isinstance(b, date): + return a.date() == b + if isinstance(a, date) and isinstance(b, datetime): + return a == b.date() + if isinstance(a, date) and isinstance(b, date): + return a == b + + if isinstance(a, time) and isinstance(b, time): + return a == b + + return str(a) == str(b) diff --git a/compliance/harness/src/harness/runner.py b/compliance/harness/src/harness/runner.py new file mode 100644 index 00000000..46732320 --- /dev/null +++ b/compliance/harness/src/harness/runner.py @@ -0,0 +1,530 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Main test runner for the OSI compliance test suite.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time +from pathlib import Path + +import yaml + +from .db_manager import DBManager +from .models import SuiteResult, TestCase, TestResult, TestStatus +from .reporter import format_summary_console, write_reports +from .result_compare import compare_results + + +def discover_tests( + tests_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + include_planned: bool = False, +) -> list[TestCase]: + """Walk the tests directory and discover all test cases.""" + test_cases: list[TestCase] = [] + + for metadata_path in sorted(tests_dir.rglob("metadata.yaml")): + test_dir = metadata_path.parent + meta = yaml.safe_load(metadata_path.read_text()) + + test_difficulty = meta.get("difficulty", "") + test_area = meta.get("area", "") + + if difficulty and test_difficulty != difficulty: + continue + if area and test_area != area: + continue + + model_path = test_dir / "model.yaml" + query_path = test_dir / "query.json" + gold_sql_path = test_dir / "gold.sql" + + if not all(p.exists() for p in [model_path, query_path, gold_sql_path]): + print( + f"WARN: Skipping {test_dir.name} — missing required files", + file=sys.stderr, + ) + continue + + parts = test_dir.relative_to(tests_dir).parts + test_id = "/".join(parts) + + test_status = meta.get("status", "active") + if test_status == "planned" and not include_planned: + continue + + test_cases.append( + TestCase( + test_id=test_id, + name=meta.get("name", test_dir.name), + description=meta.get("description", ""), + area=test_area, + difficulty=test_difficulty, + dataset=meta.get("dataset", ""), + spec_refs=meta.get("spec_refs", []), + tags=meta.get("tags", []), + model_path=model_path, + query_path=query_path, + gold_sql_path=gold_sql_path, + test_dir=test_dir, + expected_error=bool(meta.get("expected_error", False)), + expected_error_code=meta.get("expected_error_code", ""), + conformance_level=meta.get("conformance_level", "full"), + status=test_status, + required_features=meta.get("required_features", []), + ) + ) + + return test_cases + + +def invoke_adapter( + adapter_path: Path, + model_path: Path, + query_path: Path, + dialect: str = "duckdb", + timeout: int = 60, +) -> tuple[str, str, int]: + """Invoke the adapter subprocess. Returns (stdout, stderr, returncode).""" + cmd = [ + sys.executable, + str(adapter_path), + "sql", + "--model", + str(model_path), + "--query-file", + str(query_path), + "--dialect", + dialect, + ] + + try: + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + return proc.stdout, proc.stderr, proc.returncode + except subprocess.TimeoutExpired: + return "", "Adapter timed out", 1 + + +def run_test( + test: TestCase, + adapter_path: Path, + db: DBManager, + datasets_dir: Path, +) -> TestResult: + """Run a single test case and return the result.""" + start = time.monotonic() + + try: + db.reset() + db.load_dataset(test.dataset, datasets_dir) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="dataset_load", + error_detail=str(e), + duration_ms=(time.monotonic() - start) * 1000, + ) + + stdout, stderr, rc = invoke_adapter( + adapter_path, + test.model_path, + test.query_path, + ) + + if test.expected_error: + elapsed = (time.monotonic() - start) * 1000 + if rc != 0: + # Adapter correctly rejected — check error code if specified + if test.expected_error_code and test.expected_error_code not in stderr: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="wrong_error_code", + error_detail=( + f"Expected error code '{test.expected_error_code}' " + f"in stderr but got: {stderr.strip()[:200]}" + ), + duration_ms=elapsed, + ) + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.PASS, + spec_refs=test.spec_refs, + error_detail=f"Correctly rejected: {stderr.strip()[:200]}", + duration_ms=elapsed, + ) + else: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="expected_error_missing", + error_detail=( + "Expected adapter to reject this query with a non-zero " + "exit code, but it succeeded" + ), + generated_sql=stdout.strip(), + duration_ms=elapsed, + ) + + if rc != 0: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="adapter_error", + error_detail=stderr.strip() or f"exit code {rc}", + duration_ms=(time.monotonic() - start) * 1000, + ) + + generated_sql = stdout.strip() + + try: + generated_rows = db.execute_sql(generated_sql) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="generated_sql_error", + error_detail=f"Generated SQL failed: {e}", + generated_sql=generated_sql, + duration_ms=(time.monotonic() - start) * 1000, + ) + + gold_sql = test.gold_sql_path.read_text().strip() + try: + gold_rows = db.execute_sql(gold_sql) + except Exception as e: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="gold_sql_error", + error_detail=f"Gold SQL failed: {e}", + generated_sql=generated_sql, + duration_ms=(time.monotonic() - start) * 1000, + ) + + is_ordered = test.has_order_by + match, detail = compare_results( + generated_rows, + gold_rows, + ordered=is_ordered, + ) + + elapsed = (time.monotonic() - start) * 1000 + + if match: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.PASS, + spec_refs=test.spec_refs, + generated_sql=generated_sql, + generated_rows=generated_rows, + gold_rows=gold_rows, + duration_ms=elapsed, + ) + else: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.FAIL, + spec_refs=test.spec_refs, + error_type="result_mismatch", + error_detail=detail, + generated_sql=generated_sql, + generated_rows=generated_rows, + gold_rows=gold_rows, + duration_ms=elapsed, + ) + + +def list_tests( + tests_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + conformance_level: str | None = None, + include_planned: bool = False, +) -> None: + """Print discovered tests without running them.""" + tests = discover_tests( + tests_dir, + difficulty=difficulty, + area=area, + include_planned=include_planned, + ) + if conformance_level: + tests = [t for t in tests if t.conformance_level == conformance_level] + + print(f"{'ID':<60} {'Area':<25} {'Diff':<12} {'Level':<10} {'Status':<8} {'Err?'}") + print("-" * 125) + for t in tests: + err = "yes" if t.expected_error else "" + status = t.status if t.status != "active" else "" + print( + f"{t.test_id:<60} {t.area:<25} {t.difficulty:<12} {t.conformance_level:<10} {status:<8} {err}" + ) + print(f"\nTotal: {len(tests)} test(s)") + + +def run_suite( + adapter_path: Path, + tests_dir: Path, + datasets_dir: Path, + output_dir: Path, + *, + difficulty: str | None = None, + area: str | None = None, + conformance_level: str | None = None, + include_planned: bool = False, + verbose: bool = False, + adapter_features: set[str] | None = None, +) -> SuiteResult: + """Run the full test suite and generate reports.""" + tests = discover_tests( + tests_dir, + difficulty=difficulty, + area=area, + include_planned=include_planned, + ) + if conformance_level: + tests = [t for t in tests if t.conformance_level == conformance_level] + + skipped_by_feature: list[TestCase] = [] + runnable_tests: list[TestCase] = tests + if adapter_features is not None: + runnable_tests = [] + for t in tests: + if t.required_features and not set(t.required_features).issubset( + adapter_features + ): + skipped_by_feature.append(t) + continue + runnable_tests.append(t) + + if not runnable_tests and not skipped_by_feature: + print("No test cases found.", file=sys.stderr) + return SuiteResult(adapter=str(adapter_path)) + + print(f"Discovered {len(tests)} test(s)") + if skipped_by_feature: + print( + f" skipping {len(skipped_by_feature)} test(s) " + "with unsupported proposals" + ) + print(f"Adapter: {adapter_path}") + print(f"Datasets: {datasets_dir}") + print() + + db = DBManager() + suite = SuiteResult( + adapter=str(adapter_path.name), + adapter_features=( + frozenset(adapter_features) if adapter_features is not None else None + ), + ) + + for t in skipped_by_feature: + missing = sorted(set(t.required_features) - set(adapter_features or ())) + suite.results.append( + TestResult( + test_id=t.test_id, + area=t.area, + difficulty=t.difficulty, + status=TestStatus.SKIP, + spec_refs=t.spec_refs, + error_type="unsupported_proposal", + error_detail=f"required proposals not advertised: {', '.join(missing)}", + required_features=list(t.required_features), + ) + ) + + for i, test in enumerate(runnable_tests, 1): + label = f"[{i}/{len(runnable_tests)}] {test.test_id}" + result = run_test(test, adapter_path, db, datasets_dir) + result.required_features = list(test.required_features) + suite.results.append(result) + + if result.status == TestStatus.PASS: + if test.expected_error: + print(f" PASS {label} (expected error)") + else: + print(f" PASS {label}") + elif result.status == TestStatus.FAIL: + print(f" FAIL {label}") + if verbose: + print(f" {result.error_detail}") + elif result.status == TestStatus.ERROR: + print(f" ERR {label}") + if verbose: + print(f" [{result.error_type}] {result.error_detail}") + else: + print(f" SKIP {label}") + + db.close() + + csv_path, md_path = write_reports(suite, output_dir) + print(format_summary_console(suite)) + print("\nReports written to:") + print(f" {csv_path}") + print(f" {md_path}") + + return suite + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="harness.runner", + description="OSI Compliance Test Suite Runner", + ) + parser.add_argument( + "--adapter", + help="Path to the adapter script (e.g., adapters/python_adapter.py)", + ) + parser.add_argument( + "--tests", + required=True, + help="Path to the tests directory", + ) + parser.add_argument( + "--datasets", + help="Path to the datasets directory", + ) + parser.add_argument( + "--output", + default="results/latest", + help=( + "Output directory for reports (default: results/latest). " + "Per-run artifacts (failures.csv, summary.md) go here. " + "The curated baseline at results/REPORT.md is committed and " + "must not be overwritten — choose a subdirectory of results/ " + "(e.g. results//) or another path entirely." + ), + ) + parser.add_argument( + "--difficulty", + choices=["easy", "moderate", "hard", "conversion"], + help="Filter tests by difficulty", + ) + parser.add_argument( + "--area", + help="Filter tests by area (e.g., grain_and_lod, filters)", + ) + parser.add_argument( + "--conformance-level", + choices=["core", "full", "extended"], + help="Filter tests by conformance level", + ) + parser.add_argument( + "--include-planned", + action="store_true", + help="Include tests with status: planned (normally skipped)", + ) + parser.add_argument( + "--list", + action="store_true", + dest="list_only", + help="List discovered tests without running them", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Show error details for failures", + ) + parser.add_argument( + "--adapter-features", + "--proposals", + nargs="*", + default=None, + dest="adapter_features", + help="Proposal IDs the adapter implements (from proposals.yaml). " + "Tests whose required_features aren't a subset of this set are " + "recorded as SKIP. Alias: --proposals.", + ) + + args = parser.parse_args() + + if args.list_only: + list_tests( + Path(args.tests), + difficulty=args.difficulty, + area=args.area, + conformance_level=args.conformance_level, + include_planned=args.include_planned, + ) + return 0 + + if not args.adapter: + parser.error("--adapter is required when running tests") + if not args.datasets: + parser.error("--datasets is required when running tests") + + feat = set(args.adapter_features) if args.adapter_features is not None else None + suite = run_suite( + adapter_path=Path(args.adapter), + tests_dir=Path(args.tests), + datasets_dir=Path(args.datasets), + output_dir=Path(args.output), + difficulty=args.difficulty, + area=args.area, + conformance_level=args.conformance_level, + include_planned=args.include_planned, + verbose=args.verbose, + adapter_features=feat, + ) + + if suite.failed + suite.errors > 0: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compliance/harness/src/harness/tests/__init__.py b/compliance/harness/src/harness/tests/__init__.py new file mode 100644 index 00000000..a595c844 --- /dev/null +++ b/compliance/harness/src/harness/tests/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + diff --git a/compliance/harness/src/harness/tests/test_proposals_check.py b/compliance/harness/src/harness/tests/test_proposals_check.py new file mode 100644 index 00000000..39b1d7a1 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_proposals_check.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for ``harness.proposals_check``. + +The check runs in CI and guards that every ``required_features`` entry in +test metadata is backed by a real proposal in ``proposals.yaml``. These +tests use a throwaway fixture tree rather than the live suite, so they +stay deterministic as tests come and go. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness import proposals_check + + +@pytest.fixture +def make_tree(tmp_path: Path): + """Build a minimal suite layout under ``tmp_path`` and return a builder.""" + + def _build(proposals_yaml: str, metadata: dict[str, str]) -> Path: + (tmp_path / "proposals.yaml").write_text(proposals_yaml) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + for rel, body in metadata.items(): + target = tests_dir / rel / "metadata.yaml" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + return tmp_path + + return _build + + +VALID_PROPOSALS = """\ +proposals: + - id: dataset_filters + status: proposed + title: Dataset filters + - id: non_equijoin + status: proposed + title: Non-equijoin + - id: pervasive_scope + status: proposed + title: Pervasive scope +""" + + +def test_clean_tree_returns_zero(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + { + "a/test_ok": "required_features: [dataset_filters]\n", + "b/test_multi": "required_features: [dataset_filters, non_equijoin]\n", + "c/test_no_features": "name: plain\n", + }, + ) + assert proposals_check.main([str(root)]) == 0 + assert "OK:" in capsys.readouterr().out + + +def test_unknown_feature_returns_one(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_bad": "required_features: [made_up_feature]\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + err = capsys.readouterr().err + assert "made_up_feature" in err + assert "x/test_bad" in err + + +def test_typo_is_caught(make_tree) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_typo": "required_features: [dataset_filterz]\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + + +def test_non_list_features_is_reported(make_tree, capsys) -> None: + root = make_tree( + VALID_PROPOSALS, + {"x/test_wrong_type": "required_features: dataset_filters\n"}, + ) + assert proposals_check.main([str(root)]) == 1 + assert "not-a-list" in capsys.readouterr().err + + +def test_duplicate_proposal_id_fails_fast(make_tree, capsys) -> None: + root = make_tree( + """\ +proposals: + - id: dupe + status: proposed + - id: dupe + status: proposed +""", + {}, + ) + assert proposals_check.main([str(root)]) == 2 + assert "duplicate" in capsys.readouterr().err + + +def test_invalid_status_fails_fast(make_tree, capsys) -> None: + root = make_tree( + """\ +proposals: + - id: foo + status: maybe +""", + {}, + ) + assert proposals_check.main([str(root)]) == 2 + assert "invalid status" in capsys.readouterr().err + + +def test_missing_top_level_key_fails_fast(make_tree, capsys) -> None: + root = make_tree("proposalz:\n - id: foo\n", {}) + assert proposals_check.main([str(root)]) == 2 + assert "top-level 'proposals:'" in capsys.readouterr().err + + +def test_live_registry_validates() -> None: + """The real ``proposals.yaml`` in the repo passes the check. + + Post-migration, ``proposals.yaml`` and the test corpus live under + ``compliance/foundation/``. We resolve the path relative to + this file so the test stays correct regardless of where the + harness package itself happens to be installed. + """ + foundation_root = ( + Path(__file__).resolve().parents[4] / "foundation" + ) + assert (foundation_root / "proposals.yaml").exists(), ( + f"proposals.yaml not found at {foundation_root} — the suite " + "layout under compliance/foundation/ has changed." + ) + assert proposals_check.main([str(foundation_root)]) == 0 diff --git a/compliance/harness/src/harness/tests/test_registry_yaml.py b/compliance/harness/src/harness/tests/test_registry_yaml.py new file mode 100644 index 00000000..89050f35 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_registry_yaml.py @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cleanliness gate for the compliance registry YAML files. + +``decisions.yaml`` is the source of truth that ties Appendix B decisions +in the Foundation spec to runnable witness tests. ``proposals.yaml`` +plays the same role for §10 deferred features. Both files have already +broken at least once because an unquoted ``:`` inside a title was +parsed by YAML as a mapping value (Phase 4 compliance review, finding +B1). This test pins both files so a future edit cannot reintroduce +that class of bug. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +_FOUNDATION_DIR = ( + Path(__file__).resolve().parents[4] / "foundation" +) + + +@pytest.mark.parametrize( + "yaml_path", + [ + _FOUNDATION_DIR / "decisions.yaml", + _FOUNDATION_DIR / "proposals.yaml", + _FOUNDATION_DIR / "conformance.yaml", + ], + ids=lambda p: p.name, +) +def test_registry_yaml_is_parseable(yaml_path: Path) -> None: + """Every registry YAML file must load as a top-level mapping. + + A future edit that introduces an unquoted ``:`` or a stray ``-`` + breaks the entire coverage-by-decision report — silently in CI + unless this test exists. The assertion is intentionally weak (only + ``isinstance(..., dict)``) because the harness loaders enforce the + shape; here we just want the bare YAML parse to succeed. + """ + assert yaml_path.exists(), f"registry file missing: {yaml_path}" + raw = yaml_path.read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + assert isinstance(loaded, dict), ( + f"{yaml_path.name} did not parse as a top-level YAML mapping; " + "every registry file is required to be a mapping with either a " + "``decisions`` or ``proposals`` top-level key." + ) + + +# Decision IDs that are intentionally absent from decisions.yaml because +# the spec has demoted them. Each entry must come with a one-line +# reference so a future reviewer can verify the demotion is still +# accurate. Update this set in lockstep with the spec. +_INTENTIONALLY_ABSENT_DECISIONS: dict[str, str] = { + # D-013 is reserved in the spec but has no Appendix-B row (the + # number is intentionally skipped — see Proposed_OSI_Semantics.md). + "D-013": "Reserved in Appendix B (number skipped).", + # D-015 is struck in Appendix B (~~D-015~~ Deferred — moved to a + # separate proposal). The compilation-strategy equivalence for + # field-level cross-grain aggregation is moot at the Foundation + # level because field-level aggregation itself is deferred per + # D-003. The Foundation surface is exercised by the D-003 + # rejection witness; the strategy equivalence returns alongside + # §10's grain-aware-functions proposal. + "D-015": "Struck in Appendix B; depends on the deferred D-003.", + # D-017 is deferred — semi-join filtering moved to a follow-up + # proposal. The negative test for EXISTS_IN is a rejection test. + "D-017": "Deferred — Proposed_OSI_Semantics.md table row marked", +} + + +def test_decisions_yaml_has_all_decision_ids() -> None: + """Every active Appendix-B decision must appear exactly once. + + Pinning the ID set here surfaces both: + + - a duplicate ID (``yaml.safe_load`` collapses duplicate keys + silently inside a mapping; we count list entries instead). + - a missing decision (Phase 4 compliance review I7 — several + decisions had no row at all). + + Decisions that the spec has intentionally demoted live in + :data:`_INTENTIONALLY_ABSENT_DECISIONS` with a rationale. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + decisions = loaded.get("decisions", []) + ids = [d["id"] for d in decisions] + + duplicates = {x for x in ids if ids.count(x) > 1} + assert not duplicates, ( + f"decisions.yaml has duplicate ids: {sorted(duplicates)}. Each " + "decision must appear exactly once." + ) + + expected = {f"D-{i:03d}" for i in range(1, 34)} + present = set(ids) + missing = expected - present - set(_INTENTIONALLY_ABSENT_DECISIONS) + assert not missing, ( + f"decisions.yaml is missing rows for: {sorted(missing)}. Every " + "active Appendix-B decision must have a registry row, even if " + "its ``tests:`` list is empty pending witness work. If a " + "decision was intentionally demoted, add it to " + "_INTENTIONALLY_ABSENT_DECISIONS with a rationale." + ) + + # Inverse check: an entry in _INTENTIONALLY_ABSENT_DECISIONS that + # actually appears in decisions.yaml is also drift. + accidental = present & set(_INTENTIONALLY_ABSENT_DECISIONS) + assert not accidental, ( + f"decisions.yaml has rows for IDs marked as intentionally " + f"absent: {sorted(accidental)}. Update " + "_INTENTIONALLY_ABSENT_DECISIONS or remove the row." + ) + + +def test_decisions_yaml_paths_exist_on_disk() -> None: + """Every ``tests:`` path in decisions.yaml must point to a real test. + + Phase 4 review B2 — pre-migration paths still littered the file + and the coverage-by-decision report was reading a fictional disk. + A path that doesn't resolve to a directory with a ``metadata.yaml`` + is a regression: either the test was renamed, moved, or deleted + and decisions.yaml wasn't updated. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + decisions = loaded.get("decisions", []) + + missing: list[tuple[str, str]] = [] + for row in decisions: + for test_rel in row.get("tests") or (): + test_dir = _FOUNDATION_DIR / test_rel + if not (test_dir / "metadata.yaml").exists(): + missing.append((row["id"], test_rel)) + + assert not missing, ( + "decisions.yaml references paths that don't exist on disk:\n" + + "\n".join(f" {d}: {p}" for d, p in missing) + + "\nRegenerate the tests: lists from disk metadata, or move " + "the renamed test back." + ) + + +def test_every_disk_test_pins_a_known_decision() -> None: + """The inverse: every metadata.yaml's ``decision`` must be in + decisions.yaml (or :data:`_INTENTIONALLY_ABSENT_DECISIONS`). + + Closes the second half of the drift gap — without this, a test + can pin a decision that no longer has a registry row and the + coverage report silently misses it. + """ + raw = (_FOUNDATION_DIR / "decisions.yaml").read_text(encoding="utf-8") + loaded = yaml.safe_load(raw) + known = {row["id"] for row in loaded.get("decisions", [])} + known.update(_INTENTIONALLY_ABSENT_DECISIONS) + + unknown_pins: list[tuple[Path, str]] = [] + tests_dir = _FOUNDATION_DIR / "tests" + for meta_path in sorted(tests_dir.rglob("metadata.yaml")): + metadata = yaml.safe_load(meta_path.read_text()) + decision = metadata.get("decision") or metadata.get("decisions") + decisions = ( + [decision] + if isinstance(decision, str) + else (decision or []) + ) + for d in decisions: + if d not in known: + unknown_pins.append((meta_path, d)) + + assert not unknown_pins, ( + "Test metadata pins decisions that don't appear in " + "decisions.yaml:\n" + + "\n".join( + f" {p.relative_to(_FOUNDATION_DIR)}: {d}" + for p, d in unknown_pins + ) + ) diff --git a/compliance/harness/src/harness/tests/test_reporter_proposals.py b/compliance/harness/src/harness/tests/test_reporter_proposals.py new file mode 100644 index 00000000..b833e2ea --- /dev/null +++ b/compliance/harness/src/harness/tests/test_reporter_proposals.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Proposals-status reporter section.""" + +from __future__ import annotations + +import io + +from harness.models import SuiteResult, TestResult +from harness.models import TestStatus as Status +from harness.reporter import ( + _write_proposals_status, + format_summary_console, +) + + +def _res(status: TestStatus, *, features: list[str], error_type: str = "") -> TestResult: + return TestResult( + test_id="x", + area="a", + difficulty="easy", + status=status, + required_features=features, + error_type=error_type, + ) + + +def test_section_groups_results_by_proposal() -> None: + suite = SuiteResult( + adapter="adapter.py", + adapter_features=frozenset({"non_equijoin"}), + results=[ + _res(Status.PASS, features=["non_equijoin"]), + _res(Status.FAIL, features=["non_equijoin"]), + _res( + Status.SKIP, + features=["grain_modes"], + error_type="unsupported_proposal", + ), + _res( + Status.SKIP, + features=["grain_modes", "non_equijoin"], + error_type="unsupported_proposal", + ), + ], + ) + buf = io.StringIO() + _write_proposals_status(buf, suite) + md = buf.getvalue() + + assert "## Proposals Status" in md + # Skipped-due-to-proposal counts are attributed to every required feature, + # even ones the adapter DOES have, because the test still didn't run. + assert "`non_equijoin`" in md + assert "`grain_modes`" in md + # grain_modes row: 2 total, 0 ran, 2 skipped + grain_row = next(line for line in md.splitlines() if line.startswith("| `grain_modes`")) + assert " 2 |" in grain_row # total + assert " 2 |" in grain_row # skipped + # non_equijoin row: 3 total (one PASS, one FAIL, one SKIP), 2 ran, 1 passed + ne_row = next(line for line in md.splitlines() if line.startswith("| `non_equijoin`")) + assert "50%" in ne_row + + +def test_no_filter_applied_marker() -> None: + suite = SuiteResult( + adapter="x", + adapter_features=None, + results=[_res(Status.PASS, features=["non_equijoin"])], + ) + buf = io.StringIO() + _write_proposals_status(buf, suite) + md = buf.getvalue() + assert "implicitly enabled" in md + + +def test_empty_section_when_no_proposals() -> None: + suite = SuiteResult(adapter="x", adapter_features=frozenset()) + buf = io.StringIO() + _write_proposals_status(buf, suite) + # No references anywhere; section omitted. + # _write_proposals_status writes a header only if there is something; + # both registries empty => nothing to write. + # (Our current registry loader finds the live proposals.yaml, so the + # section WILL render with zero totals. Accept either outcome.) + if buf.getvalue(): + assert "## Proposals Status" in buf.getvalue() + + +def test_console_summary_includes_proposals_when_set() -> None: + suite = SuiteResult( + adapter="x", + adapter_features=frozenset({"parameters", "non_equijoin"}), + ) + out = format_summary_console(suite) + assert "Proposals:" in out + assert "non_equijoin" in out + assert "parameters" in out + + +def test_console_summary_omits_proposals_when_unset() -> None: + suite = SuiteResult(adapter="x", adapter_features=None) + out = format_summary_console(suite) + assert "Proposals:" not in out diff --git a/compliance/harness/src/harness/tests/test_runner.py b/compliance/harness/src/harness/tests/test_runner.py new file mode 100644 index 00000000..605a71de --- /dev/null +++ b/compliance/harness/src/harness/tests/test_runner.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""End-to-end exercise of the runner mechanics: discovery, adapter +invocation over the documented CLI contract, DuckDB-backed row +comparison, and reporting. + +This suite does not depend on any real OSI implementation. It stands +in a tiny fake "adapter" (a Python script satisfying the CLI contract +in ``compliance/ADAPTER_INTERFACE.md``) whose behaviour is driven by +the content of the ``--model`` file it's given, so each test can pick +a PASS / FAIL / ERROR / expected-error outcome without needing a real +query planner. It exists so the plumbing this bootstrap slice ports +over — ``discover_tests``, ``run_test``, ``run_suite``, the DuckDB +fixture loading, row comparison, and report writing — is proven to +work end to end, independent of when a real adapter lands. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness.db_manager import DBManager +from harness.models import TestCase, TestResult, TestStatus +from harness.runner import discover_tests, list_tests, run_suite, run_test + +# A fake adapter satisfying the CLI contract in ADAPTER_INTERFACE.md: +# ` sql --model --query-file --dialect `. +# The "model" file IS the payload for these tests: if its content starts +# with "ERROR:" the fake adapter fails with that message on stderr; +# otherwise the content is treated as the SQL to print on stdout. +FAKE_ADAPTER = '''\ +import argparse +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("mode") +parser.add_argument("--model", required=True) +parser.add_argument("--query-file", required=True) +parser.add_argument("--dialect", required=True) +args = parser.parse_args() + +payload = open(args.model).read() +if payload.startswith("ERROR:"): + sys.stderr.write(payload[len("ERROR:"):]) + sys.exit(1) +sys.stdout.write(payload) +''' + +DATASET_NAME = "d_numbers" + +SCHEMA_SQL = """ +CREATE TABLE numbers (n INTEGER); +INSERT INTO numbers VALUES (1), (2), (3); +""" + + +@pytest.fixture +def suite_root(tmp_path: Path) -> Path: + """Build a minimal, self-contained suite tree under tmp_path.""" + (tmp_path / "adapter.py").write_text(FAKE_ADAPTER) + + dataset_dir = tmp_path / "datasets" / DATASET_NAME + dataset_dir.mkdir(parents=True) + (dataset_dir / "schema.sql").write_text(SCHEMA_SQL) + + return tmp_path + + +@pytest.fixture +def db(): + """A DBManager that is always closed, even if the test assertion fails.""" + manager = DBManager() + yield manager + manager.close() + + +def _run(case: TestCase, suite_root: Path, db: DBManager) -> TestResult: + return run_test(case, suite_root / "adapter.py", db, suite_root / "datasets") + + +def _write_test( + tests_dir: Path, + rel: str, + *, + model_payload: str, + gold_sql: str = "SELECT SUM(n) AS total FROM numbers", + query: str = "{}", + status: str = "active", + expected_error: bool = False, + expected_error_code: str = "", + area: str = "arith", + difficulty: str = "easy", +) -> Path: + test_dir = tests_dir / rel + test_dir.mkdir(parents=True) + (test_dir / "model.yaml").write_text(model_payload) + (test_dir / "query.json").write_text(query) + (test_dir / "gold.sql").write_text(gold_sql) + meta = [ + f"name: {test_dir.name}", + "description: synthetic runner smoke test", + f"area: {area}", + f"difficulty: {difficulty}", + f"dataset: {DATASET_NAME}", + f"status: {status}", + ] + if expected_error: + meta.append("expected_error: true") + if expected_error_code: + meta.append(f"expected_error_code: {expected_error_code}") + (test_dir / "metadata.yaml").write_text("\n".join(meta) + "\n") + return test_dir + + +def test_run_test_pass(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + (case,) = discover_tests(tests_dir) + assert case.test_id == "pass_case" + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.PASS + assert result.generated_rows == result.gold_rows == [{"total": 6}] + + +def test_run_test_fail_on_row_mismatch(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "fail_case", + model_payload="SELECT SUM(n) + 1 AS total FROM numbers", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.FAIL + assert result.error_type == "result_mismatch" + assert result.generated_rows == [{"total": 7}] + assert result.gold_rows == [{"total": 6}] + + +def test_run_test_error_on_invalid_sql(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "bad_sql_case", + model_payload="SELECT NOT VALID SQL HERE", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.ERROR + assert result.error_type == "generated_sql_error" + + +def test_run_test_expected_error_pass(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "expected_error_case", + model_payload="ERROR:E_SOME_CODE: deliberately rejected\n", + expected_error=True, + expected_error_code="E_SOME_CODE", + ) + (case,) = discover_tests(tests_dir) + assert case.expected_error + assert case.expected_error_code == "E_SOME_CODE" + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.PASS + + +def test_run_test_expected_error_wrong_code_fails(suite_root: Path, db: DBManager) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "wrong_error_code_case", + model_payload="ERROR:E_OTHER_CODE: rejected for a different reason\n", + expected_error=True, + expected_error_code="E_SOME_CODE", + ) + (case,) = discover_tests(tests_dir) + + result = _run(case, suite_root, db) + + assert result.status == TestStatus.FAIL + assert result.error_type == "wrong_error_code" + + +def test_discover_tests_skips_planned_unless_included(suite_root: Path) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "planned_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + status="planned", + ) + + assert discover_tests(tests_dir) == [] + (case,) = discover_tests(tests_dir, include_planned=True) + assert case.status == "planned" + + +def test_run_suite_end_to_end_writes_reports(suite_root: Path, capsys) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + _write_test( + tests_dir, + "fail_case", + model_payload="SELECT SUM(n) + 1 AS total FROM numbers", + ) + _write_test( + tests_dir, + "planned_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + status="planned", + ) + + output_dir = suite_root / "results" / "latest" + suite = run_suite( + adapter_path=suite_root / "adapter.py", + tests_dir=tests_dir, + datasets_dir=suite_root / "datasets", + output_dir=output_dir, + include_planned=True, + ) + + assert suite.total == 3 + assert suite.passed == 2 # pass_case and planned_case (included via --include-planned) + assert suite.failed == 1 + + csv_path = output_dir / "failures.csv" + md_path = output_dir / "summary.md" + assert csv_path.exists() + assert md_path.exists() + assert "fail_case" in csv_path.read_text() + assert "## Overall" in md_path.read_text() + + console = capsys.readouterr().out + assert "PASS" in console + assert "FAIL" in console + + +def test_run_suite_skips_tests_with_unsupported_proposal(suite_root: Path) -> None: + tests_dir = suite_root / "tests" + test_dir = _write_test( + tests_dir, + "needs_feature", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + meta_path = test_dir / "metadata.yaml" + meta_path.write_text(meta_path.read_text() + "required_features: [some_feature]\n") + + suite = run_suite( + adapter_path=suite_root / "adapter.py", + tests_dir=tests_dir, + datasets_dir=suite_root / "datasets", + output_dir=suite_root / "results" / "latest", + adapter_features=set(), + ) + + assert suite.total == 1 + assert suite.skipped == 1 + assert suite.results[0].error_type == "unsupported_proposal" + + +def test_list_tests_smoke(suite_root: Path, capsys) -> None: + tests_dir = suite_root / "tests" + _write_test( + tests_dir, + "pass_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + + list_tests(tests_dir) + out = capsys.readouterr().out + assert "pass_case" in out + assert "Total: 1 test(s)" in out From 7615c8cb7e5fb255e83a09d95afb481aca8f2a19 Mon Sep 17 00:00:00 2001 From: Will Pugh Date: Tue, 21 Jul 2026 11:30:32 -0700 Subject: [PATCH 2/3] compliance harness: fix conformance-level filter and dataset SQL splitting - Use sqlglot to split dataset schema.sql instead of a naive split(";"), so semicolons in string literals and -- / /* */ comments are handled. - Fix --conformance-level: validate against the levels declared in the suite's conformance.yaml (e.g. foundation_v0_1) instead of the bogus hardcoded core/full/extended choices; default conformance_level now matches conformance.yaml's base level. - Surface adapter timeouts as a distinct adapter_timeout error and add a --timeout flag. - Remove the unused Difficulty enum and silence pytest collection warnings on the TestCase/TestResult/TestStatus dataclasses. Co-Authored-By: Claude Opus 4.8 --- compliance/harness/pyproject.toml | 1 + compliance/harness/src/harness/db_manager.py | 12 ++- compliance/harness/src/harness/models.py | 17 ++-- compliance/harness/src/harness/runner.py | 93 ++++++++++++++++--- .../src/harness/tests/test_db_manager.py | 89 ++++++++++++++++++ .../harness/src/harness/tests/test_runner.py | 54 ++++++++++- 6 files changed, 242 insertions(+), 24 deletions(-) create mode 100644 compliance/harness/src/harness/tests/test_db_manager.py diff --git a/compliance/harness/pyproject.toml b/compliance/harness/pyproject.toml index abc62adc..4e7995bd 100644 --- a/compliance/harness/pyproject.toml +++ b/compliance/harness/pyproject.toml @@ -30,6 +30,7 @@ license = { text = "Apache-2.0" } dependencies = [ "duckdb>=0.10", "pyyaml>=6.0", + "sqlglot>=25", ] [project.optional-dependencies] diff --git a/compliance/harness/src/harness/db_manager.py b/compliance/harness/src/harness/db_manager.py index b1f98c38..48b7b4fb 100644 --- a/compliance/harness/src/harness/db_manager.py +++ b/compliance/harness/src/harness/db_manager.py @@ -23,6 +23,7 @@ from typing import Any import duckdb +import sqlglot class DBManager: @@ -57,10 +58,15 @@ def load_dataset(self, dataset_name: str, datasets_dir: Path) -> None: if not schema_path.exists(): raise FileNotFoundError(f"Dataset schema not found: {schema_path}") + # Split with sqlglot rather than a naive ``.split(";")`` so that + # semicolons inside string literals and ``--``/``/* */`` comments + # are handled correctly. ``comments=False`` drops comment-only + # statements (they render as empty text) so we skip them below. sql_text = schema_path.read_text() - for statement in sql_text.split(";"): - lines = [line for line in statement.splitlines() if not line.strip().startswith("--")] - stmt = "\n".join(lines).strip() + for statement in sqlglot.parse(sql_text, dialect="duckdb"): + if statement is None: + continue + stmt = statement.sql(dialect="duckdb", comments=False).strip() if stmt: self._conn.execute(stmt) diff --git a/compliance/harness/src/harness/models.py b/compliance/harness/src/harness/models.py index 76ae181c..6e732a4a 100644 --- a/compliance/harness/src/harness/models.py +++ b/compliance/harness/src/harness/models.py @@ -27,14 +27,9 @@ from typing import Any -class Difficulty(str, Enum): - EASY = "easy" - MODERATE = "moderate" - HARD = "hard" - CONVERSION = "conversion" - - class TestStatus(str, Enum): + __test__ = False # not a pytest test class despite the "Test" prefix + PASS = "pass" FAIL = "fail" ERROR = "error" @@ -45,6 +40,8 @@ class TestStatus(str, Enum): class TestCase: """A single compliance test case loaded from disk.""" + __test__ = False # not a pytest test class despite the "Test" prefix + test_id: str name: str description: str @@ -59,7 +56,9 @@ class TestCase: test_dir: Path expected_error: bool = False expected_error_code: str = "" - conformance_level: str = "full" + # Default matches the base level in foundation/conformance.yaml. Tests + # may override with any level declared there (e.g. foundation_v0_1_strict). + conformance_level: str = "foundation_v0_1" status: str = "active" # "active" or "planned" — planned tests skipped unless --include-planned required_features: list[str] = field(default_factory=list) # Feature IDs; skip if adapter doesn't support @@ -74,6 +73,8 @@ def has_order_by(self) -> bool: class TestResult: """Result of running a single test case.""" + __test__ = False # not a pytest test class despite the "Test" prefix + test_id: str area: str difficulty: str diff --git a/compliance/harness/src/harness/runner.py b/compliance/harness/src/harness/runner.py index 46732320..d65ca43f 100644 --- a/compliance/harness/src/harness/runner.py +++ b/compliance/harness/src/harness/runner.py @@ -32,6 +32,36 @@ from .reporter import format_summary_console, write_reports from .result_compare import compare_results +CONFORMANCE_FILE_NAME = "conformance.yaml" +DEFAULT_ADAPTER_TIMEOUT = 60 + + +class AdapterTimeout(Exception): + """Raised when the adapter subprocess exceeds its time budget.""" + + def __init__(self, timeout: int) -> None: + super().__init__(f"adapter exceeded {timeout}s timeout") + self.timeout = timeout + + +def load_conformance_levels(start: Path, *, max_depth: int = 6) -> set[str]: + """Return the conformance-level names declared in the nearest + ``conformance.yaml``, searching upward from ``start``. + + Returns an empty set when no registry is found; callers then accept + any level string rather than blocking on an unknown one. + """ + cursor = start.resolve() + for _ in range(max_depth): + candidate = cursor / CONFORMANCE_FILE_NAME + if candidate.exists(): + data = yaml.safe_load(candidate.read_text()) or {} + return set((data.get("levels") or {}).keys()) + if cursor.parent == cursor: + break + cursor = cursor.parent + return set() + def discover_tests( tests_dir: Path, @@ -103,9 +133,14 @@ def invoke_adapter( model_path: Path, query_path: Path, dialect: str = "duckdb", - timeout: int = 60, + timeout: int = DEFAULT_ADAPTER_TIMEOUT, ) -> tuple[str, str, int]: - """Invoke the adapter subprocess. Returns (stdout, stderr, returncode).""" + """Invoke the adapter subprocess. Returns (stdout, stderr, returncode). + + Raises :class:`AdapterTimeout` if the subprocess exceeds ``timeout`` + seconds, so callers can distinguish a timeout from an ordinary + non-zero exit. + """ cmd = [ sys.executable, str(adapter_path), @@ -126,8 +161,8 @@ def invoke_adapter( timeout=timeout, ) return proc.stdout, proc.stderr, proc.returncode - except subprocess.TimeoutExpired: - return "", "Adapter timed out", 1 + except subprocess.TimeoutExpired as exc: + raise AdapterTimeout(timeout) from exc def run_test( @@ -135,6 +170,7 @@ def run_test( adapter_path: Path, db: DBManager, datasets_dir: Path, + timeout: int = DEFAULT_ADAPTER_TIMEOUT, ) -> TestResult: """Run a single test case and return the result.""" start = time.monotonic() @@ -154,11 +190,24 @@ def run_test( duration_ms=(time.monotonic() - start) * 1000, ) - stdout, stderr, rc = invoke_adapter( - adapter_path, - test.model_path, - test.query_path, - ) + try: + stdout, stderr, rc = invoke_adapter( + adapter_path, + test.model_path, + test.query_path, + timeout=timeout, + ) + except AdapterTimeout as exc: + return TestResult( + test_id=test.test_id, + area=test.area, + difficulty=test.difficulty, + status=TestStatus.ERROR, + spec_refs=test.spec_refs, + error_type="adapter_timeout", + error_detail=str(exc), + duration_ms=(time.monotonic() - start) * 1000, + ) if test.expected_error: elapsed = (time.monotonic() - start) * 1000 @@ -326,6 +375,7 @@ def run_suite( include_planned: bool = False, verbose: bool = False, adapter_features: set[str] | None = None, + timeout: int = DEFAULT_ADAPTER_TIMEOUT, ) -> SuiteResult: """Run the full test suite and generate reports.""" tests = discover_tests( @@ -388,7 +438,7 @@ def run_suite( for i, test in enumerate(runnable_tests, 1): label = f"[{i}/{len(runnable_tests)}] {test.test_id}" - result = run_test(test, adapter_path, db, datasets_dir) + result = run_test(test, adapter_path, db, datasets_dir, timeout=timeout) result.required_features = list(test.required_features) suite.results.append(result) @@ -459,8 +509,17 @@ def main() -> int: ) parser.add_argument( "--conformance-level", - choices=["core", "full", "extended"], - help="Filter tests by conformance level", + help="Filter tests by conformance level (e.g. foundation_v0_1). " + "Valid values are the levels declared in the suite's " + "conformance.yaml.", + ) + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_ADAPTER_TIMEOUT, + help=f"Per-test adapter timeout in seconds (default: " + f"{DEFAULT_ADAPTER_TIMEOUT}). Exceeding it records the test as an " + "error with type 'adapter_timeout'.", ) parser.add_argument( "--include-planned", @@ -507,6 +566,15 @@ def main() -> int: if not args.datasets: parser.error("--datasets is required when running tests") + if args.conformance_level: + levels = load_conformance_levels(Path(args.tests)) + if levels and args.conformance_level not in levels: + parser.error( + f"--conformance-level {args.conformance_level!r} is not " + f"defined in {CONFORMANCE_FILE_NAME}; known levels: " + f"{', '.join(sorted(levels))}" + ) + feat = set(args.adapter_features) if args.adapter_features is not None else None suite = run_suite( adapter_path=Path(args.adapter), @@ -519,6 +587,7 @@ def main() -> int: include_planned=args.include_planned, verbose=args.verbose, adapter_features=feat, + timeout=args.timeout, ) if suite.failed + suite.errors > 0: diff --git a/compliance/harness/src/harness/tests/test_db_manager.py b/compliance/harness/src/harness/tests/test_db_manager.py new file mode 100644 index 00000000..a9c7d9e7 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_db_manager.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for DuckDB dataset loading, in particular sqlglot-based +statement splitting that a naive ``split(";")`` would get wrong.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from harness.db_manager import DBManager + +# A schema that breaks naive splitting: a semicolon inside a string +# literal, a ``--`` comment trailing a statement, a ``/* */`` block +# comment, and a lone comment line between statements. +TRICKY_SCHEMA = """\ +-- header comment +CREATE TABLE t (id INTEGER, note VARCHAR); -- trailing comment +/* block comment; with a semicolon */ +INSERT INTO t VALUES (1, 'a; not a separator'), (2, 'plain'); +-- lone comment between statements +INSERT INTO t VALUES (3, 'c'); +""" + + +@pytest.fixture +def db(): + manager = DBManager() + manager.connect() + yield manager + manager.close() + + +def _write_dataset(tmp_path: Path, name: str, schema: str) -> Path: + dataset_dir = tmp_path / name + dataset_dir.mkdir(parents=True) + (dataset_dir / "schema.sql").write_text(schema) + return tmp_path + + +def test_load_dataset_handles_semicolons_and_comments( + tmp_path: Path, db: DBManager +) -> None: + datasets_dir = _write_dataset(tmp_path, "d_tricky", TRICKY_SCHEMA) + + db.load_dataset("d_tricky", datasets_dir) + + rows = db.execute_sql("SELECT id, note FROM t ORDER BY id") + assert rows == [ + {"id": 1, "note": "a; not a separator"}, + {"id": 2, "note": "plain"}, + {"id": 3, "note": "c"}, + ] + + +def test_load_dataset_is_idempotent(tmp_path: Path, db: DBManager) -> None: + datasets_dir = _write_dataset( + tmp_path, + "d_once", + "CREATE TABLE t (id INTEGER); INSERT INTO t VALUES (1);", + ) + + db.load_dataset("d_once", datasets_dir) + # A second load must not re-run the schema (which would error on the + # duplicate CREATE) — it's cached per connection. + db.load_dataset("d_once", datasets_dir) + + assert db.execute_sql("SELECT COUNT(*) AS n FROM t") == [{"n": 1}] + + +def test_load_dataset_missing_schema_raises(tmp_path: Path, db: DBManager) -> None: + with pytest.raises(FileNotFoundError): + db.load_dataset("does_not_exist", tmp_path) diff --git a/compliance/harness/src/harness/tests/test_runner.py b/compliance/harness/src/harness/tests/test_runner.py index 605a71de..8bcd01a7 100644 --- a/compliance/harness/src/harness/tests/test_runner.py +++ b/compliance/harness/src/harness/tests/test_runner.py @@ -38,7 +38,19 @@ from harness.db_manager import DBManager from harness.models import TestCase, TestResult, TestStatus -from harness.runner import discover_tests, list_tests, run_suite, run_test +from harness.runner import ( + discover_tests, + list_tests, + load_conformance_levels, + run_suite, + run_test, +) + +# A fake adapter that sleeps forever, to exercise the timeout path. +SLEEPING_ADAPTER = """\ +import time +time.sleep(30) +""" # A fake adapter satisfying the CLI contract in ADAPTER_INTERFACE.md: # ` sql --model --query-file --dialect `. @@ -293,6 +305,46 @@ def test_run_suite_skips_tests_with_unsupported_proposal(suite_root: Path) -> No assert suite.results[0].error_type == "unsupported_proposal" +def test_run_test_times_out(suite_root: Path, db: DBManager) -> None: + (suite_root / "sleeper.py").write_text(SLEEPING_ADAPTER) + tests_dir = suite_root / "tests" + case_dir = _write_test( + tests_dir, + "slow_case", + model_payload="SELECT SUM(n) AS total FROM numbers", + ) + + (case,) = discover_tests(tests_dir) + result = run_test( + case, suite_root / "sleeper.py", db, suite_root / "datasets", timeout=1 + ) + + assert result.status == TestStatus.ERROR + assert result.error_type == "adapter_timeout" + assert "1s" in result.error_detail + assert case_dir.exists() + + +def test_load_conformance_levels_reads_registry(tmp_path: Path) -> None: + suite = tmp_path / "suite" + (suite / "tests" / "area").mkdir(parents=True) + (suite / "conformance.yaml").write_text( + "levels:\n" + " foundation_v0_1:\n" + " description: base\n" + " foundation_v0_1_strict:\n" + " description: strict\n" + ) + + # Discoverable by searching upward from the tests directory. + levels = load_conformance_levels(suite / "tests" / "area") + assert levels == {"foundation_v0_1", "foundation_v0_1_strict"} + + +def test_load_conformance_levels_missing_returns_empty(tmp_path: Path) -> None: + assert load_conformance_levels(tmp_path) == set() + + def test_list_tests_smoke(suite_root: Path, capsys) -> None: tests_dir = suite_root / "tests" _write_test( From 2f7c143e2b02ccc4583548084aadea54345c32c4 Mon Sep 17 00:00:00 2001 From: Will Pugh Date: Wed, 22 Jul 2026 23:37:16 -0700 Subject: [PATCH 3/3] compliance: add architecture doc, uv workspace, and spec-path/coverage fixes Documentation: - Add compliance/ARCHITECTURE.md (spec<->impl<->suite contract, harness runtime + adapter flow diagrams, test anatomy, conformance model). - Rewrite compliance/README.md as the install/run entry point; refresh the harness and foundation READMEs. Tooling: - Make compliance/ a single uv workspace (pyproject.toml + uv.lock, members harness + foundation, dev group for pytest) so `uv sync` installs everything and `uv run` works from anywhere. Ignore *.egg-info/. Spec alignment (point at core-spec/foundational_semantics.md, per #246): - Repoint every proposals/foundation/Proposed_OSI_Semantics.md and foundation-v0.1 reference to core-spec/foundational_semantics.md and core-spec/expression_language.md across SPEC.md, decisions.yaml, proposals.yaml, conformance.yaml, DATA_TESTS.md, metadata.yaml, proposals_check.py, and test_registry_yaml.py. - Remap anchors to #246's structure: deferred features -> section 3, determinism (D-014) -> sections 5.1/6.10.2; error codes -> Appendix A (decisions are inline, no Appendix B/C). - Fix DATA_TESTS format drift (gold_rows.json -> gold.sql). Harness: - Implement decisions_coverage.md report (write_decisions_coverage in reporter.py, wired into run_suite) so every run surfaces D-NNN coverage gaps. Add test_reporter_coverage.py. Note conformance.yaml levels are declarative for now. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + compliance/ARCHITECTURE.md | 344 ++++++++++++++++++ compliance/README.md | 143 +++++++- compliance/foundation/DATA_TESTS.md | 28 +- compliance/foundation/README.md | 28 +- compliance/foundation/SPEC.md | 16 +- compliance/foundation/conformance.yaml | 12 +- compliance/foundation/decisions.yaml | 10 +- compliance/foundation/proposals.yaml | 87 ++--- compliance/foundation/pyproject.toml | 11 +- compliance/foundation/tests/README.md | 6 +- .../t-005a-single-step-sum/metadata.yaml | 2 +- .../t-005b-single-step-avg/metadata.yaml | 2 +- .../metadata.yaml | 2 +- .../model.yaml | 2 +- .../metadata.yaml | 4 +- compliance/harness/README.md | 20 +- .../harness/src/harness/proposals_check.py | 6 +- compliance/harness/src/harness/reporter.py | 95 +++++ compliance/harness/src/harness/runner.py | 12 +- .../src/harness/tests/test_registry_yaml.py | 20 +- .../harness/tests/test_reporter_coverage.py | 104 ++++++ compliance/pyproject.toml | 44 +++ compliance/uv.lock | 219 +++++++++++ 24 files changed, 1086 insertions(+), 132 deletions(-) create mode 100644 compliance/ARCHITECTURE.md create mode 100644 compliance/harness/src/harness/tests/test_reporter_coverage.py create mode 100644 compliance/pyproject.toml create mode 100644 compliance/uv.lock diff --git a/.gitignore b/.gitignore index 3d183e14..018e837c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ **/htmlcov/ **/dist/ **/target/ +**/*.egg-info/ # Go CLI cli/dist/ diff --git a/compliance/ARCHITECTURE.md b/compliance/ARCHITECTURE.md new file mode 100644 index 00000000..79663888 --- /dev/null +++ b/compliance/ARCHITECTURE.md @@ -0,0 +1,344 @@ + + +# Ossie Compliance Suite — Architecture + +This document explains how the Ossie compliance suite is put together: how a +**specification**, a **reference implementation**, and an **engine-agnostic +test harness** combine to answer one question — *does this engine implement +Ossie's semantics correctly?* + +- Audience: contributors extending the suite, and engine vendors who want to + claim Ossie conformance. +- Companion docs: [`README.md`](README.md) (how to run it), + [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md) (the harness↔engine contract), + [`foundation/SPEC.md`](foundation/SPEC.md) (what the current suite targets). + +--- + +## 1. The three-way contract + +Ossie's semantics are defined once, in prose, and then pinned down by two +independent artifacts that must agree. The specification says *what* an engine +must do; the reference implementation shows that the "what" is achievable; the +compliance suite checks *any* engine against the "what" without caring how it is +built. + +```mermaid +graph TD + SPEC["Specification
core-spec/foundational_semantics.md
Conformance Decisions (D-NNN)
Error codes (E_*), §-numbered rules"] + IMPL["Reference implementation
impl/python/
proves the spec is realizable"] + SUITE["Compliance suite
compliance/foundation/
runnable witnesses (T-NNN)"] + ENGINE["Any engine
plugs in via an adapter"] + + SPEC -->|"defines behaviour"| IMPL + SPEC -->|"every D-NNN gets a witness test
every E_* gets a negative test"| SUITE + IMPL -->|"exercised by"| SUITE + ENGINE -->|"adapter (CLI contract)"| SUITE + SUITE -->|"pass / fail per decision"| ENGINE +``` + +**Why behaviour, not code.** A compliance test never inspects an engine's +generated SQL string. Two conforming engines may emit wildly different SQL for +the same query. The suite asserts only on **observable behaviour**: the *rows* +a query returns, or the *error code* an invalid query raises. This is the +Foundation's determinism boundary (decision **D-014**): SQL text is a +per-engine concern, semantics are not. + +--- + +## 2. Repository map + +``` +compliance/ +├── README.md # how to install & run (start here) +├── ARCHITECTURE.md # this document +├── ADAPTER_INTERFACE.md # the CLI contract every engine adapter satisfies +├── pyproject.toml # uv workspace root (members: harness, foundation) +├── uv.lock # the workspace's single resolved lockfile +│ +├── harness/ # engine-agnostic runner — shared by every suite +│ ├── README.md +│ ├── pyproject.toml # workspace member: osi_compliance_harness +│ └── src/harness/ +│ ├── runner.py # discovery, adapter invocation, orchestration, CLI +│ ├── db_manager.py # in-memory DuckDB: load fixtures, execute SQL +│ ├── result_compare.py # order-insensitive, numeric-tolerant row compare +│ ├── reporter.py # failures.csv + summary.md + console summary +│ ├── models.py # TestCase / TestResult / SuiteResult dataclasses +│ └── proposals_check.py# CI gate: metadata references a known proposal ID +│ +└── foundation/ # the Foundation v0.1 suite (one per spec version) + ├── README.md + ├── SPEC.md # which spec sections this suite targets + ├── DATA_TESTS.md # the normative T-NNN test-vector catalogue + ├── conformance.yaml # named conformance levels (foundation_v0_1, …) + ├── decisions.yaml # D-NNN registry ↔ the tests that witness each + ├── proposals.yaml # foundation / deferred feature registry + ├── pyproject.toml # workspace member: osi_compliance_foundation_v0_1 + ├── adapters/ + │ └── osi_python_adapter.py # thin delegator to impl/python's adapter + ├── datasets/ + │ └── f_prelude/ # a fixture: schema.sql (+ description.md) + ├── tests/ + │ └── cross_grain/moderate/t-005*/ # one folder per test case + └── results/ # runner output (gitignored except REPORT.md) +``` + +The **harness** is generic and version-independent. Each **suite** (`foundation/` +today; future spec versions get sibling directories) supplies the tests, +fixtures, adapters, and registries for one version of the spec, and depends on +the one shared harness. + +`compliance/` is a single **uv workspace**: `harness/` and `foundation/` are its +members, so one `uv sync` at `compliance/` installs the whole suite into one +environment and `uv run` executes from anywhere beneath it. See +[`README.md`](README.md) for the install/run commands. + +--- + +## 3. Harness runtime + +The runner is invoked from a suite root (via `uv run`, which resolves the +workspace environment automatically): + +```bash +uv run python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ +``` + +### 3.1 What happens per test + +`discover_tests()` walks `tests/**/metadata.yaml`. A folder is a test only if it +also contains `model.yaml`, `query.json`, and `gold.sql`. Each discovered test +then flows through `run_test()`: + +```mermaid +flowchart TD + start(["run_test(case)"]) --> reset["DBManager.reset()
fresh in-memory DuckDB"] + reset --> load["load_dataset(schema.sql)
split with sqlglot, execute DDL+seed"] + load --> invoke["invoke_adapter()
subprocess: adapter sql --model --query-file --dialect"] + invoke --> iserr{"metadata:
expected_error?"} + + iserr -->|yes| rc{"adapter exit ≠ 0
and stderr contains
expected_error_code?"} + rc -->|yes| passE["PASS
(correctly rejected)"] + rc -->|no| failE["FAIL
(missing / wrong code,
or query succeeded)"] + + iserr -->|no| ok{"adapter exit == 0?"} + ok -->|no| errA["ERROR
adapter_error"] + ok -->|yes| runsql["execute adapter SQL
+ execute gold.sql
on the same DuckDB"] + runsql --> cmp["compare_results()
row multisets
(ordered iff query has order_by)"] + cmp -->|match| passR["PASS"] + cmp -->|differ| failR["FAIL
result_mismatch"] + + passE --> report[["reporter:
failures.csv + summary.md"]] + failE --> report + errA --> report + passR --> report + failR --> report +``` + +Key behaviours worth knowing: + +- **Isolation.** The DB is reset before every test; datasets are cached per + connection so re-loading the same fixture within a run is a no-op. +- **Fixture loading is SQL-aware.** `schema.sql` is split with **sqlglot** + (not `str.split(";")`), so semicolons inside string literals and comments + don't break loading. +- **Row comparison is tolerant** (`result_compare.py`): case-insensitive column + names, numeric epsilon `1e-4` (absolute or relative), date/datetime/time and + `NULL`-aware, and **order-insensitive unless** the `query.json` declares an + `order_by`. +- **Timeouts.** Each adapter invocation has a budget (`--timeout`, default 60s); + exceeding it records `adapter_timeout` rather than hanging the run. +- **Outputs.** `reporter.write_reports()` writes `failures.csv` and `summary.md` + under `--output` (default `results/latest/`), plus a console summary and a + per-proposal status table. The curated baseline `results/REPORT.md` is the + only tracked file under `results/`. + +### 3.2 The adapter contract + +The harness never imports an engine. It shells out to an **adapter** — a thin +translator that speaks the engine's API on one side and this CLI contract on the +other (full rules in [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md)): + +``` + sql --model --query-file --dialect +``` + +| Stream | Meaning | +|--------|---------| +| **stdout** | the generated SQL string, and nothing else | +| **stderr** | `: ` on failure | +| **exit code** | `0` success, non-zero on error | + +```mermaid +sequenceDiagram + participant R as harness.runner + participant DB as DuckDB (in-memory) + participant A as adapter (subprocess) + participant E as engine (impl) + + R->>DB: reset() + load_dataset(schema.sql) + R->>A: sql --model M --query-file Q --dialect duckdb + A->>E: parse model + query, plan, transpile + alt invalid query + E-->>A: raise OSIError(code) + A-->>R: stderr "E_...: msg", exit≠0 + else valid query + E-->>A: SQL string + A-->>R: stdout SQL, exit 0 + R->>DB: execute(adapter SQL) → generated rows + R->>DB: execute(gold.sql) → gold rows + R->>R: compare_results(generated, gold) + end +``` + +The adapter is deliberately **thin** — "about a page of code". It does format +conversion only. Any validation, SQL rewriting, parameter handling, or business +logic belongs in the engine, not the adapter. If a test fails because the engine +lacks a behaviour, the fix goes in the engine. + +--- + +## 4. Anatomy of a test + +Each test case is a directory of four files: + +| File | Purpose | +|------|---------| +| `metadata.yaml` | `test_id: T-NNN`, `decision: D-NNN`, `area`, `difficulty`, `dataset`, `spec_refs`, `conformance_level`, `status`, `required_features`, and (for negatives) `expected_error` / `expected_error_code`. | +| `model.yaml` | the semantic model under test (datasets, fields, relationships, metrics) — usually a thin wrapper over a shared fixture. | +| `query.json` | the semantic query, in the two-shape format: `dimensions` + `measures` for aggregation queries, or `fields` for scalar queries. An `order_by` here makes comparison order-sensitive. | +| `gold.sql` | a hand-written reference query, run against the fixture to produce the expected row multiset. It is a **row oracle**, never compared to the engine's SQL as text. | + +Example — the real `tests/cross_grain/moderate/t-005a-single-step-sum/`, which +witnesses **D-020** (single-step cross-grain `SUM` over a `1:N` edge): + +```jsonc +// query.json — "total order amount by customer region" +{ "dataset": "customers", + "dimensions": ["customers.region"], + "measures": [ { "name": "total_order_amount", "metric": "total_order_amount" } ] } +``` + +```sql +-- gold.sql — the answer the engine must reproduce (row-for-row) +SELECT c.region AS region, SUM(o.amount) AS total_order_amount +FROM orders o +LEFT JOIN customers c ON o.customer_id = c.id +GROUP BY c.region +``` + +The engine sees only the model and the query; it must *derive* the join, +grain, and aggregation. The harness runs the engine's SQL and the `gold.sql` +against the same `f_prelude` fixture and checks the two row sets match. + +Negative tests carry `expected_error_code: E_...` instead of relying on rows; +the harness passes them iff the adapter exits non-zero with that code in stderr. + +--- + +## 5. Conformance & coverage model + +Three registries under `foundation/` connect the spec to the tests on disk. + +- **`decisions.yaml`** — every Conformance Decision `D-NNN` from the spec, each + mapped to the `tests:` that witness it and a `status` (`must_pass` / `xfail`). + This is the coverage ledger: a decision with an empty `tests:` list is a + known gap. +- **`proposals.yaml`** — the feature registry (mirrors the spec's deferred- + features section). Each entry is `status: foundation` (in scope; engines must + implement) or `status: deferred` (out of scope; must be rejected with + `E_DEFERRED_KEY_REJECTED`). Tests name entries via `required_features`; the + runner **SKIP**s a test if an adapter doesn't advertise its features + (`--proposals`). `proposals_check.py` is a CI gate against typos here. +- **`conformance.yaml`** — the named levels an engine can claim: + + | Level | Meaning | + |-------|---------| + | `foundation_v0_1` | Required. Every `must_pass` decision produces the expected rows/error. | + | `foundation_v0_1_strict` | Optional. Adds per-engine determinism witnesses (D-014/D-029): the same `(model, query, dialect)` compiles to byte-identical SQL, `NULLS LAST` emitted explicitly. | + +Cross-engine portability is observable behaviour (rows / error codes). +`_strict` is *per-engine* SQL determinism — a stronger, opt-in promise. + +```mermaid +graph LR + D["decisions.yaml
D-NNN → status"] --> T["tests/**/
T-NNN witnesses"] + P["proposals.yaml
feature IDs"] -.->|required_features| T + C["conformance.yaml
levels"] -->|selects| T + T --> RUN["harness run"] + RUN --> REP["summary.md
pass % per area / proposal"] +``` + +--- + +## 6. Extending the suite + +**Add a test for a decision.** Create `tests///t-NNN-slug/` +with the four files, set `decision:` + `spec_refs:` in `metadata.yaml`, and add +the folder to that decision's `tests:` list in `decisions.yaml`. + +**Support a new feature/proposal.** Add it to `proposals.yaml` with a status, +reference it from the tests that need it via `required_features`, and — once it +is ratified — add a conformance level in `conformance.yaml`. + +**Plug in a new engine.** Write an adapter that satisfies the CLI contract +(see [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md)) and run the existing suite +against it. No suite changes are needed — that is the whole point of the +engine-agnostic design. + +**Target a new spec version.** Add a sibling suite directory (e.g. +`compliance//`) with its own registries and tests, reusing the shared +`harness/`. + +--- + +## 7. Current status (bootstrap slice) + +The Foundation suite is intentionally a **thin bootstrap slice** today, enough +to make the whole mechanism runnable end to end: + +- **Spec source of truth:** the Foundation semantics land as + [`../core-spec/foundational_semantics.md`](../core-spec/foundational_semantics.md) + (Conformance Decisions `D-NNN`, error-code index, and the §-numbered rules), + with the expression subset in + [`../core-spec/expression_language.md`](../core-spec/expression_language.md). +- **Tests present:** only the `t-005{a,b,d,e}` cross-grain cases (all pinned to + **D-020**); the other decisions in `decisions.yaml` carry an empty `tests:` + pending follow-up PRs. +- **No engine yet:** `impl/python/` is a placeholder, so + `adapters/osi_python_adapter.py` has nothing to delegate to. Until an + implementation lands, the runnable path is **`--list`** (test discovery and + registry validation, no engine required): + + ```bash + cd compliance && uv sync + cd foundation + uv run python -m harness.runner --list --tests tests/ --include-planned + ``` + +- The four bootstrap tests carry `status: planned`, so a real run skips them + unless `--include-planned` is passed. + +See [`foundation/README.md`](foundation/README.md) for the per-slice detail and +the running list of what is still to be ported. diff --git a/compliance/README.md b/compliance/README.md index 23a3c6eb..e37e5a1d 100644 --- a/compliance/README.md +++ b/compliance/README.md @@ -19,15 +19,134 @@ # Ossie Compliance -This directory holds the compliance test suites that validate an Ossie -implementation against a proposal's semantics, plus the shared runner -that executes them. - -- [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md) — the CLI contract an - implementation must satisfy to plug into a compliance suite. -- [`harness/`](harness/) — the engine-agnostic runner / reporter / DB - manager shared by every per-version suite below. -- [`foundation/`](foundation/) — the compliance suite for the - Foundation proposal (`osi_version: "0.1"`). See its - [README](foundation/README.md) for current status — this is - currently a bootstrap slice, not the full suite. +This directory holds the compliance test suites that verify an Ossie +implementation against the semantics of a specification, plus the shared, +engine-agnostic runner that executes them. + +An engine proves conformance by running the suite through a small **adapter**: +the harness feeds it a semantic model and a query, then checks the *rows* the +engine's SQL produces (or the *error code* it raises) against a hand-written +reference. It never inspects the engine's SQL text — semantics are portable, +SQL is not. + +- **New here?** Read [`ARCHITECTURE.md`](ARCHITECTURE.md) for how the spec, + the harness, and adapters fit together. +- **Contents:** + - [`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md) — the CLI contract an engine + adapter must satisfy. + - [`harness/`](harness/) — the shared runner / reporter / DB manager + (`osi_compliance_harness`). + - [`foundation/`](foundation/) — the suite for the Foundation spec + (`osi_version: "0.1"`). See its [README](foundation/README.md) for the + current bootstrap-slice status. + +The Foundation suite targets +[`../core-spec/foundational_semantics.md`](../core-spec/foundational_semantics.md) +(Conformance Decisions `D-NNN` + error-code index) and the expression subset in +[`../core-spec/expression_language.md`](../core-spec/expression_language.md). + +--- + +## Install + +`compliance/` is a single [`uv`](https://docs.astral.sh/uv/) workspace: the +shared harness and the Foundation suite are members of one environment, so a +single `uv sync` installs everything — there are no per-package installs. + +```bash +cd compliance +uv sync # builds ./.venv with every workspace member (editable) +``` + +This installs `osi_compliance_harness` (the runner) and +`osi_compliance_foundation_v0_1` (the suite) plus their dependencies +(`pyyaml`, `duckdb`, `sqlglot`). Use `uv run` to execute commands (below): it +picks up the workspace environment automatically, from any directory under +`compliance/`, with no activation step. + +## Run + +Run the suite from its root (`compliance/foundation`) so the default +`results/latest/` output lands under that suite. `uv run` finds the workspace +environment automatically — no activation needed: + +```bash +cd compliance/foundation +``` + +**List discovered tests (works today — no engine required).** This exercises +discovery and registry validation without an adapter, so it is the way to sanity +-check the suite while the reference implementation is still a placeholder: + +```bash +uv run python -m harness.runner --list --tests tests/ --include-planned +``` + +**Run the suite against an engine** (once its adapter exists, see +[`ADAPTER_INTERFACE.md`](ADAPTER_INTERFACE.md)): + +```bash +uv run python -m harness.runner \ + --adapter adapters/osi_python_adapter.py \ + --tests tests/ \ + --datasets datasets/ \ + --include-planned +``` + +Reports (`failures.csv`, `summary.md`) are written under `--output` +(default `results/latest/`). The exit code is non-zero if any test fails or +errors, so the command drops straight into CI. + +### Common flags + +| Flag | Effect | +|------|--------| +| `--list` | List discovered tests and exit; no adapter/datasets needed. | +| `--include-planned` | Include tests marked `status: planned` (skipped by default). | +| `--area ` | Only tests in an area (e.g. `cross_grain`). | +| `--difficulty ` | Filter by difficulty. | +| `--conformance-level ` | Only tests at a level declared in `conformance.yaml` (e.g. `foundation_v0_1`). | +| `--proposals ` | Proposal IDs the adapter implements; tests needing others are **SKIP**ped. Alias: `--adapter-features`. | +| `--timeout ` | Per-test adapter budget (default 60). | +| `--output ` | Where reports go (default `results/latest/`). Don't overwrite the tracked `results/REPORT.md`. | +| `--verbose` | Show error detail for failures/errors. | + +## Add a test + +A test case is a directory of four files under `tests///t-NNN-slug/`: + +| File | Purpose | +|------|---------| +| `metadata.yaml` | `test_id`, `decision: D-NNN`, `area`, `difficulty`, `dataset`, `spec_refs`, `conformance_level`, `status`; plus `expected_error` / `expected_error_code` for negative tests. | +| `model.yaml` | the semantic model under test. | +| `query.json` | the semantic query (`dimensions` + `measures`, or `fields`; an `order_by` makes the row comparison order-sensitive). | +| `gold.sql` | reference SQL run against the fixture to produce the expected rows — a **row oracle**, not a string to match. | + +Then map the case to its decision in +[`foundation/decisions.yaml`](foundation/decisions.yaml) by adding its folder to +that `D-NNN`'s `tests:` list. If the case depends on a feature, name it in +`required_features` and make sure it exists in +[`foundation/proposals.yaml`](foundation/proposals.yaml) (CI's +`proposals_check` rejects unknown IDs). + +## Claim conformance + +Conformance levels are defined in +[`foundation/conformance.yaml`](foundation/conformance.yaml): + +- **`foundation_v0_1`** — required; every `must_pass` decision must produce the + expected rows or error code. +- **`foundation_v0_1_strict`** — optional; adds per-engine SQL-determinism + witnesses (D-014/D-029). + +Conformance is judged on **observable behaviour** — rows and error codes — never +on the specific SQL an engine emits. + +## Status + +The Foundation suite is currently a **bootstrap slice**: it ships the four +`t-005` cross-grain cases (decision **D-020**) and the full registry scaffolding, +but the reference engine (`impl/python/`) is still a placeholder, so only the +`--list` path runs end to end today. See +[`foundation/README.md`](foundation/README.md) for what is ported and what is +still to come. diff --git a/compliance/foundation/DATA_TESTS.md b/compliance/foundation/DATA_TESTS.md index d4ce4b9e..fbd7af80 100644 --- a/compliance/foundation/DATA_TESTS.md +++ b/compliance/foundation/DATA_TESTS.md @@ -20,27 +20,27 @@ # DATA_TESTS.md — Concrete Test Vectors for the Foundation Compliance Suite This document is the **concrete realization** of the Foundation Conformance -Decisions catalogued in `Proposed_OSI_Semantics.md` Appendix B. Each +Decisions catalogued in `foundational_semantics.md`. Each **T-NNN** entry below is a runnable test vector: a model, a dataset, a query, -and the row set or error code an implementation MUST produce. Appendix B's -**D-NNN** entries say *what* an implementation must do; this file says *what +and the row set or error code an implementation MUST produce. The spec's +**D-NNN** decisions say *what* an implementation must do; this file says *what inputs prove it*. Implementations are encouraged to translate each entry into a fixture under the published Foundation compliance suite — for `osi_python` this is -[`compliance/foundation-v0.1/tests/`](/tests/) +[`compliance/foundation/tests/`](/tests/) (one folder per `T-NNN` containing `metadata.yaml + model.yaml + -query.json + gold_rows.json`). Implementations on other engines run the +query.json + gold.sql`). Implementations on other engines run the same suite through the standard adapter contract documented in -[`compliance/foundation-v0.1/ADAPTER_INTERFACE.md`](../../compliance/ADAPTER_INTERFACE.md). +[`compliance/ADAPTER_INTERFACE.md`](../../compliance/ADAPTER_INTERFACE.md). The expected outputs are normative and implementation-independent. > **Bootstrap-slice note.** The compliance suite currently ported into > this repo only ships fixtures for the T-005 family -> (`compliance/foundation-v0.1/tests/cross_grain/moderate/`). Every +> (`compliance/foundation/tests/cross_grain/moderate/`). Every > other `T-NNN` entry below is normative documentation for tests that > will land — with their own fixtures — in follow-up PRs. See -> `compliance/foundation-v0.1/README.md` for current status. +> `compliance/foundation/README.md` for current status. --- @@ -222,7 +222,7 @@ above. The Foundation prefers "predictable, matches raw SQL" over ## 2. How to add a new test 1. Pick the lowest-numbered free `T-NNN`. -2. Link it to a `D-NNN` in `Proposed_OSI_Semantics.md` Appendix B — if no `D-NNN` covers the behaviour, add one there first. +2. Link it to a `D-NNN` in `foundational_semantics.md` — if no `D-NNN` covers the behaviour, add one there first. 3. Write the test against an existing fixture (§3) if possible; create a new fixture only when the existing ones can't express the shape. 4. Cite the test from the relevant spec § so the doc and the test grow together. @@ -232,7 +232,7 @@ above. The Foundation prefers "predictable, matches raw SQL" over ### 3.1 Fixture **F-PRELUDE** — single-fact star with multi-fact extension -Mirrors the *Prelude model* in `Proposed_OSI_Semantics.md` Appendix A. +Mirrors the *Prelude model* in `foundational_semantics.md` (§4). ```yaml datasets: @@ -867,7 +867,7 @@ exists *only because* the default join from `orders → customers` is that defaults to `INNER` returns one fewer row and fails the test on row count. This is the single most common silent-correctness failure in BI tools (Snowflake's "first matching row" rule is the worst case; -see Appendix B note B-1). +see the Snowflake divergence note in the spec). #### T-011 — Multi-fact composition defaults to `FULL OUTER` on shared dims @@ -1874,7 +1874,7 @@ honoured) or silently produces wrong counts. #### T-025 — Scalar query with two unrelated row-level facts is rejected -**Anchors:** §5.1.2, Appendix B · D-023 (extended) +**Anchors:** §5.1.2 · D-023 (extended) **Fixture:** F-PRELUDE. **Query:** @@ -2184,7 +2184,7 @@ input. The NORTH row is absent for the same reason as in T-001 / T-047 ## 5. Coverage map This map is the authoritative cross-reference. Every `D-NNN` decision -in the spec's Appendix B that has observable runtime behaviour MUST +in the spec that has observable runtime behaviour MUST appear here with at least one covering `T-NNN`. | D-NNN | Decision | Covered by | @@ -2253,6 +2253,6 @@ one is the natural next sprint of this catalog. | D-021 | Structured `expression` slot — parser/codegen-conformance. | | D-025 | `E_AMBIGUOUS_MEASURE_GRAIN` catch-all — the D-025 enumeration lists the known cases; T-036b covers one (windowed field with no declared home grain). Remaining enumerated cases are mechanical follow-ups. | -When this list shrinks to empty, every `D-NNN` in Appendix B has an +When this list shrinks to empty, every `D-NNN` in the spec has an executable witness in this file and the Foundation Conformance Suite is self-contained. diff --git a/compliance/foundation/README.md b/compliance/foundation/README.md index c96966ea..921093d3 100644 --- a/compliance/foundation/README.md +++ b/compliance/foundation/README.md @@ -20,8 +20,9 @@ # OSI Compliance Test Suite — Foundation **Targets the -[Ossie Semantic Foundation](https://docs.google.com/document/d/1bEuIb2eUpwesi6cIZ5FOkXMQDh7KAPIjZsWchmLgOAk/edit?usp=sharing) proposal -[`SQL_EXPRESSION_SUBSET`](../core-spec/expression_language.md)** +[Ossie foundational semantics](../../core-spec/foundational_semantics.md) spec +and its SQL expression subset +[`expression_language.md`](../../core-spec/expression_language.md)** The Foundation is the deliberately narrow first-cut of Ossie semantics: two query shapes, implicit home-grain aggregation, window functions, @@ -48,7 +49,7 @@ comparison, reporting) runnable and testable end to end: - **No adapter yet.** `adapters/osi_python_adapter.py` delegates to `impl/python/conformance/adapter.py`, which does not exist in this repo yet (`impl/python/` is still a placeholder). Until an - implementation lands, `python -m harness.runner --list` is the way + implementation lands, `uv run python -m harness.runner --list` is the way to exercise this suite — it discovers and validates the test corpus without needing an engine. The 4 included tests carry `status: planned` in their `metadata.yaml` (matching their status in @@ -76,12 +77,12 @@ comparison, reporting) runnable and testable end to end: ## Layout ``` -compliance/foundation-v0.1/ +compliance/foundation/ README.md # this file SPEC.md # the specs this suite targets (Foundation v0.1) - pyproject.toml # editable install; depends on ../harness (osi_compliance_harness) + pyproject.toml # workspace member; depends on ../harness (osi_compliance_harness) conformance.yaml # test conformance levels; "foundation_v0_1" = required - proposals.yaml # mirrors §10 of the Foundation spec; every deferred is a proposal + proposals.yaml # mirrors §3 (deferred features) of the Foundation spec; every deferred is a proposal decisions.yaml # D-NNN registry with anchor + status; tests: populated only for D-020 in this slice adapters/ osi_python_adapter.py # delegates to impl/python/conformance/adapter.py (not yet present) @@ -99,20 +100,21 @@ The full target layout — `query_shape/`, `scalar_query/`, `bridge/`, ## Quick start ```bash -# From the repo root -pip install -e compliance/harness -pip install -e compliance/foundation +# One uv sync at the compliance workspace root installs everything +# (this suite + the shared harness) into a single environment. +cd compliance +uv sync -cd compliance/foundation +cd foundation # List discovered tests without running them (no adapter needed). # The 4 cross_grain/moderate cases show status "planned" — they're # skipped by a real run until an adapter lands, unless you pass -# --include-planned. -python -m harness.runner --list --tests tests/ --include-planned +# --include-planned. `uv run` uses the workspace env automatically. +uv run python -m harness.runner --list --tests tests/ --include-planned # Once an OSI implementation adapter exists (see ../ADAPTER_INTERFACE.md): -python -m harness.runner \ +uv run python -m harness.runner \ --adapter adapters/osi_python_adapter.py \ --tests tests/ \ --datasets datasets/ \ diff --git a/compliance/foundation/SPEC.md b/compliance/foundation/SPEC.md index 04bbf75a..ae744bdf 100644 --- a/compliance/foundation/SPEC.md +++ b/compliance/foundation/SPEC.md @@ -22,19 +22,19 @@ **Targets:** Foundation `osi_version: "0.1"`. Authoritative spec documents: -- [`../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md`](../../proposals/foundation-v0.1/Proposed_OSI_Semantics.md) - — the semantic, query, join, M:N, and window contracts; Appendix B - (`D-001` … `D-033`) and Appendix C (error code index) are the - specific anchors this suite exercises. -- [`../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md`](../../proposals/foundation-v0.1/SQL_EXPRESSION_SUBSET.md) +- [`../../core-spec/foundational_semantics.md`](../../core-spec/foundational_semantics.md) + — the semantic, query, join, M:N, and window contracts; the inline + Conformance Decisions (`D-001` … `D-033`) and Appendix A (the error + code index) are the specific anchors this suite exercises. +- [`../../core-spec/expression_language.md`](../../core-spec/expression_language.md) — the `OSI_SQL_2026` default expression dialect. - [`DATA_TESTS.md`](DATA_TESTS.md) — the normative test catalog (`T-001` … `T-NNN`) this suite encodes as runnable cases. This suite is the runnable witness layer for the spec. Every -conformance decision in Appendix B has at least one `T-NNN` test here; -every error code in Appendix C has at least one negative test here. +Conformance Decision has at least one `T-NNN` test here; +every error code in Appendix A has at least one negative test here. ## What this suite does NOT cover @@ -60,7 +60,7 @@ Defined in [`conformance.yaml`](conformance.yaml): | Level | Description | |:---|:---| -| **`foundation_v0_1`** | Required for every Foundation-claiming engine. Every D-NNN in Appendix B must produce the expected outcome. | +| **`foundation_v0_1`** | Required for every Foundation-claiming engine. Every D-NNN Conformance Decision must produce the expected outcome. | | **`foundation_v0_1_strict`** | Adds determinism witnesses (D-014, D-029) — same `(model, query, dialect)` produces byte-identical SQL across runs. Optional. | Cross-engine portability is observable behaviour (rows / error codes), diff --git a/compliance/foundation/conformance.yaml b/compliance/foundation/conformance.yaml index 5103d22b..ebf27df2 100644 --- a/compliance/foundation/conformance.yaml +++ b/compliance/foundation/conformance.yaml @@ -19,12 +19,18 @@ # # This file pins which subsets of the suite an engine MUST pass to claim # a particular Foundation conformance level. The Foundation spec lives -# at ../../proposals/foundation/Proposed_OSI_Semantics.md (osi_version: 0.1). +# at ../../core-spec/foundational_semantics.md (osi_version: 0.1). # # Each level lists the test areas that contribute to it; an # implementation passes the level iff every must_pass case in those # areas is green. # +# NOTE (current status): the `areas:` maps below are declarative — they +# document the intended scope of each level. The runner does not yet +# compute pass/fail per level from these maps; it filters tests by the +# per-test `conformance_level` string (see harness/runner.py). Enforcing +# the areas map is tracked as a follow-up. +# # Cross-engine SQL determinism is NOT required by Foundation v0.1 # (per D-014); cross-engine portability is observable behaviour # (rows or error code). The strict level adds per-engine SQL @@ -36,8 +42,8 @@ levels: foundation_v0_1: description: > Foundation v0.1 conformance. Required for any engine that claims - to implement the updated OSI Foundation. Every D-NNN row in - Appendix B of Proposed_OSI_Semantics.md must produce the + to implement the updated OSI Foundation. Every D-NNN Conformance + Decision in foundational_semantics.md must produce the expected outcome (row set or error code). pass_threshold: 100% areas: diff --git a/compliance/foundation/decisions.yaml b/compliance/foundation/decisions.yaml index c3c84fb7..ea250a6f 100644 --- a/compliance/foundation/decisions.yaml +++ b/compliance/foundation/decisions.yaml @@ -17,9 +17,9 @@ # Foundation Conformance Decisions — runnable test mapping. # -# Mirrors the D-NNN decisions called out throughout -# ../../proposals/foundation/Proposed_OSI_Semantics.md. Every D-NNN row -# from the source proposal is kept here (harness's registry-consistency +# Mirrors the D-NNN Conformance Decisions called out inline throughout +# ../../core-spec/foundational_semantics.md. Every D-NNN row +# from the source spec is kept here (harness's registry-consistency # tests require full D-NNN coverage), but this bootstrap slice only ships # witness tests for D-020 — every other decision's tests: is intentionally # empty pending its tests landing in a follow-up PR. The runner emits @@ -85,7 +85,7 @@ decisions: - id: D-009 title: Deferred relationship-level keys rejected with E_DEFERRED_KEY_REJECTED in default mode - spec_ref: §11 + spec_ref: §3 tests: [] status: must_pass @@ -109,7 +109,7 @@ decisions: - id: D-014 title: Per-engine determinism — same (model, query, dialect) → byte-identical SQL - spec_ref: §11 + spec_ref: "§5.1, §6.10.2" tests: [] status: must_pass diff --git a/compliance/foundation/proposals.yaml b/compliance/foundation/proposals.yaml index 59fc4a4a..e638f864 100644 --- a/compliance/foundation/proposals.yaml +++ b/compliance/foundation/proposals.yaml @@ -17,7 +17,8 @@ # OSI Foundation v0.1 — Proposal Registry # ----------------------------------------------------------------------- -# Mirrors §10 of ../../proposals/foundation/Proposed_OSI_Semantics.md. Every +# Mirrors the deferred-features catalogue (§3 "What is In / What is Out") +# of ../../core-spec/foundational_semantics.md. Every # entry is either: # # * status: foundation — IN scope for v0.1; engines MUST implement @@ -39,193 +40,193 @@ proposals: - id: two_query_shapes status: foundation title: Aggregation vs Scalar query shapes (Fields clause) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + spec_ref: ../../core-spec/foundational_semantics.md#5.1 decisions: [D-001, D-010, D-011, D-023] - id: routing_by_expression_shape status: foundation title: Predicate routing by resolved expression shape (no role:) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.3 + spec_ref: ../../core-spec/foundational_semantics.md#6.3 decisions: [D-005, D-012] - id: implicit_home_grain_aggregation_rejection status: foundation title: Implicit home-grain aggregation in field expressions is rejected (E_AGGREGATE_IN_FIELD) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.3 + spec_ref: ../../core-spec/foundational_semantics.md#4.3 rejection_code: E_AGGREGATE_IN_FIELD decisions: [D-003] - # D-003 and D-015 are both struck/deferred in Appendix B; the - # Foundation's active surface is the rejection rule (any aggregate - # in a field body MUST raise E_AGGREGATE_IN_FIELD per §4.3 / - # Appendix C). Per-strategy equivalence (D-015) returns when - # §10's grain-aware-functions proposal lands. + # D-003 and D-015 are both struck/deferred in the spec's decision + # set; the Foundation's active surface is the rejection rule (any + # aggregate in a field body MUST raise E_AGGREGATE_IN_FIELD per §4.3 / + # Appendix A). Per-strategy equivalence (D-015) returns when the + # deferred grain-aware-functions proposal (§3) lands. - id: cross_grain_aggregates status: foundation title: Single-step cross-grain aggregates over 1:N (nested form is deferred per D-020(c)/D-027(d)) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.5 + spec_ref: ../../core-spec/foundational_semantics.md#4.5 decisions: [D-020, D-024] - id: default_join_shapes status: foundation title: Default LEFT for N:1, FULL OUTER stitch for incompatible-root multi-fact - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.6 + spec_ref: ../../core-spec/foundational_semantics.md#6.6 decisions: [D-001, D-004, D-008] - id: bridge_dedup status: foundation title: Bridge resolution materializes distinct (fact, group-key) in a single-pass aggregate (every category) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8.1 + spec_ref: ../../core-spec/foundational_semantics.md#6.8.1 decisions: [D-026, D-027] - id: decomposition_safety status: foundation title: E_UNSAFE_REAGGREGATION fires only when the chosen plan forces decomposition the aggregate cannot survive (chasm pre-agg / stitch — bridge plan is single-pass and not in scope) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.2 + spec_ref: ../../core-spec/foundational_semantics.md#6.2 decisions: [D-022] - id: identifier_resolution status: foundation title: Global / dataset-scoped namespace, ambiguous-path / no-path errors - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#4.6 + spec_ref: ../../core-spec/foundational_semantics.md#4.6 decisions: [D-006, D-018, D-019] - id: window_functions_standard status: foundation title: Standard SQL window functions (ranking / navigation / aggregate-windows; ROWS / RANGE; integer-literal bounds) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10 + spec_ref: ../../core-spec/foundational_semantics.md#6.10 decisions: [D-028, D-030, D-031, D-032] - id: nulls_last_default status: foundation title: Default NULLS LAST emission for outer Order By + OVER ORDER BY - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#5.1 + spec_ref: ../../core-spec/foundational_semantics.md#5.1 decisions: [D-014, D-029] - id: empty_aggregate_standard_sql status: foundation title: Standard SQL empty / NULL aggregate behaviour - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.11 + spec_ref: ../../core-spec/foundational_semantics.md#6.11 decisions: [D-033] - id: osi_sql_2026_dialect status: foundation title: OSI_SQL_2026 default dialect with per-dialect expression form - spec_ref: ../../proposals/foundation/SQL_EXPRESSION_SUBSET.md + spec_ref: ../../core-spec/expression_language.md decisions: [D-021, D-016] - id: windowed_metric_composition status: foundation title: Composing a metric on top of a windowed metric is rejected with E_WINDOWED_METRIC_COMPOSITION - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.10.5 + spec_ref: ../../core-spec/foundational_semantics.md#6.10.5 rejection_code: E_WINDOWED_METRIC_COMPOSITION decisions: [D-031] - # -------- Deferred (§10 of the Foundation spec) --------------------- + # -------- Deferred (§3 "What is In / What is Out" of the spec) ------ # Each deferred entry MUST be rejected at parse time with # E_DEFERRED_KEY_REJECTED (D-009). The compliance suite ships exactly - id: explicit_grain_overrides status: deferred title: Explicit grain overrides (FIXED / INCLUDE / EXCLUDE / TABLE) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: filter_context_propagation status: deferred title: Filter context propagation (reset, filter.expression on metrics) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: metric_composition_grain_filter status: deferred title: Metric composition with grain or filter inheritance - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: natural_grain_top_level status: deferred title: Model-level natural_grain declaration - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: path_disambiguation_using_relationships status: deferred title: Per-metric joins.using_relationships path disambiguation - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: per_metric_joins_type status: deferred title: Per-metric joins.type override (INNER / LEFT / FULL) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: non_equijoin_relationships status: deferred title: condition / cardinality on relationships (non-equijoin) - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: asof_range_relationships status: deferred title: ASOF and Range relationships - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: referential_integrity_keys status: deferred title: referential_integrity / from_all_rows_match / to_all_rows_match - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: semi_additive_measures status: deferred title: Semi-additive measures over snapshot facts - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: grouping_sets status: deferred title: GROUPING SETS / ROLLUP / CUBE - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: pivot_operator status: deferred title: PIVOT / UNPIVOT - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: parameterized_window_frame_bounds status: deferred title: Parameterized window frame bounds (e.g. ROWS BETWEEN :n PRECEDING ...) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_FRAME_MODE - id: groups_frame_mode status: deferred title: GROUPS frame mode (non-portable) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_FRAME_MODE - id: ordered_set_aggregates status: deferred title: WITHIN GROUP ordered-set aggregates (LISTAGG, PERCENTILE_CONT) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED # ----- moved up: this is a Foundation rejection rule, not deferral # See the foundation block above for the actual entry. - id: named_filters status: deferred title: Reusable named boolean filters referenced by name - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: dataset_scope_filters status: deferred title: Dataset-level filters with scope-based propagation - spec_ref: "../../proposals/foundation/Proposed_OSI_Semantics.md#10" + spec_ref: "../../core-spec/foundational_semantics.md#3" rejection_code: E_DEFERRED_KEY_REJECTED - id: semi_join_filter_form status: deferred title: EXISTS_IN / NOT EXISTS_IN semi-join filter (separate proposal pending) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#6.8 + spec_ref: ../../core-spec/foundational_semantics.md#6.8 rejection_code: E_DEFERRED_KEY_REJECTED - id: role_keyword status: deferred title: "role: keyword on fields/datasets (replaced by routing-by-shape D-005)" - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: legacy_attr_unsafe_agg_grainagg status: deferred title: Legacy ATTR / UNSAFE / AGG / GRAIN_AGG keyword set - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: multi_hop_bridge status: deferred title: Multi-hop bridge resolution (more than one bridge between the same endpoints) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: symmetric_aggregates status: deferred title: Looker-style symmetric aggregates (codegen-side, not a correctness mechanism) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#10 + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED - id: snowflake_partition_by_excluding status: deferred title: Snowflake ``PARTITION BY ... EXCLUDING`` window extension (vendor-specific, outside OSI_SQL_2026) - spec_ref: ../../proposals/foundation/Proposed_OSI_Semantics.md#12a + spec_ref: ../../core-spec/foundational_semantics.md#3 rejection_code: E_DEFERRED_KEY_REJECTED diff --git a/compliance/foundation/pyproject.toml b/compliance/foundation/pyproject.toml index 15aa728f..cd8d6016 100644 --- a/compliance/foundation/pyproject.toml +++ b/compliance/foundation/pyproject.toml @@ -37,13 +37,14 @@ dev = [ "pytest>=7.4", ] -# The compliance runner harness is vendored at compliance/harness/ inside -# this OSI repo. To install everything needed to run the suite: +# This suite and the shared runner harness (compliance/harness/) are members of +# the compliance/ uv workspace, so a single `uv sync` at the workspace root +# installs everything needed to run the suite: # -# pip install -e ../harness -# pip install -e . +# cd compliance && uv sync # -# Both packages are editable so iterating on either is one-step. +# Both members are installed editable, so iterating on either is one-step, and +# `uv run` uses the workspace environment from anywhere under compliance/. [tool.setuptools.packages.find] where = ["."] diff --git a/compliance/foundation/tests/README.md b/compliance/foundation/tests/README.md index feb149b1..26c5b22f 100644 --- a/compliance/foundation/tests/README.md +++ b/compliance/foundation/tests/README.md @@ -57,8 +57,8 @@ byte-for-byte (per D-014 that's a per-engine concern). | `null_ordering/` | NULLS LAST default emission, per-engine determinism witnesses | D-014, D-029 | | `empty_inputs/` | Standard SQL empty / NULL aggregate behaviour | D-033 | | `deferred/` | One negative test per `E_DEFERRED_KEY_REJECTED` family | D-009 | -| `validation_errors/` | Structural validation errors that are NOT deferred features (e.g. `E_FIELD_DEPENDENCY_CYCLE` — fields must form a DAG). | Appendix C | -| `error_taxonomy/` | Remaining Appendix C codes (E_PRIMARY_KEY_REQUIRED, E_AMBIGUOUS_MEASURE_GRAIN, etc.) | Appendix C | +| `validation_errors/` | Structural validation errors that are NOT deferred features (e.g. `E_FIELD_DEPENDENCY_CYCLE` — fields must form a DAG). | Appendix A | +| `error_taxonomy/` | Remaining Appendix A codes (E_PRIMARY_KEY_REQUIRED, E_AMBIGUOUS_MEASURE_GRAIN, etc.) | Appendix A | ## Sprint timeline @@ -71,7 +71,7 @@ byte-for-byte (per D-014 that's a per-engine concern). ## Negative tests A test is negative when its `metadata.yaml` carries -`expected_error_code: E_` (no `gold_rows.json`). The runner +`expected_error_code: E_` (no `gold.sql`). The runner asserts that the adapter exited non-zero AND that stderr contains the named error code. Negative tests do NOT carry `required_features:` unless the rejection itself depends on a Foundation-level feature; the diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml index c2e07555..a0cbdd2b 100644 --- a/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml +++ b/compliance/foundation/tests/cross_grain/moderate/t-005a-single-step-sum/metadata.yaml @@ -4,7 +4,7 @@ area: cross_grain difficulty: moderate dataset: f_prelude spec_refs: - - "Proposed_OSI_Semantics.md#D-020" + - "core-spec/foundational_semantics.md#D-020" tags: [cross-grain, 1:N, distributive] conformance_level: foundation_v0_1 decision: D-020 diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml index 700f35a9..8ae3e0f7 100644 --- a/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml +++ b/compliance/foundation/tests/cross_grain/moderate/t-005b-single-step-avg/metadata.yaml @@ -4,7 +4,7 @@ area: cross_grain difficulty: moderate dataset: f_prelude spec_refs: - - "Proposed_OSI_Semantics.md#D-020" + - "core-spec/foundational_semantics.md#D-020" tags: [cross-grain, 1:N, non-distributive] conformance_level: foundation_v0_1 decision: D-020 diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml index a46fdd62..7930fa84 100644 --- a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/metadata.yaml @@ -4,7 +4,7 @@ area: cross_grain difficulty: moderate dataset: f_prelude spec_refs: - - "Proposed_OSI_Semantics.md#D-020" + - "core-spec/foundational_semantics.md#D-020" tags: [cross-grain, 1:N, distributive, equivalence] conformance_level: foundation_v0_1 decision: D-020 diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml index 7dbe616f..eb413178 100644 --- a/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml +++ b/compliance/foundation/tests/cross_grain/moderate/t-005d-single-step-vs-nested-sum/model.yaml @@ -36,4 +36,4 @@ metrics: - {name: order_count, expression: "COUNT(orders.id)"} - {name: sum_single, expression: "SUM(orders.amount)"} # sum_nested intentionally removed — nested aggregation in a metric - # body is deferred (D-027(d)); see Proposed_OSI_Semantics.md §10. + # body is deferred (D-027(d)); see foundational_semantics.md §3. diff --git a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml index 5979567a..734eb380 100644 --- a/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml +++ b/compliance/foundation/tests/cross_grain/moderate/t-005e-count-distinct-single-step/metadata.yaml @@ -4,8 +4,8 @@ area: cross_grain difficulty: moderate dataset: f_prelude spec_refs: - - "Proposed_OSI_Semantics.md#D-020" - - "Proposed_OSI_Semantics.md#D-022" + - "core-spec/foundational_semantics.md#D-020" + - "core-spec/foundational_semantics.md#D-022" tags: [cross-grain, 1:N, count-distinct, holistic] conformance_level: foundation_v0_1 decision: D-020 diff --git a/compliance/harness/README.md b/compliance/harness/README.md index cee3a148..0b8433f7 100644 --- a/compliance/harness/README.md +++ b/compliance/harness/README.md @@ -29,12 +29,18 @@ implements the CLI contract documented in ## Install +This package (`osi_compliance_harness`, imported as `harness`) is a member of +the [`compliance/`](../) [`uv`](https://docs.astral.sh/uv/) workspace, so you do +**not** install it on its own. A single `uv sync` at the workspace root installs +it alongside every suite: + ```bash -pip install -e . +cd .. # compliance/ (the workspace root) +uv sync ``` -This installs the `harness` package. The compliance suites then depend on -this package to run their tests. +The compliance suites depend on this package to run their tests; `uv run` picks +it up automatically from any directory under `compliance/`. ## Run @@ -44,11 +50,13 @@ under ``/results/latest/`` by default): ```bash cd ../foundation -python -m harness.runner \ +uv run python -m harness.runner \ --adapter adapters/osi_python_adapter.py \ --tests tests/ \ --datasets datasets/ ``` -See [`../foundation-v0.1/README.md`](../foundation/README.md) for -the suite-level entry point and reporting layout. +See [`../foundation/README.md`](../foundation/README.md) for +the suite-level entry point and reporting layout, and +[`../ARCHITECTURE.md`](../ARCHITECTURE.md) for how the harness, adapters, and +spec fit together. diff --git a/compliance/harness/src/harness/proposals_check.py b/compliance/harness/src/harness/proposals_check.py index 32b525c2..66d95dcd 100644 --- a/compliance/harness/src/harness/proposals_check.py +++ b/compliance/harness/src/harness/proposals_check.py @@ -34,9 +34,9 @@ TESTS_DIR = "tests" METADATA_FILE = "metadata.yaml" # ``foundation`` denotes a proposal that is in scope for Foundation v0.1 -# (the proposal at proposals/foundation/); ``deferred`` denotes a -# §10 deferred feature; ``proposed`` / ``thin_slice`` were the -# pre-Foundation rollout statuses. New rows should use ``foundation`` +# (defined in core-spec/foundational_semantics.md); ``deferred`` denotes a +# §3 "What is In / What is Out" deferred feature; ``proposed`` / ``thin_slice`` +# were the pre-Foundation rollout statuses. New rows should use ``foundation`` # or ``deferred`` — the older two are retained so legacy proposal rows # parse without churn. VALID_STATUS = {"foundation", "thin_slice", "proposed", "deferred"} diff --git a/compliance/harness/src/harness/reporter.py b/compliance/harness/src/harness/reporter.py index 0966eefb..1396d243 100644 --- a/compliance/harness/src/harness/reporter.py +++ b/compliance/harness/src/harness/reporter.py @@ -214,6 +214,101 @@ def _write_proposals_status(buf: io.StringIO, suite: SuiteResult) -> None: buf.write("\n") +def _normalize_decision_test_path(raw: str) -> str: + """Normalize a ``decisions.yaml`` test path to a runner ``test_id``. + + ``decisions.yaml`` lists tests as e.g. + ``tests/cross_grain/moderate/t-005a-single-step-sum/``; the runner's + ``test_id`` for the same case is + ``cross_grain/moderate/t-005a-single-step-sum``. Strip the leading + ``tests/`` and any surrounding slashes so the two line up. + """ + p = raw.strip().strip("/") + if p.startswith("tests/"): + p = p[len("tests/"):] + return p + + +def write_decisions_coverage( + suite: SuiteResult, + decisions_path: Path, + output_dir: Path, +) -> Path | None: + """Write ``decisions_coverage.md`` mapping each ``D-NNN`` to its tests. + + Reads the suite's ``decisions.yaml`` registry and, for every decision, + reports whether it has witness tests and how those tests fared in this + run. Decisions with an empty ``tests:`` list surface as coverage gaps — + the suite's primary PR-review signal. Returns the written path, or + ``None`` if no registry is found (the run still succeeds without it). + """ + if decisions_path is None or not decisions_path.exists(): + return None + + data = yaml.safe_load(decisions_path.read_text()) or {} + decisions = data.get("decisions", []) or [] + + status_by_id = {r.test_id: r.status for r in suite.results} + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / "decisions_coverage.md" + + rows: list[tuple[str, str, str, str]] = [] + n_total = 0 + n_with_tests = 0 + n_passing = 0 + n_gap = 0 + + for decision in decisions: + did = decision.get("id", "?") + status = decision.get("status", "") + raw_tests = decision.get("tests", []) or [] + n_total += 1 + + test_ids = [_normalize_decision_test_path(t) for t in raw_tests] + present = [t for t in test_ids if t in status_by_id] + passed = [t for t in present if status_by_id[t] == TestStatus.PASS] + + if not test_ids: + outcome = "⚠️ no tests" + n_gap += 1 + elif not present: + outcome = f"— not run ({len(test_ids)} defined)" + elif len(passed) == len(present) == len(test_ids): + outcome = f"✅ pass ({len(passed)}/{len(test_ids)})" + n_with_tests += 1 + n_passing += 1 + else: + outcome = f"❌ {len(passed)}/{len(test_ids)} passing" + n_with_tests += 1 + + rows.append((did, status, str(len(test_ids)), outcome)) + + buf = io.StringIO() + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + buf.write("# OSI Compliance — Decision Coverage\n\n") + buf.write(f"**Adapter**: {suite.adapter} \n") + buf.write(f"**Date**: {now} \n") + buf.write(f"**Source**: {decisions_path.name} \n\n") + + buf.write("## Overall\n\n") + buf.write("| Metric | Count |\n") + buf.write("|--------|-------|\n") + buf.write(f"| Decisions (total) | {n_total} |\n") + buf.write(f"| With a witness test | {n_total - n_gap} |\n") + buf.write(f"| Uncovered (no tests) | {n_gap} |\n") + buf.write(f"| Passing this run | {n_passing} |\n\n") + + buf.write("## By decision\n\n") + buf.write("| Decision | Reg. status | Tests | This run |\n") + buf.write("|----------|-------------|-------|----------|\n") + for did, status, count, outcome in rows: + buf.write(f"| {did} | {status} | {count} | {outcome} |\n") + buf.write("\n") + + path.write_text(buf.getvalue()) + return path + + def format_summary_console(suite: SuiteResult) -> str: """Format a brief console summary.""" pct = (suite.passed / suite.total * 100) if suite.total > 0 else 0 diff --git a/compliance/harness/src/harness/runner.py b/compliance/harness/src/harness/runner.py index d65ca43f..c2aa3a1a 100644 --- a/compliance/harness/src/harness/runner.py +++ b/compliance/harness/src/harness/runner.py @@ -29,7 +29,11 @@ from .db_manager import DBManager from .models import SuiteResult, TestCase, TestResult, TestStatus -from .reporter import format_summary_console, write_reports +from .reporter import ( + format_summary_console, + write_decisions_coverage, + write_reports, +) from .result_compare import compare_results CONFORMANCE_FILE_NAME = "conformance.yaml" @@ -461,10 +465,16 @@ def run_suite( db.close() csv_path, md_path = write_reports(suite, output_dir) + # decisions.yaml sits at the suite root, one level above tests/. + coverage_path = write_decisions_coverage( + suite, tests_dir.parent / "decisions.yaml", output_dir + ) print(format_summary_console(suite)) print("\nReports written to:") print(f" {csv_path}") print(f" {md_path}") + if coverage_path is not None: + print(f" {coverage_path}") return suite diff --git a/compliance/harness/src/harness/tests/test_registry_yaml.py b/compliance/harness/src/harness/tests/test_registry_yaml.py index 89050f35..c9f41c93 100644 --- a/compliance/harness/src/harness/tests/test_registry_yaml.py +++ b/compliance/harness/src/harness/tests/test_registry_yaml.py @@ -17,9 +17,9 @@ """Cleanliness gate for the compliance registry YAML files. -``decisions.yaml`` is the source of truth that ties Appendix B decisions -in the Foundation spec to runnable witness tests. ``proposals.yaml`` -plays the same role for §10 deferred features. Both files have already +``decisions.yaml`` is the source of truth that ties the Foundation spec's +Conformance Decisions to runnable witness tests. ``proposals.yaml`` +plays the same role for §3 deferred features. Both files have already broken at least once because an unquoted ``:`` inside a title was parsed by YAML as a mapping value (Phase 4 compliance review, finding B1). This test pins both files so a future edit cannot reintroduce @@ -71,20 +71,20 @@ def test_registry_yaml_is_parseable(yaml_path: Path) -> None: # reference so a future reviewer can verify the demotion is still # accurate. Update this set in lockstep with the spec. _INTENTIONALLY_ABSENT_DECISIONS: dict[str, str] = { - # D-013 is reserved in the spec but has no Appendix-B row (the - # number is intentionally skipped — see Proposed_OSI_Semantics.md). - "D-013": "Reserved in Appendix B (number skipped).", - # D-015 is struck in Appendix B (~~D-015~~ Deferred — moved to a + # D-013 is reserved in the spec but has no decision row (the + # number is intentionally skipped — see foundational_semantics.md). + "D-013": "Reserved in the spec (number skipped).", + # D-015 is struck in the spec (~~D-015~~ Deferred — moved to a # separate proposal). The compilation-strategy equivalence for # field-level cross-grain aggregation is moot at the Foundation # level because field-level aggregation itself is deferred per # D-003. The Foundation surface is exercised by the D-003 # rejection witness; the strategy equivalence returns alongside - # §10's grain-aware-functions proposal. - "D-015": "Struck in Appendix B; depends on the deferred D-003.", + # the deferred grain-aware-functions proposal. + "D-015": "Struck in the spec; depends on the deferred D-003.", # D-017 is deferred — semi-join filtering moved to a follow-up # proposal. The negative test for EXISTS_IN is a rejection test. - "D-017": "Deferred — Proposed_OSI_Semantics.md table row marked", + "D-017": "Deferred — foundational_semantics.md table row marked", } diff --git a/compliance/harness/src/harness/tests/test_reporter_coverage.py b/compliance/harness/src/harness/tests/test_reporter_coverage.py new file mode 100644 index 00000000..0eda25c6 --- /dev/null +++ b/compliance/harness/src/harness/tests/test_reporter_coverage.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the decisions-coverage reporter (decisions_coverage.md).""" + +from __future__ import annotations + +from pathlib import Path + +from harness.models import SuiteResult, TestResult, TestStatus +from harness.reporter import write_decisions_coverage + +_DECISIONS_YAML = """\ +version: "0.1" +decisions: + - id: D-001 + title: Covered and passing + tests: + - tests/cross_grain/moderate/t-005a-single-step-sum/ + status: must_pass + - id: D-002 + title: No witness tests yet + tests: [] + status: must_pass + - id: D-003 + title: Has a test that did not run this session + tests: + - tests/windows/t-099-not-run/ + status: must_pass +""" + + +def _res(test_id: str, status: TestStatus) -> TestResult: + return TestResult(test_id=test_id, area="a", difficulty="moderate", status=status) + + +def _write_decisions(tmp_path: Path) -> Path: + p = tmp_path / "decisions.yaml" + p.write_text(_DECISIONS_YAML) + return p + + +def test_coverage_report_classifies_each_decision(tmp_path: Path) -> None: + decisions_path = _write_decisions(tmp_path) + suite = SuiteResult( + adapter="adapter.py", + results=[ + # test_id has no leading "tests/" — matches the runner's form. + _res("cross_grain/moderate/t-005a-single-step-sum", TestStatus.PASS), + ], + ) + + out = write_decisions_coverage(suite, decisions_path, tmp_path / "results") + assert out is not None and out.exists() + md = out.read_text() + + # D-001: its one test ran and passed. + d1 = next(line for line in md.splitlines() if line.startswith("| D-001 ")) + assert "✅ pass (1/1)" in d1 + # D-002: no tests => flagged as a gap. + d2 = next(line for line in md.splitlines() if line.startswith("| D-002 ")) + assert "no tests" in d2 + # D-003: defines a test that wasn't in this run's results. + d3 = next(line for line in md.splitlines() if line.startswith("| D-003 ")) + assert "not run" in d3 + + # Overall counts: 3 total, 1 uncovered, 1 passing. + assert "| Decisions (total) | 3 |" in md + assert "| Uncovered (no tests) | 1 |" in md + assert "| Passing this run | 1 |" in md + + +def test_coverage_report_marks_failures(tmp_path: Path) -> None: + decisions_path = _write_decisions(tmp_path) + suite = SuiteResult( + adapter="x", + results=[ + _res("cross_grain/moderate/t-005a-single-step-sum", TestStatus.FAIL), + ], + ) + out = write_decisions_coverage(suite, decisions_path, tmp_path / "results") + assert out is not None + d1 = next(line for line in out.read_text().splitlines() if line.startswith("| D-001 ")) + assert "❌ 0/1 passing" in d1 + + +def test_returns_none_when_registry_missing(tmp_path: Path) -> None: + suite = SuiteResult(adapter="x") + out = write_decisions_coverage(suite, tmp_path / "nope.yaml", tmp_path / "results") + assert out is None diff --git a/compliance/pyproject.toml b/compliance/pyproject.toml new file mode 100644 index 00000000..d2f05d1e --- /dev/null +++ b/compliance/pyproject.toml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# --------------------------------------------------------------------------- +# Ossie compliance — uv workspace root. +# +# `compliance/` is a single uv workspace so the whole suite installs and runs +# as one unit: `uv sync` here builds ONE environment (./.venv) containing every +# member below, and `uv run` works from anywhere inside compliance/. +# +# This is the compliance-scoped realization of the project-wide single +# environment described in the Ossie dev guide. When the repo root later +# becomes the workspace root, these members roll up there and this file drops +# its [tool.uv.workspace] table (becoming a plain aggregator). +# +# This is a *virtual* root (no [project] table): it is not itself a +# distributable package, only the workspace definition. The installable +# packages are the members: +# * harness/ -> osi_compliance_harness (the shared runner) +# * foundation/ -> osi_compliance_foundation_v0_1 (the Foundation suite) +# --------------------------------------------------------------------------- + +[tool.uv.workspace] +members = ["harness", "foundation"] + +# Dev tooling for the whole workspace. `uv sync` installs this group by +# default, so `uv run pytest` runs the harness + suite tests in the one +# workspace environment. +[dependency-groups] +dev = ["pytest>=7.4"] diff --git a/compliance/uv.lock b/compliance/uv.lock new file mode 100644 index 00000000..fb39eb6b --- /dev/null +++ b/compliance/uv.lock @@ -0,0 +1,219 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[manifest] +members = [ + "osi-compliance-foundation-v0-1", + "osi-compliance-harness", +] + +[manifest.dependency-groups] +dev = [{ name = "pytest", specifier = ">=7.4" }] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "duckdb" +version = "1.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, + { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, + { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, + { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "osi-compliance-foundation-v0-1" +version = "0.1.0" +source = { editable = "foundation" } +dependencies = [ + { name = "duckdb" }, + { name = "pyyaml" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "duckdb", specifier = ">=0.10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4" }, + { name = "pyyaml", specifier = ">=6.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "osi-compliance-harness" +version = "0.1.0" +source = { editable = "harness" } +dependencies = [ + { name = "duckdb" }, + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "duckdb", specifier = ">=0.10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=25" }, +] +provides-extras = ["dev"] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/62/6d5bd3169478b7f09e08ab3e50175d486a9e7f3a419b88fc9280ba564ab1/sqlglot-30.13.0.tar.gz", hash = "sha256:f0a6eb79de2fd6efe2689f8cf197caa4f08bfe77c7880315616fa4420b8ba2bf", size = 5932385, upload-time = "2026-07-20T20:16:54.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/2f/2076eca54f6a8ed1c86301bdb4bb2ae4181b0c1c4dbc041062ca997dc1b2/sqlglot-30.13.0-py3-none-any.whl", hash = "sha256:08f87ff7b052246d61b731628c8c2db0bc91f2c9e69f5ba68a1d160a9f5b49b1", size = 719120, upload-time = "2026-07-20T20:16:53.248Z" }, +]