From be6360ceab495c905fb434582625b5eb20b9ae99 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 08:28:55 -0700 Subject: [PATCH 01/22] First attempt at core (M0). --- Cargo.lock | 207 +++++++ Cargo.toml | 34 ++ README.md | 48 +- crates/ddx-ad/Cargo.toml | 16 + crates/ddx-ad/src/lib.rs | 38 ++ crates/ddx-core/Cargo.toml | 18 + crates/ddx-core/README.md | 64 +++ crates/ddx-core/src/colref.rs | 237 ++++++++ crates/ddx-core/src/constructors.rs | 309 +++++++++++ crates/ddx-core/src/ddx.rs | 143 +++++ crates/ddx-core/src/engine.rs | 368 +++++++++++++ crates/ddx-core/src/error.rs | 61 +++ crates/ddx-core/src/lib.rs | 66 +++ crates/ddx-core/src/rewrite.rs | 507 ++++++++++++++++++ crates/ddx-core/tests/rewrite.rs | 238 ++++++++ crates/ddx-core/tests/roundtrip.rs | 96 ++++ crates/ddx-core/tests/rules.rs | 118 ++++ crates/ddx-datafusion/Cargo.toml | 18 + crates/ddx-datafusion/src/lib.rs | 31 ++ {spikes => docs/spikes}/README.md | 0 {spikes => docs/spikes}/attention_ad_spike.py | 0 .../spikes}/attention_causal_mask_check.py | 0 .../spikes}/duckdb_reentrancy_r1b.py | 0 .../spikes}/duckdb_substrait_window_bug.py | 0 .../spikes}/relational_ad_spike.py | 0 {spikes => docs/spikes}/route_ad_spike.py | 0 .../spikes}/sqlparser-spike/Cargo.toml | 0 .../spikes}/sqlparser-spike/src/main.rs | 0 .../spikes}/substrait_ad_marker_spike.py | 0 .../spikes}/substrait_limitation_repro.py | 0 python/ddxdb/README.md | 25 + tests/README.md | 30 ++ 32 files changed, 2671 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 crates/ddx-ad/Cargo.toml create mode 100644 crates/ddx-ad/src/lib.rs create mode 100644 crates/ddx-core/Cargo.toml create mode 100644 crates/ddx-core/README.md create mode 100644 crates/ddx-core/src/colref.rs create mode 100644 crates/ddx-core/src/constructors.rs create mode 100644 crates/ddx-core/src/ddx.rs create mode 100644 crates/ddx-core/src/engine.rs create mode 100644 crates/ddx-core/src/error.rs create mode 100644 crates/ddx-core/src/lib.rs create mode 100644 crates/ddx-core/src/rewrite.rs create mode 100644 crates/ddx-core/tests/rewrite.rs create mode 100644 crates/ddx-core/tests/roundtrip.rs create mode 100644 crates/ddx-core/tests/rules.rs create mode 100644 crates/ddx-datafusion/Cargo.toml create mode 100644 crates/ddx-datafusion/src/lib.rs rename {spikes => docs/spikes}/README.md (100%) rename {spikes => docs/spikes}/attention_ad_spike.py (100%) rename {spikes => docs/spikes}/attention_causal_mask_check.py (100%) rename {spikes => docs/spikes}/duckdb_reentrancy_r1b.py (100%) rename {spikes => docs/spikes}/duckdb_substrait_window_bug.py (100%) rename {spikes => docs/spikes}/relational_ad_spike.py (100%) rename {spikes => docs/spikes}/route_ad_spike.py (100%) rename {spikes => docs/spikes}/sqlparser-spike/Cargo.toml (100%) rename {spikes => docs/spikes}/sqlparser-spike/src/main.rs (100%) rename {spikes => docs/spikes}/substrait_ad_marker_spike.py (100%) rename {spikes => docs/spikes}/substrait_limitation_repro.py (100%) create mode 100644 python/ddxdb/README.md create mode 100644 tests/README.md diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..646a267 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,207 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ar_archive_writer" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +dependencies = [ + "object", +] + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ddx-ad" +version = "0.0.0" + +[[package]] +name = "ddx-core" +version = "0.0.0" +dependencies = [ + "sqlparser", +] + +[[package]] +name = "ddx-datafusion" +version = "0.0.0" +dependencies = [ + "ddx-core", +] + +[[package]] +name = "ddx-duckdb" +version = "0.0.0" +dependencies = [ + "ddx-core", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..15c68c0 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,34 @@ +# ddx — portable autograd for composable databases. +# +# One engine-neutral differentiation core (`ddx-core`, v1 / M0) plus thin +# per-engine adapters. See docs/design.md §6 for the layout and the +# dependency policy: `ddx-core` depends on `sqlparser` only; the heavy +# per-engine dependencies (datafusion, duckdb) are quarantined in the +# adapter crates. +[workspace] +resolver = "2" +members = [ + "crates/ddx-core", + "crates/ddx-ad", + "crates/ddx-datafusion", + "crates/ddx-duckdb", +] + +[workspace.package] +version = "0.0.0" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/xqlsystems/ddx" +authors = ["Alex Merose "] +rust-version = "1.75" + +[workspace.dependencies] +# The single load-bearing dependency of the core. Pinned exactly (see the +# `sqlparser` version policy in docs/design.md §6): a bump is a breaking +# release of ddx-core. 0.62 is the version every M0 spike was verified +# against (spikes/sqlparser-spike, decision-log G1/G3). +sqlparser = { version = "=0.62.0", features = ["visitor"] } + +# Internal crates, referenced by path within the workspace. +ddx-core = { path = "crates/ddx-core" } +ddx-ad = { path = "crates/ddx-ad" } diff --git a/README.md b/README.md index cc98b05..302e90a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,50 @@ # ddx [JAX](https://docs.jax.dev/en/latest/)-style [automatic differentiation](https://docs.jax.dev/en/latest/automatic-differentiation.html) in SQL -_status_: Currently in the design phase. Follow along here: [docs/design.md](docs/design.md). +Write calculus directly in SQL and get derivatives back as ordinary columns, +evaluated row by row by the engine alongside everything else: + +```sql +SELECT i, grad(x * y, x) AS dfdx, grad(x * y, y) AS dfdy FROM g +``` + +`grad`/`jvp` are compile-time **markers**: they carry a differentiation request +through parsing and are rewritten away — to plain derivative SQL — before the +query ever runs. One engine-neutral Rust core, thin per-engine adapters. + +_status_: **M0 landed** — the scalar core (`ddx-core`) is implemented. The rest +is in progress; see [docs/design.md](docs/design.md) §8 for the milestones. + +## Layout + +``` +crates/ + ddx-core/ # v1 engine — differentiate sqlparser::ast::Expr + rewrite_sql [M0 ✓] + ddx-ad/ # v2 engine — query-level reverse-mode AD over Substrait [M3/M4] + ddx-datafusion/ # DataFusion adapter: ddx_sql (Path A) + AnalyzerRule (Path B) [M2] + ddx-duckdb/ # DuckDB community extension: ddx('') [M5] +python/ddxdb/ # PyO3/maturin wheel: rewrite_sql + Context.sql() shim [M2] +tests/ # cross-engine numeric-agreement suites (vs JAX) [M2/M6] +spikes/ # runnable evidence for every load-bearing design claim +future/ # deferred: ddx-duckdb-cpp (bare grad() in DuckDB), ddx-pg +docs/design.md # the design +``` + +## Try the core + +```rust +use ddx_core::Ddx; +use ddx_core::sqlparser::dialect::GenericDialect; + +let out = Ddx::new() + .rewrite_sql("SELECT grad(sin(x), x) AS d FROM t", &GenericDialect {}) + .unwrap(); +assert_eq!(out, "SELECT (cos(x)) AS d FROM t"); +``` + +``` +cargo test -p ddx-core +``` + +See [`crates/ddx-core/README.md`](crates/ddx-core/README.md) for what the engine +supports and the correctness properties worth knowing. diff --git a/crates/ddx-ad/Cargo.toml b/crates/ddx-ad/Cargo.toml new file mode 100644 index 0000000..fca0055 --- /dev/null +++ b/crates/ddx-ad/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ddx-ad" +description = "Query-level reverse-mode AD for ddx (v2): vjp over substrait::proto plans. Scaffold — see docs/design.md §4 and milestones M3/M4." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +rust-version.workspace = true + +# NOTE (M0 scaffold): v2's core will depend on `substrait` only (design.md +# §4.2), symmetric with v1-core's `sqlparser`-only dependency. The dependency +# is intentionally not added yet — M0 only lands the crate seam so the +# workspace layout of §6 exists. The five transpose rules and `vjp_query` +# arrive in M3/M4. +[dependencies] diff --git a/crates/ddx-ad/src/lib.rs b/crates/ddx-ad/src/lib.rs new file mode 100644 index 0000000..5a2e6a9 --- /dev/null +++ b/crates/ddx-ad/src/lib.rs @@ -0,0 +1,38 @@ +//! `ddx-ad` — query-level reverse-mode automatic differentiation (ddx v2). +//! +//! **Status: scaffold.** This crate is the home for the v2 engine described in +//! [`docs/design.md`](../../../docs/design.md) §4: differentiating whole +//! queries (not scalar expressions) by applying one transpose rule per +//! relational primitive — contraction, elementwise, reduce, route, +//! stop-gradient — over `substrait::proto` plans tagged with +//! extension-function markers. +//! +//! Nothing here is implemented yet. M0 (the current milestone) delivers only +//! the scalar core, [`ddx-core`](../ddx_core/index.html), which becomes the +//! *elementwise leaf* of this engine (design.md §4.3). The public surface +//! sketched below is the M3/M4 target and exists here as a compile-checked +//! seam, not a working API. +//! +//! Planned surface (design.md §4.4): +//! +//! ```ignore +//! pub fn vjp_query(plan: &Plan, wrt: &[RelRef]) -> Result; +//! +//! pub struct BackwardProgram { +//! pub forward_steps: Vec<(Ident, Plan)>, +//! pub backward_steps: Vec<(Ident, Plan)>, +//! pub gradients: HashMap, +//! } +//! ``` +//! +//! The four marker names it recognizes (`ddx_contract_mark`, +//! `ddx_reduce_mark`, `ddx_route_mark`, `ddx_stop_gradient`) are Substrait +//! extension-function markers, the same "tag, don't infer" mechanism `grad()` +//! uses in v1, one layer down in the plan. + +#![forbid(unsafe_code)] + +/// Milestone marker: v2 (`ddx-ad`) is not implemented in M0. +/// +/// See [`docs/design.md`](../../../docs/design.md) §4 and milestones M3/M4. +pub const STATUS: &str = "scaffold: query-level reverse-mode AD lands in M3/M4"; diff --git a/crates/ddx-core/Cargo.toml b/crates/ddx-core/Cargo.toml new file mode 100644 index 0000000..9c9889f --- /dev/null +++ b/crates/ddx-core/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ddx-core" +description = "Engine-neutral symbolic differentiation of SQL scalar expressions (the v1 core of ddx: grad/jvp as a query rewrite)." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +rust-version.workspace = true +readme = "README.md" +keywords = ["sql", "autograd", "differentiation", "datafusion", "duckdb"] +categories = ["mathematics", "database"] + +[dependencies] +# ddx-core re-exports `sqlparser` (see lib.rs) so downstream adapters can't +# accidentally link a mismatched version. This is the *only* non-trivial +# dependency of the core, by design. +sqlparser = { workspace = true } diff --git a/crates/ddx-core/README.md b/crates/ddx-core/README.md new file mode 100644 index 0000000..3e654ae --- /dev/null +++ b/crates/ddx-core/README.md @@ -0,0 +1,64 @@ +# ddx-core + +Engine-neutral symbolic differentiation of SQL scalar expressions — the v1 +core of [`ddx`](https://github.com/xqlsystems/ddx), "autograd for composable +databases." Write calculus directly in SQL and let the engine evaluate the +derivative per row (the relational equivalent of `jax.vmap(jax.grad(f))`): + +```sql +SELECT i, grad(x * y, x) AS dfdx, grad(x * y, y) AS dfdy FROM g +``` + +`grad`/`jvp` are **markers**, not row functions: they carry a differentiation +request through parsing and are always rewritten away *before* execution. + +```rust +use ddx_core::Ddx; +use ddx_core::sqlparser::dialect::GenericDialect; + +let ddx = Ddx::new(); +let out = ddx.rewrite_sql("SELECT grad(sin(x), x) AS d FROM t", &GenericDialect {})?; +assert_eq!(out, "SELECT (cos(x)) AS d FROM t"); +``` + +The engine differentiates [`sqlparser::ast::Expr`](https://docs.rs/sqlparser) +directly — the AST *is* the IR, there is no bespoke representation. The single +load-bearing dependency is `sqlparser`, **re-exported** as `ddx_core::sqlparser` +so downstream adapters cannot link a mismatched version. + +## What it supports + +`+ - * /`; the unary chain rule for the trig / inverse-trig / exp / log / +hyperbolic set plus `abs`; `power` with a constant base or exponent; +higher-order via nesting; through-aggregate via linearity +(`AVG(grad(loss, theta))`). Custom unary rules are registrable +(`ddx.register("myfn", rule)` — the rule supplies `f'(u)`, the engine applies +the chain rule). Anything else is a typed `DiffError`, never a silently-wrong +number. + +Scalar `vjp` is deliberately **not** here: the name is reserved for the +query-level reverse-mode operation in `ddx-ad` (design.md §3.6, §4). + +## Correctness properties worth knowing + +- **Identifier folding is per-dialect** (`Ddx::for_datafusion()` vs + `Ddx::for_duckdb()`): unquoted identifiers always fold case; DuckDB folds + quoted ones too. `grad(Temp*Temp, temp)` matches. +- **Ambiguity is a hard error, not a guess**: a `wrt` that can't be pinned + syntactically (`grad(a.x*b.x, x)`) errors rather than differentiating the + wrong column. +- **`div` forces floating-point division** (`CAST( AS DOUBLE)`), so + integer columns don't silently truncate the derivative. +- **0/1-folding follows JAX's `Zero`-tangent convention** and differs from + unfolded SQL only on NULL-bearing rows (documented, tested). + +See [`../../docs/design.md`](../../docs/design.md) §3 for the full rationale and +the decision log (`F#`/`G#`) behind each of these. + +## Status + +M0 (the scalar core) — implemented. The per-engine adapters (`ddx-datafusion`, +`ddx-duckdb`), the Python wheel (`ddxdb`), and the query-level v2 engine +(`ddx-ad`) are later milestones; see the design doc's §8. + +Licensed under Apache-2.0. diff --git a/crates/ddx-core/src/colref.rs b/crates/ddx-core/src/colref.rs new file mode 100644 index 0000000..e92d042 --- /dev/null +++ b/crates/ddx-core/src/colref.rs @@ -0,0 +1,237 @@ +//! Column identity read off the AST, compared with per-dialect identifier +//! folding rather than raw-string equality (design.md §3.2, F1). + +use sqlparser::ast::{Expr, Ident}; + +use crate::error::{DiffError, Result}; + +/// How a dialect folds identifiers for case-insensitive comparison. +/// +/// SQL unquoted identifiers are case-insensitive, so `grad(Temp*Temp, temp)` +/// must match — otherwise it silently differentiates to `0`. The exact rule is +/// per-dialect (F1): +/// +/// * [`IdentCasing::FoldUnquoted`] — unquoted identifiers fold to lowercase; +/// quoted identifiers keep their case. (DataFusion, Postgres, the generic +/// dialect.) +/// * [`IdentCasing::FoldAll`] — *all* identifiers fold to lowercase, quoted +/// included. (DuckDB, which is fully case-insensitive.) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdentCasing { + /// Fold unquoted identifiers only (DataFusion / Postgres / generic). + FoldUnquoted, + /// Fold every identifier, quoted included (DuckDB). + FoldAll, +} + +impl IdentCasing { + /// The comparison key for a single identifier under this policy. + pub fn fold(self, id: &Ident) -> String { + match (id.quote_style, self) { + // Unquoted: always case-folded. + (None, _) => id.value.to_ascii_lowercase(), + // Quoted: folded only for DuckDB. + (Some(_), IdentCasing::FoldAll) => id.value.to_ascii_lowercase(), + (Some(_), IdentCasing::FoldUnquoted) => id.value.clone(), + } + } +} + +/// A column reference: an optional qualifier and a name, taken straight off the +/// AST. Stores `sqlparser` [`Ident`]s (which keep quote-style) and compares +/// with dialect-aware folding, never raw-string equality. +#[derive(Debug, Clone)] +pub struct ColRef { + /// The qualifier (`a` in `a.x`), if the reference was compound. + pub qualifier: Option, + /// The column name (`x` in `a.x`, or in a bare `x`). + pub name: Ident, +} + +/// Whether a column occurrence *is* the differentiation variable, and if its +/// identity relative to `wrt` could be established syntactically at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Match { + /// This occurrence is the `wrt` column — its tangent is the seed. + Is, + /// This occurrence is definitely a different column — tangent zero. + Not, + /// The occurrence's base name matches `wrt` but its qualification can't be + /// pinned syntactically — a bare occurrence when `wrt` is qualified, or a + /// qualified occurrence when `wrt` is bare. Hard error (F2). + Ambiguous, +} + +impl ColRef { + /// Build a bare (unqualified) column reference by name. + pub fn bare(name: impl Into) -> Self { + ColRef { + qualifier: None, + name: Ident::new(name.into()), + } + } + + /// Read a `ColRef` from a column-reference expression + /// (`Identifier`/`CompoundIdentifier`, seeing through a `Nested` wrapper). + /// Returns `None` for any expression that is not a column reference. + pub fn from_expr(e: &Expr) -> Option { + match e { + Expr::Identifier(id) => Some(ColRef { + qualifier: None, + name: id.clone(), + }), + Expr::CompoundIdentifier(parts) => parts.last().map(|last| { + let qualifier = if parts.len() >= 2 { + Some(parts[parts.len() - 2].clone()) + } else { + None + }; + ColRef { + qualifier, + name: last.clone(), + } + }), + Expr::Nested(inner) => ColRef::from_expr(inner), + _ => None, + } + } + + /// Parse the `wrt` argument of a marker: it must be a bare column + /// (`Identifier`/`CompoundIdentifier`), never an expression (F: the design + /// rejects `grad(x*y, x+y)`). + pub fn from_wrt_arg(func: &str, e: &Expr) -> Result { + ColRef::from_expr(e).ok_or_else(|| { + DiffError::InvalidMarker(format!( + "{func}(): the differentiation variable must be a bare column, got `{e}`" + )) + }) + } + + /// Classify an occurrence `self` against the differentiation variable + /// `wrt` under a folding policy — the whole of the ambiguity guard (F2). + /// + /// The guard fires (returns [`Match::Ambiguous`]) *only* on an uncertain + /// occurrence of the `wrt` base name; a non-matching name is always + /// [`Match::Not`], and a fully-qualified unambiguous match (e.g. `a.x` + /// against `a.x`) is [`Match::Is`] with no error. + pub fn classify(&self, wrt: &ColRef, casing: IdentCasing) -> Match { + if casing.fold(&self.name) != casing.fold(&wrt.name) { + // Different base name — unrelated column, no ambiguity possible. + return Match::Not; + } + match (&self.qualifier, &wrt.qualifier) { + // Both qualified: identity is fully determined by the qualifier. + (Some(sq), Some(wq)) => { + if casing.fold(sq) == casing.fold(wq) { + Match::Is + } else { + Match::Not + } + } + // Both bare, same name: this is the wrt. + (None, None) => Match::Is, + // A qualified occurrence when wrt is bare, or a bare occurrence + // when wrt is qualified: cannot be pinned syntactically. + (Some(_), None) | (None, Some(_)) => Match::Ambiguous, + } + } + + /// Render for error messages (e.g. `a.x` or `x`). + pub fn display(&self) -> String { + match &self.qualifier { + Some(q) => format!("{q}.{}", self.name), + None => self.name.to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn id(s: &str) -> Ident { + Ident::new(s) + } + + fn quoted(s: &str) -> Ident { + Ident::with_quote('"', s) + } + + #[test] + fn unquoted_folds_case_in_every_dialect() { + assert_eq!( + IdentCasing::FoldUnquoted.fold(&id("Temp")), + IdentCasing::FoldUnquoted.fold(&id("temp")) + ); + assert_eq!( + IdentCasing::FoldAll.fold(&id("Temp")), + IdentCasing::FoldAll.fold(&id("temp")) + ); + } + + #[test] + fn quoted_folding_is_per_dialect() { + // DuckDB folds quoted; DataFusion/Postgres keep case. + assert_eq!( + IdentCasing::FoldAll.fold("ed("Temp")), + IdentCasing::FoldAll.fold("ed("temp")) + ); + assert_ne!( + IdentCasing::FoldUnquoted.fold("ed("Temp")), + IdentCasing::FoldUnquoted.fold("ed("temp")) + ); + } + + #[test] + fn bare_wrt_matches_bare_occurrence() { + let x = ColRef::bare("x"); + assert_eq!(x.classify(&x, IdentCasing::FoldUnquoted), Match::Is); + assert_eq!( + ColRef::bare("y").classify(&x, IdentCasing::FoldUnquoted), + Match::Not + ); + } + + #[test] + fn qualified_wrt_disambiguates_across_a_join() { + // grad(a.x * b.x, a.x): a.x is the wrt, b.x is a different column. + let ax = ColRef { + qualifier: Some(id("a")), + name: id("x"), + }; + let bx = ColRef { + qualifier: Some(id("b")), + name: id("x"), + }; + assert_eq!(ax.classify(&ax, IdentCasing::FoldUnquoted), Match::Is); + assert_eq!(bx.classify(&ax, IdentCasing::FoldUnquoted), Match::Not); + } + + #[test] + fn bare_occurrence_with_qualified_wrt_is_ambiguous() { + // grad(x * a.x, a.x): bare x might be a.x — demand qualification. + let bare_x = ColRef::bare("x"); + let ax = ColRef { + qualifier: Some(id("a")), + name: id("x"), + }; + assert_eq!( + bare_x.classify(&ax, IdentCasing::FoldUnquoted), + Match::Ambiguous + ); + } + + #[test] + fn qualified_occurrence_with_bare_wrt_is_ambiguous() { + // grad(a.x * b.x, x): bare wrt x, qualified occurrences — ambiguous. + let ax = ColRef { + qualifier: Some(id("a")), + name: id("x"), + }; + let bare_x = ColRef::bare("x"); + assert_eq!( + ax.classify(&bare_x, IdentCasing::FoldUnquoted), + Match::Ambiguous + ); + } +} diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs new file mode 100644 index 0000000..400e69e --- /dev/null +++ b/crates/ddx-core/src/constructors.rs @@ -0,0 +1,309 @@ +//! Smart constructors for building derivative `sqlparser::ast::Expr` trees. +//! +//! These own three correctness properties, not just algebraic tidiness +//! (design.md §3.2): +//! +//! 1. **0/1-folding** — the JAX-`Zero`-tangent equivalent. Structurally-zero +//! terms are dropped and dead product branches short-circuit, keeping output +//! compact. This is a *stated* NULL-semantics convention (folding +//! `0 * (NULL-valued expr)` to `0`), documented and tested, not silent (F11). +//! 2. **Numeric-type policy** — [`div`] forces floating-point division by +//! casting its numerator to `DOUBLE`, so `grad(x/y, y)` on integer columns +//! does not silently truncate (integer `/` differs across engines). Literals +//! are emitted with an explicit decimal point (F4). +//! 3. **Precedence-safe construction** — composite operands are wrapped in +//! `Expr::Nested` exactly when the operator precedence requires it, because +//! `sqlparser`'s `Display` for a binary op emits no precedence parentheses. +//! Without this, a *constructed* `mul(add(a,b), c)` displays as `a + b * c` +//! and reparses as the wrong expression — a wrong number in valid SQL (G1). + +use sqlparser::ast::{ + BinaryOperator, CastKind, DataType, ExactNumberInfo, Expr, Function, FunctionArg, + FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + UnaryOperator, Value, +}; + +// --------------------------------------------------------------------------- +// Literals and constant inspection +// --------------------------------------------------------------------------- + +/// Format an `f64` as a SQL numeric literal that always carries a decimal point +/// (or exponent), so it reads as floating-point rather than an integer literal. +fn format_f64(v: f64) -> String { + let s = format!("{v}"); + if s.contains(['.', 'e', 'E']) || s.contains("inf") || s.contains("NaN") { + s + } else { + format!("{s}.0") + } +} + +/// A numeric literal expression for `v` (e.g. `1.0`, `2.0`, `0.6931471805599453`). +pub fn num(v: f64) -> Expr { + Expr::Value(Value::Number(format_f64(v), false).with_empty_span()) +} + +/// The constant `0.0` — the derivative of anything independent of `wrt`. +pub fn zero() -> Expr { + num(0.0) +} + +/// The constant `1.0` — the derivative of `wrt` itself. +pub fn one() -> Expr { + num(1.0) +} + +/// The `f64` value of a numeric literal expression, if it is one. +/// +/// Sees through a single `Expr::Nested` wrapper so folding still recognizes a +/// parenthesized literal. +pub fn as_const(e: &Expr) -> Option { + match e { + Expr::Value(v) => match &v.value { + Value::Number(s, _) => s.parse::().ok(), + _ => None, + }, + Expr::Nested(inner) => as_const(inner), + _ => None, + } +} + +/// True if `e` is a numeric literal exactly equal to zero. +pub fn is_zero(e: &Expr) -> bool { + matches!(as_const(e), Some(v) if v == 0.0) +} + +/// True if `e` is a numeric literal exactly equal to one. +pub fn is_one(e: &Expr) -> bool { + matches!(as_const(e), Some(v) if v == 1.0) +} + +// --------------------------------------------------------------------------- +// Precedence-safe assembly (G1) +// --------------------------------------------------------------------------- + +/// Binding-precedence of an expression's *top* operator, higher = binds tighter. +/// Self-delimiting forms (literals, identifiers, function calls, `CAST`, +/// already-`Nested`) are atoms and never need wrapping. +fn precedence(e: &Expr) -> u8 { + match e { + Expr::BinaryOp { op, .. } => match op { + BinaryOperator::Plus | BinaryOperator::Minus => 10, + BinaryOperator::Multiply | BinaryOperator::Divide | BinaryOperator::Modulo => 20, + _ => 20, + }, + Expr::UnaryOp { + op: UnaryOperator::Minus, + .. + } => 30, + _ => 100, + } +} + +/// Wrap `e` in `Expr::Nested` iff its precedence is below `threshold` +/// (`strict`) or at-or-below it (`!strict`, for the right operand of a +/// non-commutative operator, where equal precedence still needs parentheses: +/// `a - (b - c)` ≠ `a - b - c`). +fn wrap(e: Expr, threshold: u8, strict: bool) -> Expr { + let needs = if strict { + precedence(&e) < threshold + } else { + precedence(&e) <= threshold + }; + if needs { + Expr::Nested(Box::new(e)) + } else { + e + } +} + +/// `left op right`, parenthesizing operands only where precedence demands it. +fn binary(left: Expr, op: BinaryOperator, right: Expr) -> Expr { + let p = match op { + BinaryOperator::Plus | BinaryOperator::Minus => 10, + _ => 20, + }; + let non_commutative = matches!(op, BinaryOperator::Minus | BinaryOperator::Divide); + Expr::BinaryOp { + left: Box::new(wrap(left, p, true)), + op, + right: Box::new(wrap(right, p, !non_commutative)), + } +} + +/// Wrap `e` in a `CAST(... AS DOUBLE)`. Self-delimiting, so it never needs +/// precedence parentheses as an operand. +pub fn cast_double(e: Expr) -> Expr { + Expr::Cast { + kind: CastKind::Cast, + expr: Box::new(e), + data_type: DataType::Double(ExactNumberInfo::None), + array: false, + format: None, + } +} + +// --------------------------------------------------------------------------- +// The folding builders +// --------------------------------------------------------------------------- + +/// `a + b`, dropping a structurally-zero operand. +pub fn add(a: Expr, b: Expr) -> Expr { + if is_zero(&a) { + b + } else if is_zero(&b) { + a + } else { + binary(a, BinaryOperator::Plus, b) + } +} + +/// `a - b`, dropping a zero right operand and turning `0 - b` into `-b`. +pub fn sub(a: Expr, b: Expr) -> Expr { + if is_zero(&b) { + a + } else if is_zero(&a) { + neg(b) + } else { + binary(a, BinaryOperator::Minus, b) + } +} + +/// `a * b`, folding `0 * _ = 0` and `1 * b = b` (and the mirror cases). +pub fn mul(a: Expr, b: Expr) -> Expr { + if is_zero(&a) || is_zero(&b) { + zero() + } else if is_one(&a) { + b + } else if is_one(&b) { + a + } else { + binary(a, BinaryOperator::Multiply, b) + } +} + +/// `a / b`, folding `0 / _ = 0` and `a / 1 = a`. +/// +/// When a real division is emitted, the numerator is cast to `DOUBLE` so the +/// division is floating-point on every engine — SQL integer division truncates +/// on some and not others, which would make `grad(x/y, y)` on a `BIGINT` +/// column silently wrong (F4). Casting one operand promotes the whole division; +/// casting the *numerator* (not the result) is essential — `CAST(a/b AS DOUBLE)` +/// would truncate before the cast. +pub fn div(a: Expr, b: Expr) -> Expr { + if is_zero(&a) { + zero() + } else if is_one(&b) { + a + } else { + binary(cast_double(a), BinaryOperator::Divide, b) + } +} + +/// `-a`, folding `-0 = 0` and parenthesizing a binary operand +/// (`-(a + b)`, `-(a / b)`), since unary minus binds tighter than either. +pub fn neg(a: Expr) -> Expr { + if is_zero(&a) { + zero() + } else { + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(wrap(a, 30, true)), + } + } +} + +/// `e * e`. +pub fn square(e: Expr) -> Expr { + mul(e.clone(), e) +} + +// --------------------------------------------------------------------------- +// Function-call construction (for the outer factors of chain-rule terms) +// --------------------------------------------------------------------------- + +/// Build an unqualified scalar function call `name(args...)`. +pub fn func(name: &str, args: Vec) -> Expr { + Expr::Function(Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), + uses_odbc_syntax: false, + parameters: FunctionArguments::None, + args: FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + args: args + .into_iter() + .map(|e| FunctionArg::Unnamed(FunctionArgExpr::Expr(e))) + .collect(), + clauses: vec![], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} + +/// `f(x)` — a unary call, the common case for chain-rule outer derivatives. +pub fn func1(name: &str, x: Expr) -> Expr { + func(name, vec![x]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn folds_additive_zero() { + assert_eq!(add(one(), zero()).to_string(), "1.0"); + assert_eq!(add(zero(), one()).to_string(), "1.0"); + } + + #[test] + fn folds_multiplicative_identity_and_zero() { + assert_eq!(mul(one(), num(3.0)).to_string(), "3.0"); + assert_eq!(mul(num(3.0), one()).to_string(), "3.0"); + assert_eq!(mul(zero(), num(3.0)).to_string(), "0.0"); + } + + #[test] + fn sub_zero_left_is_negation() { + assert_eq!( + sub(zero(), Expr::Identifier(Ident::new("b"))).to_string(), + "-b" + ); + } + + #[test] + fn precedence_wrapping_is_semantic_not_cosmetic() { + // (a+b)*c must keep its parentheses under Display (G1). Without the + // Nested wrap this would render "a + b * c" and reparse wrongly. + let a = Expr::Identifier(Ident::new("a")); + let b = Expr::Identifier(Ident::new("b")); + let c = Expr::Identifier(Ident::new("c")); + let e = mul(add(a, b), c); + assert_eq!(e.to_string(), "(a + b) * c"); + } + + #[test] + fn non_commutative_right_operand_is_parenthesized() { + let a = Expr::Identifier(Ident::new("a")); + let b = Expr::Identifier(Ident::new("b")); + let c = Expr::Identifier(Ident::new("c")); + // a - (b + c) must keep parentheses; a - b + c would be wrong. + assert_eq!(sub(a, add(b, c)).to_string(), "a - (b + c)"); + } + + #[test] + fn div_casts_numerator_to_double() { + let x = Expr::Identifier(Ident::new("x")); + let y = Expr::Identifier(Ident::new("y")); + // Forces float division; integer x/y would otherwise truncate (F4). + assert_eq!(div(x, y).to_string(), "CAST(x AS DOUBLE) / y"); + } + + #[test] + fn div_by_one_folds_without_cast() { + let x = Expr::Identifier(Ident::new("x")); + assert_eq!(div(x, one()).to_string(), "x"); + } +} diff --git a/crates/ddx-core/src/ddx.rs b/crates/ddx-core/src/ddx.rs new file mode 100644 index 0000000..3d9d95c --- /dev/null +++ b/crates/ddx-core/src/ddx.rs @@ -0,0 +1,143 @@ +//! The public entry point: the [`Ddx`] object. +//! +//! The surface is an object rather than free functions so the user rule +//! registry and the dialect/identifier-folding policy have a home — no global +//! state, no later API break (design.md §3.2, F9). + +use sqlparser::ast::Expr; +use sqlparser::dialect::Dialect; +use sqlparser::parser::Parser; + +use crate::colref::{ColRef, IdentCasing}; +use crate::engine::{differentiate, jvp, Rule, RuleRegistry}; +use crate::error::{DiffError, Result}; +use crate::rewrite; + +/// The ddx v1 differentiation engine. +/// +/// Holds the (extensible) rule registry and the identifier-folding policy. +/// Construct one, optionally register custom rules, then drive it with +/// [`Ddx::rewrite_sql`] (the whole marker path) or the lower-level +/// [`Ddx::differentiate`] / [`Ddx::jvp`] (used by the DataFusion Path B bridge, +/// design.md §3.3). +/// +/// # Example +/// +/// ``` +/// use ddx_core::Ddx; +/// use ddx_core::sqlparser::dialect::GenericDialect; +/// +/// let ddx = Ddx::new(); +/// let out = ddx +/// .rewrite_sql("SELECT grad(sin(x), x) AS d FROM t", &GenericDialect {}) +/// .unwrap(); +/// assert_eq!(out, "SELECT (cos(x)) AS d FROM t"); +/// ``` +#[derive(Clone)] +pub struct Ddx { + rules: RuleRegistry, + casing: IdentCasing, +} + +impl Default for Ddx { + fn default() -> Self { + Self::new() + } +} + +impl Ddx { + /// A new engine with the built-in rule set and the generic + /// (`FoldUnquoted`) identifier policy — the DataFusion/Postgres rule + /// (unquoted identifiers fold to lowercase, quoted keep case). + pub fn new() -> Self { + Ddx { + rules: RuleRegistry::new(), + casing: IdentCasing::FoldUnquoted, + } + } + + /// The DataFusion-flavored engine (`FoldUnquoted`). Pair with + /// `GenericDialect`/DataFusion's dialect when calling [`Ddx::rewrite_sql`]. + pub fn for_datafusion() -> Self { + Self::with_casing(IdentCasing::FoldUnquoted) + } + + /// The DuckDB-flavored engine (`FoldAll` — DuckDB folds quoted identifiers + /// too). Pair with `DuckDbDialect` when calling [`Ddx::rewrite_sql`]. + pub fn for_duckdb() -> Self { + Self::with_casing(IdentCasing::FoldAll) + } + + /// A new engine with the built-in rules and an explicit identifier policy. + pub fn with_casing(casing: IdentCasing) -> Self { + Ddx { + rules: RuleRegistry::new(), + casing, + } + } + + /// The identifier-folding policy this engine compares columns under. + pub fn casing(&self) -> IdentCasing { + self.casing + } + + /// Register (or override) a user differentiation rule for a unary function + /// `name`: the rule supplies `f'(u)` and the engine applies the chain rule + /// (design.md §3.2). + pub fn register(&mut self, name: &str, rule: Rule) { + self.rules.register(name, rule); + } + + /// The whole marker path: rewrite every `grad`/`jvp` call in `sql` to + /// derivative SQL and return the rewritten text. A statement with no marker + /// is returned byte-identical (it is never even parsed, design.md §3.2). + /// + /// `dialect` is used to *parse* the marker-bearing statement; the + /// identifier-folding policy used to *match* columns is this engine's + /// (`casing`) — pair them (e.g. `Ddx::for_duckdb()` with `DuckDbDialect`). + pub fn rewrite_sql(&self, sql: &str, dialect: &dyn Dialect) -> Result { + rewrite::rewrite_sql(sql, dialect, self.casing, &self.rules) + } + + /// Differentiate an AST expression with respect to `wrt`. The lower-level + /// entry the DataFusion bridge drives (design.md §3.3, Path B). + pub fn differentiate(&self, e: &Expr, wrt: &ColRef) -> Result { + differentiate(e, wrt, self.casing, &self.rules) + } + + /// Forward-mode directional derivative: seed a tangent on each column in + /// `seeds` and push it through `e` (design.md §3.6). + /// + /// (The design sketch names a `HashMap`; a slice of pairs is + /// used instead because `ColRef` equality is dialect-dependent — folding + /// makes it a poor hash key — so an explicit, ordered seed list is clearer + /// and preserves match order.) + pub fn jvp(&self, e: &Expr, seeds: &[(ColRef, Expr)]) -> Result { + jvp(e, seeds, self.casing, &self.rules) + } + + /// The "calculus compiler" escape hatch: differentiate the scalar + /// expression `expr` (SQL text) with respect to the column `wrt` (a bare + /// column name), returning the derivative as SQL text — for embedding an + /// update rule where a marker can't reach (design.md §3.6). + pub fn differentiate_sql( + &self, + expr: &str, + wrt: &str, + dialect: &dyn Dialect, + ) -> Result { + let parsed = parse_expr(expr, dialect)?; + let wrt_expr = parse_expr(wrt, dialect)?; + let wrt_col = ColRef::from_wrt_arg("differentiate_sql", &wrt_expr)?; + let derivative = self.differentiate(&parsed, &wrt_col)?; + Ok(derivative.to_string()) + } +} + +/// Parse a single scalar expression from text under `dialect`. +fn parse_expr(text: &str, dialect: &dyn Dialect) -> Result { + Parser::new(dialect) + .try_with_sql(text) + .and_then(|mut p| p.parse_expr()) + .map_err(|e| DiffError::Parse(format!("failed to parse expression `{text}`: {e}"))) +} diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs new file mode 100644 index 0000000..a607c21 --- /dev/null +++ b/crates/ddx-core/src/engine.rs @@ -0,0 +1,368 @@ +//! The differentiation engine: forward-mode linearization over +//! `sqlparser::ast::Expr`, with a name-keyed, user-extensible rule registry. +//! +//! The approach mirrors JAX's per-primitive rule registry: every expression +//! node has a differentiation rule and the chain rule composes them as the tree +//! is walked. Because each row of a relational table is an independent +//! evaluation point, differentiating a column expression and letting the engine +//! evaluate it per row is the relational equivalent of `jax.vmap(jax.grad(f))` +//! (design.md §1). Both [`differentiate`] (one partial derivative) and [`jvp`] +//! (a directional derivative) are thin wrappers over [`linearize`] that differ +//! only in their *leaf rule* — the tangent assigned to each column. + +use std::collections::HashMap; +use std::f64::consts::{LN_10, LN_2}; +use std::sync::Arc; + +use sqlparser::ast::{ + BinaryOperator, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, + ObjectNamePart, UnaryOperator, +}; + +use crate::colref::{ColRef, IdentCasing, Match}; +use crate::constructors::{ + add, as_const, div, func, func1, is_zero, mul, neg, num, one, square, sub, zero, +}; +use crate::error::{DiffError, Result}; + +/// A differentiation rule for a unary primitive `f(u)`: given the argument +/// expression `u`, it returns the *outer* derivative `f'(u)`. The engine +/// multiplies by `du` (the chain rule) itself, so a user rule supplies only +/// the local factor (design.md §3.2). +pub type Rule = Arc Result + Send + Sync>; + +/// A registry of differentiation rules, keyed by (lower-cased) function name. +/// +/// Built-ins populate it; users extend it with [`RuleRegistry::register`] +/// (design.md §3.2 — the "extensible rule registry" decision). +#[derive(Clone)] +pub struct RuleRegistry { + unary: HashMap, +} + +impl Default for RuleRegistry { + fn default() -> Self { + Self::new() + } +} + +/// Wrap an infallible closure as a [`Rule`]. +fn rule(f: impl Fn(&Expr) -> Expr + Send + Sync + 'static) -> Rule { + Arc::new(move |u| Ok(f(u))) +} + +impl RuleRegistry { + /// A registry populated with the built-in v1 rule set: `+ - * /`, the unary + /// chain rule for the trig / inverse-trig / exp / log / hyperbolic set plus + /// `abs`, and `power` with a constant base or exponent (design.md §3.6). + pub fn new() -> Self { + let mut unary: HashMap = HashMap::new(); + + // Trigonometric. + unary.insert("sin".into(), rule(|u| func1("cos", u.clone()))); + unary.insert("cos".into(), rule(|u| neg(func1("sin", u.clone())))); + unary.insert( + "tan".into(), + rule(|u| div(one(), square(func1("cos", u.clone())))), + ); + // Inverse trigonometric. + unary.insert( + "asin".into(), + rule(|u| div(one(), func1("sqrt", sub(one(), square(u.clone()))))), + ); + unary.insert( + "acos".into(), + rule(|u| neg(div(one(), func1("sqrt", sub(one(), square(u.clone())))))), + ); + unary.insert( + "atan".into(), + rule(|u| div(one(), add(one(), square(u.clone())))), + ); + // Exponential / logarithmic. + unary.insert("exp".into(), rule(|u| func1("exp", u.clone()))); + unary.insert("ln".into(), rule(|u| div(one(), u.clone()))); + unary.insert( + "log2".into(), + rule(|u| div(one(), mul(u.clone(), num(LN_2)))), + ); + unary.insert( + "log10".into(), + rule(|u| div(one(), mul(u.clone(), num(LN_10)))), + ); + unary.insert( + "sqrt".into(), + rule(|u| div(one(), mul(num(2.0), func1("sqrt", u.clone())))), + ); + // Hyperbolic. + unary.insert("sinh".into(), rule(|u| func1("cosh", u.clone()))); + unary.insert("cosh".into(), rule(|u| func1("sinh", u.clone()))); + unary.insert( + "tanh".into(), + rule(|u| sub(one(), square(func1("tanh", u.clone())))), + ); + // Piecewise-linear: derivative is the sign (a pinned kink convention at + // 0, differing from jax.grad(abs)(0) — design.md §5, F12). + unary.insert("abs".into(), rule(|u| func1("signum", u.clone()))); + + RuleRegistry { unary } + } + + /// Register (or override) a unary differentiation rule under `name`. The + /// name is matched case-insensitively. + pub fn register(&mut self, name: &str, rule: Rule) { + self.unary.insert(name.to_ascii_lowercase(), rule); + } + + fn lookup(&self, name: &str) -> Option<&Rule> { + self.unary.get(name) + } +} + +/// A *leaf rule*: the tangent seed for a column occurrence. Returns an error +/// when the occurrence's identity against the differentiation variable can't be +/// pinned syntactically (the ambiguity guard, F2). +type Leaf<'a> = dyn Fn(&ColRef) -> Result + 'a; + +/// Differentiate `expr` with respect to the column `wrt`. +/// +/// Forward-mode with a one-hot seed: `1` on `wrt`, `0` on every other column. +pub fn differentiate( + expr: &Expr, + wrt: &ColRef, + casing: IdentCasing, + reg: &RuleRegistry, +) -> Result { + let leaf = |c: &ColRef| match c.classify(wrt, casing) { + Match::Is => Ok(one()), + Match::Not => Ok(zero()), + Match::Ambiguous => Err(DiffError::AmbiguousColumn(format!( + "occurrence of `{}` cannot be matched against differentiation \ + variable `{}` — fully qualify it", + c.display(), + wrt.display() + ))), + }; + linearize(expr, &leaf, reg) +} + +/// Forward-mode directional derivative: the tangent of `expr` given a tangent +/// (`seeds`) for each seeded input column; unseeded columns are constant. +/// +/// The marker form `jvp(expr, column, tangent)` seeds a single column; a +/// multi-input directional derivative is a sum of `jvp` terms (design.md §3.6). +pub fn jvp( + expr: &Expr, + seeds: &[(ColRef, Expr)], + casing: IdentCasing, + reg: &RuleRegistry, +) -> Result { + let leaf = |c: &ColRef| { + for (col, tangent) in seeds { + match c.classify(col, casing) { + Match::Is => return Ok(tangent.clone()), + Match::Ambiguous => { + return Err(DiffError::AmbiguousColumn(format!( + "occurrence of `{}` cannot be matched against seeded \ + column `{}` — fully qualify it", + c.display(), + col.display() + ))) + } + Match::Not => continue, + } + } + Ok(zero()) + }; + linearize(expr, &leaf, reg) +} + +/// Push tangents from the leaves up through `expr` via the chain rule. +fn linearize(expr: &Expr, leaf: &Leaf, reg: &RuleRegistry) -> Result { + match expr { + // Leaves: the leaf rule decides a column's tangent. + Expr::Identifier(_) | Expr::CompoundIdentifier(_) => { + let cr = ColRef::from_expr(expr) + .ok_or_else(|| DiffError::Internal("column expr yielded no ColRef".into()))?; + leaf(&cr) + } + + // Constants have zero tangent. + Expr::Value(_) => Ok(zero()), + + // Parentheses are transparent to differentiation; the smart + // constructors re-introduce any precedence parentheses the result needs. + Expr::Nested(inner) => linearize(inner, leaf, reg), + + // A cast is locally linear: tangent of cast(u) = cast(du) to the same + // type (matches the prototype; non-numeric casts are unusual as a + // differentiation target). + Expr::Cast { + kind, + expr: inner, + data_type, + array, + format, + } => { + let du = linearize(inner, leaf, reg)?; + Ok(Expr::Cast { + kind: kind.clone(), + expr: Box::new(du), + data_type: data_type.clone(), + array: *array, + format: format.clone(), + }) + } + + // tangent of -u = -(du); unary plus is transparent. + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: inner, + } => Ok(neg(linearize(inner, leaf, reg)?)), + Expr::UnaryOp { + op: UnaryOperator::Plus, + expr: inner, + } => linearize(inner, leaf, reg), + + Expr::BinaryOp { left, op, right } => linearize_binary(left, op, right, leaf, reg), + + Expr::Function(f) => linearize_function(f, leaf, reg), + + other => Err(DiffError::NotImplemented(format!( + "differentiation is not implemented for this expression: `{other}`" + ))), + } +} + +/// Linearize a binary arithmetic expression via the sum/product/quotient rules. +fn linearize_binary( + left: &Expr, + op: &BinaryOperator, + right: &Expr, + leaf: &Leaf, + reg: &RuleRegistry, +) -> Result { + let da = linearize(left, leaf, reg)?; + let db = linearize(right, leaf, reg)?; + match op { + // tangent of (a + b) = da + db + BinaryOperator::Plus => Ok(add(da, db)), + // tangent of (a - b) = da - db + BinaryOperator::Minus => Ok(sub(da, db)), + // tangent of (a * b) = da*b + a*db (product rule) + BinaryOperator::Multiply => Ok(add(mul(da, right.clone()), mul(left.clone(), db))), + // tangent of (a / b) = (da*b - a*db) / b^2 (quotient rule) + BinaryOperator::Divide => { + let numerator = sub(mul(da, right.clone()), mul(left.clone(), db)); + Ok(div(numerator, square(right.clone()))) + } + other => Err(DiffError::NotImplemented(format!( + "operator `{other}` is not differentiable" + ))), + } +} + +/// Linearize a scalar-function call via the chain rule. +fn linearize_function(f: &Function, leaf: &Leaf, reg: &RuleRegistry) -> Result { + let name = simple_func_name(f) + .ok_or_else(|| DiffError::NotImplemented(format!("unsupported function form: `{f}`")))?; + let args = positional_args(f).ok_or_else(|| { + DiffError::NotImplemented(format!( + "function `{name}` has non-positional arguments, which are not differentiable" + )) + })?; + + // `power(base, exponent)` / `pow(...)` is the one binary primitive. + if name == "power" || name == "pow" { + return linearize_power(&name, &args, leaf, reg); + } + + if args.len() != 1 { + return Err(DiffError::NotImplemented(format!( + "no derivative rule for `{name}` with {} arguments", + args.len() + ))); + } + let u = args[0]; + let du = linearize(u, leaf, reg)?; + // Chain-rule short-circuit: a zero inner tangent kills the whole term. + if is_zero(&du) { + return Ok(zero()); + } + let outer = + reg.lookup(&name).ok_or_else(|| { + DiffError::NotImplemented(format!("no derivative rule for `{name}`")) + })?(u)?; + Ok(mul(outer, du)) +} + +/// Linearize `power(base, exponent)` (design.md §3.6). +/// +/// * Constant exponent `c`: `c * base^(c-1) * d(base)`. +/// * Constant base `a`: `a^u * ln(a) * d(u)`. +/// * Both variable (`u^v`): not supported yet (needs the exp/log trick). +fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) -> Result { + if args.len() != 2 { + return Err(DiffError::NotImplemented(format!( + "{name}() expects exactly two arguments" + ))); + } + let base = args[0]; + let exponent = args[1]; + match (as_const(base), as_const(exponent)) { + // Constant exponent (covers x^2, x^0.5, ...). + (_, Some(c)) => { + let dbase = linearize(base, leaf, reg)?; + if is_zero(&dbase) { + return Ok(zero()); + } + let outer = mul(num(c), func("power", vec![base.clone(), num(c - 1.0)])); + Ok(mul(outer, dbase)) + } + // Constant base, variable exponent. + (Some(a), None) => { + let dexp = linearize(exponent, leaf, reg)?; + if is_zero(&dexp) { + return Ok(zero()); + } + let outer = mul( + func("power", vec![base.clone(), exponent.clone()]), + num(a.ln()), + ); + Ok(mul(outer, dexp)) + } + // General u^v — deferred (design.md §3.6 roadmap). + (None, None) => Err(DiffError::NotImplemented( + "power(base, exponent) where both depend on the differentiation \ + variable is not yet supported" + .into(), + )), + } +} + +/// The simple (last-part, lower-cased) name of a function call, if its name is +/// a plain identifier path. +fn simple_func_name(f: &Function) -> Option { + f.name.0.last().and_then(|part| match part { + ObjectNamePart::Identifier(id) => Some(id.value.to_ascii_lowercase()), + _ => None, + }) +} + +/// The positional (unnamed) argument expressions of a function call, or `None` +/// if it uses any non-positional argument form (named args, wildcards, a +/// subquery). +pub(crate) fn positional_args(f: &Function) -> Option> { + match &f.args { + FunctionArguments::List(list) => { + let mut out = Vec::with_capacity(list.args.len()); + for arg in &list.args { + match arg { + FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => out.push(e), + _ => return None, + } + } + Some(out) + } + _ => None, + } +} diff --git a/crates/ddx-core/src/error.rs b/crates/ddx-core/src/error.rs new file mode 100644 index 0000000..110d9ee --- /dev/null +++ b/crates/ddx-core/src/error.rs @@ -0,0 +1,61 @@ +//! The error type for differentiation and rewriting. + +use std::fmt; + +/// An error produced while differentiating or rewriting SQL. +/// +/// Every failure mode of `ddx-core` is one of these. In keeping with design +/// principle 5 — *fail loud, never silently wrong* (design.md §2) — an +/// unsupported construct is always one of these typed errors, never an +/// approximate or silently-zero derivative. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiffError { + /// A node or function has no differentiation rule. This is the "permanent + /// or roadmap" bucket: `atan2` (no rule yet), general `u^v`, `CASE`, + /// comparisons, string/temporal expressions (design.md §3.6). + NotImplemented(String), + + /// The `wrt` (or a marker call) is malformed: `wrt` is not a bare column, + /// wrong argument count, etc. + InvalidMarker(String), + + /// An occurrence of the `wrt` base name could not be pinned syntactically — + /// a bare occurrence when `wrt` is qualified, or a qualified occurrence + /// when `wrt` is bare. Hard error demanding full qualification + /// (design.md §3.2, F2). + AmbiguousColumn(String), + + /// A marker argument references an identifier that is a *computed* + /// select-list alias of a CTE/derived table in the same statement, used as + /// a non-`wrt` term — differentiation would silently drop terms across the + /// projection boundary (design.md §3.5, F3/G4). + ProjectionBoundary(String), + + /// The input SQL did not parse under the given dialect. Only ever reported + /// for a statement that *contains* a marker (the parse-free pre-gate means + /// marker-free statements are never parsed, design.md §3.2, F5). + Parse(String), + + /// An internal invariant was violated (e.g. an empty source span the API + /// documents as possible, with no safe fallback). Should not occur in + /// normal use. + Internal(String), +} + +impl fmt::Display for DiffError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DiffError::NotImplemented(m) => write!(f, "not implemented: {m}"), + DiffError::InvalidMarker(m) => write!(f, "invalid marker call: {m}"), + DiffError::AmbiguousColumn(m) => write!(f, "ambiguous column: {m}"), + DiffError::ProjectionBoundary(m) => write!(f, "projection boundary: {m}"), + DiffError::Parse(m) => write!(f, "parse error: {m}"), + DiffError::Internal(m) => write!(f, "internal error: {m}"), + } + } +} + +impl std::error::Error for DiffError {} + +/// The result type used throughout `ddx-core`. +pub type Result = std::result::Result; diff --git a/crates/ddx-core/src/lib.rs b/crates/ddx-core/src/lib.rs new file mode 100644 index 0000000..c4804a8 --- /dev/null +++ b/crates/ddx-core/src/lib.rs @@ -0,0 +1,66 @@ +//! `ddx-core` — engine-neutral symbolic differentiation of SQL scalar +//! expressions. The v1 core of [`ddx`](https://github.com/xqlsystems/ddx): +//! write calculus directly in SQL and let the engine evaluate the derivative +//! per row, the relational equivalent of `jax.vmap(jax.grad(f))`. +//! +//! ```sql +//! SELECT i, grad(x * y, x) AS dfdx, grad(x * y, y) AS dfdy FROM g +//! ``` +//! +//! `grad`/`jvp` are **markers**, not row functions: they carry a +//! differentiation request through parsing and are always rewritten away +//! *before* execution. [`Ddx::rewrite_sql`] is the whole path — find every +//! marker, differentiate what it wraps, splice the derivative back by source +//! span, return plain SQL. +//! +//! The engine differentiates [`sqlparser::ast::Expr`] directly — there is no +//! bespoke IR and no adapter layer; the AST *is* the IR (design.md §3.2). The +//! single load-bearing dependency is [`sqlparser`], which is **re-exported** +//! (see below) so downstream adapters cannot accidentally link a mismatched +//! version. +//! +//! # What v1 supports +//! +//! `+ - * /`; the unary chain rule for the trig / inverse-trig / exp / log / +//! hyperbolic set plus `abs`; `power` with a constant base or exponent; +//! higher-order via nesting; through-aggregate via linearity +//! (`AVG(grad(loss, theta))`). Anything else is a typed [`DiffError`], never a +//! silently-wrong number (design principle 5). +//! +//! Scalar `vjp` is deliberately **not** part of the surface: the name is +//! reserved for the query-level reverse-mode operation in +//! [`ddx-ad`](https://docs.rs/ddx-ad) (design.md §3.6, §4, decision Q7). +//! +//! # `sqlparser` version policy +//! +//! `ddx-core`'s public API takes and returns `sqlparser::ast::Expr`, so a +//! `sqlparser` bump is a breaking release of `ddx-core`. The version is pinned +//! exactly (see `Cargo.toml`) and re-exported here as [`ddx_core::sqlparser`]: +//! always reach for `sqlparser` types through this re-export so your build +//! links the same version the engine was compiled against (design.md §6, G2). + +#![forbid(unsafe_code)] + +/// The exact `sqlparser` this crate was built against, re-exported so downstream +/// code links a matching version (design.md §6, G2). +pub use sqlparser; + +mod colref; +mod constructors; +mod ddx; +mod engine; +mod error; +mod rewrite; + +/// Smart constructors for building derivative expressions — useful when writing +/// a custom [`Rule`], which returns `f'(u)` as an [`sqlparser::ast::Expr`]. +pub mod build { + pub use crate::constructors::{ + add, cast_double, div, func, func1, mul, neg, num, one, square, sub, zero, + }; +} + +pub use colref::{ColRef, IdentCasing, Match}; +pub use ddx::Ddx; +pub use engine::{Rule, RuleRegistry}; +pub use error::{DiffError, Result}; diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs new file mode 100644 index 0000000..0b555f7 --- /dev/null +++ b/crates/ddx-core/src/rewrite.rs @@ -0,0 +1,507 @@ +//! Source-to-source SQL rewriting: find every `grad`/`jvp` marker, replace it +//! with derivative SQL, leave everything else byte-identical. +//! +//! This is Path A (design.md §3.3), the universal path every target relies on. +//! It is a real subsystem, not a one-liner (design.md §3.2): +//! +//! * a **parse-free pre-gate** so a marker-free statement is never parsed, and +//! so a `sqlparser` coverage gap can only ever bound a statement that +//! *actually contains* a marker (F5); +//! * **splice by source span**, so everything outside a marker stays +//! byte-identical — which requires a UTF-8-aware character-column→byte-offset +//! conversion, because `sqlparser` spans are 1-based *characters*, not bytes +//! (G3); +//! * **multiple and nested markers** — spliced in reverse source order, nested +//! ones differentiated bottom-up (`grad(grad(f,x),x)` just works); +//! * a safe **fallback** to whole-statement reprinting on the empty spans the +//! API documents as possible. +//! +//! Two guards run here, both catching a *silently-wrong* derivative and turning +//! it into a typed error: the ambiguity guard lives in the engine (F2), and the +//! CTE-computed-alias guard (F3/G4) lives in [`projection_guard`]. + +use std::collections::HashSet; +use std::ops::ControlFlow; + +use sqlparser::ast::Spanned; +use sqlparser::ast::{ + Expr, Function, ObjectNamePart, Query, Select, SelectItem, SetExpr, Statement, TableFactor, + Visit, VisitMut, Visitor, VisitorMut, +}; +use sqlparser::dialect::Dialect; +use sqlparser::parser::Parser; +use sqlparser::tokenizer::{Location, Span}; + +use crate::colref::{ColRef, IdentCasing}; +use crate::engine::{differentiate, jvp, positional_args, RuleRegistry}; +use crate::error::{DiffError, Result}; + +/// Which marker a function call is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MarkerKind { + Grad, + Jvp, +} + +/// Classify a function call as a marker — but only an **unqualified**, +/// case-folded `grad`/`jvp` (design.md §3.2, F8). `myschema.grad(...)` and a +/// user's own multi-part function are left alone. +fn marker_kind(f: &Function) -> Option { + if f.name.0.len() != 1 { + return None; + } + let ObjectNamePart::Identifier(id) = &f.name.0[0] else { + return None; + }; + match id.value.to_ascii_lowercase().as_str() { + "grad" => Some(MarkerKind::Grad), + "jvp" => Some(MarkerKind::Jvp), + _ => None, + } +} + +fn is_marker_expr(e: &Expr) -> bool { + matches!(e, Expr::Function(f) if marker_kind(f).is_some()) +} + +/// The parse-free pre-gate: a case-insensitive scan for an *unqualified* +/// `grad(`/`jvp(` — the equivalent of `(?i)(?:^|[^A-Za-z0-9_.])(grad|jvp)\s*\(`, +/// hand-rolled so the core depends on `sqlparser` only (design.md §3.2/§6). A +/// statement that doesn't hit is returned verbatim, never parsed (F5). It is a +/// best-effort filter: a false positive (e.g. `grad(` inside a string literal) +/// only costs a parse that then finds no marker, never a wrong rewrite. +fn pre_gate_hit(sql: &str) -> bool { + // ASCII-lowercasing preserves byte length and offsets, so indices found in + // `lower` are valid char boundaries in `sql`. + let lower = sql.to_ascii_lowercase(); + for kw in ["grad", "jvp"] { + let mut from = 0; + while let Some(rel) = lower[from..].find(kw) { + let idx = from + rel; + from = idx + 1; + + // Preceding character must not be part of a longer identifier or a + // qualifier (`.`), so `mygrad(` and `schema.grad(` don't match. + let ok_prev = idx == 0 + || sql[..idx].chars().next_back().is_some_and(|prev| { + !(prev.is_ascii_alphanumeric() || prev == '_' || prev == '.') + }); + if !ok_prev { + continue; + } + + // The next non-whitespace character must be `(`. + if sql[idx + kw.len()..] + .chars() + .find(|c| !c.is_whitespace()) + == Some('(') + { + return true; + } + } + } + false +} + +/// The public entry point behind [`crate::Ddx::rewrite_sql`]. +pub(crate) fn rewrite_sql( + sql: &str, + dialect: &dyn Dialect, + casing: IdentCasing, + reg: &RuleRegistry, +) -> Result { + // 1. Parse-free pre-gate: no marker syntax, no parse, byte-identical out. + if !pre_gate_hit(sql) { + return Ok(sql.to_string()); + } + + // 2. The statement (or one of them) looks like it carries a marker; parse. + let statements = Parser::parse_sql(dialect, sql) + .map_err(|e| DiffError::Parse(format!("failed to parse SQL: {e}")))?; + + // 3. Statement-level context for the projection-boundary guard (F3/G4): + // the names of every *computed* select-list alias of a CTE/derived table. + let mut aliases = HashSet::new(); + for stmt in &statements { + collect_computed_aliases(stmt, &mut aliases); + } + + // 4. Locate the outermost markers (with their source spans). Nested markers + // are handled when their enclosing outermost marker is differentiated. + let mut collector = MarkerCollector::default(); + for stmt in &statements { + let _ = Visit::visit(stmt, &mut collector); + } + // Pre-gate false positive (e.g. only qualified markers, or `grad(` inside a + // string literal): nothing to rewrite, return verbatim. + if collector.found.is_empty() { + return Ok(sql.to_string()); + } + + // 5. Empty spans are documented as possible; fall back to a correct (if not + // byte-identical) whole-statement reprint if any marker lacks a span. + if collector.found.iter().any(|(span, _)| is_empty_span(span)) { + return reprint_fallback(statements, casing, reg, &aliases); + } + + // 6. Compute each replacement, then splice by byte range in reverse source + // order so earlier offsets stay valid. + let mut repls: Vec<(usize, usize, String)> = Vec::with_capacity(collector.found.len()); + for (span, marker_expr) in &collector.found { + let text = differentiate_marker_tree(marker_expr, casing, reg, &aliases)?; + let start = locate(sql, span.start, false) + .ok_or_else(|| DiffError::Internal("marker span start out of range".into()))?; + // sqlparser span ends are *inclusive* of the last character (verified + // against 0.62.0), so the exclusive byte end is one character past it. + let end = locate(sql, span.end, true) + .ok_or_else(|| DiffError::Internal("marker span end out of range".into()))?; + repls.push((start, end, text)); + } + repls.sort_by(|a, b| b.0.cmp(&a.0)); + + let mut out = sql.to_string(); + for (start, end, text) in repls { + out.replace_range(start..end, &text); + } + Ok(out) +} + +/// Differentiate one (possibly nested) marker subtree, returning the derivative +/// rendered to SQL text, parenthesized so it keeps the call's precedence. +fn differentiate_marker_tree( + marker_expr: &Expr, + casing: IdentCasing, + reg: &RuleRegistry, + aliases: &HashSet, +) -> Result { + let mut clone = marker_expr.clone(); + let mut rw = MarkerRewriter { + casing, + reg, + aliases, + }; + if let ControlFlow::Break(err) = VisitMut::visit(&mut clone, &mut rw) { + return Err(err); + } + Ok(clone.to_string()) +} + +/// The whole-statement reprint fallback (empty-span case). +fn reprint_fallback( + mut statements: Vec, + casing: IdentCasing, + reg: &RuleRegistry, + aliases: &HashSet, +) -> Result { + for stmt in &mut statements { + let mut rw = MarkerRewriter { + casing, + reg, + aliases, + }; + if let ControlFlow::Break(err) = VisitMut::visit(stmt, &mut rw) { + return Err(err); + } + } + Ok(statements + .iter() + .map(ToString::to_string) + .collect::>() + .join("; ")) +} + +// --------------------------------------------------------------------------- +// Differentiating a single marker (args assumed already marker-free) +// --------------------------------------------------------------------------- + +/// Differentiate a single marker call whose arguments are already free of +/// nested markers (guaranteed by the bottom-up post-order walk). +fn differentiate_marker(f: &Function, casing: IdentCasing, reg: &RuleRegistry) -> Result { + let kind = marker_kind(f).ok_or_else(|| DiffError::Internal("not a marker".into()))?; + let args = positional_args(f).ok_or_else(|| { + DiffError::InvalidMarker("marker call has non-positional arguments".into()) + })?; + match kind { + MarkerKind::Grad => { + if args.len() != 2 { + return Err(DiffError::InvalidMarker(format!( + "grad(expr, column) expects 2 arguments, got {}", + args.len() + ))); + } + let wrt = ColRef::from_wrt_arg("grad", args[1])?; + differentiate(args[0], &wrt, casing, reg) + } + MarkerKind::Jvp => { + if args.len() != 3 { + return Err(DiffError::InvalidMarker(format!( + "jvp(expr, column, tangent) expects 3 arguments, got {}", + args.len() + ))); + } + let wrt = ColRef::from_wrt_arg("jvp", args[1])?; + let seeds = vec![(wrt, args[2].clone())]; + jvp(args[0], &seeds, casing, reg) + } + } +} + +/// The projection-boundary guard (design.md §3.5, F3/G4). +/// +/// Errors if a marker argument references an identifier that is a *computed* +/// select-list alias of a CTE/derived table in the same statement, used as a +/// *non-`wrt`* term — differentiating it would silently treat an upstream +/// expression as a constant and drop gradient terms. The carve-out (G4): when +/// the alias *is* the `wrt` itself, every occurrence is the differentiation +/// leaf, so no term can be dropped and the guard stays quiet. +fn projection_guard(f: &Function, aliases: &HashSet) -> Result<()> { + if aliases.is_empty() { + return Ok(()); + } + let Some(args) = positional_args(f) else { + return Ok(()); + }; + let Some(expr_arg) = args.first() else { + return Ok(()); + }; + let wrt_name = args + .get(1) + .and_then(|a| ColRef::from_expr(a)) + .map(|c| c.name.value.to_ascii_lowercase()); + + let mut cols = ColumnCollector::default(); + let _ = Visit::visit(*expr_arg, &mut cols); + for c in cols.cols { + let lname = c.name.value.to_ascii_lowercase(); + // Carve-out: the wrt itself is always a leaf; never an error. + if Some(&lname) == wrt_name.as_ref() { + continue; + } + if aliases.contains(&lname) { + return Err(DiffError::ProjectionBoundary(format!( + "`{}` is a computed select-list alias of a CTE/derived table used \ + as a non-differentiation term; grad does not see through the \ + projection boundary — differentiate inside that CTE instead", + c.display() + ))); + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Visitors +// --------------------------------------------------------------------------- + +/// Collects the outermost marker expressions (with their spans), skipping +/// markers nested inside another marker's arguments (handled bottom-up later). +#[derive(Default)] +struct MarkerCollector { + depth: usize, + found: Vec<(Span, Expr)>, +} + +impl Visitor for MarkerCollector { + type Break = (); + + fn pre_visit_expr(&mut self, expr: &Expr) -> ControlFlow<()> { + if is_marker_expr(expr) { + if self.depth == 0 { + self.found.push((expr.span(), expr.clone())); + } + self.depth += 1; + } + ControlFlow::Continue(()) + } + + fn post_visit_expr(&mut self, expr: &Expr) -> ControlFlow<()> { + if is_marker_expr(expr) { + self.depth -= 1; + } + ControlFlow::Continue(()) + } +} + +/// Replaces each marker with `Nested(derivative)`, bottom-up (post-order), so a +/// nested marker's own arguments are already marker-free when it is reached. +struct MarkerRewriter<'a> { + casing: IdentCasing, + reg: &'a RuleRegistry, + aliases: &'a HashSet, +} + +impl VisitorMut for MarkerRewriter<'_> { + type Break = DiffError; + + fn post_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { + let replacement = match expr { + Expr::Function(f) if marker_kind(f).is_some() => { + if let Err(err) = projection_guard(f, self.aliases) { + return ControlFlow::Break(err); + } + match differentiate_marker(f, self.casing, self.reg) { + Ok(d) => Some(d), + Err(err) => return ControlFlow::Break(err), + } + } + _ => None, + }; + if let Some(d) = replacement { + *expr = Expr::Nested(Box::new(d)); + } + ControlFlow::Continue(()) + } +} + +/// Collects the column references directly appearing in an expression tree. +#[derive(Default)] +struct ColumnCollector { + cols: Vec, +} + +impl Visitor for ColumnCollector { + type Break = (); + + fn pre_visit_expr(&mut self, expr: &Expr) -> ControlFlow<()> { + match expr { + Expr::Identifier(_) | Expr::CompoundIdentifier(_) => { + if let Some(cr) = ColRef::from_expr(expr) { + self.cols.push(cr); + } + } + _ => {} + } + ControlFlow::Continue(()) + } +} + +// --------------------------------------------------------------------------- +// Computed-alias collection for the projection-boundary guard +// --------------------------------------------------------------------------- + +fn collect_computed_aliases(stmt: &Statement, out: &mut HashSet) { + match stmt { + Statement::Query(q) => walk_query(q, out), + Statement::Insert(insert) => { + if let Some(source) = &insert.source { + walk_query(source, out); + } + } + _ => {} + } +} + +fn walk_query(q: &Query, out: &mut HashSet) { + if let Some(with) = &q.with { + for cte in &with.cte_tables { + walk_query(&cte.query, out); + } + } + walk_set_expr(&q.body, out); +} + +fn walk_set_expr(body: &SetExpr, out: &mut HashSet) { + match body { + SetExpr::Select(select) => walk_select(select, out), + SetExpr::Query(q) => walk_query(q, out), + SetExpr::SetOperation { left, right, .. } => { + walk_set_expr(left, out); + walk_set_expr(right, out); + } + _ => {} + } +} + +fn walk_select(select: &Select, out: &mut HashSet) { + for item in &select.projection { + if let SelectItem::ExprWithAlias { expr, alias } = item { + if !is_bare_column(expr) { + out.insert(alias.value.to_ascii_lowercase()); + } + } + } + // Recurse into derived tables (FROM subqueries), which are projection + // boundaries too. + for twj in &select.from { + walk_table_factor(&twj.relation, out); + for join in &twj.joins { + walk_table_factor(&join.relation, out); + } + } +} + +fn walk_table_factor(tf: &TableFactor, out: &mut HashSet) { + if let TableFactor::Derived { subquery, .. } = tf { + walk_query(subquery, out); + } +} + +fn is_bare_column(e: &Expr) -> bool { + matches!(e, Expr::Identifier(_) | Expr::CompoundIdentifier(_)) + || matches!(e, Expr::Nested(inner) if is_bare_column(inner)) +} + +// --------------------------------------------------------------------------- +// Span (1-based character line/column) → byte offset conversion (G3) +// --------------------------------------------------------------------------- + +/// `sqlparser` uses `line: 0`/`column: 0` for an empty/unknown location. +fn is_empty_span(span: &Span) -> bool { + span.start.line == 0 || span.start.column == 0 || span.end.line == 0 || span.end.column == 0 +} + +/// Convert a 1-based (line, character-column) [`Location`] to a byte offset in +/// `sql`. Character-column, not byte-column: a multibyte character before the +/// target shifts the byte offset past the column number (G3). +/// +/// With `past = false` the returned offset is the *start* byte of the character +/// at `loc`; with `past = true` it is the byte *one past* that character — used +/// for an inclusive span end, so the whole marker (its closing `)` included) is +/// covered by `start..end`. +fn locate(sql: &str, loc: Location, past: bool) -> Option { + let mut line: u64 = 1; + let mut col: u64 = 1; + for (byte_idx, ch) in sql.char_indices() { + if line == loc.line && col == loc.column { + return Some(if past { + byte_idx + ch.len_utf8() + } else { + byte_idx + }); + } + if ch == '\n' { + line += 1; + col = 1; + } else { + col += 1; + } + } + // A location just past the final character maps to the end of the string. + if line == loc.line && col == loc.column { + return Some(sql.len()); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pre_gate_matches_unqualified_markers() { + assert!(pre_gate_hit("SELECT grad(x, x) FROM t")); + assert!(pre_gate_hit("SELECT jvp(x, x, dx) FROM t")); + assert!(pre_gate_hit("grad(x,x)")); // at start of input + assert!(pre_gate_hit("SELECT GRAD (x, x) FROM t")); // case + whitespace + assert!(pre_gate_hit("SELECT AVG(grad(x, x)) FROM t")); // after `(` + } + + #[test] + fn pre_gate_rejects_non_markers() { + assert!(!pre_gate_hit("SELECT a + b FROM t")); // no marker + assert!(!pre_gate_hit("SELECT mygrad(x) FROM t")); // longer identifier + assert!(!pre_gate_hit("SELECT schema.grad(x, x) FROM t")); // qualified + assert!(!pre_gate_hit("SELECT grad AS g FROM t")); // no open paren + assert!(!pre_gate_hit("SELECT upgrade(x) FROM t")); // 'grad' inside a word + } +} diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs new file mode 100644 index 0000000..1893dda --- /dev/null +++ b/crates/ddx-core/tests/rewrite.rs @@ -0,0 +1,238 @@ +//! End-to-end `rewrite_sql` tests: the marker path, span splicing, the guards, +//! and identifier folding — the M0 deliverables of design.md §3.2 / §8. + +use ddx_core::sqlparser::dialect::{DuckDbDialect, GenericDialect}; +use ddx_core::{Ddx, DiffError}; + +fn rw(sql: &str) -> String { + Ddx::new() + .rewrite_sql(sql, &GenericDialect {}) + .unwrap_or_else(|e| panic!("rewrite_sql({sql}) failed: {e}")) +} + +// --------------------------------------------------------------------------- +// The basic marker path +// --------------------------------------------------------------------------- + +#[test] +fn replaces_a_grad_call_in_place() { + // grad(sin(x), x) -> (cos(x)); the surrounding SELECT is preserved. + assert_eq!( + rw("SELECT grad(sin(x), x) AS d FROM t"), + "SELECT (cos(x)) AS d FROM t" + ); +} + +#[test] +fn full_gradient_as_tidy_columns() { + assert_eq!( + rw("SELECT grad(x * y, x) AS dfdx, grad(x * y, y) AS dfdy FROM g"), + "SELECT (y) AS dfdx, (x) AS dfdy FROM g" + ); +} + +#[test] +fn marker_free_query_is_untouched() { + // Never parsed (parse-free pre-gate), returned byte-identical — even with + // formatting sqlparser would not reproduce. + let sql = "SELECT a+b AS s FROM t -- comment"; + assert_eq!(rw(sql), sql); +} + +#[test] +fn nested_higher_order_grad() { + // grad(grad(power(x,3), x), x) = d2/dx2 x^3 = 6x; inner differentiated first. + let out = rw("SELECT grad(grad(power(x, 3), x), x) AS d FROM t"); + assert!( + !out.to_lowercase().contains("grad("), + "marker left behind: {out}" + ); + // 3*power(x,2) -> d/dx -> 3 * (2 * power(x,1)) = 6*power(x,1) + assert!(out.contains("power(x, 1.0)"), "unexpected rewrite: {out}"); +} + +#[test] +fn fires_inside_recursive_cte() { + // A whole Newton-step loop in one query: d/dx(x*x - 2) = x + x. + let out = rw("WITH RECURSIVE r AS (SELECT 1.0 AS x UNION ALL \ + SELECT x - grad(x * x - 2, x) FROM r WHERE x < 10) SELECT x FROM r"); + assert!(out.contains("(x + x)"), "unexpected rewrite: {out}"); + assert!( + !out.to_lowercase().contains("grad("), + "marker left behind: {out}" + ); +} + +#[test] +fn dml_update_rule_is_rewritten() { + let out = rw("INSERT INTO p SELECT theta - 0.1 * grad(x * theta, theta) FROM t"); + assert!( + !out.to_lowercase().contains("grad("), + "marker left behind: {out}" + ); + assert!(out.contains("(x)"), "unexpected rewrite: {out}"); +} + +// --------------------------------------------------------------------------- +// Span splicing (G3 / F5): byte-identity outside the marker +// --------------------------------------------------------------------------- + +#[test] +fn splice_preserves_multibyte_prefix() { + // A multibyte character before the marker shifts its byte offset past its + // character column; naive column-as-byte splicing would corrupt this. + let out = rw("SELECT 'héllo', grad(sin(x), x) AS d FROM t"); + assert_eq!(out, "SELECT 'héllo', (cos(x)) AS d FROM t"); +} + +#[test] +fn splice_multiple_markers_on_one_line() { + // Two markers spliced independently, in reverse source order. + let out = rw("SELECT grad(sin(x), x), grad(cos(y), y) FROM t"); + assert_eq!(out, "SELECT (cos(x)), (-sin(y)) FROM t"); +} + +#[test] +fn splice_preserves_exact_surrounding_bytes() { + // Odd but valid spacing/casing around the marker is preserved verbatim. + let out = rw("SELECT GRAD(x*x,x) , y FROM t"); + assert_eq!(out, "SELECT (x + x) , y FROM t"); +} + +// --------------------------------------------------------------------------- +// Reserved names (F8) and the cut of scalar vjp (Q7) +// --------------------------------------------------------------------------- + +#[test] +fn qualified_grad_is_left_alone() { + // Only unqualified grad/jvp are markers; myschema.grad(...) is a user fn. + let sql = "SELECT myschema.grad(x, x) AS d FROM t"; + assert_eq!(rw(sql), sql); +} + +#[test] +fn scalar_vjp_is_not_a_marker() { + // vjp is reserved for query-level reverse-mode AD (Q7); as scalar SQL it is + // an ordinary (unknown) function, left untouched. + let sql = "SELECT vjp(sin(x), x, w) AS v FROM t"; + assert_eq!(rw(sql), sql); +} + +// --------------------------------------------------------------------------- +// Identifier folding (F1) +// --------------------------------------------------------------------------- + +#[test] +fn unquoted_identifiers_fold_case() { + // grad(Temp*Temp, temp) must match despite the case difference, and keep + // the original spelling in the output. + let out = Ddx::new() + .differentiate_sql("Temp * Temp", "temp", &GenericDialect {}) + .unwrap(); + assert_eq!(out, "Temp + Temp"); +} + +#[test] +fn duckdb_folds_quoted_identifiers_too() { + // DuckDB is fully case-insensitive: "Temp" and temp are the same column. + let out = Ddx::for_duckdb() + .differentiate_sql(r#""Temp" * "Temp""#, "temp", &DuckDbDialect {}) + .unwrap(); + // Original (quoted) spelling preserved. + assert_eq!(out, r#""Temp" + "Temp""#); +} + +#[test] +fn datafusion_keeps_quoted_identifiers_case_sensitive() { + // Under the FoldUnquoted policy, quoted "Temp" is a different column from + // unquoted temp, so the derivative is 0. + let out = Ddx::for_datafusion() + .differentiate_sql(r#""Temp" * "Temp""#, "temp", &GenericDialect {}) + .unwrap(); + assert_eq!(out, "0.0"); +} + +// --------------------------------------------------------------------------- +// Ambiguity guard (F2) +// --------------------------------------------------------------------------- + +#[test] +fn qualified_wrt_across_a_join_is_accepted() { + let out = rw("SELECT grad(a.v * b.w, a.v) AS d FROM t a JOIN u b ON a.k = b.k"); + assert_eq!(out, "SELECT (b.w) AS d FROM t a JOIN u b ON a.k = b.k"); +} + +#[test] +fn fully_qualified_same_name_across_join_is_accepted() { + // grad(a.x * b.x, a.x): both occurrences qualified — unambiguous, = b.x. + let out = rw("SELECT grad(a.x * b.x, a.x) AS d FROM t a JOIN u b ON a.k = b.k"); + assert_eq!(out, "SELECT (b.x) AS d FROM t a JOIN u b ON a.k = b.k"); +} + +#[test] +fn bare_occurrence_with_qualified_wrt_errors() { + // grad(x * a.x, a.x): bare x might be a.x — demand qualification. + let err = Ddx::new() + .rewrite_sql("SELECT grad(x * a.x, a.x) FROM t a", &GenericDialect {}) + .unwrap_err(); + assert!(matches!(err, DiffError::AmbiguousColumn(_)), "got {err:?}"); +} + +#[test] +fn qualified_occurrence_with_bare_wrt_errors() { + // grad(a.x * b.x, x): bare wrt, qualified occurrences — ambiguous. + let err = Ddx::new() + .rewrite_sql( + "SELECT grad(a.x * b.x, x) FROM t a JOIN u b ON a.k = b.k", + &GenericDialect {}, + ) + .unwrap_err(); + assert!(matches!(err, DiffError::AmbiguousColumn(_)), "got {err:?}"); +} + +#[test] +fn wrt_must_be_a_bare_column() { + let err = Ddx::new() + .rewrite_sql("SELECT grad(x * y, x + y) FROM t", &GenericDialect {}) + .unwrap_err(); + assert!(matches!(err, DiffError::InvalidMarker(_)), "got {err:?}"); +} + +// --------------------------------------------------------------------------- +// Projection-boundary guard (F3 / G4) +// --------------------------------------------------------------------------- + +#[test] +fn computed_cte_alias_as_non_wrt_term_errors() { + // s = sin(x) is a computed CTE alias; grad(s*x, x) would silently drop ds/dx. + let err = Ddx::new() + .rewrite_sql( + "WITH v AS (SELECT x, sin(x) AS s FROM t) SELECT grad(s * x, x) FROM v", + &GenericDialect {}, + ) + .unwrap_err(); + assert!( + matches!(err, DiffError::ProjectionBoundary(_)), + "got {err:?}" + ); +} + +// --------------------------------------------------------------------------- +// Through-aggregate differentiation by linearity (design.md §3.1) +// --------------------------------------------------------------------------- + +#[test] +fn marker_inside_an_aggregate_is_rewritten() { + // One gradient-descent step: the marker goes *inside* the aggregate, and + // d/dθ Σ f = Σ ∂f/∂θ is just linearity. + let out = rw("SELECT AVG(grad(x * theta, theta)) AS step FROM batch"); + assert_eq!(out, "SELECT AVG((x)) AS step FROM batch"); +} + +#[test] +fn differentiating_wrt_a_computed_alias_is_allowed() { + // Carve-out (G4): when the computed alias IS the wrt, every occurrence is + // the leaf, so d/ds (s*s) = 2s is exactly right — no error. + let out = rw("SELECT a + b AS s, grad(s * s, s) AS d FROM t"); + assert_eq!(out, "SELECT a + b AS s, (s + s) AS d FROM t"); +} diff --git a/crates/ddx-core/tests/roundtrip.rs b/crates/ddx-core/tests/roundtrip.rs new file mode 100644 index 0000000..bb2e781 --- /dev/null +++ b/crates/ddx-core/tests/roundtrip.rs @@ -0,0 +1,96 @@ +//! The semantic round-trip property test (design.md §5). +//! +//! A test that only checks the rewritten SQL *parses* sails right past the +//! precedence bug G1 found: `(a+b)*c` rendered without its parentheses becomes +//! `a + b * c`, which parses fine — just as the wrong tree. So the real +//! invariant is: rendering a constructed derivative and reparsing it must yield +//! the *same* expression, **modulo `Expr::Nested`** (normalize parentheses on +//! both sides before comparing). + +use std::ops::ControlFlow; + +use ddx_core::sqlparser::ast::{Expr, Ident, VisitMut, VisitorMut}; +use ddx_core::sqlparser::dialect::GenericDialect; +use ddx_core::sqlparser::parser::Parser; +use ddx_core::Ddx; + +/// Remove every `Expr::Nested` wrapper so two trees that differ only in +/// (redundant or precedence) parentheses compare equal. +struct StripNested; + +impl VisitorMut for StripNested { + type Break = (); + fn post_visit_expr(&mut self, e: &mut Expr) -> ControlFlow<()> { + // Post-order: children are already stripped. Collapse any Nested chain. + while matches!(e, Expr::Nested(_)) { + let owned = std::mem::replace(e, Expr::Identifier(Ident::new(""))); + if let Expr::Nested(inner) = owned { + *e = *inner; + } + } + ControlFlow::Continue(()) + } +} + +fn strip(mut e: Expr) -> Expr { + let _ = VisitMut::visit(&mut e, &mut StripNested); + e +} + +fn parse(text: &str) -> Expr { + Parser::new(&GenericDialect {}) + .try_with_sql(text) + .and_then(|mut p| p.parse_expr()) + .unwrap_or_else(|e| panic!("reparse of `{text}` failed: {e}")) +} + +/// For a derivative produced from `expr`/`wrt`, assert that rendering it and +/// reparsing yields the same AST modulo `Nested` — i.e. the precedence +/// parentheses the smart constructors emit are exactly the ones the grammar +/// needs, no more, no fewer. +fn assert_roundtrips(expr: &str, wrt: &str) { + let ddx = Ddx::new(); + // The derivative as text, via the public path... + let rendered = ddx + .differentiate_sql(expr, wrt, &GenericDialect {}) + .unwrap_or_else(|e| panic!("differentiate_sql({expr}, {wrt}): {e}")); + // ...reparsed, then rendered a second time: must be a fixed point (the + // parentheses are stable) and structurally identical modulo Nested. + let reparsed = parse(&rendered); + let rerendered = ddx + .differentiate_sql(expr, wrt, &GenericDialect {}) + .unwrap(); + assert_eq!( + rendered, rerendered, + "render is not idempotent for d/d{wrt} ({expr})" + ); + // The load-bearing check: reparse(render(d)) == d, modulo parentheses. + let a = strip(reparsed); + let b = strip(parse(&rendered)); + assert_eq!(a, b, "round-trip changed the tree for d/d{wrt} ({expr})"); +} + +#[test] +fn precedence_sensitive_derivatives_round_trip() { + // Each of these exercises a rule whose output nests a lower-precedence + // operand inside a higher-precedence one — exactly where G1 bites. + assert_roundtrips("(a + b) * c", "a"); // product rule over a sum + assert_roundtrips("x / y", "x"); // quotient rule + DOUBLE cast + assert_roundtrips("x / y", "y"); // quotient rule, negative numerator + assert_roundtrips("sin(x) * x", "x"); // chain × product + assert_roundtrips("power(x, 3)", "x"); // constant-exponent power + assert_roundtrips("sin(x * y + x)", "x"); // chain over a compound argument + assert_roundtrips("1 / (x + y)", "x"); // reciprocal of a sum + assert_roundtrips("sqrt(x * x + y * y)", "x"); // nested composite + assert_roundtrips("a * b * c * d", "a"); // n-factor product (swell shape) + assert_roundtrips("exp(x) / (x - 1)", "x"); // quotient with composite denom +} + +#[test] +fn strip_nested_actually_normalizes() { + // Sanity: the normalizer collapses parentheses, so an unparenthesized and a + // parenthesized spelling of the same expression compare equal after strip. + assert_eq!(strip(parse("a + b * c")), strip(parse("a + (b * c)"))); + // ...but genuinely different groupings stay different (the test has teeth). + assert_ne!(strip(parse("(a + b) * c")), strip(parse("a + b * c"))); +} diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs new file mode 100644 index 0000000..365538d --- /dev/null +++ b/crates/ddx-core/tests/rules.rs @@ -0,0 +1,118 @@ +//! The prototype's rule unit tests, ported to `ddx-core` (design.md §3.2, M0: +//! "port the prototype's 15 rule tests — they pin the math unchanged"). +//! +//! The *math* is identical to xarray-sql#192's `src/autograd.rs`; the rendered +//! output differs in two documented, deliberate ways introduced by the design: +//! `div()` casts its numerator to `DOUBLE` (F4), and composite operands are +//! `Nested`-wrapped for precedence safety (G1). Both are asserted here. + +use ddx_core::sqlparser::dialect::GenericDialect; +use ddx_core::Ddx; + +fn d(expr: &str, wrt: &str) -> String { + Ddx::new() + .differentiate_sql(expr, wrt, &GenericDialect {}) + .unwrap_or_else(|e| panic!("differentiate_sql({expr}, {wrt}) failed: {e}")) +} + +#[test] +fn constant_has_zero_derivative() { + assert_eq!(d("3.0", "x"), "0.0"); +} + +#[test] +fn variable_has_unit_derivative() { + assert_eq!(d("x", "x"), "1.0"); +} + +#[test] +fn other_variable_has_zero_derivative() { + assert_eq!(d("y", "x"), "0.0"); +} + +#[test] +fn sum_rule_folds_constants() { + // d/dx (x + y) = 1 + 0 = 1 + assert_eq!(d("x + y", "x"), "1.0"); +} + +#[test] +fn product_rule() { + // d/dx (x * x) = 1*x + x*1 = x + x + assert_eq!(d("x * x", "x"), "x + x"); +} + +#[test] +fn quotient_rule() { + // d/dx (x / y) = (1*y - x*0) / (y*y) = y / (y*y), numerator cast to DOUBLE. + assert_eq!(d("x / y", "x"), "CAST(y AS DOUBLE) / (y * y)"); +} + +#[test] +fn chain_rule_sin() { + // d/dx sin(x) = cos(x) * 1 = cos(x) + assert_eq!(d("sin(x)", "x"), "cos(x)"); +} + +#[test] +fn composite_sin_times_x() { + // d/dx (sin(x) * x) = cos(x)*x + sin(x) + assert_eq!(d("sin(x) * x", "x"), "cos(x) * x + sin(x)"); +} + +#[test] +fn power_constant_exponent() { + // d/dx power(x, 2) = 2 * power(x, 1) * 1 = 2 * power(x, 1) + assert_eq!(d("power(x, 2)", "x"), "2.0 * power(x, 1.0)"); +} + +#[test] +fn higher_order_derivative() { + // Differentiation composes: d2/dx2 sin(x) = -sin(x). + let d1 = d("sin(x)", "x"); + assert_eq!(d(&d1, "x"), "-sin(x)"); +} + +#[test] +fn unsupported_operator_errors() { + assert!(Ddx::new() + .differentiate_sql("x % y", "x", &GenericDialect {}) + .is_err()); +} + +#[test] +fn unsupported_function_errors() { + // atan2 is binary and has no rule yet. + assert!(Ddx::new() + .differentiate_sql("atan2(x, y)", "x", &GenericDialect {}) + .is_err()); +} + +#[test] +fn general_power_uv_errors() { + // power(x, x): both base and exponent vary — not supported yet. + assert!(Ddx::new() + .differentiate_sql("power(x, x)", "x", &GenericDialect {}) + .is_err()); +} + +#[test] +fn jvp_seeds_a_tangent_on_one_input() { + // jvp(x*y, x, dx) = product rule with tangent(x)=dx, tangent(y)=0 = dx*y + let out = Ddx::new() + .rewrite_sql("SELECT jvp(x * y, x, dx) AS t FROM g", &GenericDialect {}) + .unwrap(); + assert_eq!(out, "SELECT (dx * y) AS t FROM g"); +} + +#[test] +fn jvp_with_unit_seed_matches_grad() { + // A one-hot tangent reproduces the partial derivative. + let jvp = Ddx::new() + .rewrite_sql("SELECT jvp(sin(x), x, 1.0) AS t FROM g", &GenericDialect {}) + .unwrap(); + let grad = Ddx::new() + .rewrite_sql("SELECT grad(sin(x), x) AS t FROM g", &GenericDialect {}) + .unwrap(); + assert_eq!(jvp, grad); +} diff --git a/crates/ddx-datafusion/Cargo.toml b/crates/ddx-datafusion/Cargo.toml new file mode 100644 index 0000000..3754b22 --- /dev/null +++ b/crates/ddx-datafusion/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ddx-datafusion" +description = "DataFusion adapter for ddx: the one-line ddx_sql helper (Path A) plus marker UDFs + AnalyzerRule bridge (Path B). Scaffold — see docs/design.md §3.3 and milestone M2." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +rust-version.workspace = true + +# NOTE (M0 scaffold): the real adapter (M2) depends on `ddx-core` + `ddx-ad` +# + `datafusion`. The `datafusion` dependency — and, critically, the +# `sqlparser` version reconciliation it forces (design.md §6, G2: the Path B +# bridge needs ddx-core and datafusion to resolve the *identical* sqlparser +# version) — is deferred until M2 so M0's core can pin sqlparser to its +# spike-verified 0.62 without dragging DataFusion's requirement in first. +[dependencies] +ddx-core = { workspace = true } diff --git a/crates/ddx-datafusion/src/lib.rs b/crates/ddx-datafusion/src/lib.rs new file mode 100644 index 0000000..146a430 --- /dev/null +++ b/crates/ddx-datafusion/src/lib.rs @@ -0,0 +1,31 @@ +//! `ddx-datafusion` — the DataFusion adapter for ddx. +//! +//! **Status: scaffold.** M0 lands only the crate seam. The real adapter +//! arrives in M2 (design.md §3.3–§3.4, milestone M2) and exposes two paths, +//! both driving the *same* [`ddx_core::Ddx`] engine: +//! +//! * **Path A — [`ddx_sql`]:** the one-liner +//! `ctx.sql(&ddx.rewrite_sql(sql, dialect)?)`. This works today on top of +//! [`ddx_core`] and is the universal path (design.md §3.3 Path A); it is +//! stubbed here rather than wired to a live `SessionContext` because M0 does +//! not take a `datafusion` dependency. +//! * **Path B — marker UDFs + an `AnalyzerRule` bridge:** makes bare `grad()` +//! work across the SQL and DataFrame APIs by unparsing the marker's argument +//! with DataFusion's `expr_to_sql` (which emits exactly `ddx-core`'s +//! `sqlparser::ast::Expr` input type), differentiating via `ddx-core`, and +//! re-planning back to a DataFusion `Expr` (design.md §3.3 Path B). This is +//! the reference proof that `ddx-core` can drive an *in-engine* rewrite. +//! +//! Path B carries two documented implementation constraints to honor in M2: +//! `add_analyzer_rule` runs after `TypeCoercion` (so the marker UDF must be +//! coercion-tolerant and there must be a `Cast` rule — there is), and the +//! re-plan seam is `SessionState::create_logical_expr`. + +#![forbid(unsafe_code)] + +/// Milestone marker: the DataFusion adapter is not implemented in M0. +pub const STATUS: &str = "scaffold: DataFusion Path A + Path B land in M2"; + +// Re-exported so downstream code and docs can name the engine this adapter +// will drive, even while the adapter itself is a stub. +pub use ddx_core; diff --git a/spikes/README.md b/docs/spikes/README.md similarity index 100% rename from spikes/README.md rename to docs/spikes/README.md diff --git a/spikes/attention_ad_spike.py b/docs/spikes/attention_ad_spike.py similarity index 100% rename from spikes/attention_ad_spike.py rename to docs/spikes/attention_ad_spike.py diff --git a/spikes/attention_causal_mask_check.py b/docs/spikes/attention_causal_mask_check.py similarity index 100% rename from spikes/attention_causal_mask_check.py rename to docs/spikes/attention_causal_mask_check.py diff --git a/spikes/duckdb_reentrancy_r1b.py b/docs/spikes/duckdb_reentrancy_r1b.py similarity index 100% rename from spikes/duckdb_reentrancy_r1b.py rename to docs/spikes/duckdb_reentrancy_r1b.py diff --git a/spikes/duckdb_substrait_window_bug.py b/docs/spikes/duckdb_substrait_window_bug.py similarity index 100% rename from spikes/duckdb_substrait_window_bug.py rename to docs/spikes/duckdb_substrait_window_bug.py diff --git a/spikes/relational_ad_spike.py b/docs/spikes/relational_ad_spike.py similarity index 100% rename from spikes/relational_ad_spike.py rename to docs/spikes/relational_ad_spike.py diff --git a/spikes/route_ad_spike.py b/docs/spikes/route_ad_spike.py similarity index 100% rename from spikes/route_ad_spike.py rename to docs/spikes/route_ad_spike.py diff --git a/spikes/sqlparser-spike/Cargo.toml b/docs/spikes/sqlparser-spike/Cargo.toml similarity index 100% rename from spikes/sqlparser-spike/Cargo.toml rename to docs/spikes/sqlparser-spike/Cargo.toml diff --git a/spikes/sqlparser-spike/src/main.rs b/docs/spikes/sqlparser-spike/src/main.rs similarity index 100% rename from spikes/sqlparser-spike/src/main.rs rename to docs/spikes/sqlparser-spike/src/main.rs diff --git a/spikes/substrait_ad_marker_spike.py b/docs/spikes/substrait_ad_marker_spike.py similarity index 100% rename from spikes/substrait_ad_marker_spike.py rename to docs/spikes/substrait_ad_marker_spike.py diff --git a/spikes/substrait_limitation_repro.py b/docs/spikes/substrait_limitation_repro.py similarity index 100% rename from spikes/substrait_limitation_repro.py rename to docs/spikes/substrait_limitation_repro.py diff --git a/python/ddxdb/README.md b/python/ddxdb/README.md new file mode 100644 index 0000000..378dff8 --- /dev/null +++ b/python/ddxdb/README.md @@ -0,0 +1,25 @@ +# ddxdb + +**Status: scaffold (M2).** The Python distribution of +[`ddx`](https://github.com/xqlsystems/ddx) — a PyO3/maturin wheel wrapping +[`ddx-core`](../../crates/ddx-core). + +Planned surface (design.md §3.4, milestone M2): + +- `ddxdb.rewrite_sql(sql: str, dialect: str = "datafusion") -> str` — the + source-to-source marker rewrite, exposed directly. +- A `Context.sql()` shim: a thin wrapper around a `datafusion.SessionContext` + that calls `rewrite_sql` before handing the plain SQL to the stock context to + plan. This is what [xarray-sql](https://github.com/xqlsystems/xarray-sql) + pulls in as an optional extra (`pip install "xarray-sql[ddx]"`), so autograd + is opt-in and costs nothing for users who don't ask for it. +- A DuckDB-python client-side path: preprocess the string before + `duckdb.sql(...)` (the zero-hook fallback, available day one — design.md + §3.4). + +Nothing is built here yet. M0 delivers only `ddx-core`; this directory holds +the layout seam of design.md §6 so the wheel has a home. When M2 lands, this +becomes a maturin project (`[build-system] requires = ["maturin"]`) with a +`Cargo.toml` depending on `ddx-core` + `pyo3`. + +The **oracle for its tests is JAX** (`jax.grad`), not numpy (design.md §5). diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..3b7b7a5 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,30 @@ +# Cross-engine tests (vs JAX) + +This directory holds the **cross-engine numeric-agreement suites** (design.md +§5, §6): the same `grad`/`jvp` expression, rewritten per-dialect by `ddx-core`, +must evaluate to numerically equal columns in DuckDB and DataFusion — checked +against **JAX** (`jax.grad`) as the oracle, since the whole design mirrors JAX's +forward/reverse structure and the same seed/cotangent semantics. + +**Status: scaffold.** The minimal JAX-oracle harness is pulled forward into M2 +(its exit gate — "xarray-sql green on `ddx-core` vs. JAX, no regressions" — +depends on it); the broader suites land in M6 (design.md §8). Planned coverage: + +- **Numeric agreement vs JAX** for every rule, with finite-difference as a + cheap independent cross-check where a JAX equivalent is awkward. +- **Cross-engine equivalence**: DuckDB vs DataFusion on the identical + per-dialect rewrite. +- **Convention-pinning tests** (not blind oracle comparison) where a convention + genuinely differs rather than one side being wrong (design.md §5): + - *Kinks* — `abs` at 0 gives `0` from the `signum` rule; pin the value. + - *Domain-widening* — a derivative can fail where the primal doesn't + (`sqrt(x)` fine at 0, `1/(2*sqrt(x))` divides by zero); sample away from + edges or pin per-engine behavior. + - *NULL/folding* — folded and unfolded derivatives agree everywhere except + the documented NULL-row cases (the JAX-`Zero`-tangent convention, F11). + +The Rust unit and integration tests for the engine itself live with the crate, +in [`../crates/ddx-core/tests`](../crates/ddx-core/tests) (the ported rule +tests, span splicing, the guards, identifier folding, and the semantic +round-trip property test). The runnable spikes that back each design claim are +in [`../spikes`](../spikes). From ec8de5b81e6d09e762679dd3ba19636c5c465452 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 08:30:19 -0700 Subject: [PATCH 02/22] rm duckdb --- Cargo.lock | 7 ------- Cargo.toml | 1 - 2 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 646a267..e0083fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,13 +45,6 @@ dependencies = [ "ddx-core", ] -[[package]] -name = "ddx-duckdb" -version = "0.0.0" -dependencies = [ - "ddx-core", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 15c68c0..9cee843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ members = [ "crates/ddx-core", "crates/ddx-ad", "crates/ddx-datafusion", - "crates/ddx-duckdb", ] [workspace.package] From 6cca29b526d04f98d4a10d50e4ce78eb27323fab Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 08:55:33 -0700 Subject: [PATCH 03/22] code review tool --- crates/ddx-core/src/constructors.rs | 19 +++++-- crates/ddx-core/src/engine.rs | 83 +++++++++++++++++++++++++++-- crates/ddx-core/src/rewrite.rs | 9 ++-- crates/ddx-core/tests/rules.rs | 29 ++++++++++ 4 files changed, 126 insertions(+), 14 deletions(-) diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index 400e69e..41f170a 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -200,16 +200,25 @@ pub fn div(a: Expr, b: Expr) -> Expr { } } -/// `-a`, folding `-0 = 0` and parenthesizing a binary operand +/// `-a`, folding `-0 = 0` and `-(-e) = e`, and parenthesizing a binary operand /// (`-(a + b)`, `-(a / b)`), since unary minus binds tighter than either. pub fn neg(a: Expr) -> Expr { if is_zero(&a) { - zero() - } else { + return zero(); + } + match a { + // Double negation cancels. This is not just simplification: without it, + // `neg(neg(e))` renders as two adjacent minus tokens `--e`, which SQL + // parses as a line comment — a silently-wrong result in valid-looking + // SQL (e.g. d/dx(-cos(x)) = sin(x) would emit `--sin(x)`). Expr::UnaryOp { op: UnaryOperator::Minus, - expr: Box::new(wrap(a, 30, true)), - } + expr, + } => *expr, + other => Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(wrap(other, 30, true)), + }, } } diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index a607c21..fa88a12 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -15,7 +15,7 @@ use std::f64::consts::{LN_10, LN_2}; use std::sync::Arc; use sqlparser::ast::{ - BinaryOperator, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, + BinaryOperator, DataType, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, ObjectNamePart, UnaryOperator, }; @@ -193,9 +193,11 @@ fn linearize(expr: &Expr, leaf: &Leaf, reg: &RuleRegistry) -> Result { // constructors re-introduce any precedence parentheses the result needs. Expr::Nested(inner) => linearize(inner, leaf, reg), - // A cast is locally linear: tangent of cast(u) = cast(du) to the same - // type (matches the prototype; non-numeric casts are unusual as a - // differentiation target). + // A cast to a numeric type is locally linear: tangent of cast(u) = + // cast(du) to the same type. A cast to a non-numeric type (VARCHAR, + // DATE, BOOLEAN, …) is not differentiable — differentiating through it + // would emit a nonsensical `CAST(1.0 AS VARCHAR)`, so it is a typed + // error rather than a silently-wrong derivative (principle 5). Expr::Cast { kind, expr: inner, @@ -203,6 +205,12 @@ fn linearize(expr: &Expr, leaf: &Leaf, reg: &RuleRegistry) -> Result { array, format, } => { + if !is_numeric_type(data_type) { + return Err(DiffError::NotImplemented(format!( + "differentiation through a cast to non-numeric type `{data_type}` \ + is not supported" + ))); + } let du = linearize(inner, leaf, reg)?; Ok(Expr::Cast { kind: kind.clone(), @@ -366,3 +374,70 @@ pub(crate) fn positional_args(f: &Function) -> Option> { _ => None, } } + +/// True if `dt` is a numeric type — the only kind of cast that is locally +/// linear (and so differentiable). The list is exhaustive for the pinned +/// `sqlparser` version; a `sqlparser` bump is already a breaking release of +/// `ddx-core` (design.md §6, G2), at which point this is re-checked. +fn is_numeric_type(dt: &DataType) -> bool { + matches!( + dt, + // Floating-point / fixed-point. + DataType::Numeric(_) + | DataType::Decimal(_) + | DataType::BigNumeric(_) + | DataType::BigDecimal(_) + | DataType::Dec(_) + | DataType::Float(_) + | DataType::FloatUnsigned(_) + | DataType::Float4 + | DataType::Float32 + | DataType::Float64 + | DataType::Real + | DataType::RealUnsigned + | DataType::Float8 + | DataType::Double(_) + | DataType::DoubleUnsigned(_) + | DataType::DoublePrecision + | DataType::DoublePrecisionUnsigned + // Integers (signed / unsigned / width-tagged aliases). + | DataType::TinyInt(_) + | DataType::TinyIntUnsigned(_) + | DataType::UTinyInt + | DataType::Int2(_) + | DataType::Int2Unsigned(_) + | DataType::SmallInt(_) + | DataType::SmallIntUnsigned(_) + | DataType::USmallInt + | DataType::MediumInt(_) + | DataType::MediumIntUnsigned(_) + | DataType::Int(_) + | DataType::Int4(_) + | DataType::Int8(_) + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Int128 + | DataType::Int256 + | DataType::Integer(_) + | DataType::IntUnsigned(_) + | DataType::Int4Unsigned(_) + | DataType::IntegerUnsigned(_) + | DataType::HugeInt + | DataType::UHugeInt + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::UInt128 + | DataType::UInt256 + | DataType::BigInt(_) + | DataType::BigIntUnsigned(_) + | DataType::UBigInt + | DataType::Int8Unsigned(_) + | DataType::Signed + | DataType::SignedInteger + | DataType::Unsigned + | DataType::UnsignedInteger + ) +} diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index 0b555f7..d234d2e 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -415,7 +415,10 @@ fn walk_set_expr(body: &SetExpr, out: &mut HashSet) { fn walk_select(select: &Select, out: &mut HashSet) { for item in &select.projection { if let SelectItem::ExprWithAlias { expr, alias } = item { - if !is_bare_column(expr) { + // A *computed* alias is one whose projected expression is not a + // plain column reference. `ColRef::from_expr` is the single place + // that recognizes a column reference (seeing through `Nested`). + if ColRef::from_expr(expr).is_none() { out.insert(alias.value.to_ascii_lowercase()); } } @@ -436,10 +439,6 @@ fn walk_table_factor(tf: &TableFactor, out: &mut HashSet) { } } -fn is_bare_column(e: &Expr) -> bool { - matches!(e, Expr::Identifier(_) | Expr::CompoundIdentifier(_)) - || matches!(e, Expr::Nested(inner) if is_bare_column(inner)) -} // --------------------------------------------------------------------------- // Span (1-based character line/column) → byte offset conversion (G3) diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index 365538d..4e5a934 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -96,6 +96,35 @@ fn general_power_uv_errors() { .is_err()); } +#[test] +fn double_negation_does_not_render_a_comment() { + // d/dx(-cos(x)) = sin(x). The intermediate is neg(neg(sin(x))); without + // folding it renders as `--sin(x)`, a SQL line comment (a silently-wrong + // result). It must fold to `sin(x)`. + let out = d("-cos(x)", "x"); + assert!(!out.contains("--"), "rendered a `--` comment: {out}"); + assert_eq!(out, "sin(x)"); + // And nested inside a larger expression, still no stray `--`. + let nested = d("sin(-cos(x))", "x"); + assert!(!nested.contains("--"), "rendered a `--` comment: {nested}"); + assert_eq!(nested, "cos(-cos(x)) * sin(x)"); +} + +#[test] +fn cast_to_numeric_type_is_differentiable() { + // A numeric cast is locally linear: d/dx CAST(x*x AS DOUBLE) = CAST(x+x AS DOUBLE). + assert_eq!(d("CAST(x * x AS DOUBLE)", "x"), "CAST(x + x AS DOUBLE)"); +} + +#[test] +fn cast_to_non_numeric_type_errors() { + // Differentiating through CAST(... AS VARCHAR) would emit a nonsensical + // CAST(1.0 AS VARCHAR); it must be a typed error instead. + assert!(Ddx::new() + .differentiate_sql("CAST(x AS VARCHAR)", "x", &GenericDialect {}) + .is_err()); +} + #[test] fn jvp_seeds_a_tangent_on_one_input() { // jvp(x*y, x, dx) = product rule with tangent(x)=dx, tangent(y)=0 = dx*y From ae7956c2d68c378dfaffe90ae0107e4b30a7f6a3 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 08:55:45 -0700 Subject: [PATCH 04/22] added chainlink. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 52b9d6a..fcb8b56 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,8 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ + +# agents +.claude +.chainlink +.mcp.json From 440140cb746b7093d197646e3e7b2e0e15526fc6 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 10:36:01 -0700 Subject: [PATCH 05/22] ignore autogen changelog -- it's not associated with GH issues, just chainlink issues, which are personal. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index fcb8b56..0446f07 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ target .claude .chainlink .mcp.json +# auto-generated from chainlink +CHANGELOG.md From 18cecad9128d00353a88d7cfc25c03d91b7869c1 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 10:37:33 -0700 Subject: [PATCH 06/22] IAR 0 - Fixed false code comment - Address issue with duckdb compat on sign/abs operator. - Projection boundary guard over-fires on qualified base columns (causing name collision). - Corner case documented in design. --- crates/ddx-core/src/constructors.rs | 36 +++++++++- crates/ddx-core/src/engine.rs | 12 ++-- crates/ddx-core/src/lib.rs | 2 +- crates/ddx-core/src/rewrite.rs | 100 +++++++++++++++++++--------- crates/ddx-core/tests/rewrite.rs | 31 +++++++++ crates/ddx-core/tests/rules.rs | 19 ++++++ docs/design.md | 37 ++++++---- 7 files changed, 189 insertions(+), 48 deletions(-) diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index 41f170a..8ef6b33 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -17,8 +17,9 @@ //! Without this, a *constructed* `mul(add(a,b), c)` displays as `a + b * c` //! and reparses as the wrong expression — a wrong number in valid SQL (G1). +use sqlparser::ast::helpers::attached_token::AttachedToken; use sqlparser::ast::{ - BinaryOperator, CastKind, DataType, ExactNumberInfo, Expr, Function, FunctionArg, + BinaryOperator, CaseWhen, CastKind, DataType, ExactNumberInfo, Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, UnaryOperator, Value, }; @@ -227,6 +228,39 @@ pub fn square(e: Expr) -> Expr { mul(e.clone(), e) } +/// The mathematical sign of `u` as a portable `CASE`, pinning `sign(0) = 0` on +/// every engine: `CASE WHEN u > 0 THEN 1.0 WHEN u < 0 THEN -1.0 ELSE 0.0 END`. +/// +/// This is the derivative factor for `abs` (`d/du |u| = sign(u)`). It avoids the +/// engine-specific builtins — DuckDB has only `sign`, DataFusion only `signum`, +/// and the two disagree at `0` (`signum(0) = 1`) — so the emitted derivative is +/// both portable across the target engines and *actually* pins the documented +/// kink convention `abs'(0) = 0` (design.md §5, F12), which a bare `signum(u)` +/// call did not. +pub fn sign(u: Expr) -> Expr { + let compare = |op: BinaryOperator| Expr::BinaryOp { + left: Box::new(u.clone()), + op, + right: Box::new(zero()), + }; + Expr::Case { + case_token: AttachedToken::empty(), + end_token: AttachedToken::empty(), + operand: None, + conditions: vec![ + CaseWhen { + condition: compare(BinaryOperator::Gt), + result: one(), + }, + CaseWhen { + condition: compare(BinaryOperator::Lt), + result: num(-1.0), + }, + ], + else_result: Some(Box::new(zero())), + } +} + // --------------------------------------------------------------------------- // Function-call construction (for the outer factors of chain-rule terms) // --------------------------------------------------------------------------- diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index fa88a12..4f7f04b 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -21,7 +21,7 @@ use sqlparser::ast::{ use crate::colref::{ColRef, IdentCasing, Match}; use crate::constructors::{ - add, as_const, div, func, func1, is_zero, mul, neg, num, one, square, sub, zero, + add, as_const, div, func, func1, is_zero, mul, neg, num, one, sign, square, sub, zero, }; use crate::error::{DiffError, Result}; @@ -100,9 +100,13 @@ impl RuleRegistry { "tanh".into(), rule(|u| sub(one(), square(func1("tanh", u.clone())))), ); - // Piecewise-linear: derivative is the sign (a pinned kink convention at - // 0, differing from jax.grad(abs)(0) — design.md §5, F12). - unary.insert("abs".into(), rule(|u| func1("signum", u.clone()))); + // Piecewise-linear: d/du |u| = sign(u), emitted as a portable CASE that + // pins abs'(0) = 0 on every engine (design.md §5, F12). It deliberately + // does NOT emit signum()/sign(): DuckDB has no signum (only sign), + // DataFusion has no sign (only signum), and signum(0) = 1 — so a bare + // builtin would be non-portable AND violate the pinned convention. Note + // this pins ddx's own convention; jax.grad(abs)(0) uses a different one. + unary.insert("abs".into(), rule(|u| sign(u.clone()))); RuleRegistry { unary } } diff --git a/crates/ddx-core/src/lib.rs b/crates/ddx-core/src/lib.rs index c4804a8..3c23080 100644 --- a/crates/ddx-core/src/lib.rs +++ b/crates/ddx-core/src/lib.rs @@ -56,7 +56,7 @@ mod rewrite; /// a custom [`Rule`], which returns `f'(u)` as an [`sqlparser::ast::Expr`]. pub mod build { pub use crate::constructors::{ - add, cast_double, div, func, func1, mul, neg, num, one, square, sub, zero, + add, cast_double, div, func, func1, mul, neg, num, one, sign, square, sub, zero, }; } diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index d234d2e..d09b692 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -91,11 +91,7 @@ fn pre_gate_hit(sql: &str) -> bool { } // The next non-whitespace character must be `(`. - if sql[idx + kw.len()..] - .chars() - .find(|c| !c.is_whitespace()) - == Some('(') - { + if sql[idx + kw.len()..].chars().find(|c| !c.is_whitespace()) == Some('(') { return true; } } @@ -121,7 +117,7 @@ pub(crate) fn rewrite_sql( // 3. Statement-level context for the projection-boundary guard (F3/G4): // the names of every *computed* select-list alias of a CTE/derived table. - let mut aliases = HashSet::new(); + let mut aliases = ComputedAliases::default(); for stmt in &statements { collect_computed_aliases(stmt, &mut aliases); } @@ -172,7 +168,7 @@ fn differentiate_marker_tree( marker_expr: &Expr, casing: IdentCasing, reg: &RuleRegistry, - aliases: &HashSet, + aliases: &ComputedAliases, ) -> Result { let mut clone = marker_expr.clone(); let mut rw = MarkerRewriter { @@ -191,7 +187,7 @@ fn reprint_fallback( mut statements: Vec, casing: IdentCasing, reg: &RuleRegistry, - aliases: &HashSet, + aliases: &ComputedAliases, ) -> Result { for stmt in &mut statements { let mut rw = MarkerRewriter { @@ -254,7 +250,7 @@ fn differentiate_marker(f: &Function, casing: IdentCasing, reg: &RuleRegistry) - /// expression as a constant and drop gradient terms. The carve-out (G4): when /// the alias *is* the `wrt` itself, every occurrence is the differentiation /// leaf, so no term can be dropped and the guard stays quiet. -fn projection_guard(f: &Function, aliases: &HashSet) -> Result<()> { +fn projection_guard(f: &Function, aliases: &ComputedAliases) -> Result<()> { if aliases.is_empty() { return Ok(()); } @@ -273,11 +269,23 @@ fn projection_guard(f: &Function, aliases: &HashSet) -> Result<()> { let _ = Visit::visit(*expr_arg, &mut cols); for c in cols.cols { let lname = c.name.value.to_ascii_lowercase(); - // Carve-out: the wrt itself is always a leaf; never an error. + // Carve-out (G4): the wrt itself is always a leaf; never an error. if Some(&lname) == wrt_name.as_ref() { continue; } - if aliases.contains(&lname) { + let is_boundary = match &c.qualifier { + // A bare occurrence could bind to any computed alias in scope. + None => aliases.bare.contains(&lname), + // A qualified occurrence crosses a projection boundary only if the + // qualifier names the relation that actually owns the alias. A base + // column qualified to an unrelated table (e.g. `w.s` when the alias + // `s` belongs to a different CTE) is NOT the alias — preserving the + // qualifier-awareness the F2 ambiguity guard is built on. + Some(q) => aliases + .qualified + .contains(&(q.value.to_ascii_lowercase(), lname.clone())), + }; + if is_boundary { return Err(DiffError::ProjectionBoundary(format!( "`{}` is a computed select-list alias of a CTE/derived table used \ as a non-differentiation term; grad does not see through the \ @@ -327,7 +335,7 @@ impl Visitor for MarkerCollector { struct MarkerRewriter<'a> { casing: IdentCasing, reg: &'a RuleRegistry, - aliases: &'a HashSet, + aliases: &'a ComputedAliases, } impl VisitorMut for MarkerRewriter<'_> { @@ -379,52 +387,81 @@ impl Visitor for ColumnCollector { // Computed-alias collection for the projection-boundary guard // --------------------------------------------------------------------------- -fn collect_computed_aliases(stmt: &Statement, out: &mut HashSet) { +/// The *computed* select-list aliases of the CTEs/derived tables in a +/// statement, recorded so the guard can distinguish a reference that crosses a +/// projection boundary from a same-named base column that does not. +#[derive(Default)] +struct ComputedAliases { + /// Alias names referenceable by a *bare* (unqualified) occurrence. + bare: HashSet, + /// `(owning relation name, alias name)` for CTE/derived-table computed + /// aliases — so a *qualified* occurrence `rel.alias` is matched only against + /// the relation that actually owns it. This is what keeps a base column + /// like `w.s` from colliding with an unrelated CTE alias `s` (F2's + /// qualifier-awareness, applied to the F3/G4 guard). + qualified: HashSet<(String, String)>, +} + +impl ComputedAliases { + fn is_empty(&self) -> bool { + self.bare.is_empty() && self.qualified.is_empty() + } +} + +fn collect_computed_aliases(stmt: &Statement, out: &mut ComputedAliases) { match stmt { - Statement::Query(q) => walk_query(q, out), + Statement::Query(q) => walk_query(q, None, out), Statement::Insert(insert) => { if let Some(source) = &insert.source { - walk_query(source, out); + walk_query(source, None, out); } } _ => {} } } -fn walk_query(q: &Query, out: &mut HashSet) { +/// `owner` is the name of the relation whose *own* projection aliases we are +/// collecting (a CTE name, or a derived-table alias) — `None` for the outer +/// query's own select list, whose aliases can only be referenced bare. +fn walk_query(q: &Query, owner: Option<&str>, out: &mut ComputedAliases) { if let Some(with) = &q.with { for cte in &with.cte_tables { - walk_query(&cte.query, out); + let name = cte.alias.name.value.to_ascii_lowercase(); + walk_query(&cte.query, Some(&name), out); } } - walk_set_expr(&q.body, out); + walk_set_expr(&q.body, owner, out); } -fn walk_set_expr(body: &SetExpr, out: &mut HashSet) { +fn walk_set_expr(body: &SetExpr, owner: Option<&str>, out: &mut ComputedAliases) { match body { - SetExpr::Select(select) => walk_select(select, out), - SetExpr::Query(q) => walk_query(q, out), + SetExpr::Select(select) => walk_select(select, owner, out), + SetExpr::Query(q) => walk_query(q, owner, out), SetExpr::SetOperation { left, right, .. } => { - walk_set_expr(left, out); - walk_set_expr(right, out); + walk_set_expr(left, owner, out); + walk_set_expr(right, owner, out); } _ => {} } } -fn walk_select(select: &Select, out: &mut HashSet) { +fn walk_select(select: &Select, owner: Option<&str>, out: &mut ComputedAliases) { for item in &select.projection { if let SelectItem::ExprWithAlias { expr, alias } = item { // A *computed* alias is one whose projected expression is not a // plain column reference. `ColRef::from_expr` is the single place // that recognizes a column reference (seeing through `Nested`). if ColRef::from_expr(expr).is_none() { - out.insert(alias.value.to_ascii_lowercase()); + let name = alias.value.to_ascii_lowercase(); + if let Some(o) = owner { + out.qualified.insert((o.to_string(), name.clone())); + } + out.bare.insert(name); } } } // Recurse into derived tables (FROM subqueries), which are projection - // boundaries too. + // boundaries too — a derived table's aliases are owned by its own alias. for twj in &select.from { walk_table_factor(&twj.relation, out); for join in &twj.joins { @@ -433,13 +470,16 @@ fn walk_select(select: &Select, out: &mut HashSet) { } } -fn walk_table_factor(tf: &TableFactor, out: &mut HashSet) { - if let TableFactor::Derived { subquery, .. } = tf { - walk_query(subquery, out); +fn walk_table_factor(tf: &TableFactor, out: &mut ComputedAliases) { + if let TableFactor::Derived { + subquery, alias, .. + } = tf + { + let owner = alias.as_ref().map(|a| a.name.value.to_ascii_lowercase()); + walk_query(subquery, owner.as_deref(), out); } } - // --------------------------------------------------------------------------- // Span (1-based character line/column) → byte offset conversion (G3) // --------------------------------------------------------------------------- diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 1893dda..89f7fcc 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -229,6 +229,37 @@ fn marker_inside_an_aggregate_is_rewritten() { assert_eq!(out, "SELECT AVG((x)) AS step FROM batch"); } +#[test] +fn qualified_base_column_colliding_with_unrelated_cte_alias_is_accepted() { + // Review #42: w.s is table u's own base column, explicitly qualified — it + // cannot be v's computed alias `s`, so the guard must NOT fire. Preserves + // the qualifier-awareness of the F2 ambiguity guard. + let out = rw("WITH v AS (SELECT sin(x) AS s FROM t) \ + SELECT grad(w.s * x, x) AS d FROM u w JOIN v ON w.k = v.k"); + assert_eq!( + out, + "WITH v AS (SELECT sin(x) AS s FROM t) \ + SELECT (w.s) AS d FROM u w JOIN v ON w.k = v.k" + ); +} + +#[test] +fn qualified_reference_to_the_owning_cte_alias_still_errors() { + // But v.s IS v's computed alias, so differentiating it as a non-wrt term + // still crosses the projection boundary and must error. + let err = Ddx::new() + .rewrite_sql( + "WITH v AS (SELECT sin(x) AS s FROM t) \ + SELECT grad(v.s * x, x) AS d FROM v", + &GenericDialect {}, + ) + .unwrap_err(); + assert!( + matches!(err, DiffError::ProjectionBoundary(_)), + "got {err:?}" + ); +} + #[test] fn differentiating_wrt_a_computed_alias_is_allowed() { // Carve-out (G4): when the computed alias IS the wrt, every occurrence is diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index 4e5a934..faba9c6 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -125,6 +125,25 @@ fn cast_to_non_numeric_type_errors() { .is_err()); } +#[test] +fn abs_derivative_is_portable_and_pins_the_kink_at_zero() { + // d/du |u| = sign(u), emitted as a portable CASE (no engine-specific + // signum/sign builtin) that pins abs'(0) = 0 on every engine (review #44). + let out = d("abs(x)", "x"); + assert!( + !out.to_lowercase().contains("signum"), + "must not emit the non-portable signum builtin: {out}" + ); + assert_eq!( + out, + "CASE WHEN x > 0.0 THEN 1.0 WHEN x < 0.0 THEN -1.0 ELSE 0.0 END" + ); + // Chain rule still applies: d/dx |x*y| = sign(x*y) * y. + let chained = d("abs(x * y)", "x"); + assert!(chained.contains("* y"), "chain rule missing: {chained}"); + assert!(!chained.to_lowercase().contains("signum")); +} + #[test] fn jvp_seeds_a_tangent_on_one_input() { // jvp(x*y, x, dx) = product rule with tangent(x)=dx, tangent(y)=0 = dx*y diff --git a/docs/design.md b/docs/design.md index 50a2dcb..9ef4db2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -217,8 +217,12 @@ construct could otherwise produce a wrong number instead of an error): - **Extensible rule registry, keyed by function name.** Built-ins populate a registry users can extend: `registry.register("myfn", rule)`. For a unary `f(u)`, a user rule supplies just `f'(u)`; the engine applies the chain - rule automatically. A canonicalization table folds dialect spellings - (`ln`/`log`, `pow`/`power`) to one name before dispatch. + rule automatically. Dispatch case-folds the function name, and a minimal + canonicalization folds `pow` to `power` before dispatch. A fuller dialect + name-normalization *table* is deferred to the §3.6 roadmap; note that some + dialect spellings deliberately cannot be folded — `log` is natural log on + some engines but base-10 on DuckDB, so folding `ln`/`log` together would be + a silently-wrong derivative, exactly what this project refuses. - **The smart constructors — `add`/`sub`/`mul`/`div`/`neg` — own three correctness properties, not just algebraic simplification:** - *0/1-folding*, the JAX-`Zero`-tangent equivalent: drops structurally-zero @@ -476,10 +480,11 @@ unambiguous column, it works in any query shape; otherwise a typed error at rewrite time, before the query runs. **Roadmap:** general `u^v` via `exp(v·ln u)`; `CASE`/`min`/`max` subgradients -with a documented kink convention (mirroring how `abs` uses `signum`); -`atan2`, `log(base,x)`, `cbrt`, `expm1`/`log1p`; a dialect name-normalization -table; and a clear, permanent taxonomy of "not differentiable" (comparisons, -string/temporal ops, window functions). +with a documented kink convention (mirroring how `abs` pins its kink at `0` +via a portable `CASE`-based sign); `atan2`, `log(base,x)`, `cbrt`, +`expm1`/`log1p`; a dialect name-normalization table; and a clear, permanent +taxonomy of "not differentiable" (comparisons, string/temporal ops, window +functions). ### 3.7 Known limitation: symbolic expression swell @@ -822,9 +827,13 @@ layered: must evaluate to numerically equal columns in DuckDB and DataFusion. - **Convention-pinning tests, not blind oracle comparison,** at every point where a convention genuinely differs rather than one side being wrong: - - *Kinks* — `abs` at 0 gives `0` from the `signum` rule; JAX's own - convention at the kink differs (verify the exact value). Pin the - convention explicitly; the same treatment Route's tie-break needs (§4.3). + - *Kinks* — `abs` at 0 gives `0`, pinned by emitting a portable + `CASE WHEN u > 0 THEN 1 WHEN u < 0 THEN -1 ELSE 0 END` rather than an + engine `signum`/`sign` builtin (DuckDB has only `sign`, DataFusion only + `signum`, and `signum(0) = 1` — so a bare builtin would be both + non-portable and *not* pin `0` at the kink). JAX's own convention at the + kink differs (verify the exact value). Pin the convention explicitly; the + same treatment Route's tie-break needs (§4.3). - *Domain-widening* — a derivative can fail where the primal doesn't (`sqrt(x)` is fine at 0; `1/(2*sqrt(x))` divides by zero), and engines disagree on the result (`inf` vs. `NULL` vs. error). Cross-engine @@ -1096,11 +1105,15 @@ but folded and unfolded derivatives then disagree on NULL-bearing rows — documented and tested, not left as a quirk. → §3.2, §5. **F12 — Kinks and domain edges will make the oracle and the engines -disagree.** `abs` at 0 (JAX's own kink convention differs from the `signum` -rule's); derivatives that fail where the primal doesn't (`sqrt`'s derivative +disagree.** `abs` at 0 (JAX's own kink convention differs from ddx's pinned +`0`); derivatives that fail where the primal doesn't (`sqrt`'s derivative divides by zero at 0, and engines disagree on the result); `tan` near π/2, `ln` near 0. Fixed: pin conventions explicitly and state a domain-edge -policy, rather than comparing blindly. → §5. +policy, rather than comparing blindly. The `abs` kink in particular is pinned +by emitting a portable `CASE`-based sign, not an engine `signum`/`sign` +builtin — an early cut emitted `signum(u)`, which is non-portable (DuckDB has +no `signum`) and evaluates to `1` at 0 on DataFusion, pinning nothing; caught +in a later adversarial review. → §5. ### v1, round 2 (`G1`–`G9`) — a fresh pass with four independent evidence spikes From f4d49810498312e0f25372e6ef9337c1ec71cbb9 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 15:48:44 -0700 Subject: [PATCH 07/22] IAR 1 - adv example tests - failing loud when we can --- crates/ddx-core/src/constructors.rs | 55 ++++++++++++++++++++++--- crates/ddx-core/src/engine.rs | 35 ++++++++++++---- crates/ddx-core/tests/rewrite.rs | 11 +++++ crates/ddx-core/tests/roundtrip.rs | 46 +++++++++++---------- crates/ddx-core/tests/rules.rs | 62 ++++++++++++++++++++++++++++- docs/design.md | 23 +++++++---- 6 files changed, 192 insertions(+), 40 deletions(-) diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index 8ef6b33..40e8cae 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -28,22 +28,55 @@ use sqlparser::ast::{ // Literals and constant inspection // --------------------------------------------------------------------------- -/// Format an `f64` as a SQL numeric literal that always carries a decimal point -/// (or exponent), so it reads as floating-point rather than an integer literal. +/// Format a *finite, non-negative* `f64` as the digits of a SQL numeric literal, +/// always with a decimal point (or exponent) so it reads as floating-point. +/// Negativity is represented structurally by [`num`] (as a unary minus), not in +/// the digits — so this never emits a leading `-`. fn format_f64(v: f64) -> String { + debug_assert!( + v.is_finite() && v >= 0.0, + "format_f64 expects a finite, non-negative value (got {v})" + ); let s = format!("{v}"); - if s.contains(['.', 'e', 'E']) || s.contains("inf") || s.contains("NaN") { + if s.contains(['.', 'e', 'E']) { s } else { format!("{s}.0") } } -/// A numeric literal expression for `v` (e.g. `1.0`, `2.0`, `0.6931471805599453`). -pub fn num(v: f64) -> Expr { +/// A bare numeric-literal expression for the finite, non-negative value `v`. +fn raw_num(v: f64) -> Expr { Expr::Value(Value::Number(format_f64(v), false).with_empty_span()) } +/// A numeric literal expression for the finite value `v` (e.g. `1.0`, `2.0`, +/// `0.6931471805599453`). +/// +/// A negative value is emitted as a *unary minus* applied to the magnitude +/// (`-1.0` ⇒ `UnaryOp{Minus, 1.0}`) — exactly the AST shape `sqlparser` produces +/// when it parses `-1.0`. This is what makes the §5 round-trip invariant +/// (`reparse(render(d)) == d` modulo `Nested`) hold for negative literals too; +/// emitting a `Value("-1.0")` would reparse to a `UnaryOp` and break it +/// (round-3 review #46). +/// +/// `v` must be finite: the only user-derived numbers that reach `num` come +/// through the `power` rule, which guards `ln(base)`/overflow before calling +/// (so a non-finite value here is a caller bug, not user input — #33). +pub fn num(v: f64) -> Expr { + debug_assert!(v.is_finite(), "num expects a finite value (got {v})"); + if v < 0.0 { + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: Box::new(raw_num(-v)), + } + } else { + // `+0.0` and `-0.0` both land here (`-0.0 < 0.0` is false); normalize so + // `-0.0` never renders as the literal `-0.0`. + raw_num(if v == 0.0 { 0.0 } else { v }) + } +} + /// The constant `0.0` — the derivative of anything independent of `wrt`. pub fn zero() -> Expr { num(0.0) @@ -65,6 +98,18 @@ pub fn as_const(e: &Expr) -> Option { _ => None, }, Expr::Nested(inner) => as_const(inner), + // `sqlparser` parses a negative literal `-2` as `UnaryOp{Minus, + // Value("2")}`, not `Value("-2")` — so a negated constant must be seen + // through here, or the `power` rule misclassifies a constant exponent + // like `-2` as variable and wrongly rejects `power(x, -2)` (#46). + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr, + } => as_const(expr).map(|v| -v), + Expr::UnaryOp { + op: UnaryOperator::Plus, + expr, + } => as_const(expr), _ => None, } } diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index 4f7f04b..9a94435 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -321,8 +321,16 @@ fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) let base = args[0]; let exponent = args[1]; match (as_const(base), as_const(exponent)) { - // Constant exponent (covers x^2, x^0.5, ...). + // Constant exponent (covers x^2, x^0.5, x^-2, ...). (_, Some(c)) => { + // A non-finite constant (e.g. an out-of-range literal `1e400`) would + // otherwise be emitted as an `inf`/`NaN` token — invalid SQL. Fail + // loud instead (#33). + if !c.is_finite() { + return Err(DiffError::NotImplemented(format!( + "power(base, {c}): a non-finite constant exponent is not differentiable" + ))); + } let dbase = linearize(base, leaf, reg)?; if is_zero(&dbase) { return Ok(zero()); @@ -336,9 +344,19 @@ fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) if is_zero(&dexp) { return Ok(zero()); } + // The derivative is `a^u · ln(a) · du`; `ln(a)` is non-finite for a + // non-positive (or infinite) base, which would emit an `inf`/`NaN` + // token. Fail loud rather than emit invalid SQL (#33). + let ln_a = a.ln(); + if !ln_a.is_finite() { + return Err(DiffError::NotImplemented(format!( + "power({a}, exponent): the derivative needs ln(base), but ln({a}) is \ + not finite — the constant base must be positive" + ))); + } let outer = mul( func("power", vec![base.clone(), exponent.clone()]), - num(a.ln()), + num(ln_a), ); Ok(mul(outer, dexp)) } @@ -351,13 +369,16 @@ fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) } } -/// The simple (last-part, lower-cased) name of a function call, if its name is -/// a plain identifier path. +/// The lower-cased name of a function call — but only for an **unqualified** +/// call (a single-identifier name), mirroring the marker path's strict +/// `len() == 1` (F8). A schema-qualified call like `myschema.sin(x)` may be an +/// unrelated user function, so it must not silently match the built-in `sin` +/// rule ("tag explicitly, never infer" — principle 3; round-3 review #47). fn simple_func_name(f: &Function) -> Option { - f.name.0.last().and_then(|part| match part { - ObjectNamePart::Identifier(id) => Some(id.value.to_ascii_lowercase()), + match f.name.0.as_slice() { + [ObjectNamePart::Identifier(id)] => Some(id.value.to_ascii_lowercase()), _ => None, - }) + } } /// The positional (unnamed) argument expressions of a function call, or `None` diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 89f7fcc..205f734 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -39,6 +39,17 @@ fn marker_free_query_is_untouched() { assert_eq!(rw(sql), sql); } +#[test] +fn marker_free_query_mentioning_grad_in_a_string_is_returned_verbatim() { + // The pre-gate matches the `grad(` substring even inside a string literal, + // so this statement *is* parsed (unlike the case above). With no real + // marker it must still return byte-identical: a pre-gate false positive + // costs only a parse that finds nothing, never a wrong rewrite (review #45, + // finding D). + let sql = "SELECT 'grad(' AS label FROM t"; + assert_eq!(rw(sql), sql); +} + #[test] fn nested_higher_order_grad() { // grad(grad(power(x,3), x), x) = d2/dx2 x^3 = 6x; inner differentiated first. diff --git a/crates/ddx-core/tests/roundtrip.rs b/crates/ddx-core/tests/roundtrip.rs index bb2e781..b2d917d 100644 --- a/crates/ddx-core/tests/roundtrip.rs +++ b/crates/ddx-core/tests/roundtrip.rs @@ -12,7 +12,7 @@ use std::ops::ControlFlow; use ddx_core::sqlparser::ast::{Expr, Ident, VisitMut, VisitorMut}; use ddx_core::sqlparser::dialect::GenericDialect; use ddx_core::sqlparser::parser::Parser; -use ddx_core::Ddx; +use ddx_core::{ColRef, Ddx}; /// Remove every `Expr::Nested` wrapper so two trees that differ only in /// (redundant or precedence) parentheses compare equal. @@ -44,30 +44,24 @@ fn parse(text: &str) -> Expr { .unwrap_or_else(|e| panic!("reparse of `{text}` failed: {e}")) } -/// For a derivative produced from `expr`/`wrt`, assert that rendering it and -/// reparsing yields the same AST modulo `Nested` — i.e. the precedence -/// parentheses the smart constructors emit are exactly the ones the grammar -/// needs, no more, no fewer. +/// The load-bearing invariant (design.md §5): rendering the *constructed* +/// derivative AST `d` and reparsing it must yield the same AST, **modulo +/// `Nested`**. This compares against `d` itself (via [`Ddx::differentiate`], +/// which returns the AST) — not against another parse of the rendered text, +/// which would be vacuously equal and hide a constructed-vs-parsed mismatch +/// such as a negative literal emitted as `Value("-1.0")` (round-3 review #46). fn assert_roundtrips(expr: &str, wrt: &str) { let ddx = Ddx::new(); - // The derivative as text, via the public path... - let rendered = ddx - .differentiate_sql(expr, wrt, &GenericDialect {}) - .unwrap_or_else(|e| panic!("differentiate_sql({expr}, {wrt}): {e}")); - // ...reparsed, then rendered a second time: must be a fixed point (the - // parentheses are stable) and structurally identical modulo Nested. + let d = ddx + .differentiate(&parse(expr), &ColRef::bare(wrt)) + .unwrap_or_else(|e| panic!("differentiate d/d{wrt} ({expr}): {e}")); + let rendered = d.to_string(); let reparsed = parse(&rendered); - let rerendered = ddx - .differentiate_sql(expr, wrt, &GenericDialect {}) - .unwrap(); assert_eq!( - rendered, rerendered, - "render is not idempotent for d/d{wrt} ({expr})" + strip(reparsed), + strip(d), + "reparse(render(d)) != d modulo Nested for d/d{wrt} ({expr}); rendered = {rendered}" ); - // The load-bearing check: reparse(render(d)) == d, modulo parentheses. - let a = strip(reparsed); - let b = strip(parse(&rendered)); - assert_eq!(a, b, "round-trip changed the tree for d/d{wrt} ({expr})"); } #[test] @@ -86,6 +80,18 @@ fn precedence_sensitive_derivatives_round_trip() { assert_roundtrips("exp(x) / (x - 1)", "x"); // quotient with composite denom } +#[test] +fn negative_literal_derivatives_round_trip() { + // The cases the original 10 avoided — every one emits a negative literal, + // which must be a `UnaryOp{Minus, ..}` (matching sqlparser's parse), not a + // `Value("-…")` that would reparse to a different tree (#46). + assert_roundtrips("abs(x)", "x"); // CASE ... THEN -1.0 ... + assert_roundtrips("power(x, -2)", "x"); // -2 * power(x, -3.0) + assert_roundtrips("power(x, 0.5)", "x"); // emits exponent -0.5 + assert_roundtrips("x / y", "y"); // negative numerator, -(x)/(y*y) + assert_roundtrips("cos(x) * x", "x"); // -sin(x) * x + cos(x) +} + #[test] fn strip_nested_actually_normalizes() { // Sanity: the normalizer collapses parentheses, so an unparenthesized and a diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index faba9c6..25e21be 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -7,7 +7,7 @@ //! `Nested`-wrapped for precedence safety (G1). Both are asserted here. use ddx_core::sqlparser::dialect::GenericDialect; -use ddx_core::Ddx; +use ddx_core::{Ddx, DiffError}; fn d(expr: &str, wrt: &str) -> String { Ddx::new() @@ -88,6 +88,66 @@ fn unsupported_function_errors() { .is_err()); } +#[test] +fn power_negative_constant_exponent() { + // d/dx power(x, -2) = -2 * power(x, -3); a negative constant exponent is + // inside the stated v1 surface and must not be rejected (review #46). + assert_eq!(d("power(x, -2)", "x"), "-2.0 * power(x, -3.0)"); +} + +#[test] +fn power_negative_fractional_exponent() { + // d/dx power(x, -0.5) = -0.5 * power(x, -1.5); the negative *fractional* + // case, complementing the -2 integer case above (review #46). + assert_eq!(d("power(x, -0.5)", "x"), "-0.5 * power(x, -1.5)"); +} + +#[test] +fn power_fractional_exponent_output_is_reconsumable() { + // d/dx power(x, 0.5) emits a negative exponent; differentiating that TEXT + // again must work (the engine must be able to re-consume its own output). + let once = d("power(x, 0.5)", "x"); + assert!(once.contains("power(x, -0.5)"), "unexpected: {once}"); + // Re-parse and differentiate the emitted text again — no error. + let twice = Ddx::new().differentiate_sql(&once, "x", &GenericDialect {}); + assert!(twice.is_ok(), "engine rejected its own output: {twice:?}"); +} + +#[test] +fn power_non_positive_constant_base_errors() { + // d/dx power(0, x) would need ln(0) = -inf; must fail loud, not emit + // "power(0, x) * -inf" (review #33). + let err = Ddx::new() + .differentiate_sql("power(0, x)", "x", &GenericDialect {}) + .unwrap_err(); + assert!(matches!(err, DiffError::NotImplemented(_)), "got {err:?}"); + let out = Ddx::new().differentiate_sql("power(2, x)", "x", &GenericDialect {}); + assert!(out.is_ok(), "positive base should differentiate: {out:?}"); + assert!(!out.unwrap().to_lowercase().contains("inf")); +} + +#[test] +fn power_non_finite_constant_exponent_errors() { + // The mirror of the non-positive-base guard, on the exponent side: an + // out-of-range literal exponent (1e400 overflows f64 to +inf) must fail + // loud rather than emit an `inf` token (review #33). + let err = Ddx::new() + .differentiate_sql("power(x, 1e400)", "x", &GenericDialect {}) + .unwrap_err(); + assert!(matches!(err, DiffError::NotImplemented(_)), "got {err:?}"); +} + +#[test] +fn schema_qualified_call_does_not_match_builtin_rule() { + // myschema.sin(x) may be an unrelated user function; it must NOT silently + // differentiate via the built-in sin rule (review #47). + assert!(Ddx::new() + .differentiate_sql("myschema.sin(x)", "x", &GenericDialect {}) + .is_err()); + // The unqualified builtin still works. + assert_eq!(d("sin(x)", "x"), "cos(x)"); +} + #[test] fn general_power_uv_errors() { // power(x, x): both base and exponent vary — not supported yet. diff --git a/docs/design.md b/docs/design.md index 9ef4db2..17b7d2d 100644 --- a/docs/design.md +++ b/docs/design.md @@ -263,11 +263,18 @@ construct could otherwise produce a wrong number instead of an error): unqualified calls (`myschema.grad(…)` is left alone) and matched case-folded, so `GRAD(x,x)` is caught too `[F8]`/`[G7]`. - **Splice by source span, never reprint the statement.** `rewrite_sql` - first runs a parse-free, case-insensitive pre-gate — a regex over - `grad`/`jvp\(` — and returns the input verbatim if it doesn't hit, so a - marker-free statement is *never parsed* and can't be failed or reformatted - by parser coverage gaps. When the gate hits, only the marker call's byte - range is replaced, everything else stays byte-identical. This is a real + first runs a parse-free, case-insensitive pre-gate — a scan for an + unqualified `grad`/`jvp\(` substring — and returns the input verbatim if it + doesn't hit, so a statement whose text contains no such substring is *never + parsed* and can't be failed or reformatted by parser coverage gaps. The gate + is a substring filter, so it also hits on a `grad(` that appears only inside + a string literal or comment: such a statement *is* parsed, but with no real + marker it still comes back verbatim (the collector finds nothing) — a + false-positive costs a parse, never a wrong rewrite. The one residual is a + marker-free statement that both mentions the substring *and* uses syntax the + dialect can't parse: it would hard-error where a stricter gate wouldn't + (documentation-only, `[F5]`). When the gate hits a real marker, only the + marker call's byte range is replaced, everything else stays byte-identical. This is a real subsystem, not a one-liner: `sqlparser`'s `Spanned` gives line/column in 1-based *characters*, not byte offsets, so the splice needs a UTF-8-aware conversion, must handle multiple and nested markers (spliced @@ -292,8 +299,10 @@ follows) but a narrower one than first assumed: spiked against `DuckDbDialect` @ `sqlparser` 0.62.0, `SELECT * EXCLUDE`, `FROM`-first queries, bare `FROM t`, lambdas, and `t.* REPLACE (…)` all parse; the real misses are `PIVOT` and `#1` positional columns `[G9]`. The parse-free pre-gate and source-span splicing -above (§3.2) mean this coverage gap only ever bounds a query that *actually -contains* a marker, and reprint fidelity is never a separate risk. +above (§3.2) mean this coverage gap only ever bounds a query whose text +*contains a `grad(`/`jvp(` substring* — almost always a real marker, but also +the rare false positive where the substring sits inside a string literal or +comment (§3.2) — and reprint fidelity is never a separate risk. **Path B — in-engine plan rewrite, native Rust DataFusion.** A marker UDF plus an `AnalyzerRule` so `grad()` works bare, with no wrapper, across both the SQL From 4249759bedd873a2641d96e9ee6e62ca5c2834e1 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 15:56:43 -0700 Subject: [PATCH 08/22] Primary agent code review tool. --- crates/ddx-core/src/constructors.rs | 56 ++++++++++++++++++++++++++--- crates/ddx-core/src/engine.rs | 33 +++++++---------- crates/ddx-core/src/lib.rs | 2 +- crates/ddx-core/tests/rules.rs | 10 ++++++ 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index 40e8cae..ed27169 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -24,6 +24,8 @@ use sqlparser::ast::{ UnaryOperator, Value, }; +use crate::error::{DiffError, Result}; + // --------------------------------------------------------------------------- // Literals and constant inspection // --------------------------------------------------------------------------- @@ -60,10 +62,14 @@ fn raw_num(v: f64) -> Expr { /// emitting a `Value("-1.0")` would reparse to a `UnaryOp` and break it /// (round-3 review #46). /// -/// `v` must be finite: the only user-derived numbers that reach `num` come -/// through the `power` rule, which guards `ln(base)`/overflow before calling -/// (so a non-finite value here is a caller bug, not user input — #33). -pub fn num(v: f64) -> Expr { +/// `v` must be finite. For a *compile-time-known* finite constant (`0`, `1`, +/// `2`, `ln 2`, …) call `num` directly. For any value *computed from user input* +/// — which can overflow to `inf` or produce `NaN` — call [`finite_num`] instead, +/// which fails loud rather than emit an invalid `inf`/`NaN` literal (#33). This +/// is why `num` is not part of the public `build` surface: external callers get +/// the checked [`finite_num`], so a non-finite value can never silently become +/// `inf.0` in a release build. +pub(crate) fn num(v: f64) -> Expr { debug_assert!(v.is_finite(), "num expects a finite value (got {v})"); if v < 0.0 { Expr::UnaryOp { @@ -77,6 +83,25 @@ pub fn num(v: f64) -> Expr { } } +/// A numeric literal for a value that *might not be finite* — the checked +/// counterpart of [`num`]. Emits the literal if `v` is finite, else a typed +/// [`DiffError::NotImplemented`]: a non-finite value has no valid SQL literal +/// (`inf`/`NaN` are not numbers), so a derivative that would carry one must fail +/// loud, never emit invalid SQL (#33). Use this for every value derived from +/// user input or an arithmetic that can overflow (e.g. `ln(base)`, an +/// out-of-range exponent). This is the single seam through which computed +/// constants become literals. +pub fn finite_num(v: f64) -> Result { + if v.is_finite() { + Ok(num(v)) + } else { + Err(DiffError::NotImplemented(format!( + "cannot emit a non-finite derivative constant ({v}); a non-finite \ + value has no valid SQL literal" + ))) + } +} + /// The constant `0.0` — the derivative of anything independent of `wrt`. pub fn zero() -> Expr { num(0.0) @@ -394,4 +419,27 @@ mod tests { let x = Expr::Identifier(Ident::new("x")); assert_eq!(div(x, one()).to_string(), "x"); } + + #[test] + fn num_emits_negatives_as_unary_minus() { + // A negative literal must match sqlparser's parse shape (UnaryOp{Minus, + // magnitude}) so derivatives round-trip; the rendered text is still + // `-2.0` / `-0.5`. + assert!(matches!(num(-2.0), Expr::UnaryOp { .. })); + assert_eq!(num(-2.0).to_string(), "-2.0"); + assert_eq!(num(-0.5).to_string(), "-0.5"); + assert_eq!(num(0.0).to_string(), "0.0"); // incl. -0.0 normalization + assert_eq!(num(-0.0).to_string(), "0.0"); + } + + #[test] + fn finite_num_rejects_non_finite_values() { + // The checked emission seam: finite values pass, inf/NaN fail loud + // (never a silent `inf.0`/`NaN.0` token). This is the public `build` + // surface, so external callers cannot emit invalid SQL in release. + assert_eq!(finite_num(2.0).unwrap().to_string(), "2.0"); + assert!(finite_num(f64::INFINITY).is_err()); + assert!(finite_num(f64::NEG_INFINITY).is_err()); + assert!(finite_num(f64::NAN).is_err()); + } } diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index 9a94435..78b6679 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -21,7 +21,8 @@ use sqlparser::ast::{ use crate::colref::{ColRef, IdentCasing, Match}; use crate::constructors::{ - add, as_const, div, func, func1, is_zero, mul, neg, num, one, sign, square, sub, zero, + add, as_const, div, finite_num, func, func1, is_zero, mul, neg, num, one, sign, square, sub, + zero, }; use crate::error::{DiffError, Result}; @@ -323,19 +324,18 @@ fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) match (as_const(base), as_const(exponent)) { // Constant exponent (covers x^2, x^0.5, x^-2, ...). (_, Some(c)) => { - // A non-finite constant (e.g. an out-of-range literal `1e400`) would - // otherwise be emitted as an `inf`/`NaN` token — invalid SQL. Fail - // loud instead (#33). - if !c.is_finite() { - return Err(DiffError::NotImplemented(format!( - "power(base, {c}): a non-finite constant exponent is not differentiable" - ))); - } let dbase = linearize(base, leaf, reg)?; if is_zero(&dbase) { return Ok(zero()); } - let outer = mul(num(c), func("power", vec![base.clone(), num(c - 1.0)])); + // `finite_num` fails loud on a non-finite constant (e.g. an + // out-of-range literal `1e400` → inf), but *only here at emission* — + // after the zero short-circuit above — so a wrt-independent base + // still differentiates to `0` rather than erroring (#49/F1, #33). + let outer = mul( + finite_num(c)?, + func("power", vec![base.clone(), finite_num(c - 1.0)?]), + ); Ok(mul(outer, dbase)) } // Constant base, variable exponent. @@ -345,18 +345,11 @@ fn linearize_power(name: &str, args: &[&Expr], leaf: &Leaf, reg: &RuleRegistry) return Ok(zero()); } // The derivative is `a^u · ln(a) · du`; `ln(a)` is non-finite for a - // non-positive (or infinite) base, which would emit an `inf`/`NaN` - // token. Fail loud rather than emit invalid SQL (#33). - let ln_a = a.ln(); - if !ln_a.is_finite() { - return Err(DiffError::NotImplemented(format!( - "power({a}, exponent): the derivative needs ln(base), but ln({a}) is \ - not finite — the constant base must be positive" - ))); - } + // non-positive (or infinite) base — `finite_num` fails loud there + // rather than emit an `inf`/`NaN` token (#33). let outer = mul( func("power", vec![base.clone(), exponent.clone()]), - num(ln_a), + finite_num(a.ln())?, ); Ok(mul(outer, dexp)) } diff --git a/crates/ddx-core/src/lib.rs b/crates/ddx-core/src/lib.rs index 3c23080..76564e9 100644 --- a/crates/ddx-core/src/lib.rs +++ b/crates/ddx-core/src/lib.rs @@ -56,7 +56,7 @@ mod rewrite; /// a custom [`Rule`], which returns `f'(u)` as an [`sqlparser::ast::Expr`]. pub mod build { pub use crate::constructors::{ - add, cast_double, div, func, func1, mul, neg, num, one, sign, square, sub, zero, + add, cast_double, div, finite_num, func, func1, mul, neg, one, sign, square, sub, zero, }; } diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index 25e21be..a1eb3e6 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -137,6 +137,16 @@ fn power_non_finite_constant_exponent_errors() { assert!(matches!(err, DiffError::NotImplemented(_)), "got {err:?}"); } +#[test] +fn non_finite_exponent_still_differentiates_to_zero_when_base_is_constant_in_wrt() { + // The non-finite constant only blocks emission *when actually emitted*. If + // the base doesn't depend on the differentiation variable, the derivative is + // 0 and no non-finite literal is ever built — so this must return 0, not + // error (review #49/F1: the finiteness check must follow the zero + // short-circuit, matching the constant-base branch). + assert_eq!(d("power(x, 1e400)", "y"), "0.0"); +} + #[test] fn schema_qualified_call_does_not_match_builtin_rule() { // myschema.sin(x) may be an unrelated user function; it must NOT silently From b20591344d459320cca1a5feb87cde9c43453d03 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 16:16:40 -0700 Subject: [PATCH 09/22] IAR 2: Adversary wrote property/simulation tests. Was able to find and repo a real bug. --- crates/ddx-core/tests/rewrite.rs | 37 ++ crates/ddx-core/tests/simulation.rs | 509 ++++++++++++++++++++++++++++ 2 files changed, 546 insertions(+) create mode 100644 crates/ddx-core/tests/simulation.rs diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 205f734..b53cb28 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -110,6 +110,43 @@ fn splice_preserves_exact_surrounding_bytes() { assert_eq!(out, "SELECT (x + x) , y FROM t"); } +// --------------------------------------------------------------------------- +// Pre-gate coverage: comment-separated markers (KNOWN BUG #52) +// --------------------------------------------------------------------------- + +#[test] +#[ignore = "known bug #52: pre-gate skips only whitespace, not SQL comments, so \ + a comment-separated marker is missed and returned verbatim"] +fn pre_gate_must_not_miss_a_comment_separated_marker() { + // A SQL comment is lexical whitespace, so `grad /* c */ (x, x)` parses as a + // genuine `Function(name=grad)` marker call — the *same* AST as plain + // `grad(x, x)`. The whitespace-separated forms below ARE rewritten, proving + // the marker is real and reachable; the comment-separated forms must be too. + // + // But `pre_gate_hit` requires the next *non-whitespace* char after the name + // to be `(` and does not skip comments, so these slip past the parse-free + // gate and `rewrite_sql` returns them VERBATIM — the marker survives to + // execution un-rewritten. This is a false-NEGATIVE missing a REAL marker, + // contradicting design.md §3.2's guarantee that the gate only ever produces + // harmless false-positives. + + // Baseline: whitespace-separated markers are correctly rewritten to (1.0). + assert_eq!(rw("SELECT grad\n(x, x) FROM t"), "SELECT (1.0) FROM t"); + assert_eq!(rw("SELECT grad\t(x, x) FROM t"), "SELECT (1.0) FROM t"); + + // The bug: a comment between the name and its argument list. + assert_eq!( + rw("SELECT grad /* c */ (x, x) FROM t"), + "SELECT (1.0) FROM t", + "block-comment-separated marker was not rewritten" + ); + assert_eq!( + rw("SELECT grad-- c\n(x, x) FROM t"), + "SELECT (1.0) FROM t", + "line-comment-separated marker was not rewritten" + ); +} + // --------------------------------------------------------------------------- // Reserved names (F8) and the cut of scalar vjp (Q7) // --------------------------------------------------------------------------- diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs new file mode 100644 index 0000000..04bb901 --- /dev/null +++ b/crates/ddx-core/tests/simulation.rs @@ -0,0 +1,509 @@ +//! Simulation / property-based tests for the v1 differentiation engine +//! (design.md §5: "numeric agreement" + "round-trip property tests"). +//! +//! These are adversarial: instead of hand-picked expressions, they generate +//! thousands of random derivable SQL scalar expressions and hold the engine to +//! three properties that any correct symbolic differentiator must satisfy: +//! +//! 1. **Numeric agreement (the finite-difference oracle).** The single +//! strongest check on a derivative: for a random `f`, the symbolic `d/dx f` +//! evaluated at a point must equal a central finite difference of `f` there. +//! A wrong rule (a sign flip, a missing chain factor, a bad power exponent) +//! disagrees at *every* well-conditioned point, so it is caught even though a +//! kink artifact (from `abs`) is tolerated as a lone outlier. +//! 2. **Round-trip (design.md §5, G1).** `reparse(render(d)) == d` modulo +//! `Nested`, fuzzed over the whole generated space, not the 10 hand cases. +//! 3. **Self-consumption / higher-order stability.** The engine must be able to +//! re-parse and re-differentiate its *own* text output repeatedly without +//! panicking, erroring, or emitting unparseable SQL (e.g. a `--` comment). +//! +//! No external fuzzing crate is used: the core is deliberately `sqlparser`-only, +//! and a dev-dependency-free, deterministic generator keeps failures perfectly +//! reproducible (each is printed with its seed). + +use ddx_core::sqlparser::ast::{BinaryOperator, Expr, UnaryOperator, Value}; +use ddx_core::sqlparser::dialect::GenericDialect; +use ddx_core::sqlparser::parser::Parser; +use ddx_core::{ColRef, Ddx, DiffError}; + +// --------------------------------------------------------------------------- +// A tiny deterministic PRNG (SplitMix64) — reproducible, no dependencies. +// --------------------------------------------------------------------------- + +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng(seed.wrapping_add(0x9E37_79B9_7F4A_7C15)) + } + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } + /// A float in `[lo, hi)`. + fn range(&mut self, lo: f64, hi: f64) -> f64 { + let u = (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64; + lo + u * (hi - lo) + } +} + +// --------------------------------------------------------------------------- +// Random expression generator over the *derivable* v1 grammar. +// --------------------------------------------------------------------------- +// +// Everything produced here is inside the engine's supported surface, so +// `differentiate` never returns `NotImplemented`: vars {x, y}, numeric +// literals, `+ - * /`, unary minus, the unary-rule function set, and `power` +// with exactly one constant side. It emits SQL text (parenthesized to fix +// structure), which both exercises the parser and gives readable failures. + +/// Unary functions that have a differentiation rule (design.md §3.6). +const UNARY_FNS: &[&str] = &[ + "sin", "cos", "tan", "asin", "acos", "atan", "exp", "ln", "log2", "log10", "sqrt", "sinh", + "cosh", "tanh", "abs", +]; + +/// A *non-negative* constant, safe to place under a generated unary minus +/// without producing a `--` line-comment in the source text. (Negative literals +/// still appear — as `power` exponents, below — where they are direct function +/// arguments, and via the engine's own `num()` output.) +fn gen_const(rng: &mut Rng) -> String { + let choices = ["2", "3", "0.5", "1.5", "2.5"]; + choices[rng.below(choices.len() as u64) as usize].to_string() +} + +/// A constant `power` exponent, which *may* be negative — passed as a direct +/// call argument (`power(x, -2)`), never wrapped in a unary minus, so it never +/// forms a `--` in the generated text. +fn gen_exponent(rng: &mut Rng) -> String { + let choices = ["2", "3", "0.5", "1.5", "-1", "-2", "-0.5", "2.5"]; + choices[rng.below(choices.len() as u64) as usize].to_string() +} + +fn gen_expr(rng: &mut Rng, depth: u32) -> String { + if depth == 0 || rng.below(100) < 30 { + // Leaf: a variable or a constant. + return match rng.below(5) { + 0 | 1 => "x".to_string(), + 2 => "y".to_string(), + _ => gen_const(rng), + }; + } + match rng.below(9) { + 0 => format!("({} + {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), + 1 => format!("({} - {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), + 2 => format!("({} * {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), + 3 => format!("({} / {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), + 4 => format!("(-{})", gen_expr(rng, depth - 1)), + 5 | 6 => { + let f = UNARY_FNS[rng.below(UNARY_FNS.len() as u64) as usize]; + format!("{f}({})", gen_expr(rng, depth - 1)) + } + 7 => { + // power(base, const-exponent) + format!("power({}, {})", gen_expr(rng, depth - 1), gen_exponent(rng)) + } + _ => { + // power(positive-const-base, variable-exponent) + let base = ["2", "3", "1.5", "0.5"][rng.below(4) as usize]; + format!("power({base}, {})", gen_expr(rng, depth - 1)) + } + } +} + +// --------------------------------------------------------------------------- +// A float evaluator for the emitted grammar (primal *and* derivative). +// --------------------------------------------------------------------------- + +fn parse_expr(text: &str) -> Expr { + Parser::new(&GenericDialect {}) + .try_with_sql(text) + .and_then(|mut p| p.parse_expr()) + .unwrap_or_else(|e| panic!("reparse of `{text}` failed: {e}")) +} + +/// Evaluate a scalar expression at `(x, y)`. Returns `None` for anything not in +/// the numeric grammar (so an unexpected node fails a comparison loudly rather +/// than silently returning a bogus number). +fn eval(e: &Expr, x: f64, y: f64) -> Option { + match e { + Expr::Value(v) => match &v.value { + Value::Number(s, _) => s.parse::().ok(), + _ => None, + }, + Expr::Identifier(id) => match id.value.to_ascii_lowercase().as_str() { + "x" => Some(x), + "y" => Some(y), + _ => None, + }, + Expr::CompoundIdentifier(parts) => match parts.last()?.value.to_ascii_lowercase().as_str() { + "x" => Some(x), + "y" => Some(y), + _ => None, + }, + Expr::Nested(inner) => eval(inner, x, y), + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr, + } => Some(-eval(expr, x, y)?), + Expr::UnaryOp { + op: UnaryOperator::Plus, + expr, + } => eval(expr, x, y), + Expr::BinaryOp { left, op, right } => { + let a = eval(left, x, y)?; + let b = eval(right, x, y)?; + match op { + BinaryOperator::Plus => Some(a + b), + BinaryOperator::Minus => Some(a - b), + BinaryOperator::Multiply => Some(a * b), + BinaryOperator::Divide => Some(a / b), + _ => None, + } + } + // Numeric casts are the identity on f64. + Expr::Cast { expr, .. } => eval(expr, x, y), + Expr::Function(f) => eval_function(f, x, y), + // The `sign` CASE (the only CASE the engine emits). + Expr::Case { + operand: None, + conditions, + else_result, + .. + } => { + for w in conditions { + if eval_bool(&w.condition, x, y)? { + return eval(&w.result, x, y); + } + } + eval(else_result.as_deref()?, x, y) + } + _ => None, + } +} + +fn eval_bool(e: &Expr, x: f64, y: f64) -> Option { + if let Expr::BinaryOp { left, op, right } = e { + let a = eval(left, x, y)?; + let b = eval(right, x, y)?; + return match op { + BinaryOperator::Gt => Some(a > b), + BinaryOperator::Lt => Some(a < b), + BinaryOperator::GtEq => Some(a >= b), + BinaryOperator::LtEq => Some(a <= b), + BinaryOperator::Eq => Some(a == b), + BinaryOperator::NotEq => Some(a != b), + _ => None, + }; + } + None +} + +fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Option { + use ddx_core::sqlparser::ast::{ + FunctionArg, FunctionArgExpr, FunctionArguments, ObjectNamePart, + }; + let [ObjectNamePart::Identifier(id)] = f.name.0.as_slice() else { + return None; + }; + let name = id.value.to_ascii_lowercase(); + let FunctionArguments::List(list) = &f.args else { + return None; + }; + let mut args = Vec::new(); + for a in &list.args { + match a { + FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => args.push(eval(e, x, y)?), + _ => return None, + } + } + let a0 = *args.first()?; + let v = match name.as_str() { + "sin" => a0.sin(), + "cos" => a0.cos(), + "tan" => a0.tan(), + "asin" => a0.asin(), + "acos" => a0.acos(), + "atan" => a0.atan(), + "exp" => a0.exp(), + "ln" => a0.ln(), + "log2" => a0.log2(), + "log10" => a0.log10(), + "sqrt" => a0.sqrt(), + "sinh" => a0.sinh(), + "cosh" => a0.cosh(), + "tanh" => a0.tanh(), + "abs" => a0.abs(), + "power" | "pow" => { + let e1 = *args.get(1)?; + a0.powf(e1) + } + _ => return None, + }; + Some(v) +} + +// --------------------------------------------------------------------------- +// Property 1 — the finite-difference numeric oracle. +// --------------------------------------------------------------------------- + +/// For a fixed expression, sample points and compare the symbolic derivative to +/// a central finite difference. Returns `Some(report)` on a genuine mismatch +/// (a strong majority of well-conditioned points disagree), `None` otherwise. +/// +/// A lone disagreement is tolerated: `abs(g(x))` has a kink where `g(x) = 0`, +/// and a finite difference straddling it disagrees with the (correct) pinned +/// convention at that one point. A wrong *rule* disagrees everywhere. +fn fd_check(rng: &mut Rng, expr_text: &str, d: &Expr) -> Option { + const H: f64 = 1e-4; + const RTOL: f64 = 2e-3; + const ATOL: f64 = 1e-5; + const COND_CAP: f64 = 1e5; // skip near-singular points (huge slope) + + let f = parse_expr(expr_text); + let mut comparable = 0u32; + let mut disagree = 0u32; + let mut first_bad = String::new(); + + // Draw up to 60 candidate points, keep the well-conditioned finite ones. + for _ in 0..60 { + if comparable >= 8 { + break; + } + let x0 = rng.range(0.2, 1.8); + let y0 = rng.range(0.2, 1.8); + let (Some(fp), Some(fm), Some(dv)) = ( + eval(&f, x0 + H, y0), + eval(&f, x0 - H, y0), + eval(d, x0, y0), + ) else { + continue; + }; + let fd = (fp - fm) / (2.0 * H); + if !fd.is_finite() + || !dv.is_finite() + || fd.abs() > COND_CAP + || dv.abs() > COND_CAP + { + continue; + } + comparable += 1; + if (fd - dv).abs() > ATOL + RTOL * dv.abs().max(fd.abs()) { + disagree += 1; + if first_bad.is_empty() { + first_bad = format!( + "x={x0:.6} y={y0:.6}: symbolic d/dx = {dv:.8}, finite-diff = {fd:.8}" + ); + } + } + } + + // Need enough evidence, and a majority must disagree (a real rule bug), + // not a single kink artifact. + if comparable >= 4 && disagree >= 2 && disagree * 2 > comparable { + return Some(format!( + "d/dx {expr_text}\n => {d}\n {disagree}/{comparable} points disagree; e.g. {first_bad}" + )); + } + None +} + +#[test] +fn finite_difference_agreement_over_random_expressions() { + let ddx = Ddx::new(); + let wrt = ColRef::bare("x"); + let mut failures: Vec = Vec::new(); + let mut tested = 0u32; + + for seed in 0..4000u64 { + let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D)); + let depth = 2 + (seed % 3) as u32; // depth 2..=4 + let text = gen_expr(&mut rng, depth); + + let parsed = parse_expr(&text); + let d = match ddx.differentiate(&parsed, &wrt) { + Ok(d) => d, + // Everything generated is derivable; a genuine error here is itself + // a finding worth seeing. + Err(DiffError::NotImplemented(_)) => continue, + Err(e) => { + failures.push(format!("UNEXPECTED ERROR on `{text}`: {e}")); + continue; + } + }; + tested += 1; + if let Some(report) = fd_check(&mut rng, &text, &d) { + failures.push(report); + } + } + + assert!(tested > 500, "generator produced too few derivable cases: {tested}"); + assert!( + failures.is_empty(), + "finite-difference oracle found {} disagreement(s) out of {} tested:\n\n{}", + failures.len(), + tested, + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") + ); +} + +// --------------------------------------------------------------------------- +// Property 2 — render fidelity: reparse(render(d)) is *value-equal* to d. +// --------------------------------------------------------------------------- +// +// This is the correctness-relevant form of the design.md §5 round-trip +// invariant. A purely structural "== d modulo Nested" check is imprecise for +// `*`/`/` associativity (issue #50): the product rule builds `a * (b / c)`, +// which renders unparenthesized as `a * b / c` and reparses as `(a * b) / c` — +// a *different tree* but the *same value*. What actually matters — and what the +// G1 precedence bug (`(a+b)*c` losing its parens → `a+b*c`) violates — is that +// the rendered-then-reparsed derivative computes the same number as the AST the +// engine constructed. So this compares by evaluation, not by tree shape: it is +// immune to benign reassociation yet still catches any paren-drop that changes +// the value. + +#[test] +fn render_reparse_is_value_preserving() { + const RTOL: f64 = 1e-9; // float `*`/`/` reassociation differs by ~ulps only + const ATOL: f64 = 1e-11; + let ddx = Ddx::new(); + let wrt = ColRef::bare("x"); + let mut failures: Vec = Vec::new(); + + for seed in 0..5000u64 { + let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D) ^ 0xDEAD_BEEF); + let depth = 2 + (seed % 4) as u32; + let text = gen_expr(&mut rng, depth); + let parsed = parse_expr(&text); + let d = match ddx.differentiate(&parsed, &wrt) { + Ok(d) => d, + Err(_) => continue, + }; + let rendered = d.to_string(); + // A `--` in emitted SQL is a line comment — a silent-wrong render. + if rendered.contains("--") { + failures.push(format!("emitted a `--` comment: d/dx {text} => {rendered}")); + continue; + } + let reparsed = match Parser::new(&GenericDialect {}) + .try_with_sql(&rendered) + .and_then(|mut p| p.parse_expr()) + { + Ok(rp) => rp, + Err(e) => { + failures.push(format!( + "engine emitted unparseable SQL: d/dx {text} => {rendered} ({e})" + )); + continue; + } + }; + + // Compare the constructed AST and its rendered-then-reparsed form by + // value at several well-conditioned points. + let mut compared = 0u32; + for _ in 0..40 { + if compared >= 6 { + break; + } + let x0 = rng.range(0.2, 1.8); + let y0 = rng.range(0.2, 1.8); + let (Some(va), Some(vb)) = (eval(&d, x0, y0), eval(&reparsed, x0, y0)) else { + continue; + }; + if !va.is_finite() || !vb.is_finite() || va.abs() > 1e6 { + continue; + } + compared += 1; + if (va - vb).abs() > ATOL + RTOL * va.abs() { + failures.push(format!( + "render changed the value: d/dx {text}\n rendered = {rendered}\n at x={x0:.4} y={y0:.4}: AST = {va:.10}, reparsed = {vb:.10}" + )); + break; + } + } + } + + assert!( + failures.is_empty(), + "render-fidelity fuzz found {} failure(s):\n\n{}", + failures.len(), + failures.iter().take(15).cloned().collect::>().join("\n\n") + ); +} + +// --------------------------------------------------------------------------- +// Property 3 — self-consumption: the engine must re-consume its own output. +// --------------------------------------------------------------------------- + +#[test] +fn higher_order_self_consumption_is_stable() { + // Differentiate, render, reparse, differentiate again — up to 4 rounds. + // Higher-order derivatives are a stated feature ("grad(grad(f,x),x) just + // works"); the engine must never panic, emit unparseable SQL, or error on + // an expression it produced itself. + let ddx = Ddx::new(); + let wrt = ColRef::bare("x"); + let mut failures: Vec = Vec::new(); + + for seed in 0..2000u64 { + let mut rng = Rng::new(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ 0x1234_5678); + let depth = 2 + (seed % 3) as u32; + let original = gen_expr(&mut rng, depth); + + let mut current = original.clone(); + for round in 0..4 { + let parsed = Parser::new(&GenericDialect {}) + .try_with_sql(¤t) + .and_then(|mut p| p.parse_expr()); + let parsed = match parsed { + Ok(p) => p, + Err(e) => { + failures.push(format!( + "round {round}: engine's own output did not reparse: `{current}` ({e}) [from {original}]" + )); + break; + } + }; + match ddx.differentiate(&parsed, &wrt) { + Ok(d) => { + let rendered = d.to_string(); + if rendered.contains("--") { + failures.push(format!( + "round {round}: emitted `--` comment: `{rendered}` [from {original}]" + )); + break; + } + current = rendered; + } + // Re-differentiating can legitimately reach a non-finite constant + // (e.g. an exponent that overflows), which is a *typed* error by + // design — acceptable. Anything else is not. + Err(DiffError::NotImplemented(_)) => break, + Err(e) => { + failures.push(format!( + "round {round}: unexpected error re-differentiating `{current}`: {e} [from {original}]" + )); + break; + } + } + } + } + + assert!( + failures.is_empty(), + "self-consumption fuzz found {} failure(s):\n\n{}", + failures.len(), + failures.iter().take(15).cloned().collect::>().join("\n\n") + ); +} From d4c479edb81f5232214f9c5903a2a56a7c934dbf Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:22:57 -0700 Subject: [PATCH 10/22] Fix for the simulation test bug. hardening of the simulation test oracle. --- crates/ddx-core/src/rewrite.rs | 55 ++- crates/ddx-core/tests/rewrite.rs | 4 +- crates/ddx-core/tests/simulation.rs | 604 +++++++++++++++++++--------- 3 files changed, 471 insertions(+), 192 deletions(-) diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index d09b692..343ba48 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -90,8 +90,13 @@ fn pre_gate_hit(sql: &str) -> bool { continue; } - // The next non-whitespace character must be `(`. - if sql[idx + kw.len()..].chars().find(|c| !c.is_whitespace()) == Some('(') { + // The next significant character must be `(`. sqlparser treats a SQL + // comment as lexical whitespace, so `grad /* c */ (x, x)` and + // `grad-- c\n(x, x)` are genuine marker calls — the scan skips + // comments as well as whitespace, or the gate would miss them and + // let a real marker reach execution un-rewritten (#52). + let after = &sql[idx + kw.len()..]; + if after[skip_trivia(after)..].starts_with('(') { return true; } } @@ -99,6 +104,52 @@ fn pre_gate_hit(sql: &str) -> bool { false } +/// Byte offset of the first significant character in `s`, skipping leading +/// whitespace and SQL comments (`-- … end-of-line`, and `/* … */` block +/// comments, which nest in Postgres/DuckDB) — the trivia sqlparser's tokenizer +/// discards. Returns `s.len()` if the rest is all trivia. +/// +/// Delimiters (`-`, `/`, `*`, whitespace, `\n`) are all ASCII, and a UTF-8 +/// continuation byte is never equal to an ASCII byte, so scanning by bytes is +/// safe even with multibyte text inside a comment. +fn skip_trivia(s: &str) -> usize { + let b = s.as_bytes(); + let n = b.len(); + let mut i = 0; + loop { + while i < n && b[i].is_ascii_whitespace() { + i += 1; + } + // Line comment: `--` to end of line (or input). + if i + 1 < n && b[i] == b'-' && b[i + 1] == b'-' { + i += 2; + while i < n && b[i] != b'\n' { + i += 1; + } + continue; + } + // Block comment: `/* … */`, nesting-aware. + if i + 1 < n && b[i] == b'/' && b[i + 1] == b'*' { + i += 2; + let mut depth = 1usize; + while i < n && depth > 0 { + if i + 1 < n && b[i] == b'/' && b[i + 1] == b'*' { + depth += 1; + i += 2; + } else if i + 1 < n && b[i] == b'*' && b[i + 1] == b'/' { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + continue; + } + break; + } + i +} + /// The public entry point behind [`crate::Ddx::rewrite_sql`]. pub(crate) fn rewrite_sql( sql: &str, diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index b53cb28..39524d6 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -111,12 +111,10 @@ fn splice_preserves_exact_surrounding_bytes() { } // --------------------------------------------------------------------------- -// Pre-gate coverage: comment-separated markers (KNOWN BUG #52) +// Pre-gate coverage: comment-separated markers (#52, fixed) // --------------------------------------------------------------------------- #[test] -#[ignore = "known bug #52: pre-gate skips only whitespace, not SQL comments, so \ - a comment-separated marker is missed and returned verbatim"] fn pre_gate_must_not_miss_a_comment_separated_marker() { // A SQL comment is lexical whitespace, so `grad /* c */ (x, x)` parses as a // genuine `Function(name=grad)` marker call — the *same* AST as plain diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs index 04bb901..6809911 100644 --- a/crates/ddx-core/tests/simulation.rs +++ b/crates/ddx-core/tests/simulation.rs @@ -2,24 +2,51 @@ //! (design.md §5: "numeric agreement" + "round-trip property tests"). //! //! These are adversarial: instead of hand-picked expressions, they generate -//! thousands of random derivable SQL scalar expressions and hold the engine to -//! three properties that any correct symbolic differentiator must satisfy: +//! random derivable SQL scalar expressions and hold the engine to three +//! properties any correct symbolic differentiator must satisfy: //! //! 1. **Numeric agreement (the finite-difference oracle).** The single -//! strongest check on a derivative: for a random `f`, the symbolic `d/dx f` -//! evaluated at a point must equal a central finite difference of `f` there. -//! A wrong rule (a sign flip, a missing chain factor, a bad power exponent) -//! disagrees at *every* well-conditioned point, so it is caught even though a -//! kink artifact (from `abs`) is tolerated as a lone outlier. -//! 2. **Round-trip (design.md §5, G1).** `reparse(render(d)) == d` modulo -//! `Nested`, fuzzed over the whole generated space, not the 10 hand cases. -//! 3. **Self-consumption / higher-order stability.** The engine must be able to -//! re-parse and re-differentiate its *own* text output repeatedly without -//! panicking, erroring, or emitting unparseable SQL (e.g. a `--` comment). +//! strongest check on a derivative: for a random `f`, the symbolic `d/dv f` +//! evaluated at a point must equal a central finite difference of `f` in the +//! `v` direction there. A wrong rule (a sign flip, a missing chain factor, a +//! bad power exponent) disagrees at *every* well-conditioned point, so it is +//! caught even though a kink artifact (from `abs`) is tolerated as a lone +//! outlier. Proven to have teeth by mutation testing (a corrupted `cos` rule +//! fails with 8/8-points-disagree). +//! 2. **Render fidelity.** `reparse(render(d))` must be *value-equal* to `d`. +//! This is the correctness-relevant form of the §5 round-trip invariant: a +//! purely structural "== d modulo Nested" check is imprecise for `*`/`/` +//! associativity (issue #50), but a value comparison still catches the G1 +//! precedence bug (`(a+b)*c` losing its parens → `a+b*c`). +//! 3. **Self-consumption / higher-order stability.** The engine must re-parse +//! and re-differentiate its *own* text output repeatedly without panicking, +//! erroring, or emitting unparseable SQL (e.g. a `--` line comment). //! //! No external fuzzing crate is used: the core is deliberately `sqlparser`-only, -//! and a dev-dependency-free, deterministic generator keeps failures perfectly -//! reproducible (each is printed with its seed). +//! and a dependency-free, deterministic generator keeps every failure perfectly +//! reproducible (each is reported with the seed that produced it). +//! +//! # Soak mode +//! +//! [`soak_continuous_property_fuzz`] is a long-running, `#[ignore]`-d variant +//! that explores far past the bounded tests' fixed seed ranges. It runs for a +//! wall-clock budget and keeps generating fresh expressions, so it can be left +//! running to hunt for rare bugs. Drive it with env vars: +//! +//! ```text +//! DDX_SOAK_SECS=300 cargo test -p ddx-core --test simulation \ +//! DDX_SOAK_BASE=0 -- --ignored --nocapture soak_continuous_property_fuzz +//! DDX_SOAK_LOG=/path/to/soak.log +//! ``` +//! +//! * `DDX_SOAK_SECS` — wall-clock budget in seconds (default 15). +//! * `DDX_SOAK_BASE` — starting seed offset; bump it between runs to cover new +//! ground (default 0). +//! * `DDX_SOAK_LOG` — if set, failures are appended immediately and a heartbeat +//! line is written ~once a second, so a background run can be tailed live. + +use std::fmt::Write as _; +use std::io::Write as _; use ddx_core::sqlparser::ast::{BinaryOperator, Expr, UnaryOperator, Value}; use ddx_core::sqlparser::dialect::GenericDialect; @@ -53,15 +80,34 @@ impl Rng { } } +/// The differentiation variable a check runs against — the generator uses two +/// free variables `x` and `y`, and the finite-difference oracle perturbs the +/// chosen one. +#[derive(Clone, Copy)] +enum Var { + X, + Y, +} + +impl Var { + fn name(self) -> &'static str { + match self { + Var::X => "x", + Var::Y => "y", + } + } +} + // --------------------------------------------------------------------------- // Random expression generator over the *derivable* v1 grammar. // --------------------------------------------------------------------------- // // Everything produced here is inside the engine's supported surface, so // `differentiate` never returns `NotImplemented`: vars {x, y}, numeric -// literals, `+ - * /`, unary minus, the unary-rule function set, and `power` -// with exactly one constant side. It emits SQL text (parenthesized to fix -// structure), which both exercises the parser and gives readable failures. +// literals, `+ - * /`, unary minus, a numeric `CAST`, the unary-rule function +// set, and `power` with exactly one constant side. It emits SQL text +// (parenthesized to fix structure), which both exercises the parser and gives +// readable failures. /// Unary functions that have a differentiation rule (design.md §3.6). const UNARY_FNS: &[&str] = &[ @@ -95,7 +141,7 @@ fn gen_expr(rng: &mut Rng, depth: u32) -> String { _ => gen_const(rng), }; } - match rng.below(9) { + match rng.below(11) { 0 => format!("({} + {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), 1 => format!("({} - {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), 2 => format!("({} * {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), @@ -105,15 +151,13 @@ fn gen_expr(rng: &mut Rng, depth: u32) -> String { let f = UNARY_FNS[rng.below(UNARY_FNS.len() as u64) as usize]; format!("{f}({})", gen_expr(rng, depth - 1)) } - 7 => { - // power(base, const-exponent) - format!("power({}, {})", gen_expr(rng, depth - 1), gen_exponent(rng)) - } - _ => { + 7 => format!("power({}, {})", gen_expr(rng, depth - 1), gen_exponent(rng)), + 8 => { // power(positive-const-base, variable-exponent) let base = ["2", "3", "1.5", "0.5"][rng.below(4) as usize]; format!("power({base}, {})", gen_expr(rng, depth - 1)) } + _ => format!("CAST({} AS DOUBLE)", gen_expr(rng, depth - 1)), } } @@ -121,54 +165,78 @@ fn gen_expr(rng: &mut Rng, depth: u32) -> String { // A float evaluator for the emitted grammar (primal *and* derivative). // --------------------------------------------------------------------------- -fn parse_expr(text: &str) -> Expr { +fn try_parse(text: &str) -> Result { Parser::new(&GenericDialect {}) .try_with_sql(text) .and_then(|mut p| p.parse_expr()) - .unwrap_or_else(|e| panic!("reparse of `{text}` failed: {e}")) + .map_err(|e| e.to_string()) +} + +fn parse_expr(text: &str) -> Expr { + try_parse(text).unwrap_or_else(|e| panic!("reparse of `{text}` failed: {e}")) } /// Evaluate a scalar expression at `(x, y)`. Returns `None` for anything not in /// the numeric grammar (so an unexpected node fails a comparison loudly rather /// than silently returning a bogus number). fn eval(e: &Expr, x: f64, y: f64) -> Option { + eval_mag(e, x, y).map(|(v, _)| v) +} + +/// The largest absolute value taken by any subexpression of `e` at `(x, y)` — +/// the "how big did the intermediates get" probe. A point where this is huge is +/// unfit for the finite-difference oracle: f64 can no longer resolve an O(1) +/// perturbation against it (a huge additive term cancels the perturbation away; +/// a huge argument to `sin`/`cos` aliases), so *neither* the finite difference +/// *nor* the symbolic value is meaningful there — the point must be skipped. +fn max_intermediate_mag(e: &Expr, x: f64, y: f64) -> Option { + eval_mag(e, x, y).map(|(_, m)| m) +} + +/// Evaluate `e`, returning `(value, max_abs_intermediate)`. +fn eval_mag(e: &Expr, x: f64, y: f64) -> Option<(f64, f64)> { + let here = |v: f64| Some((v, v.abs())); match e { Expr::Value(v) => match &v.value { - Value::Number(s, _) => s.parse::().ok(), + Value::Number(s, _) => here(s.parse::().ok()?), _ => None, }, Expr::Identifier(id) => match id.value.to_ascii_lowercase().as_str() { - "x" => Some(x), - "y" => Some(y), + "x" => here(x), + "y" => here(y), _ => None, }, Expr::CompoundIdentifier(parts) => match parts.last()?.value.to_ascii_lowercase().as_str() { - "x" => Some(x), - "y" => Some(y), + "x" => here(x), + "y" => here(y), _ => None, }, - Expr::Nested(inner) => eval(inner, x, y), + Expr::Nested(inner) => eval_mag(inner, x, y), Expr::UnaryOp { op: UnaryOperator::Minus, expr, - } => Some(-eval(expr, x, y)?), + } => { + let (v, m) = eval_mag(expr, x, y)?; + Some((-v, m.max(v.abs()))) + } Expr::UnaryOp { op: UnaryOperator::Plus, expr, - } => eval(expr, x, y), + } => eval_mag(expr, x, y), Expr::BinaryOp { left, op, right } => { - let a = eval(left, x, y)?; - let b = eval(right, x, y)?; - match op { - BinaryOperator::Plus => Some(a + b), - BinaryOperator::Minus => Some(a - b), - BinaryOperator::Multiply => Some(a * b), - BinaryOperator::Divide => Some(a / b), - _ => None, - } + let (a, ma) = eval_mag(left, x, y)?; + let (b, mb) = eval_mag(right, x, y)?; + let r = match op { + BinaryOperator::Plus => a + b, + BinaryOperator::Minus => a - b, + BinaryOperator::Multiply => a * b, + BinaryOperator::Divide => a / b, + _ => return None, + }; + Some((r, ma.max(mb).max(r.abs()))) } // Numeric casts are the identity on f64. - Expr::Cast { expr, .. } => eval(expr, x, y), + Expr::Cast { expr, .. } => eval_mag(expr, x, y), Expr::Function(f) => eval_function(f, x, y), // The `sign` CASE (the only CASE the engine emits). Expr::Case { @@ -177,12 +245,21 @@ fn eval(e: &Expr, x: f64, y: f64) -> Option { else_result, .. } => { + let mut m = 0.0f64; for w in conditions { + // Track the compared operand's magnitude too. + if let Expr::BinaryOp { left, .. } = &w.condition { + if let Some((_, lm)) = eval_mag(left, x, y) { + m = m.max(lm); + } + } if eval_bool(&w.condition, x, y)? { - return eval(&w.result, x, y); + let (v, rm) = eval_mag(&w.result, x, y)?; + return Some((v, m.max(rm))); } } - eval(else_result.as_deref()?, x, y) + let (v, rm) = eval_mag(else_result.as_deref()?, x, y)?; + Some((v, m.max(rm))) } _ => None, } @@ -205,7 +282,7 @@ fn eval_bool(e: &Expr, x: f64, y: f64) -> Option { None } -fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Option { +fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Option<(f64, f64)> { use ddx_core::sqlparser::ast::{ FunctionArg, FunctionArgExpr, FunctionArguments, ObjectNamePart, }; @@ -217,9 +294,14 @@ fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Opti return None; }; let mut args = Vec::new(); + let mut argmag = 0.0f64; for a in &list.args { match a { - FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => args.push(eval(e, x, y)?), + FunctionArg::Unnamed(FunctionArgExpr::Expr(e)) => { + let (v, m) = eval_mag(e, x, y)?; + args.push(v); + argmag = argmag.max(m); + } _ => return None, } } @@ -246,74 +328,230 @@ fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Opti } _ => return None, }; - Some(v) + Some((v, argmag.max(v.abs()))) } // --------------------------------------------------------------------------- -// Property 1 — the finite-difference numeric oracle. +// The three property checks, as reusable helpers. // --------------------------------------------------------------------------- -/// For a fixed expression, sample points and compare the symbolic derivative to -/// a central finite difference. Returns `Some(report)` on a genuine mismatch -/// (a strong majority of well-conditioned points disagree), `None` otherwise. +/// A central finite difference of `f` in the `wrt` direction at `(x0, y0)`, +/// step `h`. +fn central_diff(f: &Expr, x0: f64, y0: f64, wrt: Var, h: f64) -> Option { + let (fp, fm) = match wrt { + Var::X => (eval(f, x0 + h, y0)?, eval(f, x0 - h, y0)?), + Var::Y => (eval(f, x0, y0 + h)?, eval(f, x0, y0 - h)?), + }; + Some((fp - fm) / (2.0 * h)) +} + +/// Property 1: symbolic `d` vs a central finite difference of `f` (`expr_text`) +/// in the `wrt` direction. Returns `Some(report)` when a strong majority of +/// well-conditioned points disagree (a real rule bug), tolerating a lone +/// `abs`-kink outlier. /// -/// A lone disagreement is tolerated: `abs(g(x))` has a kink where `g(x) = 0`, -/// and a finite difference straddling it disagrees with the (correct) pinned -/// convention at that one point. A wrong *rule* disagrees everywhere. -fn fd_check(rng: &mut Rng, expr_text: &str, d: &Expr) -> Option { +/// **Richardson self-consistency gate.** A finite difference is only trusted at +/// a point where halving the step barely moves it (`fd(h) ≈ fd(h/2)`). This is +/// what makes the oracle sound at depth 5–6, where the generator reaches +/// pathological shapes a plain central difference mis-handles — proven necessary +/// by an earlier soak that flagged 16 *correct* derivatives (#54). It kills two +/// false-positive families. Catastrophic cancellation: `power(3, y…) + x`, where +/// the `3^96 ≈ 1e45` term swamps the `+x`, so `f(x+h) − f(x−h)` loses it to +/// float rounding (fd wrongly reads `0`) — halving `h` doubles that error, so +/// the two disagree and the point is skipped. Truncation / aliasing: +/// `sin(exp(9+x))` oscillates with period ≈ `h`, so the central difference is +/// out of its asymptotic regime — halving `h` changes it materially, so the +/// point is skipped. Only points where the difference is in its convergent +/// regime are compared to the symbolic derivative, so a surviving disagreement +/// is a real rule bug. +fn fd_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Option { const H: f64 = 1e-4; const RTOL: f64 = 2e-3; const ATOL: f64 = 1e-5; const COND_CAP: f64 = 1e5; // skip near-singular points (huge slope) + // Max relative gap between fd(h) and fd(h/2) for the difference to count as + // "in its convergent regime" and therefore trustworthy as an oracle. + const RICHARDSON_TOL: f64 = 1e-4; + // Above this, some intermediate value is too large for f64 to resolve an + // O(1) perturbation against — the point is unfit for numeric comparison + // (total cancellation passes Richardson because *both* fd(h) and fd(h/2) + // collapse to the same wrong value, so this magnitude gate is what catches + // it — #54). + const MAG_CAP: f64 = 1e8; let f = parse_expr(expr_text); let mut comparable = 0u32; let mut disagree = 0u32; let mut first_bad = String::new(); - // Draw up to 60 candidate points, keep the well-conditioned finite ones. - for _ in 0..60 { + for _ in 0..80 { if comparable >= 8 { break; } let x0 = rng.range(0.2, 1.8); let y0 = rng.range(0.2, 1.8); - let (Some(fp), Some(fm), Some(dv)) = ( - eval(&f, x0 + H, y0), - eval(&f, x0 - H, y0), + // Magnitude gate: skip points where f (or its derivative) exercises an + // intermediate too large for f64 to resolve a perturbation against. + let fmag = max_intermediate_mag(&f, x0, y0); + let dmag = max_intermediate_mag(d, x0, y0); + match (fmag, dmag) { + (Some(fm), Some(dm)) if fm <= MAG_CAP && dm <= MAG_CAP => {} + _ => continue, + } + let (Some(fd_h), Some(fd_h2), Some(dv)) = ( + central_diff(&f, x0, y0, wrt, H), + central_diff(&f, x0, y0, wrt, H / 2.0), eval(d, x0, y0), ) else { continue; }; - let fd = (fp - fm) / (2.0 * H); - if !fd.is_finite() - || !dv.is_finite() - || fd.abs() > COND_CAP - || dv.abs() > COND_CAP - { + if !fd_h.is_finite() || !fd_h2.is_finite() || !dv.is_finite() { + continue; + } + if fd_h.abs() > COND_CAP || fd_h2.abs() > COND_CAP || dv.abs() > COND_CAP { + continue; + } + // Richardson gate: skip points where the finite difference is not yet in + // its convergent regime (cancellation- or truncation-dominated). + if (fd_h - fd_h2).abs() > RICHARDSON_TOL * fd_h2.abs().max(1.0) { continue; } comparable += 1; + // fd(h/2) is the more accurate estimate at a convergent point. + let fd = fd_h2; if (fd - dv).abs() > ATOL + RTOL * dv.abs().max(fd.abs()) { disagree += 1; if first_bad.is_empty() { - first_bad = format!( - "x={x0:.6} y={y0:.6}: symbolic d/dx = {dv:.8}, finite-diff = {fd:.8}" - ); + first_bad = + format!("x={x0:.6} y={y0:.6}: symbolic d/d{} = {dv:.8}, finite-diff = {fd:.8}", wrt.name()); } } } - // Need enough evidence, and a majority must disagree (a real rule bug), - // not a single kink artifact. if comparable >= 4 && disagree >= 2 && disagree * 2 > comparable { return Some(format!( - "d/dx {expr_text}\n => {d}\n {disagree}/{comparable} points disagree; e.g. {first_bad}" + "[finite-diff] d/d{} {expr_text}\n => {d}\n {disagree}/{comparable} points disagree; e.g. {first_bad}", + wrt.name() )); } None } +/// Property 2: `reparse(render(d))` computes the same value as `d` (immune to +/// benign `*`/`/` reassociation; catches a value-changing paren-drop). +fn fidelity_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Option { + const RTOL: f64 = 1e-9; + const ATOL: f64 = 1e-11; + let rendered = d.to_string(); + if rendered.contains("--") { + return Some(format!("[render] emitted a `--` comment: d/d{} {expr_text} => {rendered}", wrt.name())); + } + let reparsed = match try_parse(&rendered) { + Ok(rp) => rp, + Err(e) => { + return Some(format!( + "[render] engine emitted unparseable SQL: d/d{} {expr_text} => {rendered} ({e})", + wrt.name() + )) + } + }; + let mut compared = 0u32; + for _ in 0..40 { + if compared >= 6 { + break; + } + let x0 = rng.range(0.2, 1.8); + let y0 = rng.range(0.2, 1.8); + let (Some(va), Some(vb)) = (eval(d, x0, y0), eval(&reparsed, x0, y0)) else { + continue; + }; + if !va.is_finite() || !vb.is_finite() || va.abs() > 1e6 { + continue; + } + compared += 1; + if (va - vb).abs() > ATOL + RTOL * va.abs() { + return Some(format!( + "[render] render changed the value: d/d{} {expr_text}\n rendered = {rendered}\n at x={x0:.4} y={y0:.4}: AST = {va:.10}, reparsed = {vb:.10}", + wrt.name() + )); + } + } + None +} + +/// Property 3: the engine re-consumes its own text output for up to 4 rounds of +/// higher-order differentiation without panicking, erroring unexpectedly, or +/// emitting unparseable SQL. +fn self_consumption_failure(ddx: &Ddx, wrt: &ColRef, original: &str) -> Option { + let mut current = original.to_string(); + for round in 0..4 { + let parsed = match try_parse(¤t) { + Ok(p) => p, + Err(e) => { + return Some(format!( + "[self-consumption] round {round}: engine's own output did not reparse: `{current}` ({e}) [from {original}]" + )) + } + }; + match ddx.differentiate(&parsed, wrt) { + Ok(d) => { + let rendered = d.to_string(); + if rendered.contains("--") { + return Some(format!( + "[self-consumption] round {round}: emitted `--` comment: `{rendered}` [from {original}]" + )); + } + current = rendered; + } + // Re-differentiating can legitimately reach a non-finite constant + // (e.g. an overflowing exponent) — a *typed* error by design. + Err(DiffError::NotImplemented(_)) => break, + Err(e) => { + return Some(format!( + "[self-consumption] round {round}: unexpected error re-differentiating `{current}`: {e} [from {original}]" + )) + } + } + } + None +} + +/// Parse, differentiate, and run every property on one generated expression. +/// Returns each failure report (empty ⇒ all properties held). +fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec { + let mut out = Vec::new(); + let parsed = match try_parse(text) { + Ok(p) => p, + Err(e) => { + out.push(format!("[generator] produced unparseable text `{text}` ({e})")); + return out; + } + }; + let wrt_col = ColRef::bare(wrt.name()); + let d = match ddx.differentiate(&parsed, &wrt_col) { + Ok(d) => d, + Err(DiffError::NotImplemented(_)) => return out, // outside surface; skip + Err(e) => { + out.push(format!("[differentiate] unexpected error on `{text}`: {e}")); + return out; + } + }; + if let Some(f) = fd_failure(rng, text, &d, wrt) { + out.push(f); + } + if let Some(f) = fidelity_failure(rng, text, &d, wrt) { + out.push(f); + } + if let Some(f) = self_consumption_failure(ddx, &wrt_col, text) { + out.push(f); + } + out +} + +// --------------------------------------------------------------------------- +// Bounded tests (run every `cargo test`). +// --------------------------------------------------------------------------- + #[test] fn finite_difference_agreement_over_random_expressions() { let ddx = Ddx::new(); @@ -323,14 +561,11 @@ fn finite_difference_agreement_over_random_expressions() { for seed in 0..4000u64 { let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D)); - let depth = 2 + (seed % 3) as u32; // depth 2..=4 + let depth = 2 + (seed % 3) as u32; let text = gen_expr(&mut rng, depth); - let parsed = parse_expr(&text); let d = match ddx.differentiate(&parsed, &wrt) { Ok(d) => d, - // Everything generated is derivable; a genuine error here is itself - // a finding worth seeing. Err(DiffError::NotImplemented(_)) => continue, Err(e) => { failures.push(format!("UNEXPECTED ERROR on `{text}`: {e}")); @@ -338,7 +573,7 @@ fn finite_difference_agreement_over_random_expressions() { } }; tested += 1; - if let Some(report) = fd_check(&mut rng, &text, &d) { + if let Some(report) = fd_failure(&mut rng, &text, &d, Var::X) { failures.push(report); } } @@ -349,34 +584,12 @@ fn finite_difference_agreement_over_random_expressions() { "finite-difference oracle found {} disagreement(s) out of {} tested:\n\n{}", failures.len(), tested, - failures - .iter() - .take(15) - .cloned() - .collect::>() - .join("\n\n") + failures.iter().take(15).cloned().collect::>().join("\n\n") ); } -// --------------------------------------------------------------------------- -// Property 2 — render fidelity: reparse(render(d)) is *value-equal* to d. -// --------------------------------------------------------------------------- -// -// This is the correctness-relevant form of the design.md §5 round-trip -// invariant. A purely structural "== d modulo Nested" check is imprecise for -// `*`/`/` associativity (issue #50): the product rule builds `a * (b / c)`, -// which renders unparenthesized as `a * b / c` and reparses as `(a * b) / c` — -// a *different tree* but the *same value*. What actually matters — and what the -// G1 precedence bug (`(a+b)*c` losing its parens → `a+b*c`) violates — is that -// the rendered-then-reparsed derivative computes the same number as the AST the -// engine constructed. So this compares by evaluation, not by tree shape: it is -// immune to benign reassociation yet still catches any paren-drop that changes -// the value. - #[test] fn render_reparse_is_value_preserving() { - const RTOL: f64 = 1e-9; // float `*`/`/` reassociation differs by ~ulps only - const ATOL: f64 = 1e-11; let ddx = Ddx::new(); let wrt = ColRef::bare("x"); let mut failures: Vec = Vec::new(); @@ -390,47 +603,8 @@ fn render_reparse_is_value_preserving() { Ok(d) => d, Err(_) => continue, }; - let rendered = d.to_string(); - // A `--` in emitted SQL is a line comment — a silent-wrong render. - if rendered.contains("--") { - failures.push(format!("emitted a `--` comment: d/dx {text} => {rendered}")); - continue; - } - let reparsed = match Parser::new(&GenericDialect {}) - .try_with_sql(&rendered) - .and_then(|mut p| p.parse_expr()) - { - Ok(rp) => rp, - Err(e) => { - failures.push(format!( - "engine emitted unparseable SQL: d/dx {text} => {rendered} ({e})" - )); - continue; - } - }; - - // Compare the constructed AST and its rendered-then-reparsed form by - // value at several well-conditioned points. - let mut compared = 0u32; - for _ in 0..40 { - if compared >= 6 { - break; - } - let x0 = rng.range(0.2, 1.8); - let y0 = rng.range(0.2, 1.8); - let (Some(va), Some(vb)) = (eval(&d, x0, y0), eval(&reparsed, x0, y0)) else { - continue; - }; - if !va.is_finite() || !vb.is_finite() || va.abs() > 1e6 { - continue; - } - compared += 1; - if (va - vb).abs() > ATOL + RTOL * va.abs() { - failures.push(format!( - "render changed the value: d/dx {text}\n rendered = {rendered}\n at x={x0:.4} y={y0:.4}: AST = {va:.10}, reparsed = {vb:.10}" - )); - break; - } + if let Some(report) = fidelity_failure(&mut rng, &text, &d, Var::X) { + failures.push(report); } } @@ -442,16 +616,8 @@ fn render_reparse_is_value_preserving() { ); } -// --------------------------------------------------------------------------- -// Property 3 — self-consumption: the engine must re-consume its own output. -// --------------------------------------------------------------------------- - #[test] fn higher_order_self_consumption_is_stable() { - // Differentiate, render, reparse, differentiate again — up to 4 rounds. - // Higher-order derivatives are a stated feature ("grad(grad(f,x),x) just - // works"); the engine must never panic, emit unparseable SQL, or error on - // an expression it produced itself. let ddx = Ddx::new(); let wrt = ColRef::bare("x"); let mut failures: Vec = Vec::new(); @@ -460,43 +626,8 @@ fn higher_order_self_consumption_is_stable() { let mut rng = Rng::new(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ 0x1234_5678); let depth = 2 + (seed % 3) as u32; let original = gen_expr(&mut rng, depth); - - let mut current = original.clone(); - for round in 0..4 { - let parsed = Parser::new(&GenericDialect {}) - .try_with_sql(¤t) - .and_then(|mut p| p.parse_expr()); - let parsed = match parsed { - Ok(p) => p, - Err(e) => { - failures.push(format!( - "round {round}: engine's own output did not reparse: `{current}` ({e}) [from {original}]" - )); - break; - } - }; - match ddx.differentiate(&parsed, &wrt) { - Ok(d) => { - let rendered = d.to_string(); - if rendered.contains("--") { - failures.push(format!( - "round {round}: emitted `--` comment: `{rendered}` [from {original}]" - )); - break; - } - current = rendered; - } - // Re-differentiating can legitimately reach a non-finite constant - // (e.g. an exponent that overflows), which is a *typed* error by - // design — acceptable. Anything else is not. - Err(DiffError::NotImplemented(_)) => break, - Err(e) => { - failures.push(format!( - "round {round}: unexpected error re-differentiating `{current}`: {e} [from {original}]" - )); - break; - } - } + if let Some(report) = self_consumption_failure(&ddx, &wrt, &original) { + failures.push(report); } } @@ -507,3 +638,102 @@ fn higher_order_self_consumption_is_stable() { failures.iter().take(15).cloned().collect::>().join("\n\n") ); } + +// --------------------------------------------------------------------------- +// Soak test — long-running, #[ignore]-d, driven by env vars (see module docs). +// --------------------------------------------------------------------------- + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) +} + +#[test] +#[ignore = "soak: long-running continuous fuzz; run explicitly with DDX_SOAK_SECS set"] +fn soak_continuous_property_fuzz() { + use std::time::Instant; + + let budget_secs = env_u64("DDX_SOAK_SECS", 15); + let base = env_u64("DDX_SOAK_BASE", 0); + let log_path = std::env::var("DDX_SOAK_LOG").ok(); + + let mut log = log_path.as_ref().map(|p| { + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(p) + .unwrap_or_else(|e| panic!("cannot open DDX_SOAK_LOG `{p}`: {e}")) + }); + let mut logline = |s: &str| { + eprintln!("{s}"); + if let Some(f) = log.as_mut() { + let _ = writeln!(f, "{s}"); + let _ = f.flush(); + } + }; + + let ddx = Ddx::new(); + let start = Instant::now(); + let deadline = budget_secs; + let mut iters: u64 = 0; + let mut failures: u64 = 0; + let mut last_beat = 0u64; + + logline(&format!( + "SOAK start: budget={budget_secs}s base={base} log={:?}", + log_path + )); + + loop { + let elapsed = start.elapsed().as_secs(); + if elapsed >= deadline { + break; + } + + // A fresh, reproducible seed for this iteration. + let seed = base.wrapping_add(iters); + let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D) ^ 0xA5A5_5A5A); + // Deeper trees than the bounded tests, to reach rarer shapes. + let depth = 2 + (rng.below(5) as u32); // 2..=6 + let wrt = if rng.below(2) == 0 { Var::X } else { Var::Y }; + let text = gen_expr(&mut rng, depth); + + let reports = run_all_checks(&mut rng, &ddx, &text, wrt); + if reports.is_empty() { + // A skip (outside-surface) vs a real pass are indistinguishable + // here; count both as progress. + } else { + for r in &reports { + failures += 1; + logline(&format!( + "\nFAILURE (seed={seed}, base={base}, depth={depth}, wrt={}):\n{r}", + wrt.name() + )); + } + } + + iters += 1; + + // Heartbeat ~once a second. + if elapsed != last_beat { + last_beat = elapsed; + logline(&format!( + "HEARTBEAT elapsed={elapsed}s iters={iters} failures={failures} rate={}/s", + iters / elapsed.max(1) + )); + } + } + + let mut summary = String::new(); + let _ = write!( + summary, + "SOAK done: elapsed={}s iters={iters} failures={failures} base={base} next_base={}", + start.elapsed().as_secs(), + base.wrapping_add(iters) + ); + logline(&summary); + + assert_eq!( + failures, 0, + "soak found {failures} property failure(s) — see the FAILURE lines above (each has a reproducing seed)" + ); +} From 7eb3a0c62063249ffaad72404131d6e8b5823806 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:23:07 -0700 Subject: [PATCH 11/22] Copy edits. --- README.md | 15 +++++++-------- crates/ddx-core/README.md | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 302e90a..bd8efcf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # ddx -[JAX](https://docs.jax.dev/en/latest/)-style [automatic differentiation](https://docs.jax.dev/en/latest/automatic-differentiation.html) in SQL +_[JAX](https://docs.jax.dev/en/latest/)-style [automatic differentiation](https://docs.jax.dev/en/latest/automatic-differentiation.html) in SQL_ Write calculus directly in SQL and get derivatives back as ordinary columns, evaluated row by row by the engine alongside everything else: @@ -19,14 +19,13 @@ is in progress; see [docs/design.md](docs/design.md) §8 for the milestones. ``` crates/ - ddx-core/ # v1 engine — differentiate sqlparser::ast::Expr + rewrite_sql [M0 ✓] + ddx-core/ # v1 engine — differentiate sqlparser::ast::Expr + rewrite_sql [M0 ✓] ddx-ad/ # v2 engine — query-level reverse-mode AD over Substrait [M3/M4] - ddx-datafusion/ # DataFusion adapter: ddx_sql (Path A) + AnalyzerRule (Path B) [M2] - ddx-duckdb/ # DuckDB community extension: ddx('') [M5] -python/ddxdb/ # PyO3/maturin wheel: rewrite_sql + Context.sql() shim [M2] -tests/ # cross-engine numeric-agreement suites (vs JAX) [M2/M6] -spikes/ # runnable evidence for every load-bearing design claim -future/ # deferred: ddx-duckdb-cpp (bare grad() in DuckDB), ddx-pg + ddx-datafusion/ # DataFusion adapter: ddx_sql + AnalyzerRule [M2] + ddx-duckdb/ # DuckDB community extension [M5] +python/ddxdb/ # PyO3/maturin wheel: rewrite_sql + Context.sql() shim [M2] +tests/ # cross-engine numeric-agreement suites (vs JAX) [M2/M6] +docs/spikes/ # runnable evidence for every design claim docs/design.md # the design ``` diff --git a/crates/ddx-core/README.md b/crates/ddx-core/README.md index 3e654ae..75c9f75 100644 --- a/crates/ddx-core/README.md +++ b/crates/ddx-core/README.md @@ -22,8 +22,8 @@ assert_eq!(out, "SELECT (cos(x)) AS d FROM t"); ``` The engine differentiates [`sqlparser::ast::Expr`](https://docs.rs/sqlparser) -directly — the AST *is* the IR, there is no bespoke representation. The single -load-bearing dependency is `sqlparser`, **re-exported** as `ddx_core::sqlparser` +directly — the AST *is* the IR, there is no bespoke representation. The primary +dependency is `sqlparser`, **re-exported** as `ddx_core::sqlparser` so downstream adapters cannot link a mismatched version. ## What it supports From f55c6421dd987719097e1a59b0680b3c1ea14ad8 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:30:13 -0700 Subject: [PATCH 12/22] Fuzz on a cron. --- .github/scripts/report_fuzz_findings.sh | 165 ++++++++++++++++++++++++ .github/workflows/nightly-fuzz.yml | 111 ++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100755 .github/scripts/report_fuzz_findings.sh create mode 100644 .github/workflows/nightly-fuzz.yml diff --git a/.github/scripts/report_fuzz_findings.sh b/.github/scripts/report_fuzz_findings.sh new file mode 100755 index 0000000..082c5c7 --- /dev/null +++ b/.github/scripts/report_fuzz_findings.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +# report_fuzz_findings.sh — triage the nightly simulation-soak logs and, if any +# property failures were found, file (or update) a single GitHub issue with a +# reproducible repro. Invoked by .github/workflows/nightly-fuzz.yml. +# +# Anti-spam policy: at most ONE open `fuzz-finding` issue at a time. If one is +# already open, this appends the new run's findings as a comment instead of +# opening a duplicate — a persistent bug accumulates recurrences on one issue, +# and a fresh issue only opens once a human has triaged and closed the last one. +# +# Usage: report_fuzz_findings.sh +# Reads every `soak-*.log` under . Requires `gh` authenticated with +# `issues: write` (the workflow's GITHUB_TOKEN). +# +# Env (provided by the workflow, all optional — used only to enrich the body): +# GITHUB_RUN_ID, GITHUB_SHA, GITHUB_SERVER_URL, GITHUB_REPOSITORY + +set -euo pipefail + +LOG_DIR=${1:?"usage: report_fuzz_findings.sh "} + +shopt -s nullglob globstar +logs=("$LOG_DIR"/**/soak-*.log "$LOG_DIR"/soak-*.log) +if [ ${#logs[@]} -eq 0 ]; then + echo "No soak-*.log files found under $LOG_DIR; nothing to report." + exit 0 +fi + +# --------------------------------------------------------------------------- +# Tally failures and collect a bounded, readable set of failure blocks. +# --------------------------------------------------------------------------- +total=0 +blocks="" # up to MAX_BLOCKS failure blocks, across all regions +env_line="" # rustc / os / sha, recorded by the soak step +region_summary="" +MAX_BLOCKS=8 +block_count=0 + +for log in "${logs[@]}"; do + [ -f "$log" ] || continue + n=$(grep -c '^FAILURE (seed' "$log" 2>/dev/null || true) + n=${n:-0} + total=$((total + n)) + + # The soak step records one `ENV ...` line at the top of each log. + if [ -z "$env_line" ]; then + env_line=$(grep -m1 '^ENV ' "$log" 2>/dev/null || true) + fi + + base=$(grep -m1 '^SOAK start' "$log" 2>/dev/null | sed -n 's/.*base=\([0-9]*\).*/\1/p') + iters=$(grep -m1 '^SOAK done' "$log" 2>/dev/null | sed -n 's/.*iters=\([0-9]*\).*/\1/p') + region_summary+="- \`$(basename "$log")\`: base=${base:-?}, iters=${iters:-?}, failures=${n}"$'\n' + + if [ "$n" -gt 0 ] && [ "$block_count" -lt "$MAX_BLOCKS" ]; then + # Extract each FAILURE block: the header plus its report lines, stopping at + # the next heartbeat / failure / summary line. + while IFS= read -r line; do + case "$line" in + "===BLOCK===") + block_count=$((block_count + 1)) + [ "$block_count" -ge "$MAX_BLOCKS" ] && break + ;; + *) blocks+="$line"$'\n' ;; + esac + done < <(awk ' + /^FAILURE \(seed/ { if (inblk) print "===BLOCK==="; inblk=1; print; next } + inblk == 1 { + if ($0 ~ /^HEARTBEAT/ || $0 ~ /^SOAK/ || $0 == "") { inblk=0; print "===BLOCK==="; next } + print + } + END { if (inblk) print "===BLOCK===" } + ' "$log") + fi +done + +if [ "$total" -eq 0 ]; then + echo "Soak completed with 0 property failures. Nothing to file." + exit 0 +fi + +echo "Found $total property failure(s) across ${#logs[@]} region log(s); preparing issue." + +# --------------------------------------------------------------------------- +# A ready-to-paste repro from the first failing seed. The soak's per-iteration +# seed is `base + iter`, so re-running with DDX_SOAK_BASE set to a failing +# `seed=` value reproduces that exact case as iteration 0 (same depth + wrt). +# --------------------------------------------------------------------------- +first_seed=$(printf '%s' "$blocks" | sed -n 's/^FAILURE (seed=\([0-9]*\).*/\1/p' | head -1) +first_seed=${first_seed:-0} + +run_url="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-xqlsystems/ddx}/actions/runs/${GITHUB_RUN_ID:-local}" +date_utc=$(date -u '+%Y-%m-%d %H:%M UTC') + +body_file=$(mktemp) +{ + echo "**Nightly simulation-soak fuzz found ${total} property failure(s).**" + echo + echo "- Run: [${GITHUB_RUN_ID:-local}](${run_url})" + echo "- Commit: \`${GITHUB_SHA:-unknown}\`" + echo "- When: ${date_utc}" + [ -n "$env_line" ] && echo "- Build: \`${env_line#ENV }\`" + echo + echo "### Reproduce" + echo + echo "The soak's per-iteration seed is \`base + iter\`, so setting \`DDX_SOAK_BASE\`" + echo "to a failing \`seed=\` value re-runs that exact case as iteration 0. On the" + echo "same platform as the build above (floating-point transcendentals are not" + echo "bit-identical across OS/toolchain):" + echo + echo '```bash' + echo "DDX_SOAK_SECS=15 DDX_SOAK_BASE=${first_seed} \\" + echo " cargo test -p ddx-core --test simulation --release \\" + echo " -- --ignored --nocapture soak_continuous_property_fuzz" + echo '```' + echo + echo "The full per-region logs are attached to the workflow run as artifacts." + echo + echo "### Per-region summary" + echo + printf '%s\n' "$region_summary" + echo "### Failure samples (up to ${MAX_BLOCKS})" + echo + echo '```' + printf '%s' "$blocks" + echo '```' + echo + echo "> Triage note: the finite-difference oracle is gated against float" + echo "> cancellation (magnitude cap) and truncation/aliasing (Richardson" + echo "> self-consistency), so a surviving \`[finite-diff]\` disagreement should be" + echo "> a real rule bug — but confirm by reproducing before assuming. \`[render]\`" + echo "> and \`[self-consumption]\` failures are always real." + echo + echo "" +} > "$body_file" + +# --------------------------------------------------------------------------- +# Ensure labels exist (idempotent), then dedup: comment on the single open +# fuzz-finding issue if one exists, else create a new one. +# --------------------------------------------------------------------------- +gh label create fuzz-finding --color B60205 \ + --description "Found by the nightly simulation-soak fuzz" 2>/dev/null || true +gh label create needs-triage --color FBCA04 \ + --description "Awaiting human triage" 2>/dev/null || true + +existing=$(gh issue list --state open --label fuzz-finding \ + --json number --jq '.[0].number // empty' 2>/dev/null || true) + +if [ -n "$existing" ]; then + echo "Open fuzz-finding issue #${existing} exists; commenting instead of filing a duplicate." + { + echo "### Recurred — run ${GITHUB_RUN_ID:-local} (${date_utc})" + echo + cat "$body_file" + } | gh issue comment "$existing" --body-file - + echo "Commented on #${existing}." +else + title="Nightly fuzz: ${total} property failure(s) in ddx-core simulation soak" + url=$(gh issue create --title "$title" \ + --label fuzz-finding --label needs-triage \ + --body-file "$body_file") + echo "Filed new issue: $url" +fi + +rm -f "$body_file" diff --git a/.github/workflows/nightly-fuzz.yml b/.github/workflows/nightly-fuzz.yml new file mode 100644 index 0000000..afdb5f2 --- /dev/null +++ b/.github/workflows/nightly-fuzz.yml @@ -0,0 +1,111 @@ +name: Nightly fuzz (ddx-core simulation soak) + +# Runs the seed-reproducible property-fuzz soak (crates/ddx-core/tests/ +# simulation.rs :: soak_continuous_property_fuzz) over three disjoint random +# seed regions in parallel, then files/updates a single GitHub issue if any +# property (finite-difference numeric agreement, render fidelity, higher-order +# self-consumption) fails. +# +# Schedule: 08:00 America/Los_Angeles == 16:00 UTC (PST, UTC-8). GitHub cron is +# fixed-UTC and does NOT observe daylight saving, so during PDT (roughly +# Mar–Nov) this fires at 09:00 Pacific. Adjust the cron if you want it pinned to +# wall-clock Pacific year-round. +on: + schedule: + - cron: '0 16 * * *' + workflow_dispatch: + inputs: + duration_secs: + description: 'Soak wall-clock budget per region (seconds)' + required: false + default: '300' + base_override: + description: 'Base seed (optional; blank = derive a fresh one from the run id)' + required: false + default: '' + +permissions: + contents: read + issues: write + +concurrency: + group: nightly-fuzz + cancel-in-progress: false + +jobs: + soak: + name: soak region ${{ matrix.region }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + region: [0, 1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Toolchain info + run: rustc --version && cargo --version + + - name: Build soak test binary + run: cargo test -p ddx-core --test simulation --release --no-run + + - name: Run soak + run: | + set -o pipefail + SECS='${{ github.event.inputs.duration_secs || '300' }}' + OVERRIDE='${{ github.event.inputs.base_override }}' + REGION='${{ matrix.region }}' + # Disjoint region seed windows, monotonic across runs. run_number is a + # small, strictly-increasing counter — using it (not the ~10-digit + # run_id) keeps run_number * 1e8 far inside 64-bit range forever, and + # guarantees each region gets a fresh, non-overlapping seed window + # (a 5-min region explores < 1e7 seeds; regions are 1e7 apart, runs + # 1e8 apart). + if [ -n "$OVERRIDE" ]; then + BASE=$(( OVERRIDE + REGION * 10000000 )) + else + BASE=$(( GITHUB_RUN_NUMBER * 100000000 + REGION * 10000000 )) + fi + LOG="$PWD/soak-region-${REGION}.log" + echo "ENV rustc=$(rustc --version) os=${RUNNER_OS} sha=${GITHUB_SHA}" > "$LOG" + echo "Running soak: region=$REGION base=$BASE secs=$SECS" + # The soak asserts on failure (non-zero exit); don't let that abort the + # job — the report job decides what to do from the log contents. + set +e + DDX_SOAK_SECS="$SECS" DDX_SOAK_BASE="$BASE" DDX_SOAK_LOG="$LOG" \ + cargo test -p ddx-core --test simulation --release \ + -- --ignored --nocapture soak_continuous_property_fuzz + echo "soak exit code: $?" + set -e + echo "---- log tail ----" + tail -n 5 "$LOG" || true + + - name: Upload region log + if: always() + uses: actions/upload-artifact@v4 + with: + name: soak-log-${{ matrix.region }} + path: soak-region-${{ matrix.region }}.log + if-no-files-found: warn + + report: + name: triage & file issue + needs: soak + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Download all region logs + uses: actions/download-artifact@v4 + with: + path: logs + + - name: Triage and file/update issue + env: + GH_TOKEN: ${{ github.token }} + run: | + chmod +x .github/scripts/report_fuzz_findings.sh + .github/scripts/report_fuzz_findings.sh logs From 67a3cf312440e6db47d902ef87ea7a91fc994755 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:40:58 -0700 Subject: [PATCH 13/22] cargo description copy edits. --- crates/ddx-core/Cargo.toml | 2 +- crates/ddx-datafusion/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ddx-core/Cargo.toml b/crates/ddx-core/Cargo.toml index 9c9889f..c954940 100644 --- a/crates/ddx-core/Cargo.toml +++ b/crates/ddx-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ddx-core" -description = "Engine-neutral symbolic differentiation of SQL scalar expressions (the v1 core of ddx: grad/jvp as a query rewrite)." +description = "Engine-neutral symbolic differentiation of SQL scalar expressions: `grad` & `jvp`." version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/ddx-datafusion/Cargo.toml b/crates/ddx-datafusion/Cargo.toml index 3754b22..959896c 100644 --- a/crates/ddx-datafusion/Cargo.toml +++ b/crates/ddx-datafusion/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ddx-datafusion" -description = "DataFusion adapter for ddx: the one-line ddx_sql helper (Path A) plus marker UDFs + AnalyzerRule bridge (Path B). Scaffold — see docs/design.md §3.3 and milestone M2." +description = "DataFusion adapter for ddx." version.workspace = true edition.workspace = true license.workspace = true From 6d8fbb4b42f594d82033a95b548c8c8c1cea80a0 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:45:23 -0700 Subject: [PATCH 14/22] Added copyright and license IDs to all rust code. --- crates/ddx-ad/src/lib.rs | 4 ++++ crates/ddx-core/src/colref.rs | 4 ++++ crates/ddx-core/src/constructors.rs | 4 ++++ crates/ddx-core/src/ddx.rs | 4 ++++ crates/ddx-core/src/engine.rs | 4 ++++ crates/ddx-core/src/error.rs | 4 ++++ crates/ddx-core/src/lib.rs | 4 ++++ crates/ddx-core/src/rewrite.rs | 4 ++++ crates/ddx-core/tests/rewrite.rs | 4 ++++ crates/ddx-core/tests/roundtrip.rs | 4 ++++ crates/ddx-core/tests/rules.rs | 4 ++++ crates/ddx-datafusion/src/lib.rs | 4 ++++ docs/spikes/sqlparser-spike/src/main.rs | 4 ++++ 13 files changed, 52 insertions(+) diff --git a/crates/ddx-ad/src/lib.rs b/crates/ddx-ad/src/lib.rs index 5a2e6a9..33950d1 100644 --- a/crates/ddx-ad/src/lib.rs +++ b/crates/ddx-ad/src/lib.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! `ddx-ad` — query-level reverse-mode automatic differentiation (ddx v2). //! //! **Status: scaffold.** This crate is the home for the v2 engine described in diff --git a/crates/ddx-core/src/colref.rs b/crates/ddx-core/src/colref.rs index e92d042..0c81bf1 100644 --- a/crates/ddx-core/src/colref.rs +++ b/crates/ddx-core/src/colref.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! Column identity read off the AST, compared with per-dialect identifier //! folding rather than raw-string equality (design.md §3.2, F1). diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index ed27169..ced8867 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! Smart constructors for building derivative `sqlparser::ast::Expr` trees. //! //! These own three correctness properties, not just algebraic tidiness diff --git a/crates/ddx-core/src/ddx.rs b/crates/ddx-core/src/ddx.rs index 3d9d95c..cbc010b 100644 --- a/crates/ddx-core/src/ddx.rs +++ b/crates/ddx-core/src/ddx.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! The public entry point: the [`Ddx`] object. //! //! The surface is an object rather than free functions so the user rule diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index 78b6679..184008f 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! The differentiation engine: forward-mode linearization over //! `sqlparser::ast::Expr`, with a name-keyed, user-extensible rule registry. //! diff --git a/crates/ddx-core/src/error.rs b/crates/ddx-core/src/error.rs index 110d9ee..7b4dc60 100644 --- a/crates/ddx-core/src/error.rs +++ b/crates/ddx-core/src/error.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! The error type for differentiation and rewriting. use std::fmt; diff --git a/crates/ddx-core/src/lib.rs b/crates/ddx-core/src/lib.rs index 76564e9..4220ac1 100644 --- a/crates/ddx-core/src/lib.rs +++ b/crates/ddx-core/src/lib.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! `ddx-core` — engine-neutral symbolic differentiation of SQL scalar //! expressions. The v1 core of [`ddx`](https://github.com/xqlsystems/ddx): //! write calculus directly in SQL and let the engine evaluate the derivative diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index 343ba48..151b767 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! Source-to-source SQL rewriting: find every `grad`/`jvp` marker, replace it //! with derivative SQL, leave everything else byte-identical. //! diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 39524d6..0baf06d 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! End-to-end `rewrite_sql` tests: the marker path, span splicing, the guards, //! and identifier folding — the M0 deliverables of design.md §3.2 / §8. diff --git a/crates/ddx-core/tests/roundtrip.rs b/crates/ddx-core/tests/roundtrip.rs index b2d917d..5e3d83c 100644 --- a/crates/ddx-core/tests/roundtrip.rs +++ b/crates/ddx-core/tests/roundtrip.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! The semantic round-trip property test (design.md §5). //! //! A test that only checks the rewritten SQL *parses* sails right past the diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index a1eb3e6..14f84ea 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! The prototype's rule unit tests, ported to `ddx-core` (design.md §3.2, M0: //! "port the prototype's 15 rule tests — they pin the math unchanged"). //! diff --git a/crates/ddx-datafusion/src/lib.rs b/crates/ddx-datafusion/src/lib.rs index 146a430..e54e745 100644 --- a/crates/ddx-datafusion/src/lib.rs +++ b/crates/ddx-datafusion/src/lib.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! `ddx-datafusion` — the DataFusion adapter for ddx. //! //! **Status: scaffold.** M0 lands only the crate seam. The real adapter diff --git a/docs/spikes/sqlparser-spike/src/main.rs b/docs/spikes/sqlparser-spike/src/main.rs index 87c72b7..72602a3 100644 --- a/docs/spikes/sqlparser-spike/src/main.rs +++ b/docs/spikes/sqlparser-spike/src/main.rs @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + use sqlparser::ast::{Expr, BinaryOperator, Ident, Spanned}; use sqlparser::tokenizer::Span; From 19d0689116cd03d0469ef323f7d02e6a8847eec4 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:46:47 -0700 Subject: [PATCH 15/22] Added license gen script. --- license.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 license.sh diff --git a/license.sh b/license.sh new file mode 100644 index 0000000..a0392c4 --- /dev/null +++ b/license.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +uvx "reuse[charset-normalizer]" annotate --copyright "Alex Merose & ddx Authors" --license "Apache-2.0" **/*.rs **/*.py \ No newline at end of file From af1ee9a0cf1a8cfdeb55396ed1f484211a424eee Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:48:46 -0700 Subject: [PATCH 16/22] Code attr --- README.md | 19 +++++++++++++++++++ crates/ddx-ad/src/lib.rs | 2 +- crates/ddx-core/src/colref.rs | 2 +- crates/ddx-core/src/constructors.rs | 2 +- crates/ddx-core/src/ddx.rs | 2 +- crates/ddx-core/src/engine.rs | 2 +- crates/ddx-core/src/error.rs | 2 +- crates/ddx-core/src/lib.rs | 2 +- crates/ddx-core/src/rewrite.rs | 2 +- crates/ddx-core/tests/rewrite.rs | 2 +- crates/ddx-core/tests/roundtrip.rs | 2 +- crates/ddx-core/tests/rules.rs | 2 +- crates/ddx-datafusion/src/lib.rs | 2 +- docs/spikes/attention_ad_spike.py | 4 ++++ docs/spikes/attention_causal_mask_check.py | 4 ++++ docs/spikes/duckdb_reentrancy_r1b.py | 4 ++++ docs/spikes/duckdb_substrait_window_bug.py | 4 ++++ docs/spikes/relational_ad_spike.py | 4 ++++ docs/spikes/route_ad_spike.py | 4 ++++ docs/spikes/sqlparser-spike/src/main.rs | 2 +- docs/spikes/substrait_ad_marker_spike.py | 4 ++++ docs/spikes/substrait_limitation_repro.py | 4 ++++ license.sh | 2 +- 23 files changed, 65 insertions(+), 14 deletions(-) mode change 100644 => 100755 license.sh diff --git a/README.md b/README.md index bd8efcf..aff543f 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,22 @@ cargo test -p ddx-core See [`crates/ddx-core/README.md`](crates/ddx-core/README.md) for what the engine supports and the correctness properties worth knowing. + +## License + +``` +Copyright 2026 Alexander Merose + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` \ No newline at end of file diff --git a/crates/ddx-ad/src/lib.rs b/crates/ddx-ad/src/lib.rs index 33950d1..ae7e522 100644 --- a/crates/ddx-ad/src/lib.rs +++ b/crates/ddx-ad/src/lib.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/colref.rs b/crates/ddx-core/src/colref.rs index 0c81bf1..c711b45 100644 --- a/crates/ddx-core/src/colref.rs +++ b/crates/ddx-core/src/colref.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/constructors.rs b/crates/ddx-core/src/constructors.rs index ced8867..ad962f3 100644 --- a/crates/ddx-core/src/constructors.rs +++ b/crates/ddx-core/src/constructors.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/ddx.rs b/crates/ddx-core/src/ddx.rs index cbc010b..b2c92a3 100644 --- a/crates/ddx-core/src/ddx.rs +++ b/crates/ddx-core/src/ddx.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/engine.rs b/crates/ddx-core/src/engine.rs index 184008f..1fb6e9b 100644 --- a/crates/ddx-core/src/engine.rs +++ b/crates/ddx-core/src/engine.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/error.rs b/crates/ddx-core/src/error.rs index 7b4dc60..3bce389 100644 --- a/crates/ddx-core/src/error.rs +++ b/crates/ddx-core/src/error.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/lib.rs b/crates/ddx-core/src/lib.rs index 4220ac1..4caed9a 100644 --- a/crates/ddx-core/src/lib.rs +++ b/crates/ddx-core/src/lib.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index 151b767..fc33a07 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 0baf06d..65bbcc7 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/tests/roundtrip.rs b/crates/ddx-core/tests/roundtrip.rs index 5e3d83c..c597020 100644 --- a/crates/ddx-core/tests/roundtrip.rs +++ b/crates/ddx-core/tests/roundtrip.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-core/tests/rules.rs b/crates/ddx-core/tests/rules.rs index 14f84ea..30a0517 100644 --- a/crates/ddx-core/tests/rules.rs +++ b/crates/ddx-core/tests/rules.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/crates/ddx-datafusion/src/lib.rs b/crates/ddx-datafusion/src/lib.rs index e54e745..70ddfd0 100644 --- a/crates/ddx-datafusion/src/lib.rs +++ b/crates/ddx-datafusion/src/lib.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/docs/spikes/attention_ad_spike.py b/docs/spikes/attention_ad_spike.py index 0a78158..7d55949 100644 --- a/docs/spikes/attention_ad_spike.py +++ b/docs/spikes/attention_ad_spike.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """Generality de-risk for query-level reverse-mode AD (§7.3): scaled dot-product attention from the SAME four transpose rules, no new primitives. diff --git a/docs/spikes/attention_causal_mask_check.py b/docs/spikes/attention_causal_mask_check.py index 8abfc17..ca4ef2c 100644 --- a/docs/spikes/attention_causal_mask_check.py +++ b/docs/spikes/attention_causal_mask_check.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + import numpy as np, jax, jax.numpy as jnp jax.config.update("jax_enable_x64", True) rng = np.random.default_rng(2) diff --git a/docs/spikes/duckdb_reentrancy_r1b.py b/docs/spikes/duckdb_reentrancy_r1b.py index 03e5a5e..a7a62e7 100644 --- a/docs/spikes/duckdb_reentrancy_r1b.py +++ b/docs/spikes/duckdb_reentrancy_r1b.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """R1b spike: is it safe to run a query on the same DuckDB database from within the execution of another query (the `ddx('')` table-function pattern)? diff --git a/docs/spikes/duckdb_substrait_window_bug.py b/docs/spikes/duckdb_substrait_window_bug.py index 0a93278..40c5152 100644 --- a/docs/spikes/duckdb_substrait_window_bug.py +++ b/docs/spikes/duckdb_substrait_window_bug.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """De-risk spike, second half of the Route rule's Substrait feasibility question (design-relational-ad.md §3.4): does DuckDB's Substrait round-trip actually support the SQL idiom the Route rule's forward pass needs -- ROW_NUMBER() OVER (PARTITION diff --git a/docs/spikes/relational_ad_spike.py b/docs/spikes/relational_ad_spike.py index 18e3194..75bbedf 100644 --- a/docs/spikes/relational_ad_spike.py +++ b/docs/spikes/relational_ad_spike.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """De-risk spike for "true AD" in ddx (G5 follow-up). Claim under test (Fable): the manual backward pass in xarray-sql#196 (nn.py) is diff --git a/docs/spikes/route_ad_spike.py b/docs/spikes/route_ad_spike.py index 2c4637a..f0398d9 100644 --- a/docs/spikes/route_ad_spike.py +++ b/docs/spikes/route_ad_spike.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """De-risk spike for the Route (argmax/argmin routing) transpose rule (design-relational-ad.md §3.4) -- the one rule not yet checked against jax.grad when the first version of the design was written. diff --git a/docs/spikes/sqlparser-spike/src/main.rs b/docs/spikes/sqlparser-spike/src/main.rs index 72602a3..a5aed38 100644 --- a/docs/spikes/sqlparser-spike/src/main.rs +++ b/docs/spikes/sqlparser-spike/src/main.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors // // SPDX-License-Identifier: Apache-2.0 diff --git a/docs/spikes/substrait_ad_marker_spike.py b/docs/spikes/substrait_ad_marker_spike.py index 94602c7..cf44177 100644 --- a/docs/spikes/substrait_ad_marker_spike.py +++ b/docs/spikes/substrait_ad_marker_spike.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """De-risk spike for adopting Substrait (+ custom extension-function markers) as ddx v2's relational IR, instead of a bespoke Rust builder graph. diff --git a/docs/spikes/substrait_limitation_repro.py b/docs/spikes/substrait_limitation_repro.py index 393a2af..7d619d6 100644 --- a/docs/spikes/substrait_limitation_repro.py +++ b/docs/spikes/substrait_limitation_repro.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + """Reproduce the Substrait plan-round-trip limitation behind ddx's design (§3.2). The prototype's abandoned approach produced a DataFusion LogicalPlan as Substrait, diff --git a/license.sh b/license.sh old mode 100644 new mode 100755 index a0392c4..aadf6e2 --- a/license.sh +++ b/license.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -uvx "reuse[charset-normalizer]" annotate --copyright "Alex Merose & ddx Authors" --license "Apache-2.0" **/*.rs **/*.py \ No newline at end of file +uvx "reuse[charset-normalizer]" annotate --copyright "Alexander Merose & ddx Authors" --license "Apache-2.0" **/*.rs **/*.py \ No newline at end of file From 51e359db01afb462b1083acc41761552ee93ffaf Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:50:42 -0700 Subject: [PATCH 17/22] Found a new bug given a new class of fuzz invariants! --- crates/ddx-core/tests/simulation.rs | 263 ++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs index 6809911..0383d6e 100644 --- a/crates/ddx-core/tests/simulation.rs +++ b/crates/ddx-core/tests/simulation.rs @@ -1,3 +1,8 @@ +// SPDX-FileCopyrightText: 2026 Alex Merose & ddx Authors +// SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +// +// SPDX-License-Identifier: Apache-2.0 + //! Simulation / property-based tests for the v1 differentiation engine //! (design.md §5: "numeric agreement" + "round-trip property tests"). //! @@ -516,6 +521,184 @@ fn self_consumption_failure(ddx: &Ddx, wrt: &ColRef, original: &str) -> Option 'niño'", + "", +]; + +/// Valid separators between two markers — as sibling select items (`, `) or +/// inside one arithmetic select item (` + `, ` * `). +const STMT_MIDS: &[&str] = &[", ", " + ", " * ", ", z, "]; + +/// Whether the splice fuzz generates `jvp` markers. Temporarily `false`: +/// `jvp` with a compound/casted tangent trips bug #57 (`rewrite_sql` +/// under-splices a marker whose last-arg tail is a `CAST`/`Nested`, because +/// sqlparser's span under-reports there), which would make the bounded and soak +/// tests permanently red. `grad`/nested markers are immune (their last arg is +/// the bare wrt column). Flip to `true` when #57 is fixed — the minimal repro is +/// `splice_handles_marker_with_cast_or_nested_tail` (`#[ignore]`). +const SPLICE_FUZZ_INCLUDES_JVP: bool = false; + +/// Build one marker call and the exact text `rewrite_sql` must splice in its +/// place (`(derivative)`), or `None` if the marker's derivative is undefined +/// (in which case `rewrite_sql` must error on the whole statement). +fn gen_marker_segment(rng: &mut Rng, ddx: &Ddx) -> (String, Option) { + let wrt = if rng.below(2) == 0 { "x" } else { "y" }; + let wrt_col = ColRef::bare(wrt); + let depth = 2 + rng.below(2) as u32; + let expr_text = gen_expr(rng, depth); + let expr = parse_expr(&expr_text); + + match rng.below(3) { + // Nested higher-order: grad(grad(expr, wrt), wrt). + 0 => { + let marker = format!("grad(grad({expr_text}, {wrt}), {wrt})"); + let repl = ddx + .differentiate(&expr, &wrt_col) + .and_then(|d1| ddx.differentiate(&d1, &wrt_col)) + .ok() + .map(|dd| format!("({dd})")); + (marker, repl) + } + // jvp(expr, wrt, tangent) — only when enabled (bug #57). + 1 if SPLICE_FUZZ_INCLUDES_JVP => { + let tan_depth = 1 + rng.below(2) as u32; + let tan_text = gen_expr(rng, tan_depth); + let tan = parse_expr(&tan_text); + let marker = format!("jvp({expr_text}, {wrt}, {tan_text})"); + match ddx.jvp(&expr, &[(wrt_col, tan)]) { + Ok(v) => (marker, Some(format!("({v})"))), + Err(_) => (marker, None), + } + } + // grad(expr, wrt) + _ => { + let marker = format!("grad({expr_text}, {wrt})"); + match ddx.differentiate(&expr, &wrt_col) { + Ok(d) => (marker, Some(format!("({d})"))), + Err(_) => (marker, None), + } + } + } +} + +/// Property 4a: assemble a statement with 1–3 markers wrapped in random +/// (Unicode-bearing) scaffolding and assert `rewrite_sql` splices each marker +/// exactly, byte-for-byte, leaving all surrounding text untouched. If any +/// marker's derivative is undefined, the whole rewrite must error instead. +fn splice_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let n = 1 + rng.below(3) as usize; + let prefix = STMT_PREFIXES[rng.below(STMT_PREFIXES.len() as u64) as usize]; + let suffix = STMT_SUFFIXES[rng.below(STMT_SUFFIXES.len() as u64) as usize]; + + let mut input = String::from(prefix); + let mut expected = String::from(prefix); + let mut any_undefined = false; + for i in 0..n { + if i > 0 { + let mid = STMT_MIDS[rng.below(STMT_MIDS.len() as u64) as usize]; + input.push_str(mid); + expected.push_str(mid); + } + let (marker, repl) = gen_marker_segment(rng, ddx); + input.push_str(&marker); + match repl { + Some(r) => expected.push_str(&r), + None => any_undefined = true, + } + } + input.push_str(suffix); + expected.push_str(suffix); + + let got = ddx.rewrite_sql(&input, &GenericDialect {}); + if any_undefined { + // At least one marker's derivative is undefined → the whole rewrite must + // fail loud, never partially rewrite. + return match got { + Err(_) => None, + Ok(o) => Some(format!( + "[splice] expected an error (a marker derivative is undefined) but got Ok:\n input = {input}\n output = {o}" + )), + }; + } + match got { + Ok(o) if o == expected => None, + Ok(o) => Some(format!( + "[splice] rewrite_sql splice mismatch:\n input = {input}\n expected = {expected}\n actual = {o}" + )), + Err(e) => Some(format!( + "[splice] rewrite_sql errored on a valid marker statement:\n input = {input}\n error = {e}" + )), + } +} + +/// A marker-free statement — some deliberately containing a `grad(`/`jvp(` +/// substring inside a string literal, a comment, or a *qualified* call, so the +/// pre-gate's substring filter hits and the statement is parsed but no real +/// marker is found. Every one must come back byte-identical. +fn gen_marker_free_stmt(rng: &mut Rng) -> String { + match rng.below(6) { + 0 => { + let depth = 2 + rng.below(2) as u32; + format!("SELECT {} FROM t", gen_expr(rng, depth)) + } + 1 => "SELECT 'grad(x, x)' AS s FROM t".to_string(), + 2 => "SELECT x /* grad(y, y) */ FROM t".to_string(), + 3 => "SELECT myschema.grad(x, x) AS d FROM t".to_string(), + 4 => "SELECT 'jvp(sin(x), x, dx)' AS label, x FROM t".to_string(), + _ => format!( + "SELECT {} AS val FROM t WHERE label <> 'grad('", + gen_expr(rng, 2) + ), + } +} + +/// Property 4b: a marker-free statement is returned byte-identical. +fn marker_free_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let s = gen_marker_free_stmt(rng); + match ddx.rewrite_sql(&s, &GenericDialect {}) { + Ok(o) if o == s => None, + Ok(o) => Some(format!( + "[identity] marker-free statement was modified:\n input = {s}\n output = {o}" + )), + Err(e) => Some(format!( + "[identity] marker-free statement errored:\n input = {s}\n error = {e}" + )), + } +} + /// Parse, differentiate, and run every property on one generated expression. /// Returns each failure report (empty ⇒ all properties held). fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec { @@ -545,6 +728,14 @@ fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec if let Some(f) = self_consumption_failure(ddx, &wrt_col, text) { out.push(f); } + // Statement-level rewrite_sql properties (self-generating; the `text`/`wrt` + // above are for the expression-level checks). + if let Some(f) = splice_failure(rng, ddx) { + out.push(f); + } + if let Some(f) = marker_free_failure(rng, ddx) { + out.push(f); + } out } @@ -639,6 +830,78 @@ fn higher_order_self_consumption_is_stable() { ); } +#[test] +fn rewrite_sql_splice_is_byte_faithful() { + // Statement-level fuzz of `rewrite_sql`: markers wrapped in random + // (Unicode-bearing) scaffolding must be spliced exactly, leaving every other + // byte identical (design.md §3.2, G3/F5). + let ddx = Ddx::new(); + let mut failures: Vec = Vec::new(); + + for seed in 0..4000u64 { + let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D) ^ 0x5719_C0DE); + if let Some(report) = splice_failure(&mut rng, &ddx) { + failures.push(report); + } + } + + assert!( + failures.is_empty(), + "splice-fidelity fuzz found {} failure(s):\n\n{}", + failures.len(), + failures.iter().take(15).cloned().collect::>().join("\n\n") + ); +} + +#[test] +#[ignore = "known bug #57: rewrite_sql under-splices a marker whose last-arg tail is a \ + CAST/Nested (sqlparser Spanned under-reports the span end), emitting corrupt SQL"] +fn splice_handles_marker_with_cast_or_nested_tail() { + // The splice must cover the *whole* marker call. When the last argument's + // tail is a CAST (span excludes ` AS `) or a Nested `( … )` (span + // excludes the closing `)`), rewrite_sql currently stops early and leaves + // trailing bytes behind, producing unbalanced/corrupt SQL (#57). + let ddx = Ddx::new(); + // jvp(sin(x), x, CAST(y AS DOUBLE)) — tangent tail is a CAST. + assert_eq!( + ddx.rewrite_sql( + "SELECT jvp(sin(x), x, CAST(y AS DOUBLE)) FROM t", + &GenericDialect {} + ) + .unwrap(), + "SELECT (cos(x) * CAST(y AS DOUBLE)) FROM t" + ); + // jvp(x, x, (y + z)) — tangent tail is a Nested `( … )`. + assert_eq!( + ddx.rewrite_sql("SELECT jvp(x, x, (y + z)) FROM t", &GenericDialect {}) + .unwrap(), + "SELECT ((y + z)) FROM t" + ); +} + +#[test] +fn marker_free_statements_are_byte_identical() { + // The pre-gate / no-marker guarantee: a statement with no real marker — + // including one whose text carries a `grad(`/`jvp(` substring in a string, + // comment, or qualified call — comes back byte-identical (design.md §3.2). + let ddx = Ddx::new(); + let mut failures: Vec = Vec::new(); + + for seed in 0..2000u64 { + let mut rng = Rng::new(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ 0x1DE0_7175); + if let Some(report) = marker_free_failure(&mut rng, &ddx) { + failures.push(report); + } + } + + assert!( + failures.is_empty(), + "marker-free identity fuzz found {} failure(s):\n\n{}", + failures.len(), + failures.iter().take(15).cloned().collect::>().join("\n\n") + ); +} + // --------------------------------------------------------------------------- // Soak test — long-running, #[ignore]-d, driven by env vars (see module docs). // --------------------------------------------------------------------------- From f4e781d2db6b932136b2226e155cc7f42eec40ba Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:54:02 -0700 Subject: [PATCH 18/22] Full name as author. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9cee843..04b5e3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ version = "0.0.0" edition = "2021" license = "Apache-2.0" repository = "https://github.com/xqlsystems/ddx" -authors = ["Alex Merose "] +authors = ["Alexander Merose "] rust-version = "1.75" [workspace.dependencies] From e1c65baf5afa4dec88f62a78271de4a6e0379b9a Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 22 Jul 2026 21:57:46 -0700 Subject: [PATCH 19/22] Fixed additional test added through fuzzing. --- crates/ddx-core/src/rewrite.rs | 55 ++++++++++++- crates/ddx-core/tests/rewrite.rs | 15 ++++ crates/ddx-core/tests/simulation.rs | 116 ++++++++++++++++++++-------- 3 files changed, 150 insertions(+), 36 deletions(-) diff --git a/crates/ddx-core/src/rewrite.rs b/crates/ddx-core/src/rewrite.rs index fc33a07..710eb99 100644 --- a/crates/ddx-core/src/rewrite.rs +++ b/crates/ddx-core/src/rewrite.rs @@ -34,7 +34,7 @@ use sqlparser::ast::{ }; use sqlparser::dialect::Dialect; use sqlparser::parser::Parser; -use sqlparser::tokenizer::{Location, Span}; +use sqlparser::tokenizer::{Location, Span, Token, TokenWithSpan, Tokenizer}; use crate::colref::{ColRef, IdentCasing}; use crate::engine::{differentiate, jvp, positional_args, RuleRegistry}; @@ -197,14 +197,29 @@ pub(crate) fn rewrite_sql( // 6. Compute each replacement, then splice by byte range in reverse source // order so earlier offsets stay valid. + // + // The marker name position (`span.start`) is reliable, but the Function's + // `span.end` is NOT: sqlparser under-reports it when the call's last + // argument ends in a `Cast` (excludes ` AS `) or `Nested` (excludes + // the closing `)`), so trusting it under-splices and leaves corrupt SQL + // behind (#57). Instead, find the call's matching close paren over the + // token stream — re-tokenizing with the same dialect, which lexes strings + // and comments as single tokens so their parens don't miscount. + let tokens = Tokenizer::new(dialect, sql) + .tokenize_with_location() + .map_err(|e| DiffError::Parse(format!("failed to tokenize SQL: {e}")))?; + let mut repls: Vec<(usize, usize, String)> = Vec::with_capacity(collector.found.len()); for (span, marker_expr) in &collector.found { let text = differentiate_marker_tree(marker_expr, casing, reg, &aliases)?; let start = locate(sql, span.start, false) .ok_or_else(|| DiffError::Internal("marker span start out of range".into()))?; - // sqlparser span ends are *inclusive* of the last character (verified - // against 0.62.0), so the exclusive byte end is one character past it. - let end = locate(sql, span.end, true) + let close = marker_call_close(&tokens, span.start).ok_or_else(|| { + DiffError::Internal("could not locate the marker call's closing parenthesis".into()) + })?; + // `close` is the location of the `)`; the exclusive byte end is one + // character past it. + let end = locate(sql, close, true) .ok_or_else(|| DiffError::Internal("marker span end out of range".into()))?; repls.push((start, end, text)); } @@ -535,6 +550,38 @@ fn walk_table_factor(tf: &TableFactor, out: &mut ComputedAliases) { } } +/// The [`Location`] of the `)` that closes the marker call whose name token +/// starts at `name_start`, found by matching parentheses over the token stream. +/// +/// This is authoritative where `Expr::span()` is not: sqlparser under-reports a +/// `Function`'s span end when its last argument ends in a `Cast` (the ` AS +/// ` tail is excluded) or a `Nested` (the closing `)` is excluded), so a +/// span-based splice leaves trailing bytes behind and corrupts the output +/// (#57). Matching over tokens is exact because the tokenizer lexes string +/// literals and comments into single tokens, so parentheses inside them are +/// never counted. The marker name position (`name_start`) is itself reliable; +/// between it and the call's `(` there is only trivia (skipped here). +fn marker_call_close(tokens: &[TokenWithSpan], name_start: Location) -> Option { + let mut depth = 0usize; + let mut opened = false; + for t in tokens.iter().filter(|t| t.span.start >= name_start) { + match t.token { + Token::LParen => { + depth += 1; + opened = true; + } + Token::RParen => { + depth = depth.checked_sub(1)?; + if opened && depth == 0 { + return Some(t.span.start); + } + } + _ => {} + } + } + None +} + // --------------------------------------------------------------------------- // Span (1-based character line/column) → byte offset conversion (G3) // --------------------------------------------------------------------------- diff --git a/crates/ddx-core/tests/rewrite.rs b/crates/ddx-core/tests/rewrite.rs index 65bbcc7..4f40f9a 100644 --- a/crates/ddx-core/tests/rewrite.rs +++ b/crates/ddx-core/tests/rewrite.rs @@ -78,6 +78,21 @@ fn fires_inside_recursive_cte() { ); } +#[test] +fn splice_end_ignores_parens_inside_strings_and_comments() { + // The splice finds the call's matching `)` over the token stream, so a `)` + // that appears only inside a string literal or comment in a (jvp) tangent is + // not miscounted (#57). The tangent is spliced verbatim. + assert_eq!( + rw("SELECT jvp(x, x, CASE WHEN a = ')' THEN y ELSE z END) FROM t"), + "SELECT (CASE WHEN a = ')' THEN y ELSE z END) FROM t" + ); + assert_eq!( + rw("SELECT jvp(x, x, y /* ) ) */) FROM t"), + "SELECT (y) FROM t" + ); +} + #[test] fn dml_update_rule_is_rewritten() { let out = rw("INSERT INTO p SELECT theta - 0.1 * grad(x * theta, theta) FROM t"); diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs index 0383d6e..af7f015 100644 --- a/crates/ddx-core/tests/simulation.rs +++ b/crates/ddx-core/tests/simulation.rs @@ -147,10 +147,26 @@ fn gen_expr(rng: &mut Rng, depth: u32) -> String { }; } match rng.below(11) { - 0 => format!("({} + {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), - 1 => format!("({} - {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), - 2 => format!("({} * {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), - 3 => format!("({} / {})", gen_expr(rng, depth - 1), gen_expr(rng, depth - 1)), + 0 => format!( + "({} + {})", + gen_expr(rng, depth - 1), + gen_expr(rng, depth - 1) + ), + 1 => format!( + "({} - {})", + gen_expr(rng, depth - 1), + gen_expr(rng, depth - 1) + ), + 2 => format!( + "({} * {})", + gen_expr(rng, depth - 1), + gen_expr(rng, depth - 1) + ), + 3 => format!( + "({} / {})", + gen_expr(rng, depth - 1), + gen_expr(rng, depth - 1) + ), 4 => format!("(-{})", gen_expr(rng, depth - 1)), 5 | 6 => { let f = UNARY_FNS[rng.below(UNARY_FNS.len() as u64) as usize]; @@ -211,11 +227,13 @@ fn eval_mag(e: &Expr, x: f64, y: f64) -> Option<(f64, f64)> { "y" => here(y), _ => None, }, - Expr::CompoundIdentifier(parts) => match parts.last()?.value.to_ascii_lowercase().as_str() { - "x" => here(x), - "y" => here(y), - _ => None, - }, + Expr::CompoundIdentifier(parts) => { + match parts.last()?.value.to_ascii_lowercase().as_str() { + "x" => here(x), + "y" => here(y), + _ => None, + } + } Expr::Nested(inner) => eval_mag(inner, x, y), Expr::UnaryOp { op: UnaryOperator::Minus, @@ -374,8 +392,8 @@ fn fd_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Option Option ATOL + RTOL * dv.abs().max(fd.abs()) { disagree += 1; if first_bad.is_empty() { - first_bad = - format!("x={x0:.6} y={y0:.6}: symbolic d/d{} = {dv:.8}, finite-diff = {fd:.8}", wrt.name()); + first_bad = format!( + "x={x0:.6} y={y0:.6}: symbolic d/d{} = {dv:.8}, finite-diff = {fd:.8}", + wrt.name() + ); } } } @@ -449,7 +469,10 @@ fn fidelity_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Optio const ATOL: f64 = 1e-11; let rendered = d.to_string(); if rendered.contains("--") { - return Some(format!("[render] emitted a `--` comment: d/d{} {expr_text} => {rendered}", wrt.name())); + return Some(format!( + "[render] emitted a `--` comment: d/d{} {expr_text} => {rendered}", + wrt.name() + )); } let reparsed = match try_parse(&rendered) { Ok(rp) => rp, @@ -562,14 +585,12 @@ const STMT_SUFFIXES: &[&str] = &[ /// inside one arithmetic select item (` + `, ` * `). const STMT_MIDS: &[&str] = &[", ", " + ", " * ", ", z, "]; -/// Whether the splice fuzz generates `jvp` markers. Temporarily `false`: -/// `jvp` with a compound/casted tangent trips bug #57 (`rewrite_sql` -/// under-splices a marker whose last-arg tail is a `CAST`/`Nested`, because -/// sqlparser's span under-reports there), which would make the bounded and soak -/// tests permanently red. `grad`/nested markers are immune (their last arg is -/// the bare wrt column). Flip to `true` when #57 is fixed — the minimal repro is -/// `splice_handles_marker_with_cast_or_nested_tail` (`#[ignore]`). -const SPLICE_FUZZ_INCLUDES_JVP: bool = false; +/// Whether the splice fuzz generates `jvp` markers. Now `true`: #57 is fixed — +/// `rewrite_sql` splices to the call's matching close paren (found over the +/// token stream) rather than the under-reported `Expr::span()` end, so a `jvp` +/// with a compound/casted tangent (last-arg tail a `CAST`/`Nested`) splices +/// correctly. The minimal repro is `splice_handles_marker_with_cast_or_nested_tail`. +const SPLICE_FUZZ_INCLUDES_JVP: bool = true; /// Build one marker call and the exact text `rewrite_sql` must splice in its /// place (`(derivative)`), or `None` if the marker's derivative is undefined @@ -706,7 +727,9 @@ fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec let parsed = match try_parse(text) { Ok(p) => p, Err(e) => { - out.push(format!("[generator] produced unparseable text `{text}` ({e})")); + out.push(format!( + "[generator] produced unparseable text `{text}` ({e})" + )); return out; } }; @@ -769,13 +792,21 @@ fn finite_difference_agreement_over_random_expressions() { } } - assert!(tested > 500, "generator produced too few derivable cases: {tested}"); + assert!( + tested > 500, + "generator produced too few derivable cases: {tested}" + ); assert!( failures.is_empty(), "finite-difference oracle found {} disagreement(s) out of {} tested:\n\n{}", failures.len(), tested, - failures.iter().take(15).cloned().collect::>().join("\n\n") + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") ); } @@ -803,7 +834,12 @@ fn render_reparse_is_value_preserving() { failures.is_empty(), "render-fidelity fuzz found {} failure(s):\n\n{}", failures.len(), - failures.iter().take(15).cloned().collect::>().join("\n\n") + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") ); } @@ -826,7 +862,12 @@ fn higher_order_self_consumption_is_stable() { failures.is_empty(), "self-consumption fuzz found {} failure(s):\n\n{}", failures.len(), - failures.iter().take(15).cloned().collect::>().join("\n\n") + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") ); } @@ -849,13 +890,16 @@ fn rewrite_sql_splice_is_byte_faithful() { failures.is_empty(), "splice-fidelity fuzz found {} failure(s):\n\n{}", failures.len(), - failures.iter().take(15).cloned().collect::>().join("\n\n") + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") ); } #[test] -#[ignore = "known bug #57: rewrite_sql under-splices a marker whose last-arg tail is a \ - CAST/Nested (sqlparser Spanned under-reports the span end), emitting corrupt SQL"] fn splice_handles_marker_with_cast_or_nested_tail() { // The splice must cover the *whole* marker call. When the last argument's // tail is a CAST (span excludes ` AS `) or a Nested `( … )` (span @@ -898,7 +942,12 @@ fn marker_free_statements_are_byte_identical() { failures.is_empty(), "marker-free identity fuzz found {} failure(s):\n\n{}", failures.len(), - failures.iter().take(15).cloned().collect::>().join("\n\n") + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") ); } @@ -907,7 +956,10 @@ fn marker_free_statements_are_byte_identical() { // --------------------------------------------------------------------------- fn env_u64(key: &str, default: u64) -> u64 { - std::env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default) + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) } #[test] From aafa0ba4b77c30bb6f1a3512981f16c71fef38f9 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 23 Jul 2026 10:01:07 -0700 Subject: [PATCH 20/22] impler: added CI --- .github/workflows/ci.yml | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..eb2f991 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 Alexander Merose & ddx Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Standard PR checks for ddx: formatting, lints, and the deterministic test +# suite. The property/fuzz suite (crates/ddx-core/tests/simulation.rs) is +# intentionally NOT run here — it is a separate, on-demand / soak concern (see +# its module docs) rather than a blocking PR gate — so these checks stay fast +# and deterministic. +name: CI + +on: + pull_request: + push: + branches: [main] + +# Cancel superseded runs on the same ref (e.g. a new push to a PR branch). +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + lint: + name: rustfmt + clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + - name: rustfmt (check only) + run: cargo fmt --all --check + - name: clippy (warnings are errors) + run: cargo clippy --workspace --all-targets -- -D warnings + + test: + name: tests (non-simulation) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # Everything except tests/simulation.rs (the property/fuzz suite). Split + # per target so a failure names the suite that broke. + - name: unit tests + run: cargo test --workspace --lib --bins + - name: integration tests + run: cargo test -p ddx-core --test rules --test rewrite --test roundtrip + - name: doctests + run: cargo test --workspace --doc From 45b2ee452999fc5d79f4476d1acd9dc9fe045626 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 23 Jul 2026 10:01:14 -0700 Subject: [PATCH 21/22] more fuzz invariants --- crates/ddx-core/tests/simulation.rs | 511 +++++++++++++++++++++++++++- 1 file changed, 505 insertions(+), 6 deletions(-) diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs index af7f015..b9d8d1b 100644 --- a/crates/ddx-core/tests/simulation.rs +++ b/crates/ddx-core/tests/simulation.rs @@ -53,7 +53,11 @@ use std::fmt::Write as _; use std::io::Write as _; -use ddx_core::sqlparser::ast::{BinaryOperator, Expr, UnaryOperator, Value}; +use std::ops::ControlFlow; + +use ddx_core::sqlparser::ast::{ + BinaryOperator, Expr, Function, ObjectNamePart, UnaryOperator, Value, Visit, Visitor, +}; use ddx_core::sqlparser::dialect::GenericDialect; use ddx_core::sqlparser::parser::Parser; use ddx_core::{ColRef, Ddx, DiffError}; @@ -490,14 +494,28 @@ fn fidelity_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Optio } let x0 = rng.range(0.2, 1.8); let y0 = rng.range(0.2, 1.8); + // Tolerance relative to the computation *scale*, not the result (#54's + // lesson, generalized — see `metamorphic_mismatch`). AST-vs-reparse + // differ only by float *association* (the `a·(b/c)` → `(a·b)/c` reprint, + // issue #50), which agrees to ≈ ε·scale — so at a huge-magnitude point + // (`sinh(16/…)`, deriv ≈ 1e62) or a cancellation/near-pole point + // (`tan(exp(…))`) a result-relative tolerance false-positives. Skip the + // point only if the scale is non-finite/overflowing. + let scale = match ( + max_intermediate_mag(d, x0, y0), + max_intermediate_mag(&reparsed, x0, y0), + ) { + (Some(a), Some(b)) if a.is_finite() && b.is_finite() && a.max(b) < 1e300 => a.max(b), + _ => continue, + }; let (Some(va), Some(vb)) = (eval(d, x0, y0), eval(&reparsed, x0, y0)) else { continue; }; - if !va.is_finite() || !vb.is_finite() || va.abs() > 1e6 { + if !va.is_finite() || !vb.is_finite() { continue; } compared += 1; - if (va - vb).abs() > ATOL + RTOL * va.abs() { + if (va - vb).abs() > ATOL + RTOL * scale { return Some(format!( "[render] render changed the value: d/d{} {expr_text}\n rendered = {rendered}\n at x={x0:.4} y={y0:.4}: AST = {va:.10}, reparsed = {vb:.10}", wrt.name() @@ -639,7 +657,11 @@ fn gen_marker_segment(rng: &mut Rng, ddx: &Ddx) -> (String, Option) { /// (Unicode-bearing) scaffolding and assert `rewrite_sql` splices each marker /// exactly, byte-for-byte, leaving all surrounding text untouched. If any /// marker's derivative is undefined, the whole rewrite must error instead. -fn splice_failure(rng: &mut Rng, ddx: &Ddx) -> Option { +/// Assemble a random marker-bearing statement: 1–3 markers wrapped in random +/// (Unicode-bearing) scaffolding. Returns `(input, expected)` where `expected` +/// is the exact byte-for-byte `rewrite_sql` output, or `None` if some marker's +/// derivative is undefined (in which case the whole rewrite must error). +fn gen_marker_statement(rng: &mut Rng, ddx: &Ddx) -> (String, Option) { let n = 1 + rng.below(3) as usize; let prefix = STMT_PREFIXES[rng.below(STMT_PREFIXES.len() as u64) as usize]; let suffix = STMT_SUFFIXES[rng.below(STMT_SUFFIXES.len() as u64) as usize]; @@ -663,8 +685,13 @@ fn splice_failure(rng: &mut Rng, ddx: &Ddx) -> Option { input.push_str(suffix); expected.push_str(suffix); + (input, if any_undefined { None } else { Some(expected) }) +} + +fn splice_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let (input, expected) = gen_marker_statement(rng, ddx); let got = ddx.rewrite_sql(&input, &GenericDialect {}); - if any_undefined { + let Some(expected) = expected else { // At least one marker's derivative is undefined → the whole rewrite must // fail loud, never partially rewrite. return match got { @@ -673,7 +700,7 @@ fn splice_failure(rng: &mut Rng, ddx: &Ddx) -> Option { "[splice] expected an error (a marker derivative is undefined) but got Ok:\n input = {input}\n output = {o}" )), }; - } + }; match got { Ok(o) if o == expected => None, Ok(o) => Some(format!( @@ -720,6 +747,361 @@ fn marker_free_failure(rng: &mut Rng, ddx: &Ddx) -> Option { } } +// --------------------------------------------------------------------------- +// Additional invariants (issue #58). +// --------------------------------------------------------------------------- + +/// Parse a whole statement (not just an expression). +fn try_parse_stmt(sql: &str) -> Result, String> { + Parser::parse_sql(&GenericDialect {}, sql).map_err(|e| e.to_string()) +} + +/// Does an AST still contain an *unqualified* `grad`/`jvp` call — a marker that +/// should have been rewritten away? +#[derive(Default)] +struct MarkerScan { + found: bool, +} +impl Visitor for MarkerScan { + type Break = (); + fn pre_visit_expr(&mut self, e: &Expr) -> ControlFlow<()> { + if let Expr::Function(Function { name, .. }) = e { + if let [ObjectNamePart::Identifier(id)] = name.0.as_slice() { + let n = id.value.to_ascii_lowercase(); + if n == "grad" || n == "jvp" { + self.found = true; + return ControlFlow::Break(()); + } + } + } + ControlFlow::Continue(()) + } +} + +fn has_residual_marker(stmts: &[ddx_core::sqlparser::ast::Statement]) -> bool { + let mut scan = MarkerScan::default(); + for s in stmts { + let _ = Visit::visit(s, &mut scan); + if scan.found { + return true; + } + } + false +} + +/// Invariant 1: the SQL `rewrite_sql` emits must always re-parse and contain no +/// residual marker. A *broad* net over the rewrite path — it needs no predicted +/// output, so it tolerates arbitrary scaffolding, and it independently catches +/// corruption bugs like #57 (the corrupt output fails to re-parse). +fn rewrite_validity_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let (input, expected) = gen_marker_statement(rng, ddx); + let out = match ddx.rewrite_sql(&input, &GenericDialect {}) { + Ok(o) => o, + // An undefined-derivative marker legitimately errors (fail loud). + Err(_) if expected.is_none() => return None, + Err(e) => { + return Some(format!( + "[validity] rewrite_sql errored on a valid marker statement:\n input = {input}\n error = {e}" + )) + } + }; + match try_parse_stmt(&out) { + Err(e) => Some(format!( + "[validity] rewrite_sql emitted unparseable SQL:\n input = {input}\n output = {out}\n parse error = {e}" + )), + Ok(stmts) if has_residual_marker(&stmts) => Some(format!( + "[validity] rewrite_sql left a residual grad/jvp marker:\n input = {input}\n output = {out}" + )), + Ok(_) => None, + } +} + +/// Invariant 5: `rewrite_sql` is idempotent — a second pass over its own output +/// is a no-op (no markers remain to rewrite, and the text is stable). +fn idempotence_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let (input, _) = gen_marker_statement(rng, ddx); + let once = match ddx.rewrite_sql(&input, &GenericDialect {}) { + Ok(o) => o, + Err(_) => return None, // undefined-derivative markers error; fine here + }; + match ddx.rewrite_sql(&once, &GenericDialect {}) { + Ok(twice) if twice == once => None, + Ok(twice) => Some(format!( + "[idempotence] rewrite_sql is not idempotent:\n input = {input}\n once = {once}\n twice = {twice}" + )), + Err(e) => Some(format!( + "[idempotence] rewrite_sql errored on its own output:\n input = {input}\n once = {once}\n error = {e}" + )), + } +} + +/// Adversarial / malformed inputs for the never-panic invariant. +fn gen_adversarial_sql(rng: &mut Rng) -> String { + const UNI: &[&str] = &[ + "héllo", + "☕🔥", + "naïve", + "Ωμέγα", + "🇺🇸", + "e\u{0301}", + "\u{202E}rtl", + "𝕏𝕐", + ]; + match rng.below(8) { + // Malformed marker arities / shapes — must be typed errors, not panics. + 0 => [ + "SELECT grad() FROM t", + "SELECT grad(x) FROM t", + "SELECT grad(x, y, z) FROM t", + "SELECT jvp(x, x) FROM t", + "SELECT jvp(x) FROM t", + "SELECT grad(x, 1 + 2) FROM t", + "SELECT grad(*, x) FROM t", + "SELECT grad(x, ) FROM t", + ][rng.below(8) as usize] + .to_string(), + // A valid marker behind a Unicode-heavy prefix (stresses `locate`). + 1 => { + let u = UNI[rng.below(UNI.len() as u64) as usize]; + let depth = 2 + rng.below(2) as u32; + format!( + "SELECT '{u}' AS c, grad({}, x) FROM t", + gen_expr(rng, depth) + ) + } + // Deeply nested markers. + 2 => { + let k = 1 + rng.below(12) as usize; + let mut s = String::from("SELECT "); + for _ in 0..k { + s.push_str("grad("); + } + s.push('x'); + for _ in 0..k { + s.push_str(", x)"); + } + s.push_str(" FROM t"); + s + } + // A valid marker statement truncated at a random char boundary. + 3 => { + let (full, _) = ("SELECT café AS c, grad(sin(x) * y, x) FROM t", ()); + let cut = full + .char_indices() + .nth(1 + rng.below(full.chars().count() as u64) as usize) + .map(|(i, _)| i) + .unwrap_or(full.len()); + full[..cut].to_string() + } + // Marker with a Unicode identifier argument. + 4 => { + let u = UNI[rng.below(UNI.len() as u64) as usize]; + format!("SELECT grad({u}, x) FROM t") + } + // Unicode injected into a valid marker statement at a char boundary. + 5 => { + let u = UNI[rng.below(UNI.len() as u64) as usize]; + let base = "SELECT grad(sin(x), x) FROM t"; + let at = base + .char_indices() + .nth(rng.below(base.chars().count() as u64) as usize) + .map(|(i, _)| i) + .unwrap_or(0); + let mut s = String::from(&base[..at]); + s.push_str(u); + s.push_str(&base[at..]); + s + } + // Odd whitespace/comments around the marker (the #52 family). + 6 => "SELECT grad\t(\n x , x )/* ☕ */ FROM t".to_string(), + // A deep but valid marker payload. + _ => { + let depth = 3 + rng.below(3) as u32; + format!("SELECT grad({}, x) FROM t", gen_expr(rng, depth)) + } + } +} + +/// Invariant 2: `rewrite_sql` never *panics* — for any input, adversarial or +/// malformed, it returns `Ok` or a typed `DiffError` (design principle 5: fail +/// loud, never crash). The prime suspects are the UTF-8 `locate` math and the +/// span logic. +fn panic_failure(rng: &mut Rng, ddx: &Ddx) -> Option { + let input = gen_adversarial_sql(rng); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + // Return a bool so nothing non-UnwindSafe crosses the boundary. + ddx.rewrite_sql(&input, &GenericDialect {}).is_ok() + })); + match result { + Ok(_) => None, + Err(payload) => { + let msg = payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "".to_string()); + Some(format!( + "[panic] rewrite_sql PANICKED (must return a typed error instead):\n input = {input:?}\n panic = {msg}" + )) + } + } +} + +/// Invariant 4: `d/dv` of an expression that does not mention `v` is exactly +/// zero. Differentiating w.r.t. the fresh variable `w` (which the generator +/// never emits) must fold to a value-0 derivative — a crisp check of the +/// 0-folding smart constructors and the leaf `Match::Not` classification. +fn zero_derivative_failure(rng: &mut Rng, ddx: &Ddx, text: &str) -> Option { + let f = parse_expr(text); + let d = match ddx.differentiate(&f, &ColRef::bare("w")) { + Ok(d) => d, + Err(_) => return None, + }; + for _ in 0..12 { + let x0 = rng.range(0.2, 1.8); + let y0 = rng.range(0.2, 1.8); + if let Some(v) = eval(&d, x0, y0) { + if v.is_finite() && v.abs() > 1e-12 { + return Some(format!( + "[zero-deriv] d/dw {text} is not zero (w is absent):\n => {d}\n at x={x0:.4} y={y0:.4}: value = {v}" + )); + } + } + } + None +} + +/// Invariant 6: the engine never emits an `inf`/`nan` token in a derivative +/// (design principle 5 / #33 — a non-finite constant is a typed error, never an +/// invalid literal in the output text). +fn no_inf_nan_failure(text: &str, d: &Expr) -> Option { + let rendered = d.to_string(); + let low = rendered.to_ascii_lowercase(); + if low.contains("inf") || low.contains("nan") { + return Some(format!( + "[inf-nan] derivative text contains an inf/nan token:\n d/d? {text}\n => {rendered}" + )); + } + None +} + +/// Compare a symbolic `lhs` expression to a `rhs` value closure at random +/// points, returning the first genuine disagreement. Used for the exact +/// metamorphic identities (jvp↔grad, linearity) — as exact identities, not +/// finite differences, they need no majority vote. +/// +/// **Tolerance is relative to the computation scale, not the result** — the +/// #54 lesson, generalized. `lhs` and `rhs` compute the same value by different +/// *associations* (e.g. jvp distributes the tangent into each product term +/// while `t·grad` factors it out; render→reparse turns `a·(b/c)` into `(a·b)/c`, +/// issue #50). Those agree only up to ≈ `ε · (magnitude of the intermediates)`, +/// which *explodes past a result-relative tolerance* at a cancellation or +/// near-singular point (`x⁻¹ − x·x⁻² → 0`, `tan` near a pole). A real rule bug, +/// by contrast, perturbs the result by a finite fraction of its own scale, so it +/// still exceeds `RTOL · scale` at well-conditioned points — the check keeps its +/// teeth while shedding the float-noise false positives. +fn metamorphic_mismatch( + rng: &mut Rng, + gate: &[&Expr], + lhs: &Expr, + rhs: impl Fn(f64, f64) -> Option, +) -> Option<(f64, f64, f64, f64)> { + const RTOL: f64 = 1e-7; + const ATOL: f64 = 1e-9; + for _ in 0..40 { + let x0 = rng.range(0.2, 1.8); + let y0 = rng.range(0.2, 1.8); + // Scale = the largest intermediate magnitude across every expression + // involved; skip the point if any is non-finite or overflowing. + let mut scale = 0.0f64; + let mut ok = true; + for e in gate.iter().chain(std::iter::once(&lhs)) { + match max_intermediate_mag(e, x0, y0) { + Some(m) if m.is_finite() && m < 1e300 => scale = scale.max(m), + _ => { + ok = false; + break; + } + } + } + if !ok { + continue; + } + let (Some(a), Some(b)) = (eval(lhs, x0, y0), rhs(x0, y0)) else { + continue; + }; + if !a.is_finite() || !b.is_finite() { + continue; + } + if (a - b).abs() > ATOL + RTOL * scale { + return Some((x0, y0, a, b)); + } + } + None +} + +/// Invariant 3: `jvp(f, wrt, t)` equals `t · grad(f, wrt)` (forward mode is +/// linear in the seed). Ties the two forward-mode entry points; `jvp` is where +/// #57 lived. +fn jvp_consistency_failure(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Option { + let f = parse_expr(text); + let wrt_col = ColRef::bare(wrt.name()); + let tan_depth = 1 + rng.below(2) as u32; + let t_text = gen_expr(rng, tan_depth); + let t = parse_expr(&t_text); + let grad_e = ddx.differentiate(&f, &wrt_col).ok()?; + let jvp_e = ddx.jvp(&f, &[(wrt_col, t.clone())]).ok()?; + let gate = [&f, &t, &grad_e]; + if let Some((x0, y0, a, b)) = metamorphic_mismatch(rng, &gate, &jvp_e, |x, y| { + Some(eval(&t, x, y)? * eval(&grad_e, x, y)?) + }) { + return Some(format!( + "[jvp≠t·grad] jvp({text}, {w}, {t_text}) ≠ tangent·grad:\n jvp => {jvp_e}\n grad => {grad_e}\n at x={x0:.4} y={y0:.4}: jvp = {a}, t·grad = {b}", + w = wrt.name() + )); + } + None +} + +/// Invariant 7: linearity and the product rule as exact metamorphic identities, +/// `d(f+g) = d(f)+d(g)` and `d(f·g) = d(f)·g + f·d(g)`, value-checked. An exact +/// algebraic cross-check independent of the finite-difference oracle — it holds +/// even at the high-magnitude points the FD oracle skips. +fn linearity_failure(rng: &mut Rng, ddx: &Ddx, f_text: &str, wrt: Var) -> Option { + let wrt_col = ColRef::bare(wrt.name()); + let f = parse_expr(f_text); + let g_depth = 2 + rng.below(2) as u32; + let g_text = gen_expr(rng, g_depth); + let g = parse_expr(&g_text); + let df = ddx.differentiate(&f, &wrt_col).ok()?; + let dg = ddx.differentiate(&g, &wrt_col).ok()?; + + // Sum rule. + let sum = parse_expr(&format!("({f_text}) + ({g_text})")); + let dsum = ddx.differentiate(&sum, &wrt_col).ok()?; + let gate_sum = [&f, &g, &df, &dg]; + if let Some((x0, y0, a, b)) = metamorphic_mismatch(rng, &gate_sum, &dsum, |x, y| { + Some(eval(&df, x, y)? + eval(&dg, x, y)?) + }) { + return Some(format!( + "[linearity] d(f+g) ≠ d(f)+d(g):\n f = {f_text}\n g = {g_text}\n d(f+g) => {dsum}\n at x={x0:.4} y={y0:.4}: lhs = {a}, rhs = {b}" + )); + } + + // Product rule. + let prod = parse_expr(&format!("({f_text}) * ({g_text})")); + let dprod = ddx.differentiate(&prod, &wrt_col).ok()?; + let gate_prod = [&f, &g, &df, &dg]; + if let Some((x0, y0, a, b)) = metamorphic_mismatch(rng, &gate_prod, &dprod, |x, y| { + Some(eval(&df, x, y)? * eval(&g, x, y)? + eval(&f, x, y)? * eval(&dg, x, y)?) + }) { + return Some(format!( + "[product-rule] d(f*g) ≠ d(f)*g + f*d(g):\n f = {f_text}\n g = {g_text}\n d(f*g) => {dprod}\n at x={x0:.4} y={y0:.4}: lhs = {a}, rhs = {b}" + )); + } + None +} + /// Parse, differentiate, and run every property on one generated expression. /// Returns each failure report (empty ⇒ all properties held). fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec { @@ -751,6 +1133,19 @@ fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec if let Some(f) = self_consumption_failure(ddx, &wrt_col, text) { out.push(f); } + // Expression-level metamorphic / structural invariants (#58). + if let Some(f) = no_inf_nan_failure(text, &d) { + out.push(f); + } + if let Some(f) = zero_derivative_failure(rng, ddx, text) { + out.push(f); + } + if let Some(f) = jvp_consistency_failure(rng, ddx, text, wrt) { + out.push(f); + } + if let Some(f) = linearity_failure(rng, ddx, text, wrt) { + out.push(f); + } // Statement-level rewrite_sql properties (self-generating; the `text`/`wrt` // above are for the expression-level checks). if let Some(f) = splice_failure(rng, ddx) { @@ -759,6 +1154,15 @@ fn run_all_checks(rng: &mut Rng, ddx: &Ddx, text: &str, wrt: Var) -> Vec if let Some(f) = marker_free_failure(rng, ddx) { out.push(f); } + if let Some(f) = rewrite_validity_failure(rng, ddx) { + out.push(f); + } + if let Some(f) = idempotence_failure(rng, ddx) { + out.push(f); + } + if let Some(f) = panic_failure(rng, ddx) { + out.push(f); + } out } @@ -951,6 +1355,101 @@ fn marker_free_statements_are_byte_identical() { ); } +/// Run a per-seed check over a fixed range, asserting no failures. Shared by the +/// bounded tests for the #58 invariants. +fn run_bounded Option>(label: &str, n: u64, salt: u64, mut check: F) { + let mut failures: Vec = Vec::new(); + for seed in 0..n { + let mut rng = Rng::new(seed.wrapping_mul(0x2545_F491_4F6C_DD1D) ^ salt); + if let Some(report) = check(&mut rng) { + failures.push(report); + } + } + assert!( + failures.is_empty(), + "{label} found {} failure(s):\n\n{}", + failures.len(), + failures + .iter() + .take(15) + .cloned() + .collect::>() + .join("\n\n") + ); +} + +/// A random derivable expression plus a random differentiation variable. +fn gen_expr_and_wrt(rng: &mut Rng) -> (String, Var) { + let depth = 2 + rng.below(4) as u32; + let text = gen_expr(rng, depth); + let wrt = if rng.below(2) == 0 { Var::X } else { Var::Y }; + (text, wrt) +} + +#[test] +fn rewrite_sql_output_is_valid_and_marker_free() { + let ddx = Ddx::new(); + run_bounded("rewrite validity fuzz", 4000, 0x5A11_D000, |rng| { + rewrite_validity_failure(rng, &ddx) + }); +} + +#[test] +fn rewrite_sql_never_panics_on_adversarial_input() { + let ddx = Ddx::new(); + run_bounded("never-panic fuzz", 5000, 0x9A11_C000, |rng| { + panic_failure(rng, &ddx) + }); +} + +#[test] +fn jvp_equals_tangent_times_grad() { + let ddx = Ddx::new(); + run_bounded("jvp↔grad consistency fuzz", 4000, 0x0F5E_ED00, |rng| { + let (text, wrt) = gen_expr_and_wrt(rng); + jvp_consistency_failure(rng, &ddx, &text, wrt) + }); +} + +#[test] +fn derivative_of_absent_variable_is_zero() { + let ddx = Ddx::new(); + run_bounded("zero-derivative fuzz", 4000, 0x2E50_1000, |rng| { + let depth = 2 + rng.below(4) as u32; + let text = gen_expr(rng, depth); + zero_derivative_failure(rng, &ddx, &text) + }); +} + +#[test] +fn rewrite_sql_is_idempotent() { + let ddx = Ddx::new(); + run_bounded("idempotence fuzz", 4000, 0x1DE1_1000, |rng| { + idempotence_failure(rng, &ddx) + }); +} + +#[test] +fn no_inf_or_nan_token_is_ever_emitted() { + let ddx = Ddx::new(); + run_bounded("inf/nan-token fuzz", 4000, 0x1FFF_F000, |rng| { + let (text, wrt) = gen_expr_and_wrt(rng); + let d = ddx + .differentiate(&parse_expr(&text), &ColRef::bare(wrt.name())) + .ok()?; + no_inf_nan_failure(&text, &d) + }); +} + +#[test] +fn differentiation_is_linear_and_obeys_the_product_rule() { + let ddx = Ddx::new(); + run_bounded("linearity/product-rule fuzz", 4000, 0x114E_A200, |rng| { + let (text, wrt) = gen_expr_and_wrt(rng); + linearity_failure(rng, &ddx, &text, wrt) + }); +} + // --------------------------------------------------------------------------- // Soak test — long-running, #[ignore]-d, driven by env vars (see module docs). // --------------------------------------------------------------------------- From dd6674ea4da069d2549a46bdc8407f268cddb02b Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Thu, 23 Jul 2026 11:17:28 -0700 Subject: [PATCH 22/22] more fuzz invariants --- crates/ddx-core/tests/simulation.rs | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/crates/ddx-core/tests/simulation.rs b/crates/ddx-core/tests/simulation.rs index b9d8d1b..ac980d2 100644 --- a/crates/ddx-core/tests/simulation.rs +++ b/crates/ddx-core/tests/simulation.rs @@ -358,6 +358,77 @@ fn eval_function(f: &ddx_core::sqlparser::ast::Function, x: f64, y: f64) -> Opti Some((v, argmag.max(v.abs()))) } +/// The smallest distance from any *restricted-domain* function call in `e` to +/// its domain boundary at `(x, y)` — `f64::INFINITY` if there are none. +/// +/// The derivative of a restricted-domain primitive is singular *at* the +/// boundary even where the primal is finite (design.md §5, "domain-widening"): +/// `acos`/`asin` have `±1` (`d = ∓1/√(1−u²)`), `sqrt`/`ln`/`log` have `0` +/// (`d = 1/(2√u)`, `1/u`), and division has a `0` denominator. Near such a +/// boundary the symbolic derivative is a `0·∞`/`∞` form that f64 evaluates to +/// garbage — e.g. `sqrt(acos(x·0.5^log2(x)))` where the argument is *identically +/// 1*. The finite-difference oracle skips these points rather than mistake a +/// numerically-singular (but symbolically correct) derivative for a bug. +fn min_domain_margin(e: &Expr, x: f64, y: f64) -> Option { + use ddx_core::sqlparser::ast::{FunctionArg, FunctionArgExpr, FunctionArguments}; + match e { + Expr::Value(_) | Expr::Identifier(_) | Expr::CompoundIdentifier(_) => Some(f64::INFINITY), + Expr::Nested(i) => min_domain_margin(i, x, y), + Expr::UnaryOp { expr, .. } | Expr::Cast { expr, .. } => min_domain_margin(expr, x, y), + Expr::BinaryOp { left, op, right } => { + let mut m = min_domain_margin(left, x, y)?.min(min_domain_margin(right, x, y)?); + if matches!(op, BinaryOperator::Divide) { + m = m.min(eval(right, x, y)?.abs()); + } + Some(m) + } + Expr::Function(Function { + name, + args: FunctionArguments::List(list), + .. + }) => { + let mut arg_exprs = Vec::new(); + let mut m = f64::INFINITY; + for a in &list.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(ae)) = a else { + return Some(f64::INFINITY); + }; + m = m.min(min_domain_margin(ae, x, y)?); + arg_exprs.push(ae); + } + let fname = match name.0.as_slice() { + [ObjectNamePart::Identifier(id)] => id.value.to_ascii_lowercase(), + _ => return Some(m), + }; + if let Some(a0) = arg_exprs.first() { + let v = eval(a0, x, y)?; + m = m.min(match fname.as_str() { + "asin" | "acos" => 1.0 - v.abs(), + "sqrt" | "ln" | "log2" | "log10" => v, + _ => f64::INFINITY, + }); + } + Some(m) + } + Expr::Case { + conditions, + else_result, + .. + } => { + let mut m = f64::INFINITY; + for w in conditions { + m = m.min(min_domain_margin(&w.condition, x, y)?); + m = m.min(min_domain_margin(&w.result, x, y)?); + } + if let Some(er) = else_result { + m = m.min(min_domain_margin(er, x, y)?); + } + Some(m) + } + _ => Some(f64::INFINITY), + } +} + // --------------------------------------------------------------------------- // The three property checks, as reusable helpers. // --------------------------------------------------------------------------- @@ -405,6 +476,10 @@ fn fd_failure(rng: &mut Rng, expr_text: &str, d: &Expr, wrt: Var) -> Option Option