From b5903fd9a9eeb34185d997359d3573842d43fa60 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sun, 1 Nov 2026 09:28:55 -0700 Subject: [PATCH] Repeated calculations need summaries that combine without replay --- CONTRIBUTING.md | 7 + docs/AGGREGATOR_DESIGN.md | 234 +++++ lib/aggregate.dart | 17 + lib/onehz.dart | 10 + lib/src/aggregate/aggregator.dart | 271 ++++++ lib/src/aggregate/checkpoint.dart | 39 + lib/src/aggregate/json_support.dart | 64 ++ lib/src/aggregate/laws.dart | 343 +++++++ lib/src/aggregate/primitives.dart | 885 ++++++++++++++++++ lib/src/onehz/incremental_core.dart | 636 +++++++++++++ lib/src/onehz/sleep/sleep.dart | 8 + test/aggregate/aggregator_core_test.dart | 242 +++++ test/aggregate/ewma_prefix_test.dart | 286 ++++++ test/aggregate/group_by_test.dart | 259 +++++ test/aggregate/last_rows_test.dart | 75 ++ test/aggregate/laws_harness_test.dart | 707 ++++++++++++++ test/aggregate/lookback_test.dart | 255 +++++ test/aggregate/primitives_test.dart | 271 ++++++ test/aggregate/sum_of_test.dart | 149 +++ .../onehz/incremental/int_histogram_test.dart | 210 +++++ test/onehz/incremental/lomb_scargle_test.dart | 99 ++ .../incremental/running_moments_test.dart | 71 ++ test/onehz/lomb_scargle_precision_test.dart | 119 +++ test/onehz/support/incremental_compare.dart | 78 ++ test/onehz/support/incremental_fixtures.dart | 113 +++ test/support/aggregator_laws.dart | 128 +++ test/support/checkpoint_support.dart | 60 ++ test/support/fold_law_support.dart | 133 +++ test/support/law_registry.dart | 124 +++ test/support/property.dart | 753 +++++++++++++++ test/support/property_test.dart | 778 +++++++++++++++ 31 files changed, 7424 insertions(+) create mode 100644 docs/AGGREGATOR_DESIGN.md create mode 100644 lib/aggregate.dart create mode 100644 lib/src/aggregate/aggregator.dart create mode 100644 lib/src/aggregate/checkpoint.dart create mode 100644 lib/src/aggregate/json_support.dart create mode 100644 lib/src/aggregate/laws.dart create mode 100644 lib/src/aggregate/primitives.dart create mode 100644 lib/src/onehz/incremental_core.dart create mode 100644 test/aggregate/aggregator_core_test.dart create mode 100644 test/aggregate/ewma_prefix_test.dart create mode 100644 test/aggregate/group_by_test.dart create mode 100644 test/aggregate/last_rows_test.dart create mode 100644 test/aggregate/laws_harness_test.dart create mode 100644 test/aggregate/lookback_test.dart create mode 100644 test/aggregate/primitives_test.dart create mode 100644 test/aggregate/sum_of_test.dart create mode 100644 test/onehz/incremental/int_histogram_test.dart create mode 100644 test/onehz/incremental/lomb_scargle_test.dart create mode 100644 test/onehz/incremental/running_moments_test.dart create mode 100644 test/onehz/lomb_scargle_precision_test.dart create mode 100644 test/onehz/support/incremental_compare.dart create mode 100644 test/onehz/support/incremental_fixtures.dart create mode 100644 test/support/aggregator_laws.dart create mode 100644 test/support/checkpoint_support.dart create mode 100644 test/support/fold_law_support.dart create mode 100644 test/support/law_registry.dart create mode 100644 test/support/property.dart create mode 100644 test/support/property_test.dart diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82abddbe..447291f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,14 @@ There is **one** sleep source (`segmentSleep`) and **one** headline readiness ## Tests +On Linux, `nix develop` supplies the pinned Flutter 3.41.6 / Dart 3.11.4 +toolchain used by Edge, plus Git, Clang, CMake, Ninja, and pkg-config. +The interactive shell enters an FHS environment for native build tools. +For a command without an interactive shell, use `nix develop -c dart test`; +`nix develop -c analytics-fhs ` also supplies the FHS environment. + ```bash +nix develop dart pub get dart analyze dart test diff --git a/docs/AGGREGATOR_DESIGN.md b/docs/AGGREGATOR_DESIGN.md new file mode 100644 index 00000000..2beab003 --- /dev/null +++ b/docs/AGGREGATOR_DESIGN.md @@ -0,0 +1,234 @@ +# Aggregators: mergeable summaries without a rigid monoid + +Goal: make it cheap to turn an analytics calculation into something that can +be computed in chunks (checkpoint/resume, segment recompute, parallel work) +and merged, with one generic law check instead of hand-written chunking tests. + +## Why not a plain monoid + +A textbook monoid on summary values asks for three things that cost speed or +honesty here: + +1. Immutable values combined one element at a time. Folding 86,400 samples a + day through `combine(summary, singleton(x))` allocated per sample: about + 180,000 µs against a 141 µs hand loop + (edge.research/experiments/dart-monoid/REPORT.md). +2. Structural equality of states. Two states can differ (float rounding + order, a field `finish` never reads) and still give the same metric. +3. One context-free operation. Several calculations depend on context known + before any data arrives: session bounds, a bin origin, heart-rate anchors. + +## The contract + +An `Aggregator` is a dictionary of four operations over slices of a +column batch `B`: + +| op | role | +|---|---| +| `S empty()` | fresh identity state | +| `S addSlice(S s, B batch, int start, int end)` | fold rows `[start, end)`; may mutate `s` and return it | +| `S combine(S left, S right)` | merge two adjacent chunks, `left` first; may mutate and return `left`; consumes both | +| `R finish(S s)` | state to result; must not mutate `s` | + +`Batch` is a run of rows stored column-wise (`length`, `copyRange`, +`join`). Missing doubles are NaN, so filtering happens inside tight loops +instead of through per-row closures. `ListBatch` wraps a plain list for +calculations over objects (hypnogram segments); `DoubleColumn` wraps one +`Float64List`. A module defines its own batch when it needs several columns +(`Beats`, `Epochs`, `TimedValues`). + +What this relaxes, each one on purpose: + +- **Slices, not elements.** The unit of `add` is a slice. Generic dispatch + then costs once per slice and the per-row work is a plain loop. +- **Mutation and consumption.** `addSlice` and `combine` reuse their inputs. + The laws are stated on fresh folds, so nothing reuses a consumed state. +- **Equality after `finish`.** Laws compare finished results with a + caller-supplied equality (exact for integer outputs, a tolerance for + floats). State layout is free. +- **Order matters.** `combine` joins a chunk with the one that follows it, so + a state can carry bounded edge context across a seam. No commutativity. +- **Context-indexed instances.** An aggregator may be built from context + (`HypnogramAggregator(sessionStart, sessionEnd)`, + `hrvTimeAggregator(originMs: …)`, `banisterTrimpAggregator(restingHr: …)`). + The laws hold per instance. + +## Laws + +`checkLaws(agg, cases, equals: …)` (lib/src/aggregate/laws.dart) checks, all +through `finish` and `equals`: + +1. left identity: `combine(empty, fold xs) ≈ fold xs` +2. right identity: `combine(fold xs, empty) ≈ fold xs` +3. split: for split points k (every k for short cases), + `combine(fold [0,k), fold [k,n)) ≈ fold xs` +4. associativity on three adjacent slices +5. singletons: left-combining one-row folds ≈ `fold xs` +6. slicing: one state fed random consecutive sub-slices ≈ `fold xs` (the + `addSlice`-after-data path, separate from `combine`) +7. finishPure: finishing a state twice gives the same result, and the state + keeps working afterwards +8. roundTrip (persistable aggregators only): see Checkpoints + +Law 3 is what the app needs: any chunking gives the same metric. The harness +tests itself against deliberately broken aggregators +(test/aggregate/laws_harness_test.dart), so a lawful verdict means something. +In tests, `expectLawful(agg, cases, equals: …)` is the one-liner. + +## Checkpoints + +A state can be stored and resumed. `encodeState` / `decodeState` turn a +state into JSON and back; `persistable` says whether an aggregator supports +it (a composite is persistable when all its children are). Doubles round-trip +exactly; NaN and ±Infinity are written as the strings `"NaN"`, `"Infinity"`, +`"-Infinity"`. Decoders throw `FormatException` on anything malformed. + +`encodeCheckpoint(agg, state, kind:, version:)` wraps a state with a kind and +version. `decodeCheckpoint` returns null, and never guesses, when the kind or +version differs or the payload is damaged; the caller then recomputes from +raw data. Bump the version whenever a module's state layout or arithmetic +changes. + +When an aggregator is persistable, `checkLaws` adds the law `roundTrip`: a state that went through JSON combines and continues exactly +like the original. So every `expectConversion` call also checks persistence. + +## Building blocks + +All lawful, so anything assembled from them is lawful too: + +- counts and sums: `Count`, `CountFinite`, `CountWhere`, `Sum` + (Neumaier-compensated), `SumSq`, `SumOf(col, f)` (sum of `f(x)`, NaN skips) +- extremes and edges: `Min`, `Max`, `First`, `Last` +- moments: `Moments` (wraps `RunningMoments`) +- composition: `both(a, b)` (mutable pair state, no per-row allocation), + `.map(f)` on the result +- grouping: `GroupBy(key, inner)` (consecutive equal keys reach `inner` as one + slice; per-key order kept) +- suffixes: `LastRows(k)` (the last k rows of the whole input, kept + verbatim) +- prefixes: `Prefix(k, head, tail)` (the first k rows of the whole input go + to `head`, kept verbatim, at most k rows; every later row goes to `tail`) +- recurrences: `Ewma(col, λ)` finishes as an `Affine` map (see below) +- neighbour context: `Lookback(k, inner)`. `inner` processes every length-k + window inside each slice it is given. `Lookback` keeps the first and last + k−1 rows and hands `inner` the seam rows on `combine` and on + `addSlice`-after-data, so each window is counted exactly once under any + chunking. `Lookback(2)` gives successive differences, `Lookback(3)` lag + pairs, and `Lookback(epochsPerDay + 1)` the SRI day-to-day comparison. + +Where no block fits, a small hand-written inner (an `Aggregator` with a tight +loop) is fine. It is still checked by the same laws. + +## Two ways to make an "unmergeable" calculation mergeable + +Earlier experiments filed some calculations as not mergeable. Two +techniques moved them across: + +- **Summarise a function, not a value.** An EWMA step + `c ← (1 − λ)c + λx` is an affine map of the running value. Affine maps + compose associatively, so a chunk's summary is the composed map + `(scale, offset)`, and `finish` applies it to the seed. CTL/ATL/TSB is + `both(Count, Prefix(prime, mean, both(Ewma, Ewma)))`. +- **Summarise a distribution exactly when its support is small.** + Time-in-zone credits each sample with `min(gap, m)`, where `m` is the + median gap of the whole stream. The state keeps exact histograms of gap + values, which are bounded by the number of distinct gaps (a handful on a + 1 Hz stream), and the median and crediting move to `finish`. The median is + bit-identical to the batch code. + +## Converting a module: recipe + +1. Pick the batch: an existing one, or a small `Batch` subclass with one typed + column per input. +2. List the sufficient statistics `finish` needs. If they're sums, counts, + extremes or moments, use the blocks; if they need neighbours, wrap them in + `Lookback(k)`; if they're per bin or per day, wrap them in `GroupBy`. +3. Write `finish` (`.map`) from the old function's formulas, including its + absent cases and notes. +4. Tests: one `expectConversion(build, cases, reference)` call checks parity + with the old function on every evaluation path (whole, chunked, + right-nested, copies, sliced, singletons) and every law, including + `roundTrip` once the aggregator is persistable. Add hand-worked value and + absent-case tests as usual. +5. Integer outputs: rewire the old function in place to run the aggregator, + and keep a frozen copy of the old body in the test. Float outputs: keep + the aggregator additive beside the old function, so no published value + moves and no kAlgoVersion bump is needed. + +## Results so far + +| module | how | parity | shape | +|---|---|---|---| +| hypnogram metrics | rewired in place | exact | hand inner over `ListBatch`; wake spans kept as a list (clipped at finish) | +| HRV time domain | additive | 1e-9 | `Lookback(2)`, `Lookback(3)`, `Moments`, `GroupBy` over `Beats` | +| Phillips SRI | rewired in place | exact | `Lookback(epochsPerDay + 1)` + `GroupBy` per day pair over `Epochs` | +| Banister TRIMP | additive | 1e-12 | `both(Count, SumOf)` | +| cosinor | additive | 1e-9 (R² 1e-7) | one inner computing cos/sin once per row; centred co-moments | +| time in zone | additive | 1e-12 | `Lookback(2)` + exact gap histograms; median at finish | +| CTL/ATL/TSB | additive | 1e-9 | `Count`, `Prefix`, `Sum`, two `Ewma` maps | +| readiness lnRMSSD | rewired in place | exact | `Count` + `LastRows(windowDays + 1)` | +| PRSA DC/AC | additive | 1e-9 | `Count` + `Lookback(2l + 1)` with a profile-sum inner | +| circadian IS/IV/M10/L5/RA | additive | 1e-12 int data, 1e-9 real | `Moments`, `GroupBy`, `Lookback(2)` + one 15-line inner | + +Every module above is persistable and resumes from a mid-stream checkpoint. +The circadian conversion, done by following the recipe, took a subagent +about nine tool calls: a 134-line aggregator and a 166-line test. + +Speed, AOT, Dart 3.11.4 on this box (`tool/bench_aggregate.dart`): + +| 30,000 beats, HRV time domain | µs | +|---|---:| +| `hrvTime()` (current batch code) | 4,956 | +| hand loop, same statistics | 639 | +| composed, one slice | 1,281 | +| composed, 24 slices + combine | 1,209 | +| composed, one-row slices | 10,106 | +| per-element composed (earlier API) | ~24,800 | + +The composed version is about 2× a hand loop and about 4× faster than the +current batch function. Chunking costs nothing extra. (It was 1.7× before the +review fixes switched the lag-1 ACF to centred co-moments, which cost one +division per row and removed a cancellation bug.) + +## Proposed next step: corrected HRV end to end + +Every HRV aggregator here starts from already-corrected NN intervals. RR +artifact correction is the upstream blocker the earlier experiments named. +The package already has a streaming `RrCorrector` +(lib/src/onehz/foundations/rr_correction_stream.dart): a left-to-right fold +whose output for a beat is final about 2H beats (H = windowBeats ~/ 2) plus +the current artifact run plus two normal beats behind the newest beat, and +whose checkpoint holds that window, not the series. + +That suggests a mergeable "sandwich" summary for raw RR: + +- raw head: the first beats of a chunk whose correction still depends on the + chunk before it (window, carried class, left normal pair); +- settled middle: corrected beats, folded straight into a downstream + aggregator (HRV time, PRSA, …); +- raw tail: the last beats whose correction still depends on the chunk after. + +`combine` re-runs correction on `left.tail ++ right.head` only, feeds the +newly settled beats to the downstream aggregator, and keeps the remainder as +the new head/tail. That is `Lookback` with a transducer in front of the +inner aggregator, and it is bounded only if artifact runs are bounded. So it +needs a cap on run length (a longer run becomes a gap: honest absence), which +changes output and needs a kAlgoVersion bump. Owner decision. + +## Known limits + +- `Lookback` copies k−1 rows per seam. With large k and tiny slices this + dominates: SRI over 7 days at 1,440 epochs/day takes 0.2 ms as one slice, + 0.5 ms in hourly slices, 17.7 ms in one-epoch slices. Feed slices of at + least k rows, or give `Lookback` an appendable edge buffer if a caller + can't. +- Still batch-only: exact medians and quantiles over floats, pairwise + statistics (sample entropy), Viterbi backtrack, and anything whose context + is unbounded (current RR artifact correction). These need a bounded variant + behind a kAlgoVersion bump, or stay batch. +- Still hard: `nocturnalRhr` slides a TIME window whose qualifying rule + depends on the stream-wide median cadence. A mergeable version would keep, + per possible valid-sample count, the lowest window mean (bounded by the + window length) plus a time-based lookback block, which doesn't exist yet. +- Float results are equal within a tolerance, not bit-identical, across + chunkings. That is why float modules stay additive. diff --git a/lib/aggregate.dart b/lib/aggregate.dart new file mode 100644 index 00000000..d6cae607 --- /dev/null +++ b/lib/aggregate.dart @@ -0,0 +1,17 @@ +/// Mergeable summaries and the building blocks used to compose them. +library aggregate; + +export 'src/aggregate/aggregator.dart'; +export 'src/aggregate/checkpoint.dart'; +export 'src/aggregate/laws.dart'; +export 'src/aggregate/primitives.dart'; + + + + + + + + + + diff --git a/lib/onehz.dart b/lib/onehz.dart index 308538be..ea5d9a40 100644 --- a/lib/onehz.dart +++ b/lib/onehz.dart @@ -18,6 +18,7 @@ export 'src/onehz/device.dart'; // Layer 0: foundations. export 'src/onehz/foundations/rr_correction.dart'; + export 'src/onehz/foundations/baseline.dart'; export 'src/onehz/foundations/ewma_baselines.dart'; export 'src/onehz/foundations/fusion.dart'; @@ -33,6 +34,8 @@ export 'src/onehz/clinical/cosinor.dart'; export 'src/onehz/clinical/load_trimp.dart'; export 'src/onehz/clinical/stress_si.dart'; export 'src/onehz/clinical/irregular_rhythm.dart'; + + export 'src/onehz/clinical/cardiac_coherence.dart'; export 'src/onehz/clinical/vo2max.dart'; @@ -43,3 +46,10 @@ export 'src/onehz/motion/motion.dart'; export 'src/onehz/workout/workout.dart'; export 'src/onehz/wellness/wellness.dart'; export 'src/onehz/human/human.dart'; + + + +// Calculation-group registry (design 06): the groups this package declares. + + +export 'src/onehz/incremental_core.dart'; diff --git a/lib/src/aggregate/aggregator.dart b/lib/src/aggregate/aggregator.dart new file mode 100644 index 00000000..7763f979 --- /dev/null +++ b/lib/src/aggregate/aggregator.dart @@ -0,0 +1,271 @@ +import 'dart:typed_data'; + +import 'json_support.dart'; + +/// A run of rows stored column-wise. +abstract class Batch> { + int get length; + B copyRange(int start, int end); + B join(B other); + + /// A JSON-encodable copy of the rows. Throws [UnsupportedError] unless the + /// batch type implements it. + Object? toJson() => + throw UnsupportedError('$runtimeType does not support toJson'); +} + +/// How a [ListBatch] writes and reads one row. [decode] must throw +/// [FormatException] on a malformed row. +class RowCodec { + const RowCodec(this.encode, this.decode); + + final Object? Function(A row) encode; + final A Function(Object? json) decode; + + /// Reads a coded batch from the JSON list written by [ListBatch.toJson]. + ListBatch batchFromJson(Object? json) => + ListBatch.coded([for (final row in decodeList(json)) decode(row)], this); +} + +/// A batch backed by a normal Dart list. +class ListBatch extends Batch> { + ListBatch(this.rows) : _codec = null; + + /// A batch whose rows can be written with [codec] (see [toJson]). + ListBatch.coded(this.rows, RowCodec codec) : _codec = codec; + + final List rows; + final RowCodec? _codec; + + @override + int get length => rows.length; + + @override + ListBatch copyRange(int start, int end) => + ListBatch._(rows.sublist(start, end), _codec); + + @override + ListBatch join(ListBatch other) => + ListBatch._([...rows, ...other.rows], _codec ?? other._codec); + + ListBatch._(this.rows, this._codec); + + /// The encoded rows. Throws [UnsupportedError] for a batch without a codec. + @override + Object? toJson() { + final codec = _codec; + if (codec == null) { + throw UnsupportedError('ListBatch has no row codec; use ListBatch.coded'); + } + return [for (final row in rows) codec.encode(row)]; + } +} + +/// A batch containing one double column. NaN represents a missing row. +class DoubleColumn extends Batch { + DoubleColumn(this.values); + + final Float64List values; + + @override + int get length => values.length; + + @override + DoubleColumn copyRange(int start, int end) { + final copied = Float64List(end - start) + ..setRange(0, end - start, values, start); + return DoubleColumn(copied); + } + + @override + DoubleColumn join(DoubleColumn other) { + final joined = Float64List(length + other.length); + joined.setRange(0, length, values); + joined.setRange(length, joined.length, other.values); + return DoubleColumn(joined); + } + + /// The values as a JSON list; NaN and infinities become strings. + @override + Object? toJson() => encodeDoubles(values); + + factory DoubleColumn.fromJson(Object? json) => DoubleColumn(decodeDoubles(json)); +} + +/// A mergeable summary over slices of a [Batch]. +abstract class Aggregator, R> { + const Aggregator(); + + S empty(); + S addSlice(S state, B batch, int start, int end); + S combine(S left, S right); + R finish(S state); + + /// Whether [encodeState] and [decodeState] are implemented. A composite is + /// persistable only when every child is. + bool get persistable => false; + + /// A JSON-encodable form of [state] (null, bool, num, String, List, or a + /// Map with String keys). Throws [UnsupportedError] unless [persistable]. + Object? encodeState(S state) => + throw UnsupportedError('$runtimeType is not persistable'); + + /// Inverse of [encodeState]. Throws [FormatException] on malformed input, + /// and [UnsupportedError] unless [persistable]. + S decodeState(Object? json) => + throw UnsupportedError('$runtimeType is not persistable'); +} + +/// Convenience operations and result adapters for [Aggregator]. +extension AggregatorOps, R> on Aggregator { + S fold(B batch, [int start = 0, int? end]) { + final actualEnd = end ?? batch.length; + return addSlice(empty(), batch, start, actualEnd); + } + + R run(B batch) => finish(fold(batch)); + + Aggregator map(R2 Function(R result) transform) => + _MappedAggregator(this, transform); +} + +/// Element-world convenience for aggregators over [ListBatch]. +extension ListAggregatorOps on Aggregator, R> { + R runList(List rows) => run(ListBatch(rows)); +} + +class _MappedAggregator, R, R2> + extends Aggregator { + const _MappedAggregator(this.inner, this.transform); + + final Aggregator inner; + final R2 Function(R result) transform; + + @override + S empty() => inner.empty(); + + @override + S addSlice(S state, B batch, int start, int end) => + inner.addSlice(state, batch, start, end); + + @override + S combine(S left, S right) => inner.combine(left, right); + + @override + R2 finish(S state) => transform(inner.finish(state)); + + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(S state) => inner.encodeState(state); + + @override + S decodeState(Object? json) => inner.decodeState(json); +} + +/// Mutable state for [both]. +class Pair { + Pair(this.first, this.second); + + A first; + B second; +} + +/// Runs two aggregators over each same slice. +Aggregator, B, (R1, R2)> + both, R1, R2>( + Aggregator first, + Aggregator second, +) => + _BothAggregator(first, second); + +/// Adds module-specific consistency checks to a persistable checkpoint. +/// +/// [validate] runs after [inner] has decoded each child state. It receives the +/// encoded state so a composed module can compare child values whose concrete +/// state types are private to their aggregators. Encoding and hot paths are +/// otherwise unchanged. +Aggregator validateCheckpoint, R>( + Aggregator inner, + void Function(Object? json) validate, +) => + _CheckpointValidatedAggregator(inner, validate); + +class _CheckpointValidatedAggregator, R> + extends Aggregator { + const _CheckpointValidatedAggregator(this.inner, this.validate); + + final Aggregator inner; + final void Function(Object? json) validate; + + @override + S empty() => inner.empty(); + + @override + S addSlice(S state, B batch, int start, int end) => + inner.addSlice(state, batch, start, end); + + @override + S combine(S left, S right) => inner.combine(left, right); + + @override + R finish(S state) => inner.finish(state); + + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(S state) => inner.encodeState(state); + + @override + S decodeState(Object? json) { + final state = inner.decodeState(json); + validate(json); + return state; + } +} + +class _BothAggregator, R1, R2> + extends Aggregator, B, (R1, R2)> { + const _BothAggregator(this.first, this.second); + + final Aggregator first; + final Aggregator second; + + @override + Pair empty() => Pair(first.empty(), second.empty()); + + @override + Pair addSlice(Pair state, B batch, int start, int end) { + state.first = first.addSlice(state.first, batch, start, end); + state.second = second.addSlice(state.second, batch, start, end); + return state; + } + + @override + Pair combine(Pair left, Pair right) { + left.first = first.combine(left.first, right.first); + left.second = second.combine(left.second, right.second); + return left; + } + + @override + (R1, R2) finish(Pair state) => + (first.finish(state.first), second.finish(state.second)); + + @override + bool get persistable => first.persistable && second.persistable; + + @override + Object? encodeState(Pair state) => + [first.encodeState(state.first), second.encodeState(state.second)]; + + @override + Pair decodeState(Object? json) { + final list = decodeList(json); + if (list.length != 2) throw const FormatException('Expected two states'); + return Pair( + first.decodeState(list[0]), second.decodeState(list[1])); + } +} diff --git a/lib/src/aggregate/checkpoint.dart b/lib/src/aggregate/checkpoint.dart new file mode 100644 index 00000000..b27dbb76 --- /dev/null +++ b/lib/src/aggregate/checkpoint.dart @@ -0,0 +1,39 @@ +import 'aggregator.dart'; + +/// Wraps the JSON form of [state] in a versioned envelope: +/// {'kind', 'version', 'state'}. Throws [UnsupportedError] if [agg] is not +/// persistable. +Map encodeCheckpoint( + Aggregator agg, + S state, { + required String kind, + required int version, +}) { + if (!agg.persistable) { + throw UnsupportedError('${agg.runtimeType} is not persistable'); + } + return {'kind': kind, 'version': version, 'state': agg.encodeState(state)}; +} + +/// Restores a state written by [encodeCheckpoint]. Returns null, never a +/// guess, when the kind or version differ or the envelope or state is +/// malformed; the caller then recomputes from raw data. +S? decodeCheckpoint( + Aggregator agg, + Object? json, { + required String kind, + required int version, +}) { + if (json is! Map || + json['kind'] != kind || + json['version'] is! int || + json['version'] != version || + !json.containsKey('state')) { + return null; + } + try { + return agg.decodeState(json['state']); + } on FormatException { + return null; + } +} diff --git a/lib/src/aggregate/json_support.dart b/lib/src/aggregate/json_support.dart new file mode 100644 index 00000000..911521b3 --- /dev/null +++ b/lib/src/aggregate/json_support.dart @@ -0,0 +1,64 @@ +// JSON helpers shared by the aggregate persistence codecs. Not exported. +// +// Non-finite doubles are written as the strings 'NaN', 'Infinity' and +// '-Infinity' (the same strings incremental_core.dart reads). Every reader +// throws FormatException, never TypeError or RangeError. + +import 'dart:typed_data'; + +/// A double as a JSON value: finite stays a number, the rest become strings. +Object encodeDouble(double value) { + if (value.isFinite) return value; + if (value.isNaN) return 'NaN'; + return value > 0 ? 'Infinity' : '-Infinity'; +} + +/// Inverse of [encodeDouble]; an int is accepted for a double. +double decodeDouble(Object? value) { + if (value is num) return value.toDouble(); + if (value == 'NaN') return double.nan; + if (value == 'Infinity') return double.infinity; + if (value == '-Infinity') return double.negativeInfinity; + throw const FormatException('Invalid number'); +} + +double? decodeDoubleOrNull(Object? value) => + value == null ? null : decodeDouble(value); + +/// A non-negative int. +int decodeCount(Object? value) { + if (value is! int || value < 0) throw const FormatException('Invalid count'); + return value; +} + +List decodeList(Object? value) { + if (value is! List) throw const FormatException('Expected a list'); + return value; +} + +/// A JSON object with string keys. +Map decodeMap(Object? value) { + if (value is! Map) throw const FormatException('Expected an object'); + for (final key in value.keys) { + if (key is! String) throw const FormatException('Expected string keys'); + } + return value.cast(); +} + +/// The value at [key]; the key must be present (null is a value). +Object? field(Map map, String key) { + if (!map.containsKey(key)) throw FormatException('Missing field $key'); + return map[key]; +} + +List encodeDoubles(Float64List values) => + [for (final value in values) encodeDouble(value)]; + +Float64List decodeDoubles(Object? value) { + final list = decodeList(value); + final result = Float64List(list.length); + for (var i = 0; i < list.length; i++) { + result[i] = decodeDouble(list[i]); + } + return result; +} diff --git a/lib/src/aggregate/laws.dart b/lib/src/aggregate/laws.dart new file mode 100644 index 00000000..ba92b232 --- /dev/null +++ b/lib/src/aggregate/laws.dart @@ -0,0 +1,343 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'aggregator.dart'; + +typedef ResultEquals = bool Function(R left, R right); + +class LawViolation { + const LawViolation(this.law, this.detail); + final String law; + final String detail; +} + +/// Checks aggregator merge and slicing laws on each batch in [cases]. +List checkLaws, R>( + Aggregator aggregator, + List cases, { + ResultEquals? equals, + int maxSplits = 64, + int seed = 0, +}) { + final equal = equals ?? (R left, R right) => left == right; + final violations = []; + + R whole(B batch) => aggregator.finish(aggregator.fold(batch)); + R joined(B batch, int split) => aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, split), + aggregator.fold(batch, split, batch.length))); + + for (var caseIndex = 0; caseIndex < cases.length; caseIndex++) { + final batch = cases[caseIndex]; + final detail = 'case $caseIndex: $batch'; + final expected = whole(batch); + final leftIdentity = aggregator.finish( + aggregator.combine(aggregator.empty(), aggregator.fold(batch))); + if (!equal(leftIdentity, expected)) { + violations.add(LawViolation('leftIdentity', detail)); + } + final rightIdentity = aggregator.finish( + aggregator.combine(aggregator.fold(batch), aggregator.empty())); + if (!equal(rightIdentity, expected)) { + violations.add(LawViolation('rightIdentity', detail)); + } + + final splits = _splitPoints(batch.length, maxSplits, seed + caseIndex); + for (final split in splits) { + if (!equal(joined(batch, split), expected)) { + violations.add(LawViolation('split', '$detail; split $split')); + } + } + for (var i = 0; i < splits.length; i++) { + for (var j = i; j < splits.length; j++) { + final first = splits[i]; + final second = splits[j]; + final left = aggregator.finish(aggregator.combine( + aggregator.combine(aggregator.fold(batch, 0, first), + aggregator.fold(batch, first, second)), + aggregator.fold(batch, second, batch.length))); + final right = aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, first), + aggregator.combine(aggregator.fold(batch, first, second), + aggregator.fold(batch, second, batch.length)))); + if (!equal(left, right)) { + violations.add( + LawViolation('associativity', '$detail; splits $first, $second')); + } + } + } + + var singletonState = aggregator.empty(); + for (var i = 0; i < batch.length; i++) { + singletonState = aggregator.combine( + singletonState, aggregator.fold(batch, i, i + 1)); + } + if (!equal(aggregator.finish(singletonState), expected)) { + violations.add(LawViolation('singletons', detail)); + } + + var sliced = aggregator.empty(); + final slicePoints = _splitPoints(batch.length, maxSplits, seed ^ caseIndex); + for (var i = 0; i + 1 < slicePoints.length; i++) { + sliced = aggregator.addSlice(sliced, batch, slicePoints[i], slicePoints[i + 1]); + } + if (!equal(aggregator.finish(sliced), expected)) { + violations.add(LawViolation('slicing', detail)); + } + + // finish must not consume or otherwise close a state. Keep separate + // states for the add and combine continuations: combine is allowed to + // consume either input. + final continuationStart = batch.length ~/ 2; + var afterFinish = aggregator.fold(batch, 0, continuationStart); + final firstFinish = aggregator.finish(afterFinish); + final secondFinish = aggregator.finish(afterFinish); + var finishIsPure = equal(firstFinish, secondFinish); + + afterFinish = aggregator.addSlice( + afterFinish, batch, continuationStart, batch.length); + var addReference = aggregator.fold(batch, 0, continuationStart); + addReference = aggregator.addSlice( + addReference, batch, continuationStart, batch.length); + if (!equal(aggregator.finish(afterFinish), aggregator.finish(addReference))) { + finishIsPure = false; + } + + var combinedAfterFinish = aggregator.fold(batch, 0, continuationStart); + aggregator.finish(combinedAfterFinish); + aggregator.finish(combinedAfterFinish); + combinedAfterFinish = aggregator.combine( + combinedAfterFinish, + aggregator.fold(batch, continuationStart, batch.length)); + var combineReference = aggregator.fold(batch, 0, continuationStart); + combineReference = aggregator.combine( + combineReference, + aggregator.fold(batch, continuationStart, batch.length)); + if (!equal(aggregator.finish(combinedAfterFinish), + aggregator.finish(combineReference))) { + finishIsPure = false; + } + if (!finishIsPure) { + violations.add(LawViolation('finishPure', detail)); + } + + if (aggregator.persistable) { + // A state written to JSON text and read back must behave like the + // original: alone, on either side of a merge, and when extended. + S roundTrip(S state) => aggregator.decodeState( + jsonDecode(jsonEncode(aggregator.encodeState(state)))); + void check(String what, R Function() actual) { + try { + if (!equal(actual(), expected)) { + violations.add(LawViolation('roundTrip', '$detail; $what')); + } + } catch (error) { + violations.add(LawViolation('roundTrip', '$detail; $what: $error')); + } + } + + check('whole state', () => aggregator.finish(roundTrip(aggregator.fold(batch)))); + for (final split in splits) { + check( + 'combine after split $split (restored left)', + () => aggregator.finish(aggregator.combine( + roundTrip(aggregator.fold(batch, 0, split)), + aggregator.fold(batch, split, batch.length)))); + check( + 'combine after split $split (restored right)', + () => aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, split), + roundTrip(aggregator.fold(batch, split, batch.length))))); + check( + 'combine after split $split (restored both)', + () => aggregator.finish(aggregator.combine( + roundTrip(aggregator.fold(batch, 0, split)), + roundTrip(aggregator.fold(batch, split, batch.length))))); + check( + 'addSlice after split $split', + () => aggregator.finish(aggregator.addSlice( + roundTrip(aggregator.fold(batch, 0, split)), + batch, + split, + batch.length))); + } + } + } + return violations; +} + +class ConversionFailure { + const ConversionFailure(this.path, this.caseIndex, this.detail); + final String path; + final int caseIndex; + final String detail; +} + +/// Checks that an aggregator reproduces [reference] on every evaluation path. +/// +/// Paths: 'whole', 'chunked' (left-to-right combine of ranges of the original +/// batch), 'chunkedRight' (same bounds, right-nested combine), 'copies' (each +/// chunk cut with copyRange first), 'sliced' (one state, consecutive slices) +/// and 'singletons'. With [laws], checkLaws violations appear as 'law:'. +/// Deterministic per [seed]. +List checkConversion, R>( + Aggregator Function(B input) build, + List cases, + R Function(B input) reference, { + ResultEquals? equals, + int chunkings = 8, + int seed = 0, + bool laws = true, +}) { + final equal = equals ?? (R left, R right) => left == right; + final failures = []; + + for (var caseIndex = 0; caseIndex < cases.length; caseIndex++) { + final batch = cases[caseIndex]; + final aggregator = build(batch); + final expected = reference(batch); + final seen = {}; + + void check(String path, R Function() actual, [String where = '']) { + if (seen.contains(path)) return; + final got = actual(); + if (equal(got, expected)) return; + seen.add(path); + failures.add(ConversionFailure(path, caseIndex, + 'case $caseIndex: $batch$where; got $got, expected $expected')); + } + + check('whole', () => aggregator.finish(aggregator.fold(batch))); + + final rng = _Lcg(seed * 7919 + caseIndex); + for (var c = 0; c < chunkings; c++) { + final bounds = _chunkBounds(batch.length, rng); + final where = '; bounds $bounds'; + List states({bool copy = false}) => [ + for (var i = 0; i + 1 < bounds.length; i++) + copy + ? aggregator.fold(batch.copyRange(bounds[i], bounds[i + 1])) + : aggregator.fold(batch, bounds[i], bounds[i + 1]), + ]; + S leftToRight(List parts) { + var acc = parts.first; + for (var i = 1; i < parts.length; i++) { + acc = aggregator.combine(acc, parts[i]); + } + return acc; + } + + check('chunked', () => aggregator.finish(leftToRight(states())), where); + check('chunkedRight', () { + final parts = states(); + var acc = parts.last; + for (var i = parts.length - 2; i >= 0; i--) { + acc = aggregator.combine(parts[i], acc); + } + return aggregator.finish(acc); + }, where); + check('copies', + () => aggregator.finish(leftToRight(states(copy: true))), where); + check('sliced', () { + var state = aggregator.empty(); + for (var i = 0; i + 1 < bounds.length; i++) { + state = aggregator.addSlice(state, batch, bounds[i], bounds[i + 1]); + } + return aggregator.finish(state); + }, where); + } + + check('singletons', () { + var state = aggregator.empty(); + for (var i = 0; i < batch.length; i++) { + state = aggregator.combine(state, aggregator.fold(batch, i, i + 1)); + } + return aggregator.finish(state); + }); + + if (laws) { + for (final v in checkLaws(aggregator, [batch], + equals: equals, seed: seed + caseIndex)) { + failures.add(ConversionFailure( + 'law:${v.law}', caseIndex, v.detail.replaceFirst('case 0', 'case $caseIndex'))); + } + } + } + return failures; +} + +class _Lcg { + _Lcg(int seed) : _state = seed & 0x7fffffff; + int _state; + int next(int bound) { + _state = (_state * 1103515245 + 12345) & 0x7fffffff; + return (_state >> 8) % bound; + } +} + +/// Sorted chunk bounds from 0 to [length]: always cuts at 1 and length - 1 and +/// a duplicated cut (an empty chunk), plus a few random cuts. +List _chunkBounds(int length, _Lcg rng) { + final cuts = [ + 0, + length, + math.min(1, length), + math.max(0, length - 1), + for (var i = rng.next(4); i > 0; i--) rng.next(length + 1), + ]; + cuts.add(cuts[2 + rng.next(cuts.length - 2)]); + return cuts..sort(); +} + +List _splitPoints(int length, int maxSplits, int seed) { + if (length == 0) return [0]; + final wanted = math.max(2, maxSplits); + if (length + 1 <= wanted) return [for (var i = 0; i <= length; i++) i]; + final points = {0, length}; + var state = seed & 0x7fffffff; + while (points.length < wanted) { + state = (state * 1103515245 + 12345) & 0x7fffffff; + points.add(1 + state % (length - 1)); + } + return points.toList()..sort(); +} + +/// Compares the listed fields pairwise; numbers use the tolerances and lists +/// compare element by element with the same rules. +ResultEquals fieldsEqual( + List Function(R value) fields, { + double relTol = 0, + double absTol = 0, +}) => + (left, right) { + final a = fields(left); + final b = fields(right); + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_fieldEqual(a[i], b[i], relTol, absTol)) return false; + } + return true; + }; + +bool _fieldEqual(Object? left, Object? right, double relTol, double absTol) { + // Converting two ints to double loses distinctions above 2^53. + if (left is int && right is int) return left == right; + if (left is List || right is List) { + if (left is! List || right is! List || left.length != right.length) { + return false; + } + for (var i = 0; i < left.length; i++) { + if (!_fieldEqual(left[i], right[i], relTol, absTol)) return false; + } + return true; + } + if (left == right) return true; + if (left == null || right == null) return false; + if (left is! num || right is! num) return false; + if (left.isNaN || right.isNaN) return left.isNaN && right.isNaN; + if (!left.isFinite || !right.isFinite) return false; + final a = left.toDouble(); + final b = right.toDouble(); + final tolerance = absTol + relTol * math.max(a.abs(), b.abs()); + return (a - b).abs() <= tolerance; +} diff --git a/lib/src/aggregate/primitives.dart b/lib/src/aggregate/primitives.dart new file mode 100644 index 00000000..d2ddb26b --- /dev/null +++ b/lib/src/aggregate/primitives.dart @@ -0,0 +1,885 @@ +// Mergeable sums, centered moments and affine EWMA composition reformulate +// the corresponding batch arithmetic. Composition is algebraically equal; +// floating-point evaluation order can differ. Count/min/max and structural +// wrappers are definitional operations, not additional physiological estimators. + +import 'dart:typed_data'; + +import '../onehz/incremental_core.dart' show RunningMoments; + +import 'aggregator.dart'; +import 'json_support.dart'; + +/// Counts every row, including rows whose column value is NaN. +class Count> extends Aggregator { + const Count(); + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) => state + end - start; + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Counts non-missing values in a double column. +class CountFinite> extends Aggregator { + const CountFinite(this.column); + final Float64List Function(B batch) column; + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) { + final values = column(batch); + var count = state; + for (var i = start; i < end; i++) { + if (!values[i].isNaN) count++; + } + return count; + } + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Counts non-missing values in a double column which pass [test]. +class CountWhere> extends Aggregator { + const CountWhere(this.column, this.test); + final Float64List Function(B batch) column; + final bool Function(double value) test; + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) { + final values = column(batch); + var count = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && test(value)) count++; + } + return count; + } + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Neumaier-compensated sum of non-missing double column values. +/// +/// Chunk-invariant results are supported while all partial sums are finite. +/// Once a partial sum overflows, IEEE-754 arithmetic can make the result +/// depend on chunking. +class Sum> extends Aggregator<_SumState, B, double> { + const Sum(this.column); + final Float64List Function(B batch) column; + @override + _SumState empty() => _SumState(); + @override + _SumState addSlice(_SumState state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) state.add(value); + } + return state; + } + @override + _SumState combine(_SumState left, _SumState right) { + left.add(right.sum); + left.add(right.correction); + return left; + } + @override + double finish(_SumState state) => state.sum + state.correction; + @override + bool get persistable => true; + + @override + Object? encodeState(_SumState state) => state.toJson(); + + @override + _SumState decodeState(Object? json) => _SumState.fromJson(json); +} + +/// Neumaier-compensated sum of a function of non-missing double column values. +/// +/// A NaN source value is skipped without calling [transform]. A NaN result from +/// [transform] is also skipped. +/// Chunk-invariant results are supported while all partial sums are finite. +/// Once a partial sum overflows, IEEE-754 arithmetic can make the result +/// depend on chunking. +class SumOf> extends Aggregator<_SumState, B, double> { + const SumOf(this.column, this.transform); + + final Float64List Function(B batch) column; + final double Function(double value) transform; + + @override + _SumState empty() => _SumState(); + + @override + _SumState addSlice(_SumState state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (value.isNaN) continue; + final transformed = transform(value); + if (!transformed.isNaN) state.add(transformed); + } + return state; + } + + @override + _SumState combine(_SumState left, _SumState right) { + left.add(right.sum); + left.add(right.correction); + return left; + } + + @override + double finish(_SumState state) => state.sum + state.correction; + @override + bool get persistable => true; + + @override + Object? encodeState(_SumState state) => state.toJson(); + + @override + _SumState decodeState(Object? json) => _SumState.fromJson(json); +} + +class _SumState { + _SumState(); + + factory _SumState.fromJson(Object? json) { + final map = decodeMap(json); + return _SumState() + ..sum = decodeDouble(field(map, 'sum')) + ..correction = decodeDouble(field(map, 'correction')); + } + + double sum = 0.0; + double correction = 0.0; + + Map toJson() => + {'sum': encodeDouble(sum), 'correction': encodeDouble(correction)}; + + void add(double value) { + final next = sum + value; + if (!next.isFinite) { + sum = next; + return; + } + if (sum.abs() >= value.abs()) { + correction += (sum - next) + value; + } else { + correction += (value - next) + sum; + } + sum = next; + } +} + +/// Sum of squares of non-missing double column values. +class SumSq> extends Aggregator { + const SumSq(this.column); + final Float64List Function(B batch) column; + @override + double empty() => 0.0; + @override + double addSlice(double state, B batch, int start, int end) { + final values = column(batch); + var sum = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) sum += value * value; + } + return sum; + } + @override + double combine(double left, double right) => left + right; + @override + double finish(double state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double state) => encodeDouble(state); + @override + double decodeState(Object? json) => decodeDouble(json); +} + +/// Smallest non-missing value in a double column. +class Min> extends Aggregator { + const Min(this.column); + final Float64List Function(B batch) column; + @override + double? empty() => null; + @override + double? addSlice(double? state, B batch, int start, int end) { + final values = column(batch); + var result = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && (result == null || value < result)) result = value; + } + return result; + } + @override + double? combine(double? left, double? right) { + if (left == null || right == null) return left ?? right; + return right < left ? right : left; + } + @override + double? finish(double? state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double? state) => + state == null ? null : encodeDouble(state); + @override + double? decodeState(Object? json) => decodeDoubleOrNull(json); +} + +/// Largest non-missing value in a double column. +class Max> extends Aggregator { + const Max(this.column); + final Float64List Function(B batch) column; + @override + double? empty() => null; + @override + double? addSlice(double? state, B batch, int start, int end) { + final values = column(batch); + var result = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && (result == null || value > result)) result = value; + } + return result; + } + @override + double? combine(double? left, double? right) { + if (left == null || right == null) return left ?? right; + return right > left ? right : left; + } + @override + double? finish(double? state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double? state) => + state == null ? null : encodeDouble(state); + @override + double? decodeState(Object? json) => decodeDoubleOrNull(json); +} + +// Delegates updates/removal to Welford 1962 and partition merging to Chan, +// Golub & LeVeque 1983; full references and clamp details: RunningMoments. +/// Centered moments of non-missing double column values. +class Moments> + extends Aggregator { + const Moments(this.column); + final Float64List Function(B batch) column; + @override + RunningMoments empty() => RunningMoments(); + @override + RunningMoments addSlice(RunningMoments state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) state.add(value); + } + return state; + } + @override + RunningMoments combine(RunningMoments left, RunningMoments right) { + left.merge(right); + return left; + } + @override + RunningMoments finish(RunningMoments state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(RunningMoments state) => state.toJson(); + @override + RunningMoments decodeState(Object? json) => + RunningMoments.fromJson(decodeMap(json)); +} + +class _EdgeState> { + B? value; +} + +Object? _encodeEdge>( + _EdgeState state, B Function(Object?)? batchFromJson) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + return {'value': state.value?.toJson()}; +} + +_EdgeState _decodeEdge>( + Object? json, B Function(Object?)? batchFromJson) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + final raw = field(decodeMap(json), 'value'); + final state = _EdgeState(); + if (raw != null) { + final row = batchFromJson(raw); + if (row.length != 1) throw const FormatException('Expected one row'); + state.value = row; + } + return state; +} + +/// The first row as a one-row copy, or null for no rows. +class First> extends Aggregator<_EdgeState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const First({this.batchFromJson}); + final B Function(Object? json)? batchFromJson; + @override + _EdgeState empty() => _EdgeState(); + @override + _EdgeState addSlice(_EdgeState state, B batch, int start, int end) { + if (state.value == null && start < end) { + state.value = batch.copyRange(start, start + 1); + } + return state; + } + @override + _EdgeState combine(_EdgeState left, _EdgeState right) { + if (left.value == null) left.value = right.value; + return left; + } + @override + B? finish(_EdgeState state) => state.value; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_EdgeState state) => _encodeEdge(state, batchFromJson); + @override + _EdgeState decodeState(Object? json) => _decodeEdge(json, batchFromJson); +} + +/// The last row as a one-row copy, or null for no rows. +class Last> extends Aggregator<_EdgeState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const Last({this.batchFromJson}); + final B Function(Object? json)? batchFromJson; + @override + _EdgeState empty() => _EdgeState(); + @override + _EdgeState addSlice(_EdgeState state, B batch, int start, int end) { + if (start < end) state.value = batch.copyRange(end - 1, end); + return state; + } + @override + _EdgeState combine(_EdgeState left, _EdgeState right) { + if (right.value != null) left.value = right.value; + return left; + } + @override + B? finish(_EdgeState state) => state.value; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_EdgeState state) => _encodeEdge(state, batchFromJson); + @override + _EdgeState decodeState(Object? json) => _decodeEdge(json, batchFromJson); +} + +class _RowsState> { + B? rows; +} + +/// The last [k] rows of the whole input as a copy (fewer when the input is +/// shorter), or null for no rows. +class LastRows> extends Aggregator<_RowsState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const LastRows(this.k, {this.batchFromJson}) : assert(k >= 1); + final int k; + final B Function(Object? json)? batchFromJson; + + B _tail(B rows) => + rows.length <= k ? rows : rows.copyRange(rows.length - k, rows.length); + + @override + _RowsState empty() => _RowsState(); + @override + _RowsState addSlice(_RowsState state, B batch, int start, int end) { + if (start >= end) return state; + final from = end - start > k ? end - k : start; + final piece = batch.copyRange(from, end); + final rows = state.rows; + state.rows = rows == null ? piece : _tail(rows.join(piece)); + return state; + } + @override + _RowsState combine(_RowsState left, _RowsState right) { + final l = left.rows; + final r = right.rows; + if (r == null) return left; + left.rows = l == null ? r : _tail(l.join(r)); + return left; + } + @override + B? finish(_RowsState state) => state.rows; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_RowsState state) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + return {'rows': state.rows?.toJson()}; + } + @override + _RowsState decodeState(Object? json) { + final fromJson = batchFromJson; + if (fromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + final raw = field(decodeMap(json), 'rows'); + final state = _RowsState(); + if (raw != null) { + final rows = fromJson(raw); + if (rows.length < 1 || rows.length > k) { + throw const FormatException('Row count out of range'); + } + state.rows = rows; + } + return state; + } +} + +/// Routes consecutive equal keys to [inner] as one slice per run. +class GroupBy, S, R> + extends Aggregator, B, Map> { + const GroupBy(this.key, this.inner); + final int Function(B batch, int row) key; + final Aggregator inner; + @override + Map empty() => {}; + @override + Map addSlice(Map state, B batch, int start, int end) { + if (start == end) return state; + var runStart = start; + var runKey = key(batch, start); + for (var i = start + 1; i < end; i++) { + final nextKey = key(batch, i); + if (nextKey != runKey) { + _addRun(state, runKey, batch, runStart, i); + runStart = i; + runKey = nextKey; + } + } + _addRun(state, runKey, batch, runStart, end); + return state; + } + void _addRun(Map state, int runKey, B batch, int start, int end) { + final previous = state.containsKey(runKey) ? state[runKey] as S : inner.empty(); + state[runKey] = inner.addSlice(previous, batch, start, end); + } + @override + Map combine(Map left, Map right) { + for (final entry in right.entries) { + if (left.containsKey(entry.key)) { + left[entry.key] = inner.combine(left[entry.key] as S, entry.value); + } else { + left[entry.key] = entry.value; + } + } + return left; + } + @override + Map finish(Map state) => { + for (final entry in state.entries) entry.key: inner.finish(entry.value), + }; + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(Map state) => { + for (final entry in state.entries) + entry.key.toString(): inner.encodeState(entry.value), + }; + + @override + Map decodeState(Object? json) { + final map = decodeMap(json); + final result = {}; + for (final entry in map.entries) { + final key = int.tryParse(entry.key); + if (key == null || key.toString() != entry.key) { + throw FormatException('Invalid group key ${entry.key}'); + } + result[key] = inner.decodeState(entry.value); + } + return result; + } +} + +/// Makes an inner length-[k] window aggregator invariant to chunk boundaries. +class Lookback, S, R> + extends Aggregator<_LookbackState, B, R> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]) and [inner] is persistable. + Lookback(this.k, this.inner, {this.batchFromJson}) : assert(k >= 1); + final int k; + final Aggregator inner; + final B Function(Object? json)? batchFromJson; + int get _edge => k - 1; + @override + _LookbackState empty() => _LookbackState(inner.empty()); + @override + _LookbackState addSlice( + _LookbackState state, B batch, int start, int end) { + if (start == end) return state; + if (state.length > 0 && _edge > 0) { + final seam = state.last!.join(batch.copyRange(start, _prefixEnd(start, end))); + state.innerState = inner.addSlice(state.innerState, seam, 0, seam.length); + } + state.innerState = inner.addSlice(state.innerState, batch, start, end); + _appendEdges(state, batch, start, end); + return state; + } + int _prefixEnd(int start, int end) => + start + (end - start < _edge ? end - start : _edge); + void _appendEdges(_LookbackState state, B batch, int start, int end) { + final sliceLength = end - start; + if (_edge == 0) { + state.length += sliceLength; + return; + } + if (state.length == 0) { + state.first = batch.copyRange(start, _prefixEnd(start, end)); + state.last = batch.copyRange( + end - (sliceLength < _edge ? sliceLength : _edge), end); + } else { + if (state.length < _edge) { + final needed = _edge - state.length; + final take = sliceLength < needed ? sliceLength : needed; + if (take > 0) state.first = state.first!.join(batch.copyRange(start, start + take)); + } + if (sliceLength >= _edge) { + state.last = batch.copyRange(end - _edge, end); + } else { + final joined = state.last!.join(batch.copyRange(start, end)); + final keepStart = joined.length > _edge ? joined.length - _edge : 0; + state.last = joined.copyRange(keepStart, joined.length); + } + } + state.length += sliceLength; + } + @override + _LookbackState combine( + _LookbackState left, _LookbackState right) { + if (left.length > 0 && right.length > 0 && _edge > 0) { + final seam = left.last!.join(right.first!); + left.innerState = inner.addSlice(left.innerState, seam, 0, seam.length); + } + left.innerState = inner.combine(left.innerState, right.innerState); + _combineEdges(left, right); + return left; + } + void _combineEdges(_LookbackState left, _LookbackState right) { + if (right.length == 0) return; + if (left.length == 0) { + left.length = right.length; + left.first = right.first; + left.last = right.last; + return; + } + if (_edge > 0) { + if (left.length < _edge) { + final needed = _edge - left.length; + final take = right.length < needed ? right.length : needed; + if (take > 0) left.first = left.first!.join(right.first!.copyRange(0, take)); + } + if (right.length >= _edge) { + left.last = right.last; + } else { + final joined = left.last!.join(right.last!); + final keepStart = joined.length > _edge ? joined.length - _edge : 0; + left.last = joined.copyRange(keepStart, joined.length); + } + } + left.length += right.length; + } + @override + R finish(_LookbackState state) => inner.finish(state.innerState); + + @override + bool get persistable => batchFromJson != null && inner.persistable; + + @override + Object? encodeState(_LookbackState state) { + _requireBatchCodec(); + return { + 'length': state.length, + 'first': state.first?.toJson(), + 'last': state.last?.toJson(), + 'inner': inner.encodeState(state.innerState), + }; + } + + @override + _LookbackState decodeState(Object? json) { + final fromJson = _requireBatchCodec(); + final map = decodeMap(json); + final length = decodeCount(field(map, 'length')); + // Edges hold the first/last min(length, k - 1) rows. + final held = length < _edge ? length : _edge; + B? edge(String name) { + final raw = field(map, name); + if (raw == null) { + if (held != 0) throw FormatException('Missing $name rows'); + return null; + } + if (held == 0) throw FormatException('Unexpected $name rows'); + final rows = fromJson(raw); + if (rows.length != held) { + throw FormatException('Expected $held $name rows'); + } + return rows; + } + + return _LookbackState(inner.decodeState(field(map, 'inner'))) + ..length = length + ..first = edge('first') + ..last = edge('last'); + } + + B Function(Object?) _requireBatchCodec() => + batchFromJson ?? + (throw UnsupportedError('Pass batchFromJson to persist Lookback')); +} + +class _LookbackState, S> { + _LookbackState(this.innerState); + S innerState; + int length = 0; + B? first; + B? last; +} + +/// An affine map x -> scale * x + offset. Composition is associative, so a +/// run of affine steps can be summarised by one map and merged in any grouping. +class Affine { + const Affine(this.scale, this.offset); + + final double scale; + final double offset; + + double apply(double x) => scale * x + offset; + + /// The map that applies this one first, then [next]. + Affine then(Affine next) => Affine( + next.scale * scale, next.scale * offset + next.offset); +} + +class _AffineState { + double scale = 1; + double offset = 0; +} + +// Algebraically equal to the EWMA recurrence: f_x(c)=a*c+lambda*x, a=1-lambda; +// f_y(f_x(c))=a*a*c+a*lambda*x+lambda*y. Floating-point order may differ. +/// An exponentially weighted moving average, as the composed affine map of the +/// running value: each row x applies c -> (1 - lambda) * c + lambda * x. +/// +/// NaN rows are skipped (they act as the identity map), and no rows finish as +/// the identity. ±Infinity input is unsupported because scale can underflow to +/// zero across a seam and yield `0*Infinity`. Apply the result to the seed +/// value to get the final average. +class Ewma> extends Aggregator<_AffineState, B, Affine> { + const Ewma(this.column, this.lambda); + + final Float64List Function(B batch) column; + final double lambda; + + @override + _AffineState empty() => _AffineState(); + + @override + _AffineState addSlice(_AffineState state, B batch, int start, int end) { + final values = column(batch); + final keep = 1 - lambda; + var scale = state.scale; + var offset = state.offset; + for (var i = start; i < end; i++) { + final x = values[i]; + if (x.isNaN) continue; + scale *= keep; + offset = keep * offset + lambda * x; + } + state.scale = scale; + state.offset = offset; + return state; + } + + @override + _AffineState combine(_AffineState left, _AffineState right) { + // Right's map runs after left's. + left.offset = right.scale * left.offset + right.offset; + left.scale = right.scale * left.scale; + return left; + } + + @override + Affine finish(_AffineState state) => Affine(state.scale, state.offset); + @override + bool get persistable => true; + + @override + Object? encodeState(_AffineState state) => { + 'scale': encodeDouble(state.scale), + 'offset': encodeDouble(state.offset), + }; + + @override + _AffineState decodeState(Object? json) { + final map = decodeMap(json); + return _AffineState() + ..scale = decodeDouble(field(map, 'scale')) + ..offset = decodeDouble(field(map, 'offset')); + } +} + +class _PrefixState, S2> { + _PrefixState(this.tail); + + /// The first rows of the input seen so far (at most k), null before any. + B? rows; + int held = 0; + S2 tail; +} + +/// Sends the first [k] rows of the whole input to [head] and every later row +/// to [tail], in order, however the input is chunked. +/// +/// The state keeps the prefix rows (a copy, at most k) and the tail state. +/// The head result is computed at finish by running [head] over the stored +/// rows, so [head] never needs a working combine. With fewer than k rows in +/// total, head sees all of them and the tail sees none. +class Prefix, S1, R1, S2, R2> + extends Aggregator<_PrefixState, B, (R1, R2)> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]) and [tail] is persistable. [head] keeps no state, + /// so it does not matter. + const Prefix(this.k, this.head, this.tail, {this.batchFromJson}); + + final int k; + final Aggregator head; + final Aggregator tail; + final B Function(Object? json)? batchFromJson; + + @override + _PrefixState empty() => _PrefixState(tail.empty()); + + /// Appends [piece] to the stored prefix rows. + void _hold(_PrefixState state, B piece) { + final rows = state.rows; + state.rows = rows == null ? piece : rows.join(piece); + state.held += piece.length; + } + + @override + _PrefixState addSlice( + _PrefixState state, B batch, int start, int end) { + final room = k - state.held; + final take = room < end - start ? room : end - start; + if (take > 0) _hold(state, batch.copyRange(start, start + take)); + if (start + take < end) { + state.tail = tail.addSlice(state.tail, batch, start + take, end); + } + return state; + } + + @override + _PrefixState combine( + _PrefixState left, _PrefixState right) { + final rightRows = right.rows; + var rest = right.tail; + if (rightRows != null) { + // Right's leading rows fill whatever room left's prefix still has; the + // remainder are tail rows that come before right's own tail. + final room = k - left.held; + final take = room < right.held ? room : right.held; + if (take > 0) _hold(left, rightRows.copyRange(0, take)); + if (take < right.held) { + final extra = + tail.addSlice(tail.empty(), rightRows, take, right.held); + rest = tail.combine(extra, rest); + } + } + left.tail = tail.combine(left.tail, rest); + return left; + } + + @override + (R1, R2) finish(_PrefixState state) { + final rows = state.rows; + final headState = rows == null + ? head.empty() + : head.addSlice(head.empty(), rows, 0, rows.length); + return (head.finish(headState), tail.finish(state.tail)); + } + + @override + bool get persistable => batchFromJson != null && tail.persistable; + + @override + Object? encodeState(_PrefixState state) { + _requireBatchCodec(); + return { + 'rows': state.rows?.toJson(), + 'tail': tail.encodeState(state.tail), + }; + } + + @override + _PrefixState decodeState(Object? json) { + final fromJson = _requireBatchCodec(); + final map = decodeMap(json); + final raw = field(map, 'rows'); + final result = _PrefixState(tail.decodeState(field(map, 'tail'))); + if (raw != null) { + final rows = fromJson(raw); + if (rows.length < 1 || rows.length > k) { + throw const FormatException('Invalid prefix row count'); + } + result.rows = rows; + result.held = rows.length; + } + return result; + } + + B Function(Object?) _requireBatchCodec() => + batchFromJson ?? + (throw UnsupportedError('Pass batchFromJson to persist Prefix')); +} diff --git a/lib/src/onehz/incremental_core.dart b/lib/src/onehz/incremental_core.dart new file mode 100644 index 00000000..08e78d25 --- /dev/null +++ b/lib/src/onehz/incremental_core.dart @@ -0,0 +1,636 @@ +import 'dart:math' as math; + +import '../aggregate/json_support.dart' show decodeDouble, encodeDouble; +import 'util.dart'; + +/// Removable, mergeable centered moments, with an origin to retain precision +/// when a series has a large offset and a small spread. +// Welford, B. P. (1962). Note on a method for calculating corrected sums of +// squares and products. Technometrics 4(3):419-420. +// add uses Welford's recurrence; remove algebraically reverses that update, +// with a nonnegative M2 clamp. Partition merging follows +// Chan, T. F., Golub, G. H., LeVeque, R. J. (1983). Algorithms for computing +// the sample variance: analysis and recommendations. The American Statistician +// 37(3):242-247. +// Removable/mergeable centered-moment recurrence. Algebraically equivalent to +// batch moments; floating-point order differs. The long-run drift fixture checks +// 1e-9, not bit identity. +class RunningMoments { + int _count = 0; + double _origin = 0, _meanOffset = 0, _m2 = 0; + + RunningMoments(); + + factory RunningMoments.fromJson(Map json) { + _checkVersion(json, 'RunningMoments'); + final result = RunningMoments(); + result._count = _readCount(json['count']); + result._origin = _readFinite(json['origin']); + result._meanOffset = _readFinite(json['meanOffset']); + result._m2 = _readM2(json['m2']); + if (result._m2 < 0 || + (result._count == 0 && + (result._origin != 0 || + result._meanOffset != 0 || + result._m2 != 0)) || + (result._count == 1 && result._m2 != 0)) { + throw const FormatException('Invalid centered moments'); + } + return result; + } + + int get count => _count; + double? get mean => _count == 0 ? null : _origin + _meanOffset; + double? get sampleSd => _count < 2 ? null : math.sqrt(_m2 / (_count - 1)); + double? get populationSd => _count == 0 ? null : math.sqrt(_m2 / _count); + + void add(double x) { + if (!x.isFinite) throw ArgumentError.value(x, 'x', 'Must be finite'); + if (_count == 0) { + _origin = x; + _count = 1; + return; + } + final centered = x - _origin; + final delta = centered - _meanOffset; + _count++; + _meanOffset += delta / _count; + _m2 += delta * (centered - _meanOffset); + } + + /// The caller must remove a value represented by this summary. + void remove(double x) { + if (!x.isFinite) throw ArgumentError.value(x, 'x', 'Must be finite'); + if (_count == 0) throw StateError('Cannot remove from empty moments'); + if (_count == 1) { + _count = 0; + _origin = _meanOffset = _m2 = 0; + return; + } + final centered = x - _origin; + final oldMean = _meanOffset; + _count--; + _meanOffset += (_meanOffset - centered) / _count; + _m2 = math.max(0, _m2 - (centered - oldMean) * (centered - _meanOffset)); + if (_count == 1) _m2 = 0; + } + + void merge(RunningMoments other) { + if (other._count == 0) return; + if (_count == 0) { + _count = other._count; + _origin = other._origin; + _meanOffset = other._meanOffset; + _m2 = other._m2; + return; + } + final otherCount = other._count; + final combinedCount = _count + otherCount; + final delta = (other._origin - _origin) + other._meanOffset - _meanOffset; + _m2 += other._m2 + delta * delta * _count * otherCount / combinedCount; + _meanOffset += delta * otherCount / combinedCount; + _count = combinedCount; + } + + Map toJson() => { + 'version': 1, + 'type': 'RunningMoments', + 'count': _count, + 'origin': encodeDouble(_origin), + 'meanOffset': encodeDouble(_meanOffset), + 'm2': encodeDouble(_m2), + }; +} + +/// Exact order statistics of integer-valued data, kept as a sorted +/// `value -> count` list. Heart rate is whole bpm with a few dozen distinct +/// values a night, so the list stays tiny however many samples it holds. Add, +/// remove and merge are exact; [percentile] returns the same double +/// `percentileSorted` returns for the expanded sorted list, not an estimate. +/// +/// Only for data that really is integral. Values that are nearly all distinct +/// (accelerometer magnitudes) gain nothing: the list is as long as the data. +// Hyndman, R. J., Fan, Y. (1996). Sample quantiles in statistical packages. +// The American Statistician 50(4):361-365. +// percentileSorted implements type 7: zero-based rank = (n - 1) * p / 100, +// linearly interpolating adjacent ranks; this histogram preserves that rule. +// Exact histogram reformulation of percentileSorted(): counts identify identical +// expanded-list ranks with the same interpolation. No new estimator. +class IntHistogram { + static const _maxInt64 = 0x7fffffffffffffff; + + List _values = []; + List _counts = []; + int _count = 0; + + IntHistogram(); + + factory IntHistogram.fromJson(Map json) { + _checkVersion(json, 'IntHistogram'); + final values = json['values'], counts = json['counts']; + if (values is! List || counts is! List || values.length != counts.length) { + throw const FormatException('Invalid histogram bins'); + } + final result = IntHistogram(); + for (var i = 0; i < values.length; i++) { + final v = values[i], c = counts[i]; + if (v is! int || c is! int || c < 1 || (i > 0 && v <= values[i - 1])) { + throw const FormatException('Invalid histogram bins'); + } + if (result._count > _maxInt64 - c) { + throw const FormatException('Invalid histogram counts'); + } + result._values.add(v); + result._counts.add(c); + result._count += c; + } + return result; + } + + /// Number of values held, counting repeats. + int get count => _count; + + /// Number of different values held. + int get distinct => _values.length; + + /// Linear-interpolated median, or null when empty. + double? get median => percentile(50); + + /// Index of [value], or the insertion point as `-(point + 1)`. + int _find(int value) { + var lo = 0, hi = _values.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if (_values[mid] < value) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo < _values.length && _values[lo] == value ? lo : -(lo + 1); + } + + /// Adds [times] copies of [value], which must be a finite whole number. + /// Throws [StateError] if the total count would overflow int64. + void add(num value, [int times = 1]) { + if (!value.isFinite || value != value.truncate()) { + throw ArgumentError.value(value, 'value', 'Must be a whole number'); + } + if (times < 1) throw ArgumentError.value(times, 'times', 'Must be positive'); + if (_count > _maxInt64 - times) { + throw StateError('Histogram count would overflow int64'); + } + final v = value.toInt(); + final at = _find(v); + if (at >= 0) { + _counts[at] += times; + } else { + _values.insert(-at - 1, v); + _counts.insert(-at - 1, times); + } + _count += times; + } + + /// Removes one copy of [value]. The caller must remove a value it added. + void remove(num value) { + final at = value.isFinite && value == value.truncate() + ? _find(value.toInt()) + : -1; + if (at < 0) throw StateError('Cannot remove a value that is not present'); + if (--_counts[at] == 0) { + _values.removeAt(at); + _counts.removeAt(at); + } + _count--; + } + + /// Merges [other] into this histogram. + /// Throws [StateError] if the total count would overflow int64. + void merge(IntHistogram other) { + if (_count > _maxInt64 - other._count) { + throw StateError('Histogram count would overflow int64'); + } + final values = List.of(other._values), counts = List.of(other._counts); + for (var i = 0; i < values.length; i++) { + add(values[i], counts[i]); + } + } + + /// The [p]th percentile (0 to 100), interpolated between the two + /// neighbouring order statistics exactly as `percentileSorted` does. Null + /// when empty, never 0. + double? percentile(double p) { + if (_count == 0) return null; + if (!(p >= 0 && p <= 100)) throw ArgumentError.value(p, 'p', '0 to 100'); + if (_count == 1) return _values[0].toDouble(); + final rank = (p / 100) * (_count - 1); + final lo = rank.floor(); + final hi = rank.ceil(); + final low = _orderStatistic(lo); + if (lo == hi) return low; + final frac = rank - lo; + return low + (_orderStatistic(hi) - low) * frac; + } + + /// The value at zero-based sorted position [k]. + double _orderStatistic(int k) { + var seen = 0; + for (var i = 0; i < _values.length; i++) { + seen += _counts[i]; + if (k < seen) return _values[i].toDouble(); + } + throw StateError('Order statistic out of range'); + } + + Map toJson() => { + 'version': 1, + 'type': 'IntHistogram', + 'values': List.of(_values), + 'counts': List.of(_counts), + }; +} + +/// Fixed-grid spectral sums. Raw snapshots establish that each sync is a +/// genuine prefix; callers may revise or remove any earlier sample. +// Press & Rybicki 1989 lombScargle() on the identical fixed grid using retained +// trigonometric sums and centered moments. Algebraically equal; long-run drift +// fixtures check 1e-9, not bit identity. +class IncrementalLombScargle { + final List _frequencies; + List _t = [], _y = []; + List> _sums; + RunningMoments _moments = RunningMoments(); + int _processedPoints = 0; + bool _incremental = true; + double _origin = 0, _yOrigin = 0, _tMin = 0, _tMax = 0; + + IncrementalLombScargle(List frequencies) + : _frequencies = List.of(frequencies), + _sums = [for (final _ in frequencies) List.filled(7, 0.0)]; + + factory IncrementalLombScargle.fromJson(Map json) { + _checkVersion(json, 'IncrementalLombScargle'); + final result = IncrementalLombScargle(_readDoubles(json['frequencies'])); + result._t = _readDoubles(json['t']); + result._y = _readDoubles(json['y']); + result._processedPoints = _readCount(json['processedPoints']); + final incremental = json['incremental']; + if (incremental is! bool || result._t.length != result._y.length) { + throw const FormatException('Invalid spectral input state'); + } + result._incremental = incremental; + result._origin = _readFinite(json['origin']); + result._yOrigin = _readFinite(json['yOrigin']); + result._tMin = _readFinite(json['tMin']); + result._tMax = _readFinite(json['tMax']); + final moments = json['moments']; + if (moments is! Map) { + throw const FormatException('Missing spectral moments'); + } + result._moments = RunningMoments.fromJson(moments); + final sums = json['sums']; + if (sums is! List || sums.length != result._frequencies.length) { + throw const FormatException('Invalid spectral sum dimensions'); + } + result._sums = [ + for (final row in sums) _readDoubles(row, finite: true), + ]; + if (result._sums.any((row) => row.length != 7) || + (incremental && + (!result._allFinite(result._t, result._y) || + result._moments.count != result._t.length || + result._processedPoints < result._t.length)) || + (!incremental && result._moments.count != 0)) { + throw const FormatException('Invalid spectral accumulator'); + } + if (result._t.isEmpty || !incremental) { + if (result._origin != 0 || + result._yOrigin != 0 || + result._tMin != 0 || + result._tMax != 0 || + result._sums.any((row) => row.any((value) => value != 0))) { + throw const FormatException('Invalid empty spectral state'); + } + } + if (incremental && result._t.isNotEmpty) { + if (result._origin != result._t.first || + result._yOrigin != result._y.first || + result._tMin != result._t.reduce(math.min) || + result._tMax != result._t.reduce(math.max)) { + throw const FormatException('Invalid spectral origin or extrema'); + } + for (final row in result._sums) { + if (row[2] < 0 || + row[3] < 0 || + ((row[2] + row[3]) - result._t.length).abs() > + 1e-8 * result._t.length) { + throw const FormatException('Invalid trigonometric sums'); + } + } + } + return result; + } + + int get processedPoints => _processedPoints; + + bool _allFinite(List t, List y) => + _frequencies.every((x) => x.isFinite) && + t.every((x) => x.isFinite) && + y.every((x) => x.isFinite); + + LombScargle? sync(List t, List y, {bool force = false}) { + if (t.length != y.length) { + _reset(); + return lombScargle(t, y, _frequencies); + } + if (!_allFinite(t, y)) { + // The batch function defines the NaN/infinity behavior. Never fold these + // values into summaries that could contaminate later finite prefixes. + final result = lombScargle(t, y, _frequencies); + _reset(); + _incremental = false; + _t = List.of(t); + _y = List.of(y); + if (t.length >= 4 && _frequencies.isNotEmpty) + _processedPoints += t.length; + return result; + } + var prefix = !force && _incremental && t.length >= _t.length; + if (prefix) { + for (var i = 0; i < _t.length; i++) { + if (_t[i] != t[i] || _y[i] != y[i]) { + prefix = false; + break; + } + } + } + if (!prefix) _reset(); + final start = _t.length; + if (start == 0 && t.isNotEmpty) { + _origin = _tMin = _tMax = t.first; + _yOrigin = y.first; + } + for (var i = start; i < t.length; i++) { + _moments.add(y[i]); + _tMin = math.min(_tMin, t[i]); + _tMax = math.max(_tMax, t[i]); + for (var j = 0; j < _frequencies.length; j++) { + final w = 2 * math.pi * _frequencies[j]; + final c = math.cos(w * (t[i] - _origin)); + final s = math.sin(w * (t[i] - _origin)); + final centeredY = y[i] - _yOrigin; + final sums = _sums[j]; + sums[0] += c; + sums[1] += s; + sums[2] += c * c; + sums[3] += s * s; + sums[4] += c * s; + sums[5] += centeredY * c; + sums[6] += centeredY * s; + } + _processedPoints++; + } + _t = List.of(t); + _y = List.of(y); + return _render(); + } + + void _reset() { + _t = []; + _y = []; + _sums = [for (final _ in _frequencies) List.filled(7, 0.0)]; + _moments = RunningMoments(); + _origin = _yOrigin = _tMin = _tMax = 0; + _incremental = true; + } + + LombScargle? _render() { + final n = _moments.count; + final span = _tMax - _tMin; + if (n < 4 || _frequencies.isEmpty || _moments._m2 <= 0 || span <= 0) { + return null; + } + final centeredMean = (_moments._origin - _yOrigin) + _moments._meanOffset; + final spectrum = []; + for (var j = 0; j < _frequencies.length; j++) { + final frequency = _frequencies[j]; + if (frequency == 0) { + spectrum.add(const LsPoint(0, 0)); + continue; + } + final sums = _sums[j]; + final phi = 0.5 * math.atan2(2 * sums[4], sums[2] - sums[3]); + final a = math.cos(phi), b = math.sin(phi); + final uc = sums[5] - centeredMean * sums[0]; + final us = sums[6] - centeredMean * sums[1]; + final cNum = a * uc + b * us, sNum = a * us - b * uc; + final cDen = a * a * sums[2] + b * b * sums[3] + 2 * a * b * sums[4]; + final sDen = a * a * sums[3] + b * b * sums[2] - 2 * a * b * sums[4]; + final term1 = cDen == 0 ? 0.0 : cNum * cNum / cDen; + final term2 = sDen == 0 ? 0.0 : sNum * sNum / sDen; + spectrum.add(LsPoint(frequency, span / (n - 1) * (term1 + term2))); + } + return LombScargle(spectrum); + } + + Map toJson() => { + 'version': 1, + 'type': 'IncrementalLombScargle', + 'frequencies': _writeDoubles(_frequencies), + 't': _writeDoubles(_t), + 'y': _writeDoubles(_y), + 'processedPoints': _processedPoints, + 'incremental': _incremental, + 'origin': _origin, + 'yOrigin': _yOrigin, + 'tMin': _tMin, + 'tMax': _tMax, + 'moments': _moments.toJson(), + 'sums': [for (final row in _sums) List.of(row)], + }; +} + +void _checkVersion(Map json, String type) { + if (json['version'] is! int || json['version'] != 1 || json['type'] != type) { + throw FormatException('Unsupported $type checkpoint'); + } +} + +int _readCount(Object? value) { + if (value is! int || value < 0) throw const FormatException('Invalid count'); + return value; +} + +double _readFinite(Object? value) { + if (value is! num || !value.isFinite) + throw const FormatException('Invalid number'); + return value.toDouble(); +} + +double _readM2(Object? value) { + final result = decodeDouble(value); + if ((!result.isFinite && value is! String) || + result.isNaN || + result.isNegative) { + throw const FormatException('Invalid number'); + } + return result; +} + +List _readDoubles(Object? value, {bool finite = false}) { + if (value is! List) throw const FormatException('Expected a numeric list'); + return [for (final item in value) _readDouble(item, finite: finite)]; +} + +double _readDouble(Object? value, {bool finite = false}) { + if (value is num && (!finite || value.isFinite)) return value.toDouble(); + if (!finite) { + if (value == 'NaN') return double.nan; + if (value == 'Infinity') return double.infinity; + if (value == '-Infinity') return double.negativeInfinity; + } + throw const FormatException('Invalid number'); +} + +List _writeDoubles(List values) => [ + for (final value in values) + if (value.isFinite) + value + else if (value.isNaN) + 'NaN' + else if (value > 0) + 'Infinity' + else + '-Infinity', + ]; + +class _CalculationEntry { + final Object? dependencies; + final Object? value; + const _CalculationEntry(this.dependencies, this.value); +} + +/// Bounded least-recently-used results with owned dependency snapshots. +class CalculationCache { + final int maxEntries; + final Map _entries = {}; + int _computations = 0, _hits = 0; + + CalculationCache({this.maxEntries = 128}) { + if (maxEntries < 1) throw ArgumentError.value(maxEntries, 'maxEntries'); + } + + int get computations => _computations; + int get hits => _hits; + + T evaluate(String key, Object? dependencies, T Function() calculate, + {bool full = false}) { + final old = _entries[key]; + if (!full && old != null && _deepEqual(old.dependencies, dependencies)) { + final result = _copyValue(old.value) as T; + _entries.remove(key); + _entries[key] = old; + _hits++; + return result; + } + // Snapshot before calling user code, which may itself mutate dependencies. + // Prepare both copies before committing, so exceptions preserve the old entry. + final snapshot = _snapshotDependency(dependencies); + final value = _copyValue(calculate()); + final result = _copyValue(value) as T; + _entries.remove(key); + _entries[key] = _CalculationEntry(snapshot, value); + while (_entries.length > maxEntries) { + _entries.remove(_entries.keys.first); + } + _computations++; + return result; + } + + void clear() => _entries.clear(); +} + +Object? _snapshotDependency(Object? value) { + if (value is List) + return [for (final item in value) _snapshotDependency(item)]; + if (value is Map) { + return { + for (final entry in value.entries) + entry.key: _snapshotDependency(entry.value) + }; + } + return value; +} + +bool _deepEqual(Object? a, Object? b) { + if (identical(a, b)) return true; + if (a is num && b is num && a.isNaN && b.isNaN) return true; + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_deepEqual(a[i], b[i])) return false; + } + return true; + } + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final key in a.keys) { + if (!b.containsKey(key) || !_deepEqual(a[key], b[key])) return false; + } + return true; + } + return a == b; +} + +Object? _copyValue(Object? value) { + if (value is List) { + // toList dispatches on the original List and keeps its reified E. + final copy = value.toList(); + for (var i = 0; i < copy.length; i++) { + copy[i] = _copyValue(value[i]); + } + return copy; + } + if (value is Map) { + // Preserve the usual JSON map types, including typed scalar/list values. + // Unlike List.toList, Map.map creates new key/value types from its callback. + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>>) + return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map) return _copyMap(value); + return _copyMap(value); + } + // Metric and other immutable domain values retain their identity and type. + return value; +} + +Map _copyMap(Map value) => { + for (final entry in value.entries) + entry.key: _copyValue(entry.value) as V, + }; diff --git a/lib/src/onehz/sleep/sleep.dart b/lib/src/onehz/sleep/sleep.dart index dc21184c..e37b330c 100644 --- a/lib/src/onehz/sleep/sleep.dart +++ b/lib/src/onehz/sleep/sleep.dart @@ -15,6 +15,9 @@ // honesty-bounded; stager.dart holds only the shared post-processing // (StagerResult, Webster rescore, consolidation) it reuses. The // autonomic-HR-only stager that used to live in stager.dart is DELETED. +// - CAUSAL (online) wake/NREM/REM stager (causal_stager.dart) — +// the same features/weights/cutoffs on past-only windows, abstains with a +// reason; for live decisions, NOT a replacement for cardioStager. // - Nonparametric circadian IS/IV/RA/L5/M10 (circadian_np.dart) // // Pure Dart, built on the package foundations (RR correction, Lomb-Scargle, @@ -27,6 +30,7 @@ export 'segment.dart'; export 'hr_fallback.dart'; export 'advanced_stager.dart'; export 'band_offset.dart'; + export 'sri.dart'; export 'accounting.dart'; // stager.dart provides StagerResult + the shared consolidation post-step. Its @@ -40,6 +44,10 @@ export 'cardio_stager.dart' cleanBeatsInWindowForTest, cleanBeatDiffsInWindowForTest, windowRmssdRkForTest; + + + + export 'circadian_np.dart'; export 'cycles.dart'; export 'night_hrv_shape.dart'; diff --git a/test/aggregate/aggregator_core_test.dart b/test/aggregate/aggregator_core_test.dart new file mode 100644 index 00000000..215f7724 --- /dev/null +++ b/test/aggregate/aggregator_core_test.dart @@ -0,0 +1,242 @@ +// Aggregator core: the Batch implementations, fold (with ranges) / run / +// runList, map and both. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +/// Sum of the int rows. State is a one-slot list so combine/addSlice can mutate. +class _SumRows extends Aggregator, ListBatch, int> { + const _SumRows(); + @override + List empty() => [0]; + @override + List addSlice(List s, ListBatch b, int start, int end) { + for (var i = start; i < end; i++) { + s[0] += b.rows[i]; + } + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + return left; + } + + @override + int finish(List s) => s[0]; +} + +/// Rows seen, in order, as a list (state mutated in place). +class _Rows extends Aggregator, ListBatch, List> { + const _Rows(); + @override + List empty() => []; + @override + List addSlice(List s, ListBatch b, int start, int end) => + s..addAll(b.rows.sublist(start, end)); + @override + List combine(List left, List right) => left..addAll(right); + @override + List finish(List s) => List.of(s); +} + +bool _same(T a, T b) => a.toString() == b.toString(); + +void main() { + group('ListBatch', () { + test('length, rows and copyRange', () { + final b = ListBatch(['a', 'b', 'c', 'd']); + expect(b.length, 4); + expect(b.rows, ['a', 'b', 'c', 'd']); + expect(b.copyRange(1, 3).rows, ['b', 'c']); + expect(b.copyRange(0, 4).rows, ['a', 'b', 'c', 'd']); + expect(b.copyRange(2, 2).length, 0); + expect(b.copyRange(4, 4).length, 0); + expect(b.copyRange(1, 3).length, 2); + }); + + test('copyRange is a copy: later changes to the source do not show', () { + final rows = [1, 2, 3, 4]; + final b = ListBatch(rows); + final c = b.copyRange(1, 4); + rows[1] = 99; + rows[3] = 77; + expect(c.rows, [2, 3, 4]); + }); + + test('join puts this before other and leaves both unchanged', () { + final a = ListBatch([1, 2]), b = ListBatch([3, 4, 5]); + final j = a.join(b); + expect(j.rows, [1, 2, 3, 4, 5]); + expect(j.length, 5); + expect(a.rows, [1, 2]); + expect(b.rows, [3, 4, 5]); + expect(a.join(ListBatch([])).rows, [1, 2]); + expect(ListBatch([]).join(b).rows, [3, 4, 5]); + expect(ListBatch([]).join(ListBatch([])).length, 0); + }); + }); + + group('DoubleColumn', () { + test('length, values and copyRange', () { + final c = col([1, 2, 3, 4]); + expect(c.length, 4); + expect(c.copyRange(1, 3).values, [2.0, 3.0]); + expect(c.copyRange(0, 4).values, [1.0, 2.0, 3.0, 4.0]); + expect(c.copyRange(3, 3).length, 0); + }); + + test('copyRange is a copy: later changes to the source do not show', () { + final v = Float64List.fromList([1, 2, 3, 4]); + final c = DoubleColumn(v).copyRange(1, 3); + v[1] = 99; + v[2] = 77; + expect(c.values, [2.0, 3.0]); + }); + + test('join, with NaN kept as a missing value', () { + final a = col([1, double.nan]), b = col([3]); + final j = a.join(b); + expect(j.length, 3); + expect(j.values[0], 1.0); + expect(j.values[1].isNaN, isTrue); + expect(j.values[2], 3.0); + expect(a.length, 2); + expect(b.values, [3.0]); + expect(a.join(col([])).length, 2); + expect(col([]).join(b).values, [3.0]); + }); + }); + + group('fold, run, runList', () { + const sum = _SumRows(); + final batch = ListBatch([for (var i = 1; i <= 5; i++) i]); + + test('run of nothing is the identity result', () { + expect(sum.run(ListBatch([])), 0); + expect(sum.runList([]), 0); + }); + + test('run equals finish(fold) and folds in order', () { + expect(sum.run(batch), 15); + expect(sum.finish(sum.fold(batch)), 15); + expect(const _Rows().run(batch), [1, 2, 3, 4, 5]); + }); + + test('runList is run over a ListBatch', () { + expect(sum.runList([4, 5, 6]), 15); + expect(const _Rows().runList([9, 8, 7]), [9, 8, 7]); + }); + + test('fold takes a half-open range; end defaults to the length', () { + expect(sum.finish(sum.fold(batch, 1, 4)), 2 + 3 + 4); + expect(sum.finish(sum.fold(batch, 0, 5)), 15); + expect(sum.finish(sum.fold(batch, 2)), 3 + 4 + 5); + expect(sum.finish(sum.fold(batch, 5)), 0); + expect(sum.finish(sum.fold(batch, 2, 2)), 0); + expect(sum.finish(sum.fold(batch, 5, 5)), 0); + expect(const _Rows().finish(const _Rows().fold(batch, 1, 3)), [2, 3]); + }); + + test('fold ranges combine back to the whole at every split', () { + final xs = ListBatch([for (var i = 1; i <= 20; i++) i]); + for (var k = 0; k <= 20; k++) { + final s = sum.combine(sum.fold(xs, 0, k), sum.fold(xs, k, 20)); + expect(sum.finish(s), 210, reason: 'split at $k'); + } + }); + + test('addSlice continues a state: several slices equal one', () { + var s = sum.empty(); + s = sum.addSlice(s, batch, 0, 2); + s = sum.addSlice(s, batch, 2, 2); + s = sum.addSlice(s, batch, 2, 5); + expect(sum.finish(s), 15); + }); + + test('empty() returns independent states', () { + final a = sum.empty(), b = sum.empty(); + sum.addSlice(a, batch, 0, 5); + expect(sum.finish(b), 0); + }); + + test('chunked and sliced runs agree with the whole run', () { + const agg = _Rows(); + final r = Rng(99); + for (final b in randomIntBatches(1)) { + for (var t = 0; t < 6; t++) { + final bounds = randomBounds(r, b.length); + expect(runChunked(agg, b, bounds), b.rows, reason: '$bounds'); + expect(runChunked(agg, b, bounds, copy: true), b.rows, + reason: 'copy $bounds'); + expect(runChunkedRight(agg, b, bounds), b.rows, reason: '$bounds'); + expect(runSliced(agg, b, bounds), b.rows, reason: '$bounds'); + } + } + }); + }); + + group('map', () { + test('transforms the result only', () { + final a = const _SumRows().map((n) => 'n=$n'); + expect(a.runList([1, 2, 3]), 'n=6'); + expect(a.runList([]), 'n=0'); + expectLawful(a, randomIntBatches(2)); + }); + + test('composes', () { + final a = Count>().map((n) => n * 2).map((n) => n + 1); + expect(a.runList([7, 7, 7]), 7); + }); + + test('keeps ranges and seams working', () { + final a = const _SumRows().map((n) => n * 10); + final b = ListBatch([1, 2, 3, 4]); + expect(a.finish(a.combine(a.fold(b, 0, 2), a.fold(b, 2, 4))), 100); + expect(a.finish(a.fold(b, 1, 3)), 50); + }); + }); + + group('both', () { + final a = both(const _SumRows(), Count>()); + + test('runs two aggregators over the same rows', () { + expect(a.runList([3, 4, 5]), (12, 3)); + expect(a.runList([]), (0, 0)); + }); + + test('nests and stays lawful', () { + final n = both(a, const _Rows()); + final r = n.runList([3, 9, 5]); // records holding lists compare by identity + expect(r.$1, (17, 3)); + expect(r.$2, [3, 9, 5]); + expectLawful(n, randomIntBatches(3), equals: _same); + }); + + test('each side has its own fresh state', () { + final p = a.addSlice(a.empty(), ListBatch([5]), 0, 1); + final q = a.empty(); + expect(a.finish(q), (0, 0)); + expect(a.finish(p), (5, 1)); + }); + + test('combines across a seam and honours fold ranges', () { + final b = ListBatch([1, 2, 3, 4]); + expect(a.finish(a.combine(a.fold(b, 0, 2), a.fold(b, 2, 4))), (10, 4)); + expect(a.finish(a.fold(b, 1, 3)), (5, 2)); + }); + + test('Pair is a mutable holder', () { + final p = Pair(1, 'a'); + p.first = 2; + p.second = 'b'; + expect((p.first, p.second), (2, 'b')); + }); + }); +} diff --git a/test/aggregate/ewma_prefix_test.dart b/test/aggregate/ewma_prefix_test.dart new file mode 100644 index 00000000..9fac2a73 --- /dev/null +++ b/test/aggregate/ewma_prefix_test.dart @@ -0,0 +1,286 @@ +// Affine, Ewma and Prefix (spec: research/r7-ewma.md). +// +// Decisions the implementation follows: +// * `const Affine(double scale, double offset)`, positional; identity is +// `Affine(1, 0)`; `apply(x) = scale * x + offset`; +// `a.then(b)` = b after a = (b.scale*a.scale, b.scale*a.offset + b.offset). +// * `Ewma(col, lambda)` is an Aggregator<_, B, Affine>, positional args. +// Its Affine maps the running value c to the value after all rows: +// one row x is c -> (1 - lambda) * c + lambda * x. NaN rows are skipped +// (identity). No rows (or only NaN rows) finish as Affine(1, 0). +// * `Prefix(k, head, tail)` (type args inferred) is an +// Aggregator<_, B, (R1, R2)>. Head sees rows [0, k) of the whole input via +// head.run on stored rows (so head.combine is never called), tail sees rows +// [k, n) in order. k = 0 gives head an empty batch. The stored prefix is a +// copy: mutating the source column after folding changes nothing. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +const nan = double.nan; + +List _aff(Affine a) => [a.scale, a.offset]; +final _affEq = fieldsEqual(_aff, relTol: 1e-12, absTol: 1e-15); + + +Ewma _ewma(double lambda) => + Ewma((b) => b.values, lambda); + +/// Reference: apply x_i one by one to the running value, skipping NaN. +double _loop(double seed, List xs, double lambda) { + var c = seed; + for (final x in xs) { + if (x.isNaN) continue; + c = c + lambda * (x - c); + } + return c; +} + +void _expectAffine(Affine got, Affine want, String tag) => expect( + _affEq(got, want), isTrue, + reason: '$tag got ${_aff(got)} want ${_aff(want)}'); + +typedef _Rows = List; + +/// Records every row it is shown, in order. Head variant never combines. +class _Rec extends Aggregator<_Rows, DoubleColumn, _Rows> { + const _Rec({this.combinable = true}); + final bool combinable; + @override + _Rows empty() => []; + @override + _Rows addSlice(_Rows s, DoubleColumn b, int start, int end) => + s..addAll([for (var i = start; i < end; i++) b.values[i]]); + @override + _Rows combine(_Rows l, _Rows r) { + if (!combinable) throw StateError('head.combine must not be called'); + return l..addAll(r); + } + + @override + _Rows finish(_Rows s) => List.of(s); +} + +DoubleColumn _iota(int n) => col([for (var i = 0; i < n; i++) i * 1.0]); + +void _is(Object? got, Object? want, {String? reason}) => + expect(got.toString(), want.toString(), reason: reason); + +bool _str(Object? a, Object? b) => a.toString() == b.toString(); + +void main() { + group('Affine', () { + test('apply, identity and then', () { + const a = Affine(2, 3); + const b = Affine(5, -1); + expect(a.apply(4), 11); + expect(const Affine(1, 0).apply(7.5), 7.5); + final ab = a.then(b); // b after a + expect(ab.scale, 10); + expect(ab.offset, 5 * 3 + -1); + for (final x in [-3.0, 0.0, 1.5, 100.0]) { + expect(ab.apply(x), b.apply(a.apply(x))); + } + _expectAffine(const Affine(1, 0).then(a), a, 'id.then(a)'); + _expectAffine(a.then(const Affine(1, 0)), a, 'a.then(id)'); + }); + + test('then is associative (seeded)', () { + final r = Rng(0xAF1); + Affine rnd() => Affine((r.nextDouble() - .5) * 4, (r.nextDouble() - .5) * 100); + for (var t = 0; t < 100; t++) { + final a = rnd(), b = rnd(), c = rnd(); + _expectAffine(a.then(b).then(c), a.then(b.then(c)), 't$t'); + } + }); + }); + + group('Ewma', () { + test('one row: (1 - lambda) c + lambda x', () { + final f = _ewma(0.25).run(col([8])); + expect(f.scale, closeTo(0.75, 1e-15)); + expect(f.offset, closeTo(2, 1e-15)); + expect(f.apply(4), closeTo(0.75 * 4 + 2, 1e-15)); + }); + + test('no rows, and only NaN rows, are the identity', () { + final e = _ewma(0.1); + _expectAffine(e.run(col([])), const Affine(1, 0), 'empty'); + _expectAffine(e.finish(e.empty()), const Affine(1, 0), 'empty state'); + _expectAffine(e.run(col([nan, nan, nan])), const Affine(1, 0), 'nans'); + }); + + test('values equal a direct loop (seeded, several lambdas)', () { + final r = Rng(0xE111); + for (final lambda in [0.5, 1 - 0.9, 0.02, 0.1331, 1.0, 0.0]) { + final e = _ewma(lambda); + for (var t = 0; t < 40; t++) { + final n = r.intIn(0, 30); + final xs = [ + for (var i = 0; i < n; i++) + r.nextBool(.2) ? nan : (r.nextDouble() - .3) * 500 + ]; + final seed = (r.nextDouble() - .5) * 300; + final got = e.run(col(xs)).apply(seed); + final want = _loop(seed, xs, lambda); + expect(got, closeTo(want, 1e-12 * (1 + want.abs())), + reason: 'lambda=$lambda t=$t'); + } + } + }); + + test('NaN rows are skipped, not decayed', () { + final e = _ewma(0.3); + final a = e.run(col([10, nan, 20, nan, nan, 5])); + final b = e.run(col([10, 20, 5])); + _expectAffine(a, b, 'nan vs removed'); + }); + + test('combine is left.then(right): chunking does not matter', () { + final r = Rng(0xE112); + final e = _ewma(0.07); + for (var t = 0; t < 60; t++) { + final n = r.intIn(0, 60); + final c = col([ + for (var i = 0; i < n; i++) r.nextBool(.1) ? nan : r.nextDouble() * 400 + ]); + final want = e.run(c); + final b = randomBounds(r, n); + _expectAffine(runChunked(e, c, b), want, 'chunked $b'); + _expectAffine(runChunked(e, c, b, copy: true), want, 'copy $b'); + _expectAffine(runChunkedRight(e, c, b), want, 'right $b'); + _expectAffine(runSliced(e, c, b), want, 'sliced $b'); + _expectAffine(runChunked(e, c, singletonBounds(n)), want, 'single'); + } + }); + + test('is lawful', () { + expectLawful(_ewma(0.05), randomColumns(0xE113, nanP: .2, maxLen: 40), + equals: _affEq); + }); + }); + + group('Prefix', () { + for (final k in [0, 1, 3, 7]) { + group('k = $k', () { + final p = Prefix(k, const _Rec(combinable: false), const _Rec()); + + (List, List) want(int n) => ( + [for (var i = 0; i < n && i < k; i++) i * 1.0], + [for (var i = k; i < n; i++) i * 1.0], + ); + + test('head sees exactly rows [0, k), tail rows [k, n), inputs of ' + 'every length', () { + for (var n = 0; n <= 20; n++) { + _is(p.run(_iota(n)), want(n), reason: 'n=$n'); + } + }); + + test('every single and double cut, all four evaluation paths', () { + for (final n in [0, 1, k, k + 1, 12]) { + final c = _iota(n); + for (var a = 0; a <= n; a++) { + for (var b = a; b <= n; b++) { + final bd = [0, a, b, n]; + _is(runChunked(p, c, bd), want(n), reason: 'chunk $bd'); + _is(runChunked(p, c, bd, copy: true), want(n), + reason: 'copy $bd'); + _is(runChunkedRight(p, c, bd), want(n), reason: 'right $bd'); + _is(runSliced(p, c, bd), want(n), reason: 'sliced $bd'); + } + } + } + }); + + test('random chunkings and slicings, empty chunks included', () { + final r = Rng(0x9F00 + k); + for (var t = 0; t < 80; t++) { + final n = r.intIn(0, 40); + final c = _iota(n); + final b = randomBounds(r, n); + _is(runChunked(p, c, b), want(n), reason: 'chunk $b n=$n'); + _is(runChunked(p, c, b, copy: true), want(n), reason: 'copy'); + _is(runChunkedRight(p, c, b), want(n), reason: 'right $b n=$n'); + _is(runSliced(p, c, b), want(n), reason: 'sliced $b n=$n'); + final s = singletonBounds(n); + _is(runChunked(p, c, s), want(n), reason: 'singletons'); + _is(runChunkedRight(p, c, s), want(n), reason: 'right singles'); + _is(runSliced(p, c, s), want(n), reason: 'sliced singles'); + } + }); + + test('is lawful with a recording head and tail', () { + expectLawful(p, [for (var n = 0; n <= 14; n++) _iota(n), _iota(40)], + equals: _str); + }); + + test('is lawful with Count and Sum', () { + final eq = fieldsEqual<(int, double)>((x) => [x.$1, x.$2], + relTol: 1e-12, absTol: 1e-9); + expectLawful( + Prefix(k, Count(), Sum((b) => b.values)), + randomColumns(0x9F10 + k, nanP: .2), + equals: eq); + final eq2 = fieldsEqual<(double, int)>((x) => [x.$1, x.$2], + relTol: 1e-12, absTol: 1e-9); + expectLawful( + Prefix(k, Sum((b) => b.values), Count()), + randomColumns(0x9F20 + k, nanP: .2), + equals: eq2); + }); + }); + } + + test('k = 3: seams of one-row chunks fill the prefix then the tail', () { + final p = Prefix(3, const _Rec(combinable: false), const _Rec()); + final c = _iota(6); + var s = p.fold(c, 0, 1); + s = p.combine(s, p.fold(c, 1, 2)); + _is(p.finish(s), ([0, 1], [])); + s = p.combine(s, p.fold(c, 2, 3)); + s = p.combine(s, p.fold(c, 3, 4)); + _is(p.finish(s), ([0, 1, 2], [3])); + // A right chunk bigger than the room left in the left prefix. + final r = p.combine(p.fold(c, 0, 2), p.fold(c, 2, 6)); + _is(p.finish(r), ([0, 1, 2], [3, 4, 5])); + // Empty left, empty right. + _is(p.finish(p.combine(p.empty(), p.fold(c, 0, 6))), ([0, 1, 2], [3, 4, 5])); + _is(p.finish(p.combine(p.fold(c, 0, 6), p.empty())), ([0, 1, 2], [3, 4, 5])); + }); + + test('the stored prefix is a copy of the rows', () { + final p = Prefix(3, const _Rec(combinable: false), const _Rec()); + final c = _iota(6); + final s = p.fold(c, 0, 6); + final s2 = p.fold(c, 0, 2); + for (var i = 0; i < 6; i++) { + c.values[i] = -99; + } + _is(p.finish(s), ([0, 1, 2], [3, 4, 5])); + _is(p.finish(s2), ([0, 1], [])); + }); + + test('composes with Ewma as the tail: mean seed, then decay', () { + final p = Prefix( + 4, Sum((b) => b.values), _ewma(0.2)); + final r = Rng(0x9F30); + for (var t = 0; t < 40; t++) { + final n = r.intIn(0, 30); + final xs = [for (var i = 0; i < n; i++) r.nextDouble() * 300]; + final c = col(xs); + final b = randomBounds(r, n); + final (sum, f) = runChunked(p, c, b); + final head = n < 4 ? n : 4; + expect(sum, closeTo(xs.take(head).fold(0.0, (a, x) => a + x), 1e-9)); + final seed = 50.0; + final got = f.apply(seed); + final want = _loop(seed, xs.skip(4).toList(), 0.2); + expect(got, closeTo(want, 1e-9 * (1 + want.abs())), reason: 't$t $b'); + } + }); + }); +} diff --git a/test/aggregate/group_by_test.dart b/test/aggregate/group_by_test.dart new file mode 100644 index 00000000..c6e064a3 --- /dev/null +++ b/test/aggregate/group_by_test.dart @@ -0,0 +1,259 @@ +// GroupBy: consecutive rows with equal key go to the inner aggregator as one +// slice; a key seen again later is combined into its existing state; finish +// maps every key through inner.finish. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart' show RunningMoments; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; +int _mod3(DoubleColumn c, int i) => c.values[i].toInt() % 3; +int _mod2(DoubleColumn c, int i) => c.values[i].toInt() % 2; +int _tens(DoubleColumn c, int i) => c.values[i] ~/ 10; + +/// Exact deep equality for (nested) maps; Map == is identity in Dart. +bool _mapsEqual(Object? a, Object? b) { + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final k in a.keys) { + if (!b.containsKey(k) || !_mapsEqual(a[k], b[k])) return false; + } + return true; + } + return a == b; +} + +List _momentFields(Map m) { + final keys = m.keys.toList()..sort(); + return [ + keys.length, + for (final k in keys) ...[k, m[k]!.count, m[k]!.mean, m[k]!.sampleSd], + ]; +} + +/// Records every slice the inner is handed, as a list of its rows. +class _Slices extends Aggregator>, DoubleColumn, List>> { + const _Slices(); + @override + List> empty() => >[]; + @override + List> addSlice( + List> s, DoubleColumn b, int start, int end) { + if (end > start) s.add(b.values.sublist(start, end)); + return s; + } + + @override + List> combine(List> l, List> r) => + l..addAll(r); + @override + List> finish(List> s) => s; +} + +DoubleColumn _ints(List xs) => col([for (final x in xs) x * 1.0]); + +void main() { + final count3 = GroupBy(_mod3, Count()); + final sum3 = GroupBy(_mod3, Sum(_v)); + + group('values', () { + test('counts per key', () { + expect(count3.run(_ints([1, 2, 3, 4, 5, 6, 7])), {0: 2, 1: 3, 2: 2}); + }); + + test('sums per key', () { + final a = GroupBy(_mod2, Sum(_v)); + expect(a.run(_ints([1, 2, 3, 4, 5])), {1: 9.0, 0: 6.0}); + }); + + test('empty input gives an empty map', () { + expect(count3.run(col([])), isEmpty); + expect(count3.finish(count3.empty()), {}); + }); + + test('a key that never occurs is absent', () { + expect(count3.run(_ints([3, 6])).keys, [0]); + }); + + test('the key function sees the batch and a row index', () { + final rows = []; + final a = GroupBy( + (b, i) { + rows.add(i); + return 0; + }, + Count()); + a.run(_ints([5, 6, 7])); + expect(rows.toSet(), {0, 1, 2}); + }); + + test('works over a ListBatch', () { + final a = GroupBy, dynamic, int>( + (b, i) => b.rows[i].length, Count>()); + expect(a.runList(['a', 'bb', 'c', 'dd', 'e']), {1: 3, 2: 2}); + }); + }); + + group('runs of equal keys', () { + const slices = _Slices(); + final g = GroupBy>>(_tens, slices); + + test('consecutive equal keys reach the inner as one slice', () { + final r = g.run(col([10, 11, 12, 20, 21])); + expect(r, { + 1: [ + [10.0, 11.0, 12.0] + ], + 2: [ + [20.0, 21.0] + ], + }); + }); + + test('a key that reappears later is combined into its existing state, ' + 'in row order', () { + final r = g.run(col([10, 11, 20, 21, 22, 12, 30, 13, 14])); + expect(r.keys.toSet(), {1, 2, 3}); + expect(r[1]!.expand((s) => s).toList(), [10.0, 11.0, 12.0, 13.0, 14.0]); + expect(r[2], [ + [20.0, 21.0, 22.0] + ]); + expect(r[3], [ + [30.0] + ]); + // each run stayed together; the two runs of key 1 stay separate slices + expect(r[1], [ + [10.0, 11.0], + [12.0], + [13.0, 14.0] + ]); + }); + + test('alternating keys give one-row slices', () { + final r = g.run(col([10, 20, 11, 21, 12])); + expect(r[1], [ + [10.0], + [11.0], + [12.0] + ]); + expect(r[2], [ + [20.0], + [21.0] + ]); + }); + + test('a fold range groups only the rows inside it', () { + final c = col([10, 11, 20, 21, 12, 13]); + final r = g.finish(g.fold(c, 1, 5)); + expect(r[1], [ + [11.0], + [12.0] + ]); + expect(r[2], [ + [20.0, 21.0] + ]); + }); + + test('a run cut by a slice boundary arrives as two slices', () { + final c = col([10, 11, 12, 13]); + var s = g.empty(); + s = g.addSlice(s, c, 0, 2); + s = g.addSlice(s, c, 2, 4); + final r = g.finish(s); + expect(r[1]!.expand((x) => x).toList(), [10.0, 11.0, 12.0, 13.0]); + }); + }); + + group('combine', () { + test('disjoint keys are adopted', () { + final s = sum3.combine(sum3.fold(_ints([3, 6])), sum3.fold(_ints([1, 4]))); + expect(sum3.finish(s), {0: 9.0, 1: 5.0}); + }); + + test('overlapping keys merge with the inner combine', () { + final s = sum3.combine( + sum3.fold(_ints([1, 2, 3])), sum3.fold(_ints([4, 5, 6]))); + expect(sum3.finish(s), {0: 9.0, 1: 5.0, 2: 7.0}); + }); + + test('mixed shared and one-sided keys', () { + final s = sum3.combine(sum3.fold(_ints([1, 3])), sum3.fold(_ints([4, 5]))); + expect(sum3.finish(s), {0: 3.0, 1: 5.0, 2: 5.0}); + }); + + test('combining with empty on either side changes nothing', () { + final xs = _ints([1, 2, 3, 4]); + expect(sum3.finish(sum3.combine(sum3.empty(), sum3.fold(xs))), + sum3.run(xs)); + expect(sum3.finish(sum3.combine(sum3.fold(xs), sum3.empty())), + sum3.run(xs)); + }); + + test('order within a key is preserved across a seam', () { + final g = GroupBy>>( + _tens, const _Slices()); + final c = col([10, 20, 11, 12, 21, 13]); + final r = g.finish(g.combine(g.fold(c, 0, 3), g.fold(c, 3, 6))); + expect(r[1]!.expand((s) => s).toList(), [10.0, 11.0, 12.0, 13.0]); + expect(r[2]!.expand((s) => s).toList(), [20.0, 21.0]); + }); + }); + + group('laws', () { + final cols = randomColumns(0x6B01, nanP: 0, value: (r) => r.intIn(-20, 20) * 1.0); + + test('GroupBy(key, Count) is lawful with exact equality', () { + expectLawful(count3, cols, equals: _mapsEqual); + }); + + test('GroupBy(key, Sum) is lawful (integer-valued sums are exact)', () { + expectLawful(sum3, cols, equals: _mapsEqual); + }); + + test('nested GroupBy', () { + final nested = GroupBy>( + _mod2, GroupBy(_mod3, Sum(_v))); + expect(nested.run(_ints([1, 2, 3, 4, 5, 6])), { + 1: {1: 1.0, 0: 3.0, 2: 5.0}, + 0: {2: 2.0, 1: 4.0, 0: 6.0}, + }); + expect(nested.run(col([])), isEmpty); + expectLawful(nested, cols, equals: _mapsEqual); + }); + + test('tolerance-compared inner (Moments per key)', () { + final g = GroupBy( + _mod3, Moments(_v)); + final r = g.run(_ints([1, 2, 3, 4, 5, 10])); + expect(r.keys.toSet(), {0, 1, 2}); + expect(r[1]!.count, 3); + expect(r[1]!.mean, closeTo(5.0, 1e-12)); // (1+4+10)/3 + expect(r[1]!.sampleSd, closeTo(4.58257569495584, 1e-9)); + expectLawful(g, cols, + equals: fieldsEqual>(_momentFields, + relTol: 1e-9, absTol: 1e-9)); + }); + + test('random chunkings and slicings agree with the whole fold', () { + final r = Rng(0x6B02); + for (var t = 0; t < 40; t++) { + final c = _ints([for (var i = r.intIn(0, 60); i > 0; i--) r.intIn(0, 9)]); + final b = randomBounds(r, c.length); + final want = sum3.run(c); + expect(_mapsEqual(runChunked(sum3, c, b), want), isTrue, + reason: 'chunked $b'); + expect(_mapsEqual(runChunked(sum3, c, b, copy: true), want), isTrue, + reason: 'copies $b'); + expect(_mapsEqual(runChunkedRight(sum3, c, b), want), isTrue, + reason: 'right $b'); + expect(_mapsEqual(runSliced(sum3, c, b), want), isTrue, + reason: 'sliced $b'); + } + }); + }); +} diff --git a/test/aggregate/last_rows_test.dart b/test/aggregate/last_rows_test.dart new file mode 100644 index 00000000..e71cfcd2 --- /dev/null +++ b/test/aggregate/last_rows_test.dart @@ -0,0 +1,75 @@ +// LastRows(k): a verbatim copy of the last min(k, n) rows of the whole input, +// null for an empty input. Persistable when batchFromJson is given. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/checkpoint_support.dart'; + +const nan = double.nan; + +List _vals(DoubleColumn? c) => [c == null, ...?c?.values]; +final _same = fieldsEqual(_vals); // NaN-aware, exact + +LastRows _last(int k) => + LastRows(k, batchFromJson: DoubleColumn.fromJson); + +void main() { + group('values', () { + final x = col([1, 2, 3, 4, 5]); + test('k = 1, 3, 8 on a longer and a shorter input', () { + expect(_last(1).run(x)!.values.toList(), [5]); + expect(_last(3).run(x)!.values.toList(), [3, 4, 5]); + expect(_last(8).run(x)!.values.toList(), [1, 2, 3, 4, 5]); // n < k + expect(_last(3).run(col([7, nan]))!.values.toList(), [7, isNaN]); + }); + test('empty input is null, also through finish(empty())', () { + for (final k in [1, 3, 8]) { + expect(_last(k).run(col([])), isNull); + expect(_last(k).finish(_last(k).empty()), isNull); + } + }); + test('honours the range and returns a copy', () { + final a = _last(3); + expect(a.finish(a.fold(x, 0, 4))!.values.toList(), [2, 3, 4]); + expect(a.finish(a.fold(x, 2, 2)), isNull); + final got = a.run(x)!; + got.values[0] = 99; + expect(x.values.toList(), [1, 2, 3, 4, 5]); + }); + test('combine keeps the last k of left ++ right', () { + final a = _last(3); + List at(int i, int j) => a + .finish(a.combine(a.fold(x, 0, i), a.fold(x, i, j)))! + .values + .toList(); + expect(at(4, 5), [3, 4, 5]); // right has 1 row: 2 from left + expect(at(2, 5), [3, 4, 5]); // right alone has k rows + expect(at(3, 4), [2, 3, 4]); + }); + }); + + group('laws', () { + for (final k in [1, 3, 8]) { + test('k=$k is lawful (slicing, finishPure, round trip)', () { + expectLawful(_last(k), randomColumns(40 + k), equals: _same); + }); + } + }); + + group('persistence', () { + test('persistable only with batchFromJson', () { + expect(LastRows(3).persistable, isFalse); + expect(_last(3).persistable, isTrue); + }); + test('a mid-stream checkpoint resumes to the one-pass result', () { + final c = randomColumns(77, count: 1, maxLen: 30).last; + for (final k in [1, 3, 8]) { + expectCheckpointResumes(_last(k), col([for (var i = 0; i < 20; i++) i * 1.5]), + equals: _same); + expectCheckpointResumes(_last(k), c, equals: _same); + } + }); + }); +} diff --git a/test/aggregate/laws_harness_test.dart b/test/aggregate/laws_harness_test.dart new file mode 100644 index 00000000..9cd219c9 --- /dev/null +++ b/test/aggregate/laws_harness_test.dart @@ -0,0 +1,707 @@ +// The law harness itself: lawful aggregators pass, deliberately broken ones are +// caught under the right law name (including 'slicing'), and fieldsEqual +// compares as documented. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; + +typedef _B = ListBatch; + +int _sliceSum(_B b, int start, int end) { + var t = 0; + for (var i = start; i < end; i++) { + t += b.rows[i]; + } + return t; +} + +/// A correct sum over int rows. +class _Sum extends Aggregator { + const _Sum(); + @override + int empty() => 0; + @override + int addSlice(int s, _B b, int start, int end) => s + _sliceSum(b, start, end); + @override + int combine(int left, int right) => left + right; + @override + int finish(int s) => s; +} + +/// Sum whose identity is 1: breaks the identity laws. +class _SumFromOne extends _Sum { + const _SumFromOne(); + @override + int empty() => 1; +} + +/// Combine drops the right side. +class _DropRight extends _Sum { + const _DropRight(); + @override + int combine(int left, int right) => left; +} + +/// addSlice ignores the state it is given (restarts from the slice), so a +/// second addSlice onto a state that already holds data loses it. fold (one +/// addSlice onto empty) and combine are both correct, so only 'slicing' sees it. +class _IgnoresState extends _Sum { + const _IgnoresState(); + @override + int addSlice(int s, _B b, int start, int end) => _sliceSum(b, start, end); +} + +/// addSlice always sums the whole batch, whatever range it is asked for. +class _IgnoresRange extends _Sum { + const _IgnoresRange(); + @override + int addSlice(int s, _B b, int start, int end) => + s + _sliceSum(b, 0, b.length); +} + +/// Mean kept as (mean, count); combine averages the two means unweighted. +class _UnweightedMean extends Aggregator, _B, double> { + const _UnweightedMean(); + @override + List empty() => [0, 0]; + @override + List addSlice(List s, _B b, int start, int end) { + for (var i = start; i < end; i++) { + final n = s[1] + 1; + s = [s[0] + (b.rows[i] - s[0]) / n, n]; + } + return s; + } + + @override + List combine(List l, List r) { + if (r[1] == 0) return l; + if (l[1] == 0) return r; + return [(l[0] + r[0]) / 2, l[1] + r[1]]; + } + + @override + double finish(List s) => s[0]; +} + +/// addSlice counts 1 per row, but joining two NON-empty chunks adds a bonus, so +/// the add path and the combine path disagree. Identities still hold. +class _JoinBonus extends Aggregator { + const _JoinBonus(); + @override + int empty() => 0; + @override + int addSlice(int s, _B b, int start, int end) => s + (end - start); + @override + int combine(int left, int right) => + left > 0 && right > 0 ? left + right + 1 : left + right; + @override + int finish(int s) => s; +} + +/// Wrong only when exactly 3 rows meet exactly 2 (a split of a 5-row batch). +class _WrongAtThreeTwo extends Aggregator, _B, int> { + const _WrongAtThreeTwo(); + @override + List empty() => [0, 0]; // sum, count + @override + List addSlice(List s, _B b, int start, int end) => + [s[0] + _sliceSum(b, start, end), s[1] + (end - start)]; + @override + List combine(List l, List r) => + [l[0] + r[0] + (l[1] == 3 && r[1] == 2 ? 1000 : 0), l[1] + r[1]]; + @override + int finish(List s) => s[0]; +} + +/// A correct sum that mutates and consumes both arguments in combine. +class _MutatingSum extends Aggregator, _B, int> { + const _MutatingSum(); + @override + List empty() => [0]; + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + right[0] = -999999; // consumed + return left; + } + + @override + int finish(List s) => s[0]; +} + +/// Same, but wipes `left` too and returns a brand new state. +class _ConsumingSum extends _MutatingSum { + const _ConsumingSum(); + @override + List combine(List left, List right) { + final out = [left[0] + right[0]]; + left[0] = -1; + right[0] = -1; + return out; + } +} + +/// A sum whose finish reads the state and then clears it. A single finish per +/// fresh state looks right, so only a law that finishes the SAME state twice +/// (or keeps using it afterwards) can see the damage. +class _DestructiveSum extends Aggregator, _B, int> { + const _DestructiveSum(); + @override + List empty() => [0]; + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + return left; + } + + @override + int finish(List s) { + final out = s[0]; + s[0] = 0; // destructive + return out; + } +} + +/// finish returns the right number the first time, but marks the state as +/// closed so later rows are ignored. Two finishes in a row agree; continuing to +/// add afterwards does not. +class _FinishFreezes extends Aggregator, _B, int> { + const _FinishFreezes(); + @override + List empty() => [0, 0]; // sum, frozen + @override + List addSlice(List s, _B b, int start, int end) { + if (s[1] == 0) s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + if (left[1] == 0) left[0] += right[0]; + return left; + } + + @override + int finish(List s) { + s[1] = 1; + return s[0]; + } +} + +/// A correct sum whose codec only breaks when the restored state is the RIGHT +/// operand of combine: a state read back from JSON carries a flag, and combine +/// drops the right side's total when that flag is set. Restoring only the left +/// operand (the old roundTrip law) never sees it. +class _RightRestoreDrops extends Aggregator, _B, int> { + const _RightRestoreDrops({this.faithful = false}); + + /// When true the same codec is lawful: combine keeps a restored right side. + final bool faithful; + @override + List empty() => [0, 0]; // total, restored flag + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) => + [left[0] + (right[1] == 1 && !faithful ? 0 : right[0]), left[1]]; + @override + int finish(List s) => s[0]; + @override + bool get persistable => true; + @override + Object? encodeState(List s) => s[0]; + @override + List decodeState(Object? json) => [json as int, 1]; +} + +Set _laws(List v) => {for (final x in v) x.law}; + +void main() { + final cases = randomIntBatches(11, lo: 1, hi: 50); + + group('lawful aggregators', () { + test('a correct sum has no violations', () { + expect(checkLaws(const _Sum(), cases), isEmpty); + }); + test('a mutating combine is judged lawful when otherwise lawful', () { + expect(checkLaws(const _MutatingSum(), cases), isEmpty); + expect(checkLaws(const _ConsumingSum(), cases), isEmpty); + }); + test('no cases is vacuously lawful', () { + expect(checkLaws(const _Sum(), <_B>[]), isEmpty); + }); + test('empty and one-row cases alone are fine', () { + expect(checkLaws(const _Sum(), [_B([]), _B([5])]), isEmpty); + }); + test('long cases are bounded by maxSplits and still pass', () { + final long = _B([for (var i = 0; i < 300; i++) i]); + expect(checkLaws(const _Sum(), [long], maxSplits: 8), isEmpty); + }); + test('the harness leaves the case batches unchanged', () { + final b = _B([5, 6, 7, 8]); + checkLaws(const _Sum(), [b]); + expect(b.rows, [5, 6, 7, 8]); + }); + test('expectLawful passes for a lawful aggregator and fails for a broken ' + 'one', () { + expectLawful(const _Sum(), cases); + expect(() => expectLawful(const _DropRight(), cases), + throwsA(isA())); + }); + }); + + group('broken aggregators are caught', () { + test('(a) non-identity empty fires the identity laws', () { + final laws = _laws(checkLaws(const _SumFromOne(), cases)); + expect(laws, containsAll(['leftIdentity', 'rightIdentity'])); + }); + + test('(b) combine that drops the right side fires split', () { + final v = checkLaws(const _DropRight(), cases); + expect(_laws(v), contains('split')); + expect(v.every((x) => x.detail.isNotEmpty), isTrue); + }); + + test('(c) unweighted mean combine fires split or associativity', () { + bool close(double a, double b) => (a - b).abs() < 1e-9; + final laws = + _laws(checkLaws(const _UnweightedMean(), cases, equals: close)); + expect(laws.intersection({'split', 'associativity'}), isNotEmpty); + }); + + test('(d) addSlice inconsistent with combine fires singletons (identities ' + 'hold)', () { + final laws = _laws(checkLaws(const _JoinBonus(), cases)); + expect(laws, contains('singletons')); + expect(laws, contains('split')); + expect(laws, isNot(contains('leftIdentity'))); + expect(laws, isNot(contains('rightIdentity'))); + }); + + test('(e) addSlice that ignores the state on a second call fires only ' + '"slicing"', () { + final laws = _laws(checkLaws(const _IgnoresState(), cases)); + expect(laws, {'slicing'}); + }); + + test('(f) addSlice that ignores its range fires split and slicing', () { + final laws = _laws(checkLaws(const _IgnoresRange(), cases)); + expect(laws, containsAll(['split', 'slicing'])); + }); + + test('every split point of a short case is tried', () { + // Only the 3|2 split of a five-row batch is wrong. + final laws = _laws(checkLaws(const _WrongAtThreeTwo(), [ + _B([1, 2, 3, 4, 5]) + ])); + expect(laws, contains('split')); + }); + + test('a lawful case alongside a broken one still reports the broken one', + () { + final v = checkLaws(const _DropRight(), [ + _B([]), + _B([5]), + _B([1, 2, 3]) + ]); + expect(_laws(v), contains('split')); + }); + + test('results are deterministic for a given seed', () { + String render(List v) => + v.map((x) => '${x.law}|${x.detail}').join('\n'); + final a = checkLaws(const _IgnoresState(), cases, seed: 5); + final b = checkLaws(const _IgnoresState(), cases, seed: 5); + expect(render(a), render(b)); + expect(a, isNotEmpty); + }); + + test('a custom equals is honoured', () { + // Everything is "equal", so even a broken aggregator passes. + expect(checkLaws(const _DropRight(), cases, equals: (a, b) => true), + isEmpty); + expect(checkLaws(const _IgnoresState(), cases, equals: (a, b) => true), + isEmpty); + }); + }); + + group('roundTrip restores the right operand too', () { + final small = [ + _B([]), + _B([1, 2, 3]), + _B([4, 5]), + ]; + + test('a codec that breaks only as the right operand is caught as exactly ' + '{roundTrip}', () { + final v = checkLaws(const _RightRestoreDrops(), small); + expect(_laws(v), {'roundTrip'}); + expect(v.every((x) => x.detail.isNotEmpty), isTrue); + }); + + test('the review case [1,2,3] is caught', () { + expect( + _laws(checkLaws(const _RightRestoreDrops(), [ + _B([1, 2, 3]) + ])), + {'roundTrip'}); + }); + + test('checkConversion reports it as law:roundTrip', () { + final f = checkConversion, _B, int>( + (_) => const _RightRestoreDrops(), + small, + (b) => b.rows.fold(0, (s, x) => s + x)); + expect({for (final x in f) x.path}, {'law:roundTrip'}); + }); + + test('expectLawful fails for it', () { + expect(() => expectLawful(const _RightRestoreDrops(), small), + throwsA(isA())); + }); + + test('the same codec with a combine that keeps a restored right side is ' + 'clean', () { + expect(checkLaws(const _RightRestoreDrops(faithful: true), small), + isEmpty); + }); + }); + + group('finishPure', () { + test('a sum whose finish clears the state is caught with exactly ' + '{finishPure}', () { + expect(_laws(checkLaws(const _DestructiveSum(), cases)), {'finishPure'}); + }); + + test('a finish that closes the state to later rows is caught with exactly ' + '{finishPure}', () { + expect(_laws(checkLaws(const _FinishFreezes(), cases)), {'finishPure'}); + }); + + test('the violation names the case', () { + final v = checkLaws(const _DestructiveSum(), [ + _B([1, 2, 3]) + ]); + expect(v, isNotEmpty); + expect(v.every((x) => x.law == 'finishPure' && x.detail.isNotEmpty), + isTrue); + }); + + test('lawful aggregators report no finishPure violation', () { + for (final agg in >[ + const _Sum(), + const _MutatingSum(), + const _ConsumingSum(), + ]) { + expect(checkLaws(agg, cases), isEmpty, reason: '$agg'); + } + }); + + test('expectLawful fails for a destructive finish', () { + expect(() => expectLawful(const _DestructiveSum(), cases), + throwsA(isA())); + }); + }); + + group('fieldsEqual', () { + List same(List x) => x; + + test('two ints compare exactly, with no double conversion', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect(eq([9007199254740992], [9007199254740993]), isFalse); + expect(eq([9007199254740993], [9007199254740993]), isTrue); + final loose = fieldsEqual>(same, relTol: 0.5, absTol: 10); + expect(loose([100], [101]), isFalse); + }); + + test('tolerance is absTol + relTol * max(|a|, |b|)', () { + final eq = fieldsEqual>(same, absTol: 0.1, relTol: 0.05); + // 0.15 <= 0.1 + 0.05 * 1.15; the max(absTol, rel) rule would say no. + expect(eq([1.0], [1.15]), isTrue); + expect(eq([1.15], [1.0]), isTrue); + expect(eq([1.0], [1.2]), isFalse); + }); + + test('a finite value never equals a non-finite one', () { + final eq = fieldsEqual>(same, absTol: 1e300, relTol: 1); + expect(eq([1.0], [double.infinity]), isFalse); + expect(eq([double.negativeInfinity], [1.0]), isFalse); + expect(eq([double.maxFinite], [double.infinity]), isFalse); + expect(eq([1.0], [double.nan]), isFalse); + expect(eq([double.nan], [double.infinity]), isFalse); + }); + + test('NaN equals NaN and each infinity only itself, tolerances or not', () { + final eq = fieldsEqual>(same, absTol: 1, relTol: 1); + expect(eq([double.nan], [double.nan]), isTrue); + expect(eq([double.infinity], [double.infinity]), isTrue); + expect(eq([double.negativeInfinity], [double.negativeInfinity]), isTrue); + expect(eq([double.infinity], [double.negativeInfinity]), isFalse); + }); + + test('an int and a double compare with tolerance', () { + final eq = fieldsEqual>(same, absTol: 0.1); + expect(eq([1], [1.05]), isTrue); + expect(eq([1], [1.5]), isFalse); + }); + + test('identical lists are equal; length mismatch is not', () { + final eq = fieldsEqual>(same); + expect(eq([1, 'a', null], [1, 'a', null]), isTrue); + expect(eq([1, 2], [1, 2, 3]), isFalse); + }); + + test('NaN equals NaN, but not a number', () { + final eq = fieldsEqual>(same); + expect(eq([double.nan], [double.nan]), isTrue); + expect(eq([double.nan], [1.0]), isFalse); + expect(eq([1.0], [double.nan]), isFalse); + }); + + test('null equals only null', () { + final eq = fieldsEqual>(same, relTol: 1, absTol: 1); + expect(eq([null], [null]), isTrue); + expect(eq([null], [0.0]), isFalse); + expect(eq([0.0], [null]), isFalse); + }); + + test('zero tolerance is exact', () { + final eq = fieldsEqual>(same); + expect(eq([1.0], [1.0]), isTrue); + expect(eq([1.0], [1.0000000000000002]), isFalse); + expect(eq([1], [1.0]), isTrue); + }); + + test('relTol scales with magnitude', () { + final eq = fieldsEqual>(same, relTol: 1e-9); + expect(eq([1000.0], [1000.0000001]), isTrue); // 1e-7 abs, 1e-10 rel + expect(eq([1000.0], [1000.00001]), isFalse); // 1e-5 abs, 1e-8 rel + expect(eq([0.0], [1e-12]), isFalse); // relTol alone gives nothing at 0 + }); + + test('absTol covers values near zero', () { + final eq = fieldsEqual>(same, absTol: 1e-6); + expect(eq([0.0], [1e-7]), isTrue); + expect(eq([0.0], [1e-5]), isFalse); + }); + + test('infinities match only themselves', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect(eq([double.infinity], [double.infinity]), isTrue); + expect(eq([double.infinity], [double.negativeInfinity]), isFalse); + }); + + test('non-numeric fields compare with ==', () { + final eq = fieldsEqual>(same, relTol: 1, absTol: 1); + expect(eq(['a'], ['a']), isTrue); + expect(eq(['a'], ['b']), isFalse); + expect(eq([true], [true]), isTrue); + expect(eq([1], ['1']), isFalse); + }); + + test('the projection is applied to both sides', () { + final eq = + fieldsEqual<({int a, double b})>((r) => [r.a, r.b], absTol: 1e-6); + expect(eq((a: 1, b: 2.0), (a: 1, b: 2.0000001)), isTrue); + expect(eq((a: 1, b: 2.0), (a: 2, b: 2.0)), isFalse); + }); + + group('list fields', () { + test('equal-valued distinct List fields are equal', () { + final eq = fieldsEqual>(same); + expect( + eq([ + [1.0, 2.5] + ], [ + [1.0, 2.5] + ]), + isTrue); + }); + + test('Float64List equals a List with the same values', () { + final eq = fieldsEqual>(same); + final typed = Float64List.fromList([1.0, 2.5]); + expect( + eq([typed], [ + [1.0, 2.5] + ]), + isTrue); + expect( + eq([ + [1.0, 2.5] + ], [typed]), + isTrue); + }); + + test('elements use the numeric tolerance', () { + final eq = fieldsEqual>(same, relTol: 1e-3); + expect( + eq([ + [1000.0, 1.0] + ], [ + [1000.5, 1.0] + ]), + isTrue); + expect( + eq([ + [1000.0, 1.0] + ], [ + [1002.0, 1.0] + ]), + isFalse); + }); + + test('different lengths are unequal (guard)', () { + final eq = fieldsEqual>(same, absTol: 1); + expect( + eq([ + [1.0, 2.0] + ], [ + [1.0] + ]), + isFalse); + }); + + test('NaN elements equal each other', () { + final eq = fieldsEqual>(same); + expect( + eq([ + [double.nan, 1.0] + ], [ + [double.nan, 1.0] + ]), + isTrue); + expect( + eq([ + [double.nan] + ], [ + [1.0] + ]), + isFalse); + }); + + test('finite never equals non-finite inside a list (guard)', () { + final eq = fieldsEqual>(same, absTol: 1e300, relTol: 1); + expect( + eq([ + [1.0] + ], [ + [double.infinity] + ]), + isFalse); + }); + + test('int elements compare exactly', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect( + eq([ + [9007199254740992] + ], [ + [9007199254740993] + ]), + isFalse); + expect( + eq([ + [9007199254740993] + ], [ + [9007199254740993] + ]), + isTrue); + }); + + test('null elements equal only null', () { + final eq = fieldsEqual>(same, absTol: 1); + expect( + eq([ + [null, 1.0] + ], [ + [null, 1.0] + ]), + isTrue); + expect( + eq([ + [null] + ], [ + [0.0] + ]), + isFalse); + }); + + test('nested lists recurse', () { + final eq = fieldsEqual>(same, absTol: 0.01); + expect( + eq([ + [ + [1.0, 2.0], + [3.0] + ] + ], [ + [ + [1.001, 2.0], + [3.0] + ] + ]), + isTrue); + expect( + eq([ + [ + [1.0, 2.0], + [3.0] + ] + ], [ + [ + [1.0, 2.0], + [3.5] + ] + ]), + isFalse); + }); + + test('a list never equals a non-list (guard)', () { + final eq = fieldsEqual>(same, absTol: 10, relTol: 1); + expect( + eq([ + [1.0] + ], [1.0]), + isFalse); + expect( + eq([1.0], [ + [1.0] + ]), + isFalse); + expect( + eq([ + [] + ], [null]), + isFalse); + }); + + test('empty lists are equal', () { + final eq = fieldsEqual>(same); + expect(eq([[]], [[]]), isTrue); + expect(eq([Float64List(0)], [[]]), isTrue); + }); + }); + }); +} diff --git a/test/aggregate/lookback_test.dart b/test/aggregate/lookback_test.dart new file mode 100644 index 00000000..4d9c45a9 --- /dev/null +++ b/test/aggregate/lookback_test.dart @@ -0,0 +1,255 @@ +// Lookback(k, inner): the inner aggregator sees every length-k window of the +// whole batch exactly once, in order, however the batch is sliced or chunked. +// +// The probe inner records each window it is shown. Per the contract, inner +// .addSlice(s, b, start, end) must itself handle the windows ending at rows +// start+k-1 .. end-1 (reading rows i-k+1..i of b), so the recorder does that +// and nothing else. Row values are their own index, so a window identifies +// itself. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/fold_law_support.dart' show foldBounds; +import '../support/property.dart' show Rng; + +typedef _Windows = List>; + +class _Record extends Aggregator<_Windows, DoubleColumn, _Windows> { + const _Record(this.k); + final int k; + @override + _Windows empty() => >[]; + @override + _Windows addSlice(_Windows s, DoubleColumn b, int start, int end) { + for (var i = start + k - 1; i < end; i++) { + s.add([for (var j = i - k + 1; j <= i; j++) b.values[j]]); + } + return s; + } + + @override + _Windows combine(_Windows left, _Windows right) => left..addAll(right); + @override + _Windows finish(_Windows s) => [for (final w in s) List.of(w)]; +} + +Lookback _lookback(int k) => + Lookback(k, _Record(k)); + +/// n rows whose values are 0, 1, 2, ... (distinct, so windows identify). +DoubleColumn _iota(int n) => col([for (var i = 0; i < n; i++) i * 1.0]); + +_Windows _expected(int n, int k) => [ + for (var i = k - 1; i < n; i++) + [for (var j = i - k + 1; j <= i; j++) j * 1.0] + ]; + +bool _same(Object? a, Object? b) => a.toString() == b.toString(); + +void main() { + for (final k in [1, 2, 3, 4]) { + group('k = $k', () { + final lb = _lookback(k); + + test('a single slice sees every window of the batch', () { + for (final n in [0, 1, k - 1, k, k + 1, 2 * k, 17]) { + if (n < 0) continue; + expect(lb.run(_iota(n)), _expected(n, k), reason: 'n=$n'); + } + }); + + test('a fold range sees only windows inside the range', () { + final c = _iota(12); + final got = lb.finish(lb.fold(c, 3, 9)); + expect(got, [ + for (var i = 3 + k - 1; i < 9; i++) + [for (var j = i - k + 1; j <= i; j++) j * 1.0] + ]); + }); + + test('every single split point (combine)', () { + final c = _iota(14); + for (var s = 0; s <= 14; s++) { + final r = lb.finish(lb.combine(lb.fold(c, 0, s), lb.fold(c, s, 14))); + expect(r, _expected(14, k), reason: 'split at $s'); + } + }); + + test('every two-slice cut on the slicing path (addSlice after data)', () { + final c = _iota(14); + for (var s = 0; s <= 14; s++) { + var st = lb.empty(); + st = lb.addSlice(st, c, 0, s); + st = lb.addSlice(st, c, s, 14); + expect(lb.finish(st), _expected(14, k), reason: 'cut at $s'); + } + }); + + test('every three-slice cut on both paths', () { + final c = _iota(10); + for (var a = 0; a <= 10; a++) { + for (var b = a; b <= 10; b++) { + final bounds = [0, a, b, 10]; + expect(runChunked(lb, c, bounds), _expected(10, k), + reason: 'chunked $bounds'); + expect(runChunkedRight(lb, c, bounds), _expected(10, k), + reason: 'right $bounds'); + expect(runSliced(lb, c, bounds), _expected(10, k), + reason: 'sliced $bounds'); + expect(runChunked(lb, c, bounds, copy: true), _expected(10, k), + reason: 'copies $bounds'); + } + } + }); + + test('random slicings and chunkings, empty chunks included', () { + final r = Rng(0x10B0 + k); + for (var t = 0; t < 80; t++) { + final n = r.intIn(0, 40); + final c = _iota(n); + final b = foldBounds(n, r.intIn(0, 9), r.nextInt(1 << 20), + special: [0, 1, 2, 3, 4]); + final want = _expected(n, k); + expect(runChunked(lb, c, b), want, reason: 'chunked $b n=$n'); + expect(runChunked(lb, c, b, copy: true), want, reason: 'copy $b n=$n'); + expect(runChunkedRight(lb, c, b), want, reason: 'right $b n=$n'); + expect(runSliced(lb, c, b), want, reason: 'sliced $b n=$n'); + } + }); + + test('one row per chunk and per slice', () { + for (final n in [0, 1, 2, 3, 9]) { + final c = _iota(n); + final b = singletonBounds(n); + expect(runChunked(lb, c, b), _expected(n, k), reason: 'chunks n=$n'); + expect(runChunked(lb, c, b, copy: true), _expected(n, k), + reason: 'copies n=$n'); + expect(runChunkedRight(lb, c, b), _expected(n, k), + reason: 'right n=$n'); + expect(runSliced(lb, c, b), _expected(n, k), reason: 'slices n=$n'); + } + }); + + test('empty chunks around and between data change nothing', () { + final c = _iota(8); + final want = _expected(8, k); + expect(runChunked(lb, c, [0, 0, 0, 3, 3, 3, 8, 8]), want); + expect(runSliced(lb, c, [0, 0, 3, 3, 8, 8]), want); + expect( + lb.finish(lb.combine(lb.combine(lb.empty(), lb.fold(c)), lb.empty())), + want); + expect( + lb.finish(lb.combine(lb.fold(c, 0, 4), + lb.combine(lb.empty(), lb.fold(c, 4, 8)))), + want); + }); + + test('is lawful', () { + expectLawful(lb, [for (var n = 0; n <= 12; n++) _iota(n), _iota(40)], + equals: _same); + }); + }); + } + + group('chunks shorter than k-1 rows', () { + test('k = 4: all chunks of 1 or 2 rows, mixed with empties', () { + final lb = _lookback(4); + final c = _iota(11); + final want = _expected(11, 4); + for (final b in [ + [0, 1, 2, 4, 5, 6, 8, 9, 11], + [0, 2, 3, 3, 4, 6, 7, 9, 10, 11], + [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + ]) { + expect(runChunked(lb, c, b), want, reason: '$b'); + expect(runChunked(lb, c, b, copy: true), want, reason: 'copy $b'); + expect(runChunkedRight(lb, c, b), want, reason: 'right $b'); + expect(runSliced(lb, c, b), want, reason: 'sliced $b'); + } + }); + + test('k = 3: a seam of two one-row chunks yields no window by itself', () { + final lb = _lookback(3); + final c = _iota(3); + expect(lb.finish(lb.combine(lb.fold(c, 0, 1), lb.fold(c, 1, 2))), isEmpty); + final s = lb.combine( + lb.combine(lb.fold(c, 0, 1), lb.fold(c, 1, 2)), lb.fold(c, 2, 3)); + expect(lb.finish(s), [ + [0.0, 1.0, 2.0] + ]); + }); + + test('k = 4: no window when the whole batch is shorter than k', () { + final lb = _lookback(4); + expect(lb.run(_iota(3)), isEmpty); + expect(runChunked(lb, _iota(3), [0, 1, 2, 3]), isEmpty); + expect(runSliced(lb, _iota(3), [0, 1, 2, 3]), isEmpty); + }); + }); + + group('seam batches are tiny', () { + test('combining two 50-row chunks feeds the inner one seam of 2*(k-1) rows', + () { + // The inner records the length of every slice it is handed. The two + // folds hand it 50 rows each; the combine hands it exactly one more + // slice: the whole seam batch, last(k-1) + first(k-1) = 4 rows. + final seen = []; + final probe = _Spy(3, seen); + final lb = Lookback(3, probe); + final c = _iota(100); + lb.combine(lb.fold(c, 0, 50), lb.fold(c, 50, 100)); + expect(seen, [50, 50, 4]); + }); + }); + + group('results of the inner pass through', () { + test('Lookback(2, absolute successive differences)', () { + final lb = Lookback, double>( + 2, _AbsDiffSum()); + final c = col([1, 5, 2, 2, 10]); + expect(lb.run(c), 4 + 3 + 0 + 8); + expect(lb.run(col([])), 0.0); + expect(lb.run(col([7])), 0.0); + expectLawful(lb, randomColumns(0x10B9, nanP: 0), + equals: fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-9)); + }); + + test('composes with both and map', () { + final a = both( + Lookback(2, _Record(2)), + Count()).map((r) => (r.$1.length, r.$2)); + expect(a.run(_iota(5)), (4, 5)); + expect(a.finish(a.combine(a.fold(_iota(5), 0, 2), a.fold(_iota(5), 2, 5))), + (4, 5)); + }); + }); +} + +class _Spy extends _Record { + _Spy(super.k, this.sizes); + final List sizes; + @override + _Windows addSlice(_Windows s, DoubleColumn b, int start, int end) { + sizes.add(end - start); + return super.addSlice(s, b, start, end); + } +} + +class _AbsDiffSum extends Aggregator, DoubleColumn, double> { + @override + List empty() => [0.0]; + @override + List addSlice(List s, DoubleColumn b, int start, int end) { + for (var i = start + 1; i < end; i++) { + s[0] += (b.values[i] - b.values[i - 1]).abs(); + } + return s; + } + + @override + List combine(List l, List r) => l..[0] += r[0]; + @override + double finish(List s) => s[0]; +} diff --git a/test/aggregate/primitives_test.dart b/test/aggregate/primitives_test.dart new file mode 100644 index 00000000..2cbc6518 --- /dev/null +++ b/test/aggregate/primitives_test.dart @@ -0,0 +1,271 @@ +// Primitives: known values (NaN skipping, empty gives the identity result), +// then the laws on seeded random columns. + +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart' show RunningMoments; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; + +const nan = double.nan; + +List _momentFields(RunningMoments m) => + [m.count, m.mean, m.sampleSd, m.populationSd]; + +// Aggregators over DoubleColumn. +final _count = Count(); +final _finite = CountFinite(_v); +final _positive = CountWhere(_v, (x) => x > 0); +final _sum = Sum(_v); +final _sumSq = SumSq(_v); +final _min = Min(_v); +final _max = Max(_v); +final _moments = Moments(_v); + +/// Every split point of [c] recombines to [want] (compared by [eq]). +void _splitsAgree(Aggregator a, DoubleColumn c, + bool Function(R, R) eq) { + final want = a.run(c); + for (var k = 0; k <= c.length; k++) { + final got = a.finish(a.combine(a.fold(c, 0, k), a.fold(c, k, c.length))); + expect(eq(got, want), isTrue, reason: 'split at $k: $got vs $want'); + } +} + +void main() { + final cols = randomColumns(11); + final pos = randomColumns(12, nanP: .1, value: (r) => 1 + r.nextDouble() * 100); + final ints = randomIntBatches(13); + + group('Count', () { + test('counts rows, NaN included', () { + expect(_count.run(col([1, nan, 3])), 3); + expect(_count.run(col([])), 0); + expect(Count>().runList(['a', 'b']), 2); + }); + test('honours the range', () { + final c = col([1, 2, 3, 4, 5]); + expect(_count.finish(_count.fold(c, 1, 4)), 3); + expect(_count.finish(_count.fold(c, 2, 2)), 0); + }); + }); + + group('CountFinite and CountWhere', () { + test('CountFinite skips NaN', () { + expect(_finite.run(col([1, nan, 3, nan])), 2); + expect(_finite.run(col([nan, nan])), 0); + expect(_finite.run(col([])), 0); + }); + test('CountWhere counts non-NaN rows passing the test', () { + expect(_positive.run(col([-1, 2, nan, 3, 0])), 2); + expect(_positive.run(col([])), 0); + // NaN is skipped before the test, even for a test NaN would pass. + final notPos = CountWhere(_v, (x) => !(x > 0)); + expect(notPos.run(col([-1, nan, 5, 0])), 2); + }); + test('both use the column getter', () { + final second = CountFinite>( + (b) => Float64List.fromList([for (final r in b.rows) r.$2])); + expect(second.runList([(1, nan), (2, 3), (3, 4)]), 2); + }); + }); + + group('Sum', () { + test('plain sums, empty is exactly 0.0', () { + expect(_sum.run(col([1.5, 2.5, -1.0])), 3.0); + final e = _sum.run(col([])); + expect(e, 0.0); + expect(e, isA()); + }); + test('skips NaN; all-NaN is 0.0', () { + expect(_sum.run(col([1, nan, 2, nan])), 3.0); + expect(_sum.run(col([nan, nan])), 0.0); + }); + test('is compensated: ten times 0.1 is exactly 1.0', () { + expect(_sum.run(col(List.filled(10, 0.1))), 1.0); + }); + test('is Neumaier compensated (big, small, minus big)', () { + expect(_sum.run(col([1e16, 1.0, -1e16])), 1.0); + expect(_sum.run(col([1.0, 1e16, -1e16])), 1.0); + }); + test('compensation survives chunking and repeated addSlice', () { + final c = col([1e16, 1.0, -1e16, 1.0]); + for (var k = 0; k <= 4; k++) { + final s = _sum.combine(_sum.fold(c, 0, k), _sum.fold(c, k, 4)); + expect(_sum.finish(s), 2.0, reason: 'split at $k'); + var one = _sum.addSlice(_sum.empty(), c, 0, k); + one = _sum.addSlice(one, c, k, 4); + expect(_sum.finish(one), 2.0, reason: 'sliced at $k'); + } + }); + test('finish does not mutate: finish twice, keep adding', () { + final c = col([1, 2, 3, 4]); + var s = _sum.fold(c, 0, 3); + expect(_sum.finish(s), 6.0); + expect(_sum.finish(s), 6.0); + s = _sum.addSlice(s, c, 3, 4); + expect(_sum.finish(s), 10.0); + }); + }); + + group('SumSq', () { + test('sums squares, skips NaN, empty is 0.0', () { + expect(_sumSq.run(col([1, -2, 3])), 14.0); + expect(_sumSq.run(col([2, nan, 3])), 13.0); + expect(_sumSq.run(col([])), 0.0); + expect(_sumSq.run(col([nan])), 0.0); + }); + }); + + group('Min and Max', () { + test('values', () { + final c = col([3, -1, 2]); + expect(_min.run(c), -1.0); + expect(_max.run(c), 3.0); + expect(_min.run(col([7])), 7.0); + expect(_max.run(col([-4, -9])), -4.0); + }); + test('null when no finite row', () { + expect(_min.run(col([])), isNull); + expect(_max.run(col([])), isNull); + expect(_min.run(col([nan, nan])), isNull); + expect(_max.run(col([nan])), isNull); + }); + test('NaN rows are skipped', () { + final c = col([nan, 5, nan, 2, 9, nan]); + expect(_min.run(c), 2.0); + expect(_max.run(c), 9.0); + }); + test('empty and all-NaN sides of a seam change nothing', () { + final full = col([4, 2]); + final nans = col([nan, nan]); + expect(_max.finish(_max.combine(_max.empty(), _max.fold(full))), 4.0); + expect(_max.finish(_max.combine(_max.fold(full), _max.empty())), 4.0); + expect(_max.finish(_max.combine(_max.fold(nans), _max.fold(full))), 4.0); + expect(_min.finish(_min.combine(_min.fold(full), _min.fold(nans))), 2.0); + expect(_max.finish(_max.combine(_max.empty(), _max.empty())), isNull); + expect(_min.finish(_min.combine(_min.fold(nans), _min.fold(nans))), isNull); + }); + }); + + group('Moments', () { + test('textbook sample', () { + final m = _moments.run(col([2, 4, 4, 4, 5, 5, 7, 9])); + expect(m.count, 8); + expect(m.mean, closeTo(5.0, 1e-12)); + expect(m.populationSd, closeTo(2.0, 1e-12)); + expect(m.sampleSd, closeTo(math.sqrt(32 / 7), 1e-12)); + }); + test('empty, one value, all NaN', () { + final e = _moments.run(col([])); + expect((e.count, e.mean, e.sampleSd), (0, null, null)); + final one = _moments.run(col([4])); + expect((one.count, one.mean, one.sampleSd), (1, 4.0, null)); + final n = _moments.run(col([nan, nan])); + expect((n.count, n.mean), (0, null)); + }); + test('skips NaN rows', () { + final m = _moments.run(col([nan, 2, 4, nan, 6])); + expect(m.count, 3); + expect(m.mean, closeTo(4.0, 1e-12)); + expect(m.sampleSd, closeTo(2.0, 1e-12)); + }); + test('merged across a seam matches the whole', () { + final c = col([for (var i = 0; i < 30; i++) math.sin(i) * 100 + 500]); + final whole = _moments.run(c); + for (final k in [0, 1, 15, 29, 30]) { + final r = _moments + .finish(_moments.combine(_moments.fold(c, 0, k), _moments.fold(c, k, 30))); + expect(r.count, whole.count); + expect(r.mean, closeTo(whole.mean!, 1e-9)); + expect(r.sampleSd, closeTo(whole.sampleSd!, 1e-9)); + } + }); + }); + + group('First and Last', () { + final first = First>(), last = Last>(); + + test('a one-row copy, null when empty', () { + final f = first.runList(['x', 'y', 'z'])!; + final l = last.runList(['x', 'y', 'z'])!; + expect(f.rows, ['x']); + expect(l.rows, ['z']); + expect(first.runList([]), isNull); + expect(last.runList([]), isNull); + }); + test('work on a column and on a range', () { + final c = col([10, 20, 30, 40]); + expect(First().run(c)!.values, [10.0]); + expect(Last().run(c)!.values, [40.0]); + final f = First(), l = Last(); + expect(f.finish(f.fold(c, 1, 3))!.values, [20.0]); + expect(l.finish(l.fold(c, 1, 3))!.values, [30.0]); + expect(f.finish(f.fold(c, 2, 2)), isNull); + }); + test('the copy does not alias the source', () { + final v = Float64List.fromList([1, 2, 3]); + final l = Last().run(DoubleColumn(v))!; + v[2] = 99; + expect(l.values, [3.0]); + }); + test('keep a seam in order, skipping empty chunks', () { + final f = First(), l = Last(); + final c = col([10, 20, 30]); + final fs = f.combine(f.combine(f.fold(c, 0, 0), f.fold(c, 0, 2)), f.fold(c, 2, 3)); + expect(f.finish(fs)!.values, [10.0]); + final ls = l.combine(l.combine(l.fold(c, 0, 2), l.fold(c, 2, 3)), l.fold(c, 3, 3)); + expect(l.finish(ls)!.values, [30.0]); + }); + }); + + group('every primitive is lawful on seeded random cases', () { + final tol = fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-6); + List batchFields(DoubleColumn? b) => + b == null ? [null] : [b.length, ...b.values]; + + test('Count', () => expectLawful(_count, cols)); + test('CountFinite', () => expectLawful(_finite, cols)); + test('CountWhere', () => expectLawful(_positive, cols)); + test('Sum', () => expectLawful(_sum, cols, equals: tol)); + test('SumSq', () => expectLawful(_sumSq, cols, equals: tol)); + test('Min', () => expectLawful(_min, cols)); + test('Max', () => expectLawful(_max, cols)); + test('Moments (tolerance)', () { + expectLawful(_moments, pos, + equals: fieldsEqual(_momentFields, + relTol: 1e-9, absTol: 1e-9)); + }); + test('First', () { + expectLawful(First(), cols, + equals: fieldsEqual(batchFields)); + expectLawful(First>(), ints, + equals: (a, b) => a?.rows.toString() == b?.rows.toString()); + }); + test('Last', () { + expectLawful(Last(), cols, + equals: fieldsEqual(batchFields)); + expectLawful(Last>(), ints, + equals: (a, b) => a?.rows.toString() == b?.rows.toString()); + }); + test('both of primitives with mixed result types', () { + expectLawful( + both(_min, _count).map((r) => [r.$1, r.$2]), cols, + equals: (a, b) => a.toString() == b.toString()); + }); + test('splits agree on a column with NaN gaps', () { + final r = Rng(5); + final c = col([ + for (var i = 0; i < 25; i++) r.nextBool(.3) ? nan : r.intIn(-9, 9) * 1.0 + ]); + _splitsAgree(_finite, c, (a, b) => a == b); + _splitsAgree(_max, c, (a, b) => a == b); + }); + }); +} diff --git a/test/aggregate/sum_of_test.dart b/test/aggregate/sum_of_test.dart new file mode 100644 index 00000000..0b9e0b88 --- /dev/null +++ b/test/aggregate/sum_of_test.dart @@ -0,0 +1,149 @@ +// SumOf(col, f): Neumaier-compensated sum of f(x) over the rows where x is +// not NaN and f(x) is not NaN. +// +// Decisions the implementation follows: +// * f is never called with NaN (a NaN input row is skipped before f). An +// infinite input IS passed to f. +// * f returning NaN skips that row (it does not poison the sum). +// * The result is a double; nothing counted (or an empty batch) is exactly 0.0. +// * Compensation is the same as Sum: combine adds the other state's sum and +// correction, so the compensation survives chunking and addSlice. +// * Exported from package:openstrap_analytics/aggregate.dart. + +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; + +const nan = double.nan; + +final _tol = fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-6); + +SumOf _sumOf(double Function(double) f) => + SumOf(_v, f); + +void main() { + final id = _sumOf((x) => x); + final square = _sumOf((x) => x * x); + // Square root of positives; NaN (skip) for everything else. + final rootPos = _sumOf((x) => x > 0 ? math.sqrt(x) : nan); + + group('values', () { + test('plain sums of f(x)', () { + expect(id.run(col([1.5, 2.5, -1.0])), 3.0); + expect(square.run(col([1, 2, 3])), 14.0); + expect(_sumOf((x) => 2 * x + 1).run(col([1, 2, 3])), 15.0); + }); + test('empty is exactly 0.0, as a double', () { + final e = square.run(col([])); + expect(e, 0.0); + expect(e, isA()); + expect(square.finish(square.empty()), 0.0); + }); + test('NaN inputs are skipped, all-NaN is 0.0', () { + expect(square.run(col([1, nan, 2, nan])), 5.0); + expect(square.run(col([nan, nan])), 0.0); + }); + test('f is never called with NaN', () { + final seen = []; + final spy = _sumOf((x) { + seen.add(x); + return x; + }); + expect(spy.run(col([1, nan, 2, nan, nan, 3])), 6.0); + expect(seen, [1.0, 2.0, 3.0]); + }); + test('f returning NaN skips the row', () { + expect(rootPos.run(col([4, -1, 9, 0, nan, 16])), 9.0); + expect(rootPos.run(col([-1, -2, 0])), 0.0); + expect(_sumOf((_) => nan).run(col([1, 2, 3])), 0.0); + }); + test('an infinite input reaches f', () { + final finiteOnly = _sumOf((x) => x.isFinite ? x : nan); + expect(finiteOnly.run(col([1, double.infinity, 2, -double.infinity])), + 3.0); + expect(id.run(col([1, double.infinity])), double.infinity); + }); + test('honours the range', () { + final c = col([1, 2, 3, 4, 5]); + expect(square.finish(square.fold(c, 1, 4)), 4.0 + 9 + 16); + expect(square.finish(square.fold(c, 2, 2)), 0.0); + }); + test('reads the column through the getter, on any batch', () { + final second = SumOf>( + (b) => Float64List.fromList([for (final r in b.rows) r.$2]), + (x) => x * 10); + expect(second.runList([(1, nan), (2, 3), (3, 4)]), 70.0); + }); + }); + + group('compensation', () { + test('ten times 0.1 is exactly 1.0', () { + expect(id.run(col(List.filled(10, 0.1))), 1.0); + }); + test('Neumaier: big, small, minus big', () { + expect(id.run(col([1e16, 1.0, -1e16])), 1.0); + expect(id.run(col([1.0, 1e16, -1e16])), 1.0); + }); + test('compensation applies to f(x), not x', () { + // f(1e8) = 1e16, f(-1e8) = -1e16; the 1.0 would vanish in a naive sum. + final signedSquare = _sumOf((x) => x >= 0 ? x * x : -(x * x)); + expect(signedSquare.run(col([1e8, 1, -1e8])), 1.0); + }); + test('survives every split and repeated addSlice', () { + final c = col([1e16, 1.0, -1e16, 1.0]); + for (var k = 0; k <= 4; k++) { + final s = id.combine(id.fold(c, 0, k), id.fold(c, k, 4)); + expect(id.finish(s), 2.0, reason: 'split at $k'); + var one = id.addSlice(id.empty(), c, 0, k); + one = id.addSlice(one, c, k, 4); + expect(id.finish(one), 2.0, reason: 'sliced at $k'); + } + }); + test('finish does not mutate: finish twice, keep adding', () { + final c = col([1, 2, 3, 4]); + var s = id.fold(c, 0, 3); + expect(id.finish(s), 6.0); + expect(id.finish(s), 6.0); + s = id.addSlice(s, c, 3, 4); + expect(id.finish(s), 10.0); + }); + }); + + group('random parity (seeded)', () { + test('matches Sum of the mapped column under random chunkings', () { + final r = Rng(0x50F); + final sum = Sum(_v); + for (final c in randomColumns(21, count: 25)) { + final mapped = DoubleColumn(Float64List.fromList([ + for (final x in c.values) x.isNaN || x <= 0 ? nan : math.sqrt(x) + ])); + final want = sum.run(mapped); + for (var t = 0; t < 3; t++) { + final b = randomBounds(r, c.length); + for (final got in [ + runChunked(rootPos, c, b), + runChunked(rootPos, c, b, copy: true), + runChunkedRight(rootPos, c, b), + runSliced(rootPos, c, b), + ]) { + expect(_tol(got, want), isTrue, reason: '$got vs $want, $b'); + } + } + } + }); + }); + + group('lawful', () { + final cols = randomColumns(22); + test('identity f', () => expectLawful(id, cols, equals: _tol)); + test('square', () => expectLawful(square, cols, equals: _tol)); + test('f that skips rows', () => expectLawful(rootPos, cols, equals: _tol)); + }); +} diff --git a/test/onehz/incremental/int_histogram_test.dart b/test/onehz/incremental/int_histogram_test.dart new file mode 100644 index 00000000..b9f2f8c3 --- /dev/null +++ b/test/onehz/incremental/int_histogram_test.dart @@ -0,0 +1,210 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; + +/// Exact equality, not a tolerance: the histogram must return the very double +/// `percentileSorted` returns for the expanded sorted list. +void _same(IntHistogram h, List values, {String? reason}) { + final sorted = [...values]..sort(); + expect(h.count, values.length, reason: reason); + for (final p in [0, 1, 5, 25, 30, 33.3, 50, 66.6, 75, 90, 95, 99, 100]) { + final expected = percentileSorted(sorted, p.toDouble()); + final actual = h.percentile(p.toDouble()); + expect(actual, expected, reason: '$reason p=$p'); + } + expect(h.median, percentileSorted(sorted, 50), reason: reason); + expect(h.median, median(values), reason: reason); +} + +List _realNightHr() => [ + for (final line in File('test/onehz/fixtures/real_night_2026_07_onehz.csv') + .readAsLinesSync() + .skip(1)) + if (line.trim().isNotEmpty) double.parse(line.split(',')[1]) + ]; + +void main() { + test('real overnight HR: exact parity at every prefix and few distinct values', + () { + final hr = _realNightHr(); + expect(hr.length, greaterThan(20000)); + final h = IntHistogram(); + for (var i = 0; i < hr.length; i++) { + h.add(hr[i].toInt()); + if (i % 997 == 0 || i == hr.length - 1) { + _same(h, hr.sublist(0, i + 1), reason: 'prefix ${i + 1}'); + } + } + expect(h.distinct, lessThan(80), reason: 'compaction is the point'); + }); + + test('real overnight HR: sliding window add/remove stays exact', () { + final hr = _realNightHr(); + const window = 1800; + final h = IntHistogram(); + for (var i = 0; i < 6000; i++) { + h.add(hr[i].toInt()); + if (i >= window) h.remove(hr[i - window].toInt()); + if (i % 251 == 0) { + _same(h, hr.sublist(math.max(0, i + 1 - window), i + 1), + reason: 'window ending $i'); + } + } + }); + + for (final seed in [1, 7, 42, 914]) { + test('randomized integer data, add/remove sequence seed=$seed', () { + final rnd = math.Random(seed); + final h = IntHistogram(); + final values = []; + for (var step = 0; step < 1500; step++) { + if (values.isNotEmpty && rnd.nextInt(10) < 4) { + final v = values.removeAt(rnd.nextInt(values.length)); + h.remove(v.toInt()); + } else { + // Includes negatives and a wide spread, not just heart rates. + final v = rnd.nextInt(400) - 100; + values.add(v.toDouble()); + h.add(v); + } + if (step % 7 == 0 || values.length < 6) { + _same(h, values, reason: 'step $step n=${values.length}'); + } + } + while (values.isNotEmpty) { + h.remove(values.removeLast().toInt()); + } + _same(h, values); + }); + } + + test('odd and even counts, single value, all-equal values', () { + for (final values in [ + [5.0], + [5.0, 5.0], + [4.0, 9.0], + [3.0, 1.0, 2.0], + [3.0, 1.0, 2.0, 10.0], + List.filled(7, 61.0), + [60.0, 60.0, 61.0], + ]) { + final h = IntHistogram(); + for (final v in values) { + h.add(v.toInt()); + } + _same(h, values, reason: '$values'); + } + }); + + test('p=0 and p=1 hit the extremes; p=100 is not a special case', () { + final h = IntHistogram(); + final values = [72.0, 55.0, 90.0, 61.0, 61.0]; + for (final v in values) { + h.add(v.toInt()); + } + expect(h.percentile(0), 55); + expect(h.percentile(100), 90); + expect(h.percentile(1), percentileSorted([...values]..sort(), 1)); + expect(h.percentile(0.5), percentileSorted([...values]..sort(), 0.5)); + }); + + test('empty histogram abstains with null, as the batch functions do', () { + final h = IntHistogram(); + expect(h.count, 0); + expect(h.distinct, 0); + expect(h.median, isNull); + expect(h.median, median(const [])); + expect(h.percentile(0), isNull); + expect(h.percentile(100), isNull); + expect(h.percentile(50), percentile(const [], 50)); + h.add(70); + h.remove(70); + expect(h.median, isNull, reason: 'emptied again'); + expect(h.distinct, 0, reason: 'no zero-count bins are retained'); + }); + + test('add with a repeat count, and removing one of several', () { + final h = IntHistogram() + ..add(60, 3) + ..add(70, 2); + _same(h, [60, 60, 60, 70, 70]); + h.remove(60); + _same(h, [60, 60, 70, 70]); + expect(h.distinct, 2); + }); + + test('removing a value that is not present throws and changes nothing', () { + final h = IntHistogram()..add(60); + expect(() => h.remove(61), throwsStateError); + expect(() => IntHistogram().remove(1), throwsStateError); + _same(h, [60]); + }); + + test('rejects values that are not whole numbers and bad arguments', () { + final h = IntHistogram(); + expect(() => h.add(double.nan), throwsArgumentError); + expect(() => h.add(double.infinity), throwsArgumentError); + expect(() => h.add(70.5), throwsArgumentError); + expect(() => h.add(70, 0), throwsArgumentError); + expect(() => h.add(70, -1), throwsArgumentError); + h.add(70.0); // a whole-number double is a whole number + expect(h.count, 1); + expect(() => h.percentile(-1), throwsArgumentError); + expect(() => h.percentile(100.5), throwsArgumentError); + expect(() => h.percentile(double.nan), throwsArgumentError); + }); + + for (final seed in [3, 11]) { + test('merge equals one histogram over the concatenation seed=$seed', () { + final rnd = math.Random(seed); + final a = [for (var i = 0; i < 300; i++) rnd.nextInt(60) + 40.0]; + final b = [for (var i = 0; i < 211; i++) rnd.nextInt(90) + 30.0]; + final ha = IntHistogram(), hb = IntHistogram(); + for (final v in a) { + ha.add(v.toInt()); + } + for (final v in b) { + hb.add(v.toInt()); + } + ha.merge(hb); + _same(ha, [...a, ...b]); + _same(hb, b, reason: 'the argument is left unchanged'); + ha.merge(IntHistogram()); + _same(ha, [...a, ...b], reason: 'merging empty is a no-op'); + final empty = IntHistogram()..merge(hb); + _same(empty, b, reason: 'merging into empty'); + empty.merge(empty); + _same(empty, [...b, ...b], reason: 'merging with itself doubles counts'); + }); + } + + test('checkpoint round-trips and continues; malformed input throws', () { + final h = IntHistogram(); + final values = []; + for (final v in [70, 70, 64, 91, 64, 64, 120]) { + h.add(v); + values.add(v.toDouble()); + } + final back = IntHistogram.fromJson(checkpoint(h.toJson())); + _same(back, values); + back.add(55); + values.add(55); + _same(back, values); + for (final bad in >[ + {}, + {'version': 2, 'type': 'IntHistogram', 'values': [], 'counts': []}, + {'version': 1, 'type': 'RunningMoments', 'values': [], 'counts': []}, + {'version': 1, 'type': 'IntHistogram', 'values': [1, 2], 'counts': [1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [2, 1], 'counts': [1, 1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1, 1], 'counts': [1, 1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1], 'counts': [0]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1.5], 'counts': [1]}, + ]) { + expect(() => IntHistogram.fromJson(bad), throwsFormatException, + reason: '$bad'); + } + }); +} diff --git a/test/onehz/incremental/lomb_scargle_test.dart b/test/onehz/incremental/lomb_scargle_test.dart new file mode 100644 index 00000000..4e9f6d44 --- /dev/null +++ b/test/onehz/incremental/lomb_scargle_test.dart @@ -0,0 +1,99 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + final frequencies = [0.0, .0033, .01, .04, .07, .1, .15, .25, .4]; + for (final seed in [1, 42, 914]) { + for (final chunk in [1, 17, 128]) { + test('Lomb prefixes and checkpoint seed=$seed chunk=$chunk', () { + final f = incrementalFixture(seed: seed, beats: 160); + final t = f.times.map((x) => x / 1000).toList(); + var state = IncrementalLombScargle(frequencies); + for (final n in prefixSizes(t.length, chunk)) { + final ts = t.sublist(0, n), y = f.nn.sublist(0, n); + spectrumClose(state.sync(ts, y), lombScargle(ts, y, frequencies)); + final work = state.processedPoints; + spectrumClose(state.sync(ts, y), lombScargle(ts, y, frequencies)); + expect(state.processedPoints, work, + reason: 'identical prefix does no math'); + state = IncrementalLombScargle.fromJson(checkpoint(state.toJson())); + } + expect(state.processedPoints, lessThanOrEqualTo(t.length + 4), + reason: 'append work is linear in newly accepted samples'); + }); + } + test('Lomb replacement, shrink, shifted origin and force seed=$seed', () { + final f = incrementalFixture(seed: seed, beats: 160); + final state = IncrementalLombScargle(frequencies); + var t = f.times.map((x) => x / 1000).toList(), y = [...f.nn]; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + y[31] += 79; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t[44] += .02; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t = t.sublist(13, 141); + y = y.sublist(13, 141); + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t = t.map((x) => x + 12345.125).toList(); + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + final work = state.processedPoints; + spectrumClose( + state.sync(t, y, force: true), lombScargle(t, y, frequencies)); + expect(state.processedPoints - work, t.length); + spectrumClose(state.sync([], []), lombScargle([], [], frequencies)); + }); + } + for (final grid in [ + [], + [0.0], + [.04, .04, .13], + [.1, .07, .4] + ]) { + test('Lomb preserves frequency grid $grid', () { + final f = incrementalFixture(beats: 31); + spectrumClose(IncrementalLombScargle(grid).sync(f.times, f.nn), + lombScargle(f.times, f.nn, grid)); + }); + } + test('Lomb constant values, collapsed time, and malformed lengths abstain', + () { + final state = IncrementalLombScargle(frequencies); + for (final input in <(List, List)>[ + ([0.0, 1, 2, 3], [7.0, 7, 7, 7]), + ([1.0, 1, 1, 1], [7.0, 8, 9, 10]), + ([1.0, 2, 3, 4], [7.0]) + ]) { + spectrumClose(state.sync(input.$1, input.$2), + lombScargle(input.$1, input.$2, frequencies)); + } + }); + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + test('Lomb nonfinite fallback $bad', () { + final f = incrementalFixture(beats: 31); + final t = f.times.map((x) => x / 1000).toList(), y = [...f.nn]; + final state = IncrementalLombScargle(frequencies); + state.sync(t, y); + y[13] = bad; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + y[13] = f.nn[13]; + t[9] = bad; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + }); + } + test('Lomb malformed checkpoint throws an explicit input error', () { + expect(() => IncrementalLombScargle.fromJson({'frequencies': 'bad'}), + throwsA(anyOf(isA(), isA()))); + }); + test('Lomb real capture irregular cleaned beat times match batch', () { + final f = realIncrementalFixture(); + expect(f.nn.length, greaterThan(300)); + final t = f.times.map((x) => x / 1000).toList(); + final state = IncrementalLombScargle(frequencies); + for (final n in prefixSizes(f.nn.length, 173)) { + final ts = t.sublist(0, n), nn = f.nn.sublist(0, n); + spectrumClose(state.sync(ts, nn), lombScargle(ts, nn, frequencies)); + } + }); +} diff --git a/test/onehz/incremental/running_moments_test.dart b/test/onehz/incremental/running_moments_test.dart new file mode 100644 index 00000000..1c2f7cc3 --- /dev/null +++ b/test/onehz/incremental/running_moments_test.dart @@ -0,0 +1,71 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void _check(RunningMoments state, List values) { + expect(state.count, values.length); + numberClose(state.mean, mean(values)); + numberClose(state.sampleSd, stddev(values)); + numberClose(state.populationSd, stddevPop(values)); +} + +void main() { + for (final seed in [1, 42, 914]) { + test('moments add/remove and JSON continuation seed=$seed', () { + var state = RunningMoments(); + final values = []; + _check(state, values); + final nn = incrementalFixture(seed: seed, beats: 100).nn; + for (final x in nn) { + values.add(x); + state.add(x); + _check(state, values); + if (values.length == 53) { + state = RunningMoments.fromJson(checkpoint(state.toJson())); + _check(state, values); + } + } + while (values.isNotEmpty) { + state.remove(values.removeAt(values.length ~/ 2)); + _check(state, values); + } + state.add(17); + _check(state, [17]); + }); + test('moments merge disjoint partitions seed=$seed', () { + final values = incrementalFixture(seed: seed, beats: 101).nn; + final left = RunningMoments(), right = RunningMoments(); + for (final x in values.take(37)) { + left.add(x); + } + for (final x in values.skip(37)) { + right.add(x); + } + left.merge(right); + _check(left, values); + _check(right, values.sublist(37)); + left.merge(RunningMoments()); + _check(left, values); + final copy = RunningMoments()..merge(left); + _check(copy, values); + }); + } + for (final values in [ + [], + [42.0], + List.filled(80, 42), + List.generate(80, (i) => 1e9 + (i % 7) * .25) + ]) { + test( + 'moments degenerate/stable data n=${values.length} first=${values.firstOrNull}', + () { + final state = RunningMoments(); + for (final x in values) { + state.add(x); + } + _check(state, values); + _check(RunningMoments.fromJson(checkpoint(state.toJson())), values); + }); + } +} diff --git a/test/onehz/lomb_scargle_precision_test.dart b/test/onehz/lomb_scargle_precision_test.dart new file mode 100644 index 00000000..0bc98758 --- /dev/null +++ b/test/onehz/lomb_scargle_precision_test.dart @@ -0,0 +1,119 @@ +// Batch Lomb–Scargle on absolute epoch seconds. Trig arguments `w * t` reach +// ~1e10 rad when `t` is ~1.8e9 s, where a double's spacing is ~1e-6 rad, so the +// periodogram used to carry ~1e-6 relative error that depends on the clock, not +// the data. `lombScargle` now measures time from the first sample. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +const _epoch = 1791028730.0; + +/// A night of beats. Times are multiples of 1/1024 s, so adding [_epoch] (a +/// whole number of seconds) is EXACT in a double: the epoch series and the +/// rebased series have identical differences, bit for bit. +({List t, List y}) _beats(int n, {int seed = 5}) { + final rnd = math.Random(seed); + final t = [], y = []; + var k = 0; + for (var i = 0; i < n; i++) { + final rr = 900 + + 90 * math.sin(i / 300) + + 45 * math.sin(i * 2 * math.pi * .25 * .9) + + rnd.nextDouble() * 30; + k += (rr / 1000 * 1024).round(); + t.add(k / 1024); + y.add(rr); + } + return (t: t, y: y); +} + +double _worstRel(LombScargle a, LombScargle b) { + expect(a.spectrum.length, b.spectrum.length); + var worst = 0.0; + for (var i = 0; i < a.spectrum.length; i++) { + final e = b.spectrum[i].psd; + final d = (a.spectrum[i].psd - e).abs(); + worst = math.max(worst, e == 0 ? d : d / e.abs()); + } + return worst; +} + +/// The same algorithm with every sum Kahan-compensated and time measured from +/// the first sample. It shares no code with `lombScargle`, so it checks the +/// shift and the summation, not just self-consistency. +LombScargle _kahanReference(List t, List y, List f) { + double ksum(Iterable xs) { + var s = 0.0, c = 0.0; + for (final x in xs) { + final v = x - c; + final u = s + v; + c = (u - s) - v; + s = u; + } + return s; + } + + final n = t.length; + final ts = [for (final v in t) v - t.first]; + final my = ksum(y) / n; + final yc = [for (final v in y) v - my]; + final span = ts.reduce(math.max) - ts.reduce(math.min); + final scale = 2.0 * span / (n - 1); + final out = []; + for (final hz in f) { + final w = 2 * math.pi * hz; + final tau = math.atan2(ksum([for (final v in ts) math.sin(2 * w * v)]), + ksum([for (final v in ts) math.cos(2 * w * v)])) / + (2 * w); + final c = [for (final v in ts) math.cos(w * (v - tau))]; + final s = [for (final v in ts) math.sin(w * (v - tau))]; + final cNum = ksum([for (var i = 0; i < n; i++) yc[i] * c[i]]); + final sNum = ksum([for (var i = 0; i < n; i++) yc[i] * s[i]]); + final cDen = ksum([for (final v in c) v * v]); + final sDen = ksum([for (final v in s) v * v]); + out.add(LsPoint(hz, 0.5 * (cNum * cNum / cDen + sNum * sNum / sDen) * scale)); + } + return LombScargle(out); +} + +void main() { + final grid = freqGrid(0.0033, 0.4, 120); + + test('epoch seconds and rebased seconds give the same spectrum', () { + final b = _beats(1200); + final epoch = [for (final v in b.t) v + _epoch]; + expect(epoch[17] - epoch[3], b.t[17] - b.t[3], reason: 'offset is exact'); + final a = lombScargle(epoch, b.y, grid)!; + final r = lombScargle(b.t, b.y, grid)!; + expect(_worstRel(a, r), lessThan(1e-11)); + }); + + test('epoch-second spectrum matches a compensated, shifted reference', () { + final b = _beats(1200, seed: 9); + final epoch = [for (final v in b.t) v + _epoch]; + final a = lombScargle(epoch, b.y, grid)!; + expect(_worstRel(a, _kahanReference(epoch, b.y, grid)), lessThan(1e-11)); + }); + + test('the shift is the first sample, so any time origin gives the same', () { + final b = _beats(400, seed: 2); + final base = lombScargle(b.t, b.y, grid)!; + for (final offset in [-1000.0, 3600.0, 1e6, 1.5e9]) { + final moved = lombScargle([for (final v in b.t) v + offset], b.y, grid)!; + expect(_worstRel(moved, base), lessThan(1e-11), reason: 'offset $offset'); + } + }); + + test('incremental state agrees with batch on epoch seconds, far inside 1e-8', + () { + final b = _beats(900, seed: 4); + final epoch = [for (final v in b.t) v + _epoch]; + final state = IncrementalLombScargle(grid); + for (final n in [10, 100, 450, 900]) { + final inc = state.sync(epoch.sublist(0, n), b.y.sublist(0, n))!; + final batch = lombScargle(epoch.sublist(0, n), b.y.sublist(0, n), grid)!; + expect(_worstRel(inc, batch), lessThan(1e-9), reason: 'n=$n'); + } + }); +} diff --git a/test/onehz/support/incremental_compare.dart b/test/onehz/support/incremental_compare.dart new file mode 100644 index 00000000..a969d77c --- /dev/null +++ b/test/onehz/support/incremental_compare.dart @@ -0,0 +1,78 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void numberClose(double? actual, double? expected, {String? reason}) { + if (expected == null) { + expect(actual, isNull, reason: reason); + } else if (expected.isNaN) { + expect(actual?.isNaN, isTrue, reason: reason); + } else if (!expected.isFinite) { + expect(actual, expected, reason: reason); + } else { + expect(actual, isNotNull, reason: reason); + expect(actual, closeTo(expected, math.max(1e-9, expected.abs() * 1e-8)), + reason: reason); + } +} + +void metricEnvelope(Metric actual, Metric expected) { + expect(actual.present, expected.present); + numberClose(actual.confidence, expected.confidence); + expect(actual.tier, expected.tier); + expect(actual.inputs_used, expected.inputs_used); + expect(actual.note, expected.note); + expect(actual.drivers?.map((d) => d.toJson()).toList(), + expected.drivers?.map((d) => d.toJson()).toList()); +} + +void hrvClose(Metric actual, Metric expected) { + metricEnvelope(actual, expected); + if (expected.value == null) return; + final a = actual.value!, e = expected.value!; + expect(a.nBeats, e.nBeats); + numberClose(a.rmssd, e.rmssd); + numberClose(a.sdnn, e.sdnn); + numberClose(a.sdann, e.sdann); + numberClose(a.sdnnIndex, e.sdnnIndex); + numberClose(a.pnn50, e.pnn50); + numberClose(a.diffAcf1, e.diffAcf1); +} + +void spectrumClose(LombScargle? actual, LombScargle? expected) { + if (expected == null) { + expect(actual, isNull); + return; + } + expect(actual, isNotNull); + expect(actual!.spectrum.length, expected.spectrum.length); + for (var i = 0; i < expected.spectrum.length; i++) { + expect(actual.spectrum[i].freqHz, expected.spectrum[i].freqHz); + numberClose(actual.spectrum[i].psd, expected.spectrum[i].psd, + reason: 'frequency ${expected.spectrum[i].freqHz}'); + } + for (final band in [(0.0, .04), (.04, .15), (.15, .4)]) { + numberClose(actual.bandPower(band.$1, band.$2), + expected.bandPower(band.$1, band.$2)); + } +} + +void enmoClose(EnmoResult actual, EnmoResult expected) { + numberClose(actual.gRef, expected.gRef); + numberClose(actual.coverage, expected.coverage); + expect(actual.minutes.length, expected.minutes.length); + for (var i = 0; i < expected.minutes.length; i++) { + final a = actual.minutes[i], e = expected.minutes[i]; + expect(a.tsMinStartMs, e.tsMinStartMs); + expect(a.nSamples, e.nSamples); + numberClose(a.enmo, e.enmo); + numberClose(a.mad, e.mad); + numberClose(a.meanMag, e.meanMag); + numberClose(a.dynAmp, e.dynAmp); + } +} + +Map checkpoint(Map json) => + jsonDecode(jsonEncode(json)) as Map; diff --git a/test/onehz/support/incremental_fixtures.dart b/test/onehz/support/incremental_fixtures.dart new file mode 100644 index 00000000..b9277cca --- /dev/null +++ b/test/onehz/support/incremental_fixtures.dart @@ -0,0 +1,113 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; + +/// Reproducible inputs; no wall clock, device, network, or unseeded randomness. +class IncrementalFixture { + final List times, nn; + final List accel; + final List minuteKeys; + final List hr; + final List cadence; + const IncrementalFixture( + this.times, this.nn, this.accel, this.minuteKeys, this.hr, this.cadence); +} + +IncrementalFixture incrementalFixture( + {int seed = 42, + int beats = 850, + int seconds = 241, + int minutes = 121, + double originMs = 0, + double cadenceHz = 1, + bool gaps = true, + bool jitter = false}) { + final random = math.Random(seed); + final nn = [], times = []; + var time = originMs; + for (var i = 0; i < beats; i++) { + final interval = jitter + ? 800 + 160 * (random.nextDouble() - .5) + : 800 + + 75 * math.sin(i * .13 + seed * .003) + + 21 * math.sin(i * .031 + seed * .007); + time += interval; + if (gaps && i % 113 == 112) time += 21000; + nn.add(interval); + times.add(time); + } + final accel = []; + for (var i = 0; i < seconds; i++) { + if (gaps && i >= 70 && i < 139) continue; + accel.add(AccelSample( + originMs + i * 1000 / cadenceHz, + .22 * math.sin(i * .29) + random.nextDouble() * .007, + .11 * math.cos(i * .07), + 1.02 + .065 * math.sin(i * .23), + valid: i % 47 != 46)); + } + const hrBranches = [54.0, 95.0, 106.799999, 106.8, 130.0, 190.0, 0.0]; + const cadBranches = [null, 99.999, 100, 110, 120, 130, 160, 0]; + return IncrementalFixture( + times, + nn, + accel, + List.generate(minutes, (i) => 28000000 + i + i ~/ 17), + List.generate(minutes, (i) => hrBranches[(i + seed) % hrBranches.length]), + List.generate( + minutes, (i) => cadBranches[(i + seed) % cadBranches.length])); +} + +List prefixSizes(int length, int chunk) => { + 0, + 1, + 2, + 3, + 4, + 29, + 30, + 31, + for (var i = chunk; i < length; i += chunk) i, + length, + }.where((n) => n <= length).toList() + ..sort(); + +/// Checked-in anonymised WHOOP-4 overnight capture, July 2026. Provenance and +/// documentation: test/onehz/real_night_cardio_stager_test.dart. Timestamps were +/// rebased to zero and only HR, axes, and RR retained. This subset is +/// read directly from the repository; missing fixtures fail, never skip. +IncrementalFixture realIncrementalFixture({int seconds = 900}) { + final accel = [], hrSeconds = []; + for (final line in File('test/onehz/fixtures/real_night_2026_07_onehz.csv') + .readAsLinesSync() + .skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(',').map(double.parse).toList(); + if (p[0] >= seconds) break; + hrSeconds.add(p[1]); + accel.add(AccelSample(p[0] * 1000, p[2], p[3], p[4])); + } + final raw = [], times = []; + for (final line in File('test/onehz/fixtures/real_night_2026_07_rr.csv') + .readAsLinesSync() + .skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(',').map(double.parse).toList(); + if (p[0] >= seconds * 1000) break; + times.add(p[0]); + raw.add(p[1]); + } + final clean = correctRr(raw, rrTsMs: times); + final hr = []; + for (var i = 0; i < hrSeconds.length; i += 60) { + hr.add(mean(hrSeconds.sublist(i, math.min(i + 60, hrSeconds.length)))!); + } + return IncrementalFixture( + clean.nnTimesMs, + clean.nn, + accel, + List.generate(hr.length, (i) => i), + hr, + List.filled(hr.length, null)); +} diff --git a/test/support/aggregator_laws.dart b/test/support/aggregator_laws.dart new file mode 100644 index 00000000..20a29c06 --- /dev/null +++ b/test/support/aggregator_laws.dart @@ -0,0 +1,128 @@ +// Shared helpers for the column-slice aggregator tests: the law assertion, +// seeded batch generation, and the three ways to evaluate an aggregator over a +// batch (whole, chunked + combined, one state sliced). + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import 'fold_law_support.dart' show foldBounds; +import 'property.dart' show Rng; + +/// Asserts that [agg] satisfies every aggregator law on [cases]. The failure +/// message lists each violated law with its detail. +void expectLawful, R>( + Aggregator agg, + List cases, { + ResultEquals? equals, +}) { + final violations = checkLaws(agg, cases, equals: equals); + expect(violations, isEmpty, + reason: violations.isEmpty + ? null + : 'aggregator laws violated:\n' + '${violations.map((v) => ' ${v.law}: ${v.detail}').join('\n')}'); +} + +/// Seeded random double columns. Always starts with the empty column, a +/// singleton and a pair, then [count] columns of length `0..maxLen`. A value +/// is NaN with probability [nanP]; [value] draws the finite ones. +List randomColumns(int seed, + {int count = 30, + int maxLen = 40, + double nanP = .15, + double Function(Rng)? value}) { + final r = Rng(seed); + final draw = value ?? (Rng r) => (r.nextDouble() - .5) * 2000; + DoubleColumn col(int n) => DoubleColumn(Float64List.fromList( + [for (var i = 0; i < n; i++) r.nextBool(nanP) ? double.nan : draw(r)])); + return [ + col(0), + col(1), + col(2), + for (var i = 0; i < count; i++) col(r.intIn(0, maxLen)), + ]; +} + +/// Seeded random int batches, same shape as [randomColumns]. +List> randomIntBatches(int seed, + {int count = 30, int maxLen = 40, int lo = -1000, int hi = 1000}) { + final r = Rng(seed); + ListBatch b(int n) => + ListBatch([for (var i = 0; i < n; i++) r.intIn(lo, hi)]); + return [b(0), b(1), b(2), for (var i = 0; i < count; i++) b(r.intIn(0, maxLen))]; +} + +/// A column of the given values. +DoubleColumn col(List xs) => DoubleColumn(Float64List.fromList(xs)); + +/// Chunk bounds `[0, ...cuts, n]` for a batch of [n] rows. Cuts may repeat or +/// sit on 0 / 1 / 2 / 3 / n-1 / n, so there are empty, singleton and very short +/// chunks. +List randomBounds(Rng r, int n) => + foldBounds(n, r.intIn(0, 6), r.nextInt(1 << 30), special: [0, 1, 2, 3]); + +/// Folds each chunk `[bounds[k], bounds[k+1])` from a fresh state and combines +/// the chunk states left to right, then finishes. With [copy] each chunk is +/// first cut out with `copyRange` and folded as a batch of its own (so the +/// inner aggregator cannot peek at rows before the chunk); otherwise chunks are +/// ranges of the one batch. +R runChunked, R>( + Aggregator agg, B batch, List bounds, + {bool copy = false}) { + S chunk(int a, int b) => copy + ? agg.fold(batch.copyRange(a, b)) + : agg.fold(batch, a, b); + var acc = chunk(bounds[0], bounds[1]); + for (var k = 1; k + 1 < bounds.length; k++) { + acc = agg.combine(acc, chunk(bounds[k], bounds[k + 1])); + } + return agg.finish(acc); +} + +/// Same chunks, combined right to left: `c0 + (c1 + (c2 + ...))`. +R runChunkedRight, R>( + Aggregator agg, B batch, List bounds) { + var acc = agg.fold(batch, bounds[bounds.length - 2], bounds.last); + for (var k = bounds.length - 3; k >= 0; k--) { + acc = agg.combine(agg.fold(batch, bounds[k], bounds[k + 1]), acc); + } + return agg.finish(acc); +} + +/// The 'slicing' path: ONE state, repeated `addSlice` over the consecutive +/// slices `[bounds[k], bounds[k+1])` (empty slices included), then finish. +R runSliced, R>( + Aggregator agg, B batch, List bounds) { + var state = agg.empty(); + for (var k = 0; k + 1 < bounds.length; k++) { + state = agg.addSlice(state, batch, bounds[k], bounds[k + 1]); + } + return agg.finish(state); +} + +/// Plain bounds for the chunk-size-one case: `[0, 1, 2, ..., n]` (`[0, 0]` for +/// an empty batch). +List singletonBounds(int n) => n == 0 ? [0, 0] : [for (var i = 0; i <= n; i++) i]; + +/// Asserts that [build] over every case in [cases] reproduces [reference] on +/// every evaluation path of [checkConversion] (and, with [laws], the aggregator +/// laws). The failure reason lists the first few failures, one per line. +void expectConversion, R>( + Aggregator Function(B input) build, + List cases, + R Function(B input) reference, { + ResultEquals? equals, + int chunkings = 8, + int seed = 0, + bool laws = true, +}) { + final failures = checkConversion(build, cases, reference, + equals: equals, chunkings: chunkings, seed: seed, laws: laws); + expect(failures, isEmpty, + reason: failures.isEmpty + ? null + : 'conversion check failed (${failures.length}):\n' + '${failures.take(5).map((f) => ' ${f.path} (case ${f.caseIndex}): ${f.detail}').join('\n')}'); +} diff --git a/test/support/checkpoint_support.dart b/test/support/checkpoint_support.dart new file mode 100644 index 00000000..97005c3e --- /dev/null +++ b/test/support/checkpoint_support.dart @@ -0,0 +1,60 @@ +// Shared by the module aggregator tests: a mid-stream checkpoint must resume. + +import 'dart:convert'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +/// Asserts that [agg] is persistable and that a mid-stream checkpoint resumes: +/// fold the first half of [batch], checkpoint it through JSON text, restore, +/// keep folding the second half, and compare with the one-pass result. +void expectCheckpointResumes, R>( + Aggregator agg, + B batch, { + ResultEquals? equals, +}) { + expect(agg.persistable, isTrue); + final half = batch.length ~/ 2; + final env = jsonDecode(jsonEncode(encodeCheckpoint( + agg, agg.fold(batch, 0, half), + kind: 'test', version: 1))); + final restored = decodeCheckpoint(agg, env, kind: 'test', version: 1); + expect(restored, isNotNull); + final got = agg.finish(agg.addSlice(restored as S, batch, half, batch.length)); + final want = agg.run(batch); + final same = equals ?? (R a, R b) => a == b; + expect(same(got, want), isTrue, reason: 'resumed $got vs one-pass $want'); +} + +/// The JSON-text form of [agg]'s state after folding all of [batch], inside a +/// checkpoint envelope (kind 'test', version 1). +Map checkpointOf, R>( + Aggregator agg, B batch) => + (jsonDecode(jsonEncode( + encodeCheckpoint(agg, agg.fold(batch), kind: 'test', version: 1))) + as Map) + .cast(); + +/// Asserts that a checkpoint whose children disagree is refused with null. +/// +/// The undamaged checkpoint of [batch] must restore; then [damage] edits a +/// deep copy of its state JSON (in place, or returning a replacement) and the +/// result must restore as null, because the damage is a relationship between +/// children that each decode on their own. +void expectDamagedCheckpointRefused, R>( + Aggregator agg, + B batch, + Object? Function(Object? state) damage, { + String? reason, +}) { + final good = checkpointOf(agg, batch); + expect(decodeCheckpoint(agg, good, kind: 'test', version: 1), isNotNull, + reason: 'the undamaged checkpoint must restore'); + final state = jsonDecode(jsonEncode(good['state'])); + final damaged = damage(state) ?? state; + expect(jsonEncode(damaged), isNot(jsonEncode(good['state'])), + reason: 'the damage must change the state'); + final env = {...good, 'state': damaged}; + expect(decodeCheckpoint(agg, env, kind: 'test', version: 1), isNull, + reason: reason ?? jsonEncode(env)); +} diff --git a/test/support/fold_law_support.dart b/test/support/fold_law_support.dart new file mode 100644 index 00000000..0443fde9 --- /dev/null +++ b/test/support/fold_law_support.dart @@ -0,0 +1,133 @@ +// Pieces the fold-law files share (design 05, pilot cluster C2A): chunk +// boundaries, the mutation outcome vocabulary, and fixed-length int-vector +// generators (a recipe is a handful of ints, so a failing input shrinks to a +// few readable numbers instead of thousands of doubles). + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// Chunk boundaries `[0, ...cuts, n]`. Cuts may repeat or sit on 0 / n, which +/// makes empty chunks; half of them are placed on [special] positions (the +/// edges of a module's windows), the rest uniformly. +List foldBounds(int n, int splits, int cutSeed, + {List special = const [0, 1, 2, 3]}) { + final r = Rng(cutSeed + 17); + final cuts = []; + for (var k = 0; k < splits; k++) { + final v = r.nextBool(.5) + ? (r.nextBool(.2) ? n - 1 + r.nextInt(2) : special[r.nextInt(special.length)]) + : r.intIn(0, n); + cuts.add(v.clamp(0, n)); + } + cuts.sort(); + return [0, ...cuts, n]; +} + +bool hasEmptyChunk(List b) { + for (var k = 0; k + 1 < b.length; k++) { + if (b[k + 1] == b[k]) return true; + } + return false; +} + +/// The seams a law checks against an expensive reference or batch parity check: first, middle, last. +Set pickSeams(int seams) => + {0, seams ~/ 2, seams - 1}..removeWhere((k) => k < 0); + +/// What a mutated checkpoint is allowed to do. +enum Must { + /// Refused with a FormatException, nothing else. + refuse, + + /// Refused, or accepted holding exactly what was written. + faithful, + + /// Refused, or accepted as the ORIGINAL (the change is ignored). + ignored, +} + +/// (kind, a, b): which mutation, and two parameters. +typedef Mut = (int, int, int); + +Object? deepCopy(Object? o) => jsonDecode(jsonEncode(o)); + +/// Checks one outcome of reading a mutated checkpoint: [outcome] is `refused`, +/// `accepted:` or `WRONG ERROR TYPE...`. +void expectMutationOutcome(String outcome, Must must, String faithfulText, + String originalText, String tag) { + expect(outcome, isNot(startsWith('WRONG')), reason: '$tag: only FormatException'); + switch (must) { + case Must.refuse: + expect(outcome, 'refused', reason: '$tag: refused whole'); + case Must.faithful: + if (outcome != 'refused') { + expect(outcome, 'accepted:$faithfulText', + reason: '$tag: accepted means exactly what was written'); + } + case Must.ignored: + if (outcome != 'refused') { + expect(outcome, 'accepted:$originalText', + reason: '$tag: an unknown key is ignored, nothing else moves'); + } + } +} + +class MutKindGen extends Gen { + MutKindGen(this.kinds); + final int kinds; + @override + int generate(Rng r, int size) => r.nextInt(kinds); + @override + Iterable shrink(int v) => G.intIn(0, kinds - 1).shrink(v); +} + +Gen mutGen(int kinds) => + G.triple(MutKindGen(kinds), G.intIn(0, 1 << 20), G.intIn(0, 1 << 20)); + +/// A size: the interesting small values first, then growing with the case +/// index up to [maxN] (cost: a property has two seconds). +class SizeGen extends Gen { + SizeGen(this.maxN, {this.pool = const [0, 1, 2, 3, 4, 5, 10]}); + final int maxN; + final List pool; + @override + int generate(Rng r, int size) { + final p = r.nextDouble(); + if (p < .04) return 0; + if (p < .29) { + final ok = [for (final x in pool) if (x <= maxN) x]; + return ok[r.nextInt(ok.length)]; + } + if (p < .40) return r.intIn(math.min(500, maxN), maxN); + final cap = math.max(120, maxN * size ~/ 100); + if (p < .62) return r.intIn(0, math.min(cap, 400)); + return r.intIn(0, math.min(cap, maxN)); + } + + @override + Iterable shrink(int v) => G.intIn(0, maxN).shrink(v); +} + +/// A fixed-length vector of ints, each slot with its own generator, each +/// shrunk on its own. +class IntsGen extends Gen> { + IntsGen(this.slots); + final List> slots; + @override + List generate(Rng r, int size) => [for (final g in slots) g.generate(r, size)]; + @override + Iterable> shrink(List v) sync* { + for (var i = 0; i < slots.length; i++) { + for (final c in slots[i].shrink(v[i])) { + yield [...v.sublist(0, i), c, ...v.sublist(i + 1)]; + } + } + } + + @override + String show(List v) => '$v'; +} diff --git a/test/support/law_registry.dart b/test/support/law_registry.dart new file mode 100644 index 00000000..7f78f3a1 --- /dev/null +++ b/test/support/law_registry.dart @@ -0,0 +1,124 @@ +// LAW REGISTRY — every law states which situations its default cases must +// reach, and one test per file measures that (design 05, generator reach). +// +// A property that never meets the situation it claims to cover is a green test +// that checks nothing. So each law is registered with a [Reach]: named +// situations, each with a share of the law's default case count it must reach, +// and an observer that names the situations of one input WITHOUT running the +// code under test (the situations come from the generator's model). The reach +// test re-draws the law's default cases (the forced examples are not counted: +// they are the part that is hand-placed) and fails when a share is not met. +// +// PROPERTY_REACH_REPORT=1 prints what each law reached, to retune the shares +// after a generator change. +// +// final laws = LawSet(); +// laws.law('L1 ...', gen, body, reach: Reach({...}, observe)); +// laws.registerReachTest(); // once, last in main() + +import 'dart:io' show Platform; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// What a law's default cases must reach. +class Reach { + const Reach(this.shares, this.observe); + + /// Situation name -> share of the default case count it must reach + /// (need = max(1, floor(cases * share))). + final Map shares; + + /// Names the situations of one input by calling `bump('name')`. + final void Function(T arg, void Function(String) bump) observe; +} + +class _Registered { + _Registered(this.name, this.cases, this.shares, this.measure); + final String name; + final int cases; + final Map shares; + final Map Function() measure; +} + +class LawSet { + final List<_Registered> _registry = []; + + /// Number of registered laws. + int get length => _registry.length; + + /// Registers [name] as a property (forced [examples] first, then [cases] + /// generated ones) and records its reach. + void law( + String name, + Gen gen, + void Function(T) body, { + required Reach reach, + List examples = const [], + int cases = 30, + String genVersion = 'g1', + Duration? budget, + Object? skip, + }) { + _registry.add(_Registered(name, cases, reach.shares, () { + final got = {for (final k in reach.shares.keys) k: 0}; + final r = runProperty( + name: name, + gen: gen, + config: PropertyConfig( + cases: cases, + // A generous guard, but PROPERTY_BUDGET_MS (the serial acceptance + // run) is the hard limit here too: drawing a law's cases is cheap. + budget: Duration( + milliseconds: int.tryParse( + Platform.environment['PROPERTY_BUDGET_MS'] ?? '') ?? + 60000)), + body: (v) => reach.observe(v, (k) => got[k] = (got[k] ?? 0) + 1), + genVersion: genVersion, + ); + if (!r.passed) fail(r.failure!.report); + return got; + })); + forAll(name, gen, body, + examples: examples, + cases: cases, + genVersion: genVersion, + // The spec's cap is 2 s per property, measured serially on the dev + // desktop. The in-test guard that fails a runaway has 2.5x headroom on + // top, so a suite running 12 test files at once does not trip it. + // PROPERTY_BUDGET_MS replaces it as the HARD limit (see + // PropertyConfig.fromEnvironment): the acceptance run is + // PROPERTY_BUDGET_MS=2000 dart test -j 1 test/properties/ + // and any property over 2 s there fails the run. + budget: budget ?? const Duration(seconds: 5), + skip: skip); + } + + /// The test that every law's default cases reach what the law claims. + void registerReachTest() { + group('generator reach', () { + test('every law sees every situation in its default cases', () { + expect(_registry, isNotEmpty); + final weak = []; + for (final law in _registry) { + final got = law.measure(); + if (Platform.environment.containsKey('PROPERTY_REACH_REPORT')) { + // ignore: avoid_print + print('REACH ${law.name.split(' ').first}/${law.cases} cases: ' + '${got.entries.map((e) => "${e.key}=${e.value}").join("; ")}'); + } + for (final e in law.shares.entries) { + final need = math.max(1, (law.cases * e.value).floor()); + if (got[e.key]! < need) { + weak.add('"${law.name}": ${e.key} in ${got[e.key]} of ' + '${law.cases} cases, need $need'); + } + } + } + expect(weak, isEmpty, reason: weak.join('\n')); + }); + }); + } +} diff --git a/test/support/property.dart b/test/support/property.dart new file mode 100644 index 00000000..89456a3c --- /dev/null +++ b/test/support/property.dart @@ -0,0 +1,753 @@ +// PROPERTY HARNESS — seeded, shrinking, replayable property tests. +// +// Port of the edge harness (edge test/support/property.dart, design 05 pilot): +// same generators, seeds, replay and budget rules, on package:test only (this +// package is pure Dart). Keep the two in step; the RNG is pinned to reference +// outputs in property_test.dart so a replay command means the same thing in +// both repos. +// +// In-house on purpose (design 05, section 3): no dependency, fully +// deterministic, and every failure prints what is needed to replay it. +// +// forAll('no two icons overlap', G.listOf(item, maxLen: 60), (items) { +// ... expect(...) ... +// }, examples: [...forced scenarios...]); +// +// How a run is built +// • SEED. Each property has its own seed, derived from its NAME (so adding or +// reordering properties never changes another property's inputs). +// PROPERTY_SEED overrides it for every property in the run. +// • CASES. Case `i` draws from its own generator stream `caseSeed(seed, i)` and +// a size that depends on `i` only, so PROPERTY_CASE=i replays exactly that +// input whatever the iteration count. Default 200 cases; PROPERTY_ITERATIONS +// overrides it (and scales the wall budget with it). +// • FORCED SCENARIOS. `examples` always run first, in order, before any +// generated case. They are never shrunk (they are explicit). Replay one with +// PROPERTY_CASE=forced:. +// • FAILURE. The first failing input is shrunk (bounded by `shrinkLimit` body +// runs) and printed with seed, case, generator version and a one-line replay +// command. Shrinkers are domain-preserving: a candidate is always something +// the generator itself could have produced. +// • BUDGET. A property that takes longer than its wall budget (default 2 s) +// fails. Shrinking counts against the same budget: when it runs out the +// best failing input so far is reported, marked "over budget (shrinking)". The budget is the ONLY +// use of a real clock; it is injectable for the harness's own tests. +// +// GENERATOR VERSION. A seed only means something for the generator that drew +// it. When a property's generator changes shape, bump its `genVersion`; the +// report prints it so an old replay command is recognised as stale. +// +// Environment: PROPERTY_SEED, PROPERTY_CASE, PROPERTY_ITERATIONS, +// PROPERTY_BUDGET_MS (fixes the budget, no scaling). + +import 'dart:io' show Platform; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +// ── randomness ────────────────────────────────────────────────────────────── + +/// True on dart2js, where `1` and `1.0` are the same value. +const bool _onJs = identical(1, 1.0); + +/// splitmix64 (Steele, Lea, Flood; the public-domain reference generator): +/// tiny, fast, and identical on every Dart VM version (unlike `dart:math` +/// Random, whose stream is not promised to be stable). The first outputs for +/// seeds 0 and 1234567 are pinned in property_test.dart against values computed +/// independently. +/// +/// ASSUMPTION: a Dart VM `int` is a 64-bit two's-complement integer whose `+` +/// and `*` wrap, and `>>>` is a logical shift. That is what the arithmetic +/// below relies on. This harness is VM-only and must not be compiled for the +/// web (ints are doubles there); the constructor refuses to run on JS. +class Rng { + Rng(int seed) : _s = seed { + if (_onJs) { + throw UnsupportedError('test/support/property.dart needs 64-bit VM ints'); + } + } + int _s; + + static const int _gamma = 0x9E3779B97F4A7C15; + + static int mix(int x) { + var z = x; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EB; + return z ^ (z >>> 31); + } + + int _next() { + _s += _gamma; + return mix(_s); + } + + /// The next raw 64-bit output (a signed Dart int; same bits as the + /// reference's unsigned value). + int next64() => _next(); + + /// A uniform int in `[0, bound)`. `bound` must be in `1 .. 2^53`. + int nextInt(int bound) { + assert(bound > 0 && bound <= (1 << 53), 'bound out of range: $bound'); + return (_next() >>> 1) % bound; + } + + /// A uniform int in `[min, max]`, both inclusive. + int intIn(int min, int max) { + assert(min <= max); + return min + nextInt(max - min + 1); + } + + /// A uniform double in `[0, 1)`. + double nextDouble() => (_next() >>> 11) / 9007199254740992.0; + + /// True with probability [p]. + bool nextBool([double p = .5]) => nextDouble() < p; +} + +/// The default seed of the property called [name] (32-bit FNV-1a). +int seedFor(String name) { + var h = 0x811C9DC5; + for (final u in name.codeUnits) { + h = ((h ^ u) * 0x01000193) & 0xFFFFFFFF; + } + return h; +} + +/// The generator stream of case [index] of a property seeded [seed]. Depends on +/// nothing else, so one case can be replayed alone. +int caseSeed(int seed, int index) => + Rng.mix(Rng.mix(seed) + index * 0x9E3779B97F4A7C15) & 0x7FFFFFFFFFFF; + +// ── generators ────────────────────────────────────────────────────────────── + +/// Draws values and shrinks them. [shrink] must only return values this +/// generator could itself have produced (domain-preserving), none equal to the +/// input, simplest first, and finitely many per call. +abstract class Gen { + const Gen(); + + /// A value; [size] grows 10..100 across the cases of a run. + T generate(Rng r, int size); + + /// Simpler candidates than [v], simplest first. May be lazy. + Iterable shrink(T v); + + /// A readable, copy-pasteable rendering for the failure report. + String show(T v) => '$v'; +} + +class _IntGen extends Gen { + _IntGen(this.min, this.max, int? origin) + : origin = (origin ?? 0).clamp(min, max).toInt() { + assert(min <= max && max - min < (1 << 53)); + } + final int min, max, origin; + + @override + int generate(Rng r, int size) { + final p = r.nextDouble(); + if (p < .06) return min; + if (p < .12) return max; + if (p < .16) return origin; + return r.intIn(min, max); + } + + @override + Iterable shrink(int v) sync* { + if (v == origin) return; + yield origin; + final d = v - origin; + var step = d ~/ 2; + final seen = {origin}; + while (step != 0) { + final c = v - step; + if (seen.add(c)) yield c; + step ~/= 2; + } + final one = v - d.sign; + if (seen.add(one) && one != v) yield one; + } +} + +class _DoubleGen extends Gen { + _DoubleGen(this.min, this.max, this.boundaries, this.specials, double? origin, + this.integerBias) + : origin = (origin ?? 0).clamp(min, max).toDouble() { + assert(min.isFinite && max.isFinite && min <= max); + } + final double min, max, origin, integerBias; + final List boundaries, specials; + + List get _pool => [ + min, + max, + if (origin != min && origin != max) origin, + for (final b in boundaries) + if (b >= min && b <= max) b, + ]; + + @override + double generate(Rng r, int size) { + final p = r.nextDouble(); + if (specials.isNotEmpty && p < .08) { + return specials[r.nextInt(specials.length)]; + } + if (p < .08 + .17) { + final pool = _pool; + return pool[r.nextInt(pool.length)]; + } + var v = min + r.nextDouble() * (max - min); + if (r.nextBool(integerBias)) v = v.roundToDouble(); + return v.clamp(min, max).toDouble(); + } + + @override + Iterable shrink(double v) sync* { + if (!v.isFinite) { + yield origin; // a special shrinks to the simplest finite value + return; + } + if (v == origin) return; + final d = v - origin; + if (d.abs() < 1e-6) { + yield origin; + return; + } + final seen = {v}; + Iterable cand() sync* { + yield origin; + for (final b in _pool) { + if ((b - origin).abs() < d.abs()) yield b; + } + yield v.truncateToDouble(); + yield v - d / 2; + yield v - d / 4; + yield v - d / 8; + } + + for (final c in cand()) { + if (c >= min && c <= max && seen.add(c)) yield c; + } + } +} + +class _BoolGen extends Gen { + @override + bool generate(Rng r, int size) => r.nextBool(); + @override + Iterable shrink(bool v) => v ? const [false] : const []; +} + +class _ElementsGen extends Gen { + _ElementsGen(this.values) { + assert(values.isNotEmpty); + } + final List values; + @override + T generate(Rng r, int size) => values[r.nextInt(values.length)]; + @override + Iterable shrink(T v) { + final i = values.indexOf(v); + return i <= 0 ? const [] : values.take(i); + } +} + +class _ListGen extends Gen> { + _ListGen(this.elem, this.minLen, this.maxLen) { + assert(minLen >= 0 && minLen <= maxLen); + } + final Gen elem; + final int minLen, maxLen; + + @override + List generate(Rng r, int size) { + final cap = minLen + ((maxLen - minLen) * (size + 1) / 101).ceil(); + final p = r.nextDouble(); + final n = p < .08 + ? minLen + : p < .16 + ? maxLen + : r.intIn(minLen, math.max(minLen, cap)); + return [for (var i = 0; i < n; i++) elem.generate(r, size)]; + } + + @override + Iterable> shrink(List v) sync* { + // Shorter first: drop chunks, big to small. + if (v.length > minLen) { + for (var chunk = math.max(1, v.length ~/ 2); chunk >= 1; chunk ~/= 2) { + if (v.length - chunk < minLen) continue; + for (var at = 0; at + chunk <= v.length; at += chunk) { + yield [...v.sublist(0, at), ...v.sublist(at + chunk)]; + } + } + } + // Then simpler elements, in place. + for (var i = 0; i < v.length; i++) { + for (final e in elem.shrink(v[i])) { + yield [...v.sublist(0, i), e, ...v.sublist(i + 1)]; + } + } + } + + @override + String show(List v) => '[${v.map(elem.show).join(', ')}]'; +} + +class _NullableGen extends Gen { + _NullableGen(this.inner, this.nullProbability); + final Gen inner; + final double nullProbability; + @override + T? generate(Rng r, int size) => + r.nextBool(nullProbability) ? null : inner.generate(r, size); + @override + Iterable shrink(T? v) sync* { + if (v == null) return; + yield null; + yield* inner.shrink(v); + } + + @override + String show(T? v) => v == null ? 'null' : inner.show(v); +} + +/// Candidate lists of the components of a tuple, shrunk SEPARATELY and then +/// TOGETHER (the k-th simpler candidate of every component at once), so a law +/// that only fails on correlated components (a == b) still shrinks. +Iterable> _shrinkTuple( + List> gens, List v) sync* { + final cands = [ + for (var i = 0; i < gens.length; i++) gens[i].shrink(v[i]).take(12).toList() + ]; + // Joint steps first: they make the biggest strides and are the only ones + // that work when the components are correlated. + final joint = cands.map((c) => c.length).reduce(math.min); + for (var k = 0; k < joint; k++) { + yield [for (final c in cands) c[k]]; + } + for (var i = 0; i < gens.length; i++) { + for (final c in gens[i].shrink(v[i])) { + yield [...v.sublist(0, i), c, ...v.sublist(i + 1)]; + } + } +} + +class _TupleGen extends Gen { + _TupleGen(this.gens, this.build, this.parts); + final List> gens; + final R Function(List) build; + final List Function(R) parts; + + @override + R generate(Rng r, int size) => + build([for (final g in gens) g.generate(r, size)]); + @override + Iterable shrink(R v) => _shrinkTuple(gens, parts(v)).map(build); + @override + String show(R v) { + final p = parts(v); + return '(${[for (var i = 0; i < gens.length; i++) gens[i].show(p[i])].join(', ')})'; + } +} + +/// The generator constructors. +abstract final class G { + /// An int in `[min, max]`; shrinks toward [origin] (default: zero, or the + /// bound nearest zero). + static Gen intIn(int min, int max, {int? origin}) => + _IntGen(min, max, origin); + + /// A double in `[min, max]` (finite). [boundaries] are drawn deliberately; + /// [specials] (NaN, infinities) are drawn occasionally and shrink to + /// [origin]. [integerBias] is the share of draws snapped to whole numbers + /// (ties and exact thresholds are where the interesting laws bend). + static Gen doubleIn(double min, double max, + {List boundaries = const [], + List specials = const [], + double? origin, + double integerBias = .25}) => + _DoubleGen(min, max, boundaries, specials, origin, integerBias); + + static Gen boolean() => _BoolGen(); + + /// One of [values]; shrinks toward the earlier ones. + static Gen elements(List values) => _ElementsGen(values); + + /// A list of `minLen .. maxLen` elements (the cap is a hard bound). + static Gen> listOf(Gen elem, + {int minLen = 0, required int maxLen}) => + _ListGen(elem, minLen, maxLen); + + static Gen nullable(Gen inner, {double nullProbability = .25}) => + _NullableGen(inner, nullProbability); + + static Gen<(A, B)> pair(Gen a, Gen b) => _TupleGen( + [a, b], (p) => (p[0] as A, p[1] as B), (r) => [r.$1, r.$2]); + + static Gen<(A, B, C)> triple(Gen a, Gen b, Gen c) => + _TupleGen([a, b, c], (p) => (p[0] as A, p[1] as B, p[2] as C), + (r) => [r.$1, r.$2, r.$3]); + + static Gen<(A, B, C, D)> quad( + Gen a, Gen b, Gen c, Gen d) => + _TupleGen( + [a, b, c, d], + (p) => (p[0] as A, p[1] as B, p[2] as C, p[3] as D), + (r) => [r.$1, r.$2, r.$3, r.$4]); +} + +// ── configuration and result ──────────────────────────────────────────────── + +class PropertyConfig { + const PropertyConfig({ + this.seed, + this.cases = 200, + this.caseOnly, + this.budget = const Duration(seconds: 2), + this.shrinkLimit = 300, + this.testFile, + }); + + /// Reads PROPERTY_SEED / PROPERTY_CASE / PROPERTY_ITERATIONS / + /// PROPERTY_BUDGET_MS from [env]. A malformed value throws: a typo must not + /// silently run the default. + factory PropertyConfig.fromEnvironment( + Map env, { + String? testFile, + int? seed, + int defaultCases = 200, + Duration defaultBudget = const Duration(seconds: 2), + int shrinkLimit = 300, + }) { + int? number(String key, {int min = 0}) { + final raw = env[key]; + if (raw == null || raw.isEmpty) return null; + final n = int.tryParse(raw); + if (n == null || n < min) { + throw FormatException('$key must be an integer >= $min', raw); + } + return n; + } + + final envSeed = number('PROPERTY_SEED', min: -(1 << 62)); + final iterations = number('PROPERTY_ITERATIONS', min: 1); + final budgetMs = number('PROPERTY_BUDGET_MS', min: 1); + final caseRaw = env['PROPERTY_CASE']; + if (caseRaw != null && + caseRaw.isNotEmpty && + !RegExp(r'^(forced:)?\d+$').hasMatch(caseRaw)) { + throw FormatException('PROPERTY_CASE must be N or forced:N', caseRaw); + } + final cases = iterations ?? defaultCases; + final scale = math.max(1.0, cases / defaultCases); + return PropertyConfig( + seed: envSeed ?? seed, + cases: cases, + caseOnly: caseRaw == null || caseRaw.isEmpty ? null : caseRaw, + budget: budgetMs != null + ? Duration(milliseconds: budgetMs) + : iterations != null + ? Duration( + milliseconds: (defaultBudget.inMilliseconds * scale).round()) + : defaultBudget, + shrinkLimit: shrinkLimit, + testFile: testFile, + ); + } + + /// null: derived from the property name. + final int? seed; + final int cases; + + /// Replay only this case: `N` or `forced:N`. null: the whole run. + final String? caseOnly; + final Duration budget; + + /// Most body runs spent shrinking one failure. + final int shrinkLimit; + + /// Repo-relative path printed in the replay command. + final String? testFile; +} + +enum FailureKind { counterexample, forcedExample, budget, generator } + +class PropertyFailure { + const PropertyFailure({ + required this.kind, + required this.seed, + required this.caseId, + required this.input, + required this.error, + required this.replay, + required this.shrinkSteps, + required this.report, + this.shrinkOverBudget = false, + }); + final FailureKind kind; + final int seed; + + /// `17` for a generated case, `forced:1` for an explicit example. + final String caseId; + + /// The (shrunk) failing input, as the generator shows it. + final String input; + final String error; + final String replay; + + /// Body runs spent shrinking. + final int shrinkSteps; + final String report; + + /// Shrinking stopped because the property's wall budget ran out; [input] is + /// the best (smallest) failing input found so far, not necessarily minimal. + final bool shrinkOverBudget; +} + +class PropertyResult { + const PropertyResult( + {required this.casesRun, required this.forcedRun, this.failure}); + final int casesRun; + final int forcedRun; + final PropertyFailure? failure; + bool get passed => failure == null; +} + +// ── running ───────────────────────────────────────────────────────────────── + +String _errorText(Object e, StackTrace st) { + final msg = e.toString(); + final text = msg.length > 900 ? '${msg.substring(0, 900)}...' : msg; + if (e is TestFailure) return text; + // A non-assertion error (RangeError, StateError, ...) needs its origin. + final frames = st.toString().split('\n').where((l) => l.isNotEmpty).take(6); + return '$text\n ${frames.join('\n ')}'; +} + +/// Runs the property and returns the outcome; never throws for a failing body. +/// [forAll] is the test-registering wrapper. +PropertyResult runProperty({ + required String name, + required Gen gen, + required void Function(T) body, + List examples = const [], + PropertyConfig config = const PropertyConfig(), + String genVersion = 'g1', + Duration Function()? elapsed, +}) { + final seed = config.seed ?? seedFor(name); + late final Duration Function() clock; + if (elapsed != null) { + clock = elapsed; + } else { + final sw = Stopwatch()..start(); + clock = () => sw.elapsed; + } + + String? attempt(T v) { + try { + body(v); + return null; + } catch (e, st) { + return _errorText(e, st); + } + } + + String replayFor(String? caseId) => + 'PROPERTY_SEED=$seed ${caseId == null ? '' : 'PROPERTY_CASE=$caseId '}' + 'dart test ' + "${config.testFile ?? ''} " + "--plain-name '${name.replaceAll("'", r"'\''")}'"; + + PropertyFailure fail(FailureKind kind, String caseId, String input, + String error, + {int shrinkSteps = 0, + int accepted = 0, + String? original, + bool overBudget = false}) { + final replay = replayFor(kind == FailureKind.budget ? null : caseId); + final b = StringBuffer() + ..writeln('Property "$name" FAILED (${kind.name})') + ..writeln(' seed: $seed case: $caseId generator: $genVersion' + ' shrink limit: ${config.shrinkLimit}'); + if (kind == FailureKind.budget) { + b.writeln(' $error'); + } else { + b.writeln(shrinkSteps > 0 + ? ' input (shrunk: $accepted accepted of $shrinkSteps tried): $input' + : ' input: $input'); + if (overBudget) { + b.writeln(' over budget (shrinking): stopped after ${config.budget.inMilliseconds} ' + 'ms; this is the best failing input so far, not necessarily minimal'); + } + if (original != null && original != input) { + b.writeln(' original input: $original'); + } + b.writeln(' error: $error'); + } + b.writeln(' replay: $replay'); + return PropertyFailure( + kind: kind, + seed: seed, + caseId: caseId, + input: input, + error: error, + replay: replay, + shrinkSteps: shrinkSteps, + report: b.toString(), + shrinkOverBudget: overBudget); + } + + PropertyFailure shrunk(String caseId, T original, String error) { + var cur = original; + var err = error; + var tried = 0, accepted = 0; + var overBudget = false; + try { + outer: + while (true) { + var progressed = false; + for (final c in gen.shrink(cur)) { + if (tried >= config.shrinkLimit) break outer; + // The wall budget covers shrinking too (a replay is exempt). + if (config.caseOnly == null && clock() > config.budget) { + overBudget = true; + break outer; + } + tried++; + final e = attempt(c); + if (e != null) { + cur = c; + err = e; + accepted++; + progressed = true; + break; + } + } + if (!progressed) break; + } + } catch (_) { + // A shrinker that throws ends shrinking; the best input so far stands. + } + return fail(FailureKind.counterexample, caseId, gen.show(cur), err, + shrinkSteps: tried, + accepted: accepted, + original: gen.show(original), + overBudget: overBudget); + } + + PropertyResult done(int cases, int forced, [PropertyFailure? f]) => + PropertyResult(casesRun: cases, forcedRun: forced, failure: f); + + // One generated case; returns a failure or null. + PropertyFailure? runCase(int i) { + final T input; + try { + input = gen.generate(Rng(caseSeed(seed, i)), math.min(100, 10 + i)); + } catch (e, st) { + return fail(FailureKind.generator, '$i', '', + _errorText(e, st)); + } + final e = attempt(input); + return e == null ? null : shrunk('$i', input, e); + } + + PropertyFailure? runForced(int i) { + final e = attempt(examples[i]); + return e == null + ? null + : fail(FailureKind.forcedExample, 'forced:$i', gen.show(examples[i]), e); + } + + final only = config.caseOnly; + if (only != null) { + if (only.startsWith('forced:')) { + final i = int.parse(only.substring(7)); + if (i >= examples.length) { + throw StateError('PROPERTY_CASE=$only: only ${examples.length} forced ' + 'examples in "$name"'); + } + return done(0, 1, runForced(i)); + } + return done(1, 0, runCase(int.parse(only))); + } + + PropertyFailure? overBudget(int cases, int forced) => clock() > config.budget + ? fail( + FailureKind.budget, + 'n/a', + '', + 'wall budget ${config.budget.inMilliseconds} ms exceeded after ' + '$cases cases + $forced forced examples ' + '(${clock().inMilliseconds} ms)') + : null; + + var forcedRun = 0, casesRun = 0; + for (var i = 0; i < examples.length; i++) { + final f = runForced(i); + forcedRun++; + if (f != null) return done(casesRun, forcedRun, f); + final b = overBudget(casesRun, forcedRun); + if (b != null) return done(casesRun, forcedRun, b); + } + for (var i = 0; i < config.cases; i++) { + final f = runCase(i); + casesRun++; + if (f != null) return done(casesRun, forcedRun, f); + final b = overBudget(casesRun, forcedRun); + if (b != null) return done(casesRun, forcedRun, b); + } + return done(casesRun, forcedRun); +} + +String? _callerTestFile() { + final m = RegExp(r'test/[\w/.\-]+\.dart'); + for (final line in StackTrace.current.toString().split('\n')) { + final hit = m.firstMatch(line)?.group(0); + if (hit != null && !RegExp(r'support/(property|law_registry)\.dart$').hasMatch(hit)) { + return hit; + } + } + return null; +} + +/// Registers one `test` that checks [body] on [examples] (forced, first) and on +/// generated cases. Replay with the command in the failure report. +/// +/// [cases], [budget] and [seed] are this property's defaults; the PROPERTY_* +/// environment wins. [skip] is for a law that currently fails because of a +/// suspected bug: say which in the reason. +void forAll( + String name, + Gen gen, + void Function(T) body, { + List examples = const [], + int? cases, + int? seed, + Duration? budget, + int? shrinkLimit, + String genVersion = 'g1', + Object? skip, +}) { + final file = _callerTestFile(); + test(name, () { + final config = PropertyConfig.fromEnvironment( + Platform.environment, + testFile: file, + seed: seed, + defaultCases: cases ?? 200, + defaultBudget: budget ?? const Duration(seconds: 2), + shrinkLimit: shrinkLimit ?? 300, + ); + final r = runProperty( + name: name, + gen: gen, + body: body, + examples: examples, + config: config, + genVersion: genVersion); + if (!r.passed) fail(r.failure!.report); + }, skip: skip); +} diff --git a/test/support/property_test.dart b/test/support/property_test.dart new file mode 100644 index 00000000..69938a31 --- /dev/null +++ b/test/support/property_test.dart @@ -0,0 +1,778 @@ +// Self-tests of the property harness (test/support/property.dart). +// +// The harness decides whether the property suites can be trusted, so its own +// guarantees are pinned here: deterministic and independent seeds, replay of a +// single case, forced examples first, domain-preserving bounded shrinking, the +// failure report (seed, case, input, generator version, replay command) and +// the per-property wall budget. + +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// Runs [gen] through the harness with a body that records every input and +/// never fails. Returns the recorded inputs (forced examples first). +List _inputs(Gen gen, + {String name = 'rec', + int cases = 30, + List examples = const [], + int? seed, + String? caseOnly}) { + final seen = []; + final r = runProperty( + name: name, + gen: gen, + body: seen.add, + examples: examples, + config: PropertyConfig(seed: seed, cases: cases, caseOnly: caseOnly), + ); + expect(r.passed, isTrue, reason: r.failure?.report); + return seen; +} + +PropertyFailure _fails(Gen gen, bool Function(T) holds, + {List examples = const [], + int cases = 200, + int shrinkLimit = 300, + String name = 'p', + int seed = 7, + String? caseOnly, + String? testFile}) { + final r = runProperty( + name: name, + gen: gen, + body: (v) { + if (!holds(v)) throw StateError('does not hold: $v'); + }, + examples: examples, + config: PropertyConfig( + seed: seed, + cases: cases, + shrinkLimit: shrinkLimit, + caseOnly: caseOnly, + testFile: testFile), + ); + expect(r.passed, isFalse, reason: 'the property should have failed'); + return r.failure!; +} + +void main() { + group('rng', () { + test('the same seed gives the same sequence, another seed another', () { + List draw(int seed) { + final r = Rng(seed); + return [for (var i = 0; i < 20; i++) r.nextInt(1 << 20)]; + } + + expect(draw(5), draw(5)); + expect(draw(5), isNot(draw(6))); + }); + + test('intIn is inclusive, nextDouble is in [0, 1), nextBool honours p', () { + final r = Rng(42); + final ints = {for (var i = 0; i < 400; i++) r.intIn(-2, 2)}; + expect(ints, {-2, -1, 0, 1, 2}); + for (var i = 0; i < 400; i++) { + final d = r.nextDouble(); + expect(d >= 0 && d < 1, isTrue); + } + expect(r.intIn(3, 3), 3); + expect([for (var i = 0; i < 50; i++) r.nextBool(0)], everyElement(false)); + expect([for (var i = 0; i < 50; i++) r.nextBool(1)], everyElement(true)); + }); + + test('next64 is SplitMix64: the first outputs match the reference values', + () { + // Computed independently with python3, masking every step to 64 bits: + // s = (s + 0x9E3779B97F4A7C15) & M + // z = ((s ^ (s >> 30)) * 0xBF58476D1CE4E5B9) & M + // z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M; out = z ^ (z >> 31) + // Seed 0 starts 0xE220A8397B1DCDAF, the published SplitMix64 test value. + // Dart's signed int holds the same 64 bits, so compare as hex. + String hex(int v) => + (v >>> 32).toRadixString(16).padLeft(8, '0') + + (v & 0xFFFFFFFF).toRadixString(16).padLeft(8, '0'); + final z = Rng(0); + expect([for (var i = 0; i < 5; i++) hex(z.next64())], [ + 'e220a8397b1dcdaf', + '6e789e6aa1b965f4', + '06c45d188009454f', + 'f88bb8a8724c81ec', + '1b39896a51a8749b', + ]); + final k = Rng(1234567); + expect([for (var i = 0; i < 5; i++) hex(k.next64())], [ + '599ed017fb08fc85', + '2c73f08458540fa5', + '883ebce5a3f27c77', + '3fbef740e9177b3f', + 'e3b8346708cb5ecd', + ]); + }); + + test('the stream is pinned: a replay command stays valid across SDKs', () { + final r = Rng(1); + expect([for (var i = 0; i < 5; i++) r.nextInt(1000)], + [232, 259, 295, 117, 380]); + expect(Rng(1).nextDouble(), 0.5665615751722809); + expect(seedFor('a law'), 1308619236); + expect(caseSeed(9, 3), 136612318154068); + }); + + test('draws are spread: a long run touches every bucket of 16', () { + final r = Rng(1); + final buckets = {for (var i = 0; i < 2000; i++) r.nextInt(16)}; + expect(buckets.length, 16); + }); + }); + + group('rng ranges', () { + forAll<(int, int)>( + 'rng draws stay in range: nextInt(bound) in [0, bound), nextDouble in [0, 1)', + G.pair( + G.intIn(-(1 << 40), 1 << 40), + G.intIn(0, 9)), + (arg) { + final (seed, pick) = arg; + const bounds = [1, 2, 3, 7, 1000, 1 << 20, (1 << 53) - 1, 1 << 53]; + final r = Rng(seed); + for (var i = 0; i < 40; i++) { + final b = bounds[(pick + i) % bounds.length]; + final n = r.nextInt(b); + expect(n >= 0 && n < b, isTrue, reason: 'nextInt($b) = $n'); + final d = r.nextDouble(); + expect(d >= 0 && d < 1, isTrue, reason: 'nextDouble() = $d'); + final k = r.intIn(0, b - 1); + expect(k >= 0 && k < b, isTrue, reason: 'intIn(0, ${b - 1}) = $k'); + } + expect(Rng(seed).nextInt(1), 0); + }, examples: const [(0, 0), (-1, 7), (1 << 40, 3)]); + }); + + group('seeds', () { + test('seedFor is stable per name and differs between names', () { + expect(seedFor('a law'), seedFor('a law')); + expect(seedFor('a law'), isNot(seedFor('another law'))); + }); + + test('case seeds are independent of each other and of the run length', () { + expect(caseSeed(9, 3), caseSeed(9, 3)); + expect(caseSeed(9, 3), isNot(caseSeed(9, 4))); + expect(caseSeed(9, 3), isNot(caseSeed(10, 3))); + }); + + test('two properties with different names draw different inputs', () { + final gen = G.listOf(G.intIn(0, 1 << 20), maxLen: 8); + expect(_inputs(gen, name: 'one', cases: 5), + isNot(_inputs(gen, name: 'two', cases: 5))); + }); + + test('the same property is reproduced exactly', () { + final gen = G.pair(G.intIn(-50, 50), G.doubleIn(0, 1)); + expect(_inputs(gen, cases: 40), _inputs(gen, cases: 40)); + }); + }); + + group('config from the environment', () { + test('defaults: 200 cases, 2 s, no seed, no single case', () { + final c = PropertyConfig.fromEnvironment(const {}); + expect(c.cases, 200); + expect(c.budget, const Duration(seconds: 2)); + expect(c.seed, isNull); + expect(c.caseOnly, isNull); + }); + + test('PROPERTY_SEED, PROPERTY_CASE and PROPERTY_ITERATIONS override', () { + final c = PropertyConfig.fromEnvironment(const { + 'PROPERTY_SEED': '12345', + 'PROPERTY_CASE': '17', + 'PROPERTY_ITERATIONS': '1000', + }, testFile: 'test/x_test.dart'); + expect(c.seed, 12345); + expect(c.caseOnly, '17'); + expect(c.cases, 1000); + expect(c.testFile, 'test/x_test.dart'); + }); + + test('raising the iterations scales the wall budget with them', () { + final c = + PropertyConfig.fromEnvironment(const {'PROPERTY_ITERATIONS': '1000'}); + expect(c.budget, const Duration(seconds: 10)); + final fixed = PropertyConfig.fromEnvironment(const { + 'PROPERTY_ITERATIONS': '1000', + 'PROPERTY_BUDGET_MS': '3000', + }); + expect(fixed.budget, const Duration(milliseconds: 3000)); + }); + + test('a malformed number is loud, not ignored', () { + expect(() => PropertyConfig.fromEnvironment(const {'PROPERTY_SEED': 'x'}), + throwsFormatException); + expect( + () => PropertyConfig.fromEnvironment( + const {'PROPERTY_ITERATIONS': '0'}), + throwsFormatException); + }); + }); + + group('running', () { + test('runs exactly the configured number of cases', () { + final r = runProperty( + name: 'n', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(cases: 37), + ); + expect(r.passed, isTrue); + expect(r.casesRun, 37); + expect(r.forcedRun, 0); + }); + + test('forced examples run first, in order, then the generated cases', () { + final seen = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10); + expect(seen.take(3), [5, 6, 7]); + expect(seen.length, 13); + final r = runProperty( + name: 'f', + gen: G.intIn(0, 9), + body: (_) {}, + examples: [1, 2], + config: const PropertyConfig(cases: 4)); + expect((r.forcedRun, r.casesRun), (2, 4)); + }); + + test('generated cases stay inside the generator, even the first ones', () { + final seen = _inputs(G.intIn(-3, 11), cases: 200); + expect(seen.every((v) => v >= -3 && v <= 11), isTrue); + }); + + test('a single case replays exactly the input it had in the full run', () { + final gen = G.pair(G.intIn(-1000, 1000), G.listOf(G.boolean(), maxLen: 9)); + final all = _inputs(gen, cases: 60, seed: 99); + for (final k in [0, 1, 17, 59]) { + expect( + _inputs(gen, cases: 60, seed: 99, caseOnly: '$k') + .map(gen.show) + .toList(), + [gen.show(all[k])], + reason: 'case $k'); + } + }); + + test('a case replays the same whatever the iteration count', () { + final gen = G.listOf(G.intIn(0, 1 << 20), maxLen: 30); + final short = _inputs(gen, cases: 25, seed: 3); + final long = _inputs(gen, cases: 80, seed: 3); + expect(long.take(25), short); + }); + + test('a forced example replays by its forced id, generated cases skip it', + () { + final seen = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10, caseOnly: 'forced:1'); + expect(seen, [6]); + final one = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10, seed: 1, caseOnly: '2'); + expect(one.length, 1); + expect(one.single, inInclusiveRange(100, 200)); + }); + + test('a replay of a case beyond the run length still runs that case', () { + final seen = _inputs(G.intIn(0, 9), cases: 5, caseOnly: '40'); + expect(seen.length, 1); + }); + + test('a body that completes normally is passed, whatever it returns', () { + final r = runProperty( + name: 'ret', gen: G.intIn(0, 3), body: (_) => 5 as dynamic); + expect(r.passed, isTrue); + }); + }); + + group('failure report', () { + test('names the seed, case, shrunk input, generator version, replay', () { + final f = _fails(G.intIn(0, 1000), (v) => v < 50, + name: 'ints stay small', seed: 4242, testFile: 'test/x_test.dart'); + expect(f.kind, FailureKind.counterexample); + expect(f.seed, 4242); + expect(f.input, '50'); + expect(f.caseId, matches(RegExp(r'^\d+$'))); + expect(f.error, contains('does not hold')); + expect( + f.replay, + 'PROPERTY_SEED=4242 PROPERTY_CASE=${f.caseId} ' + "dart test test/x_test.dart --plain-name 'ints stay small'"); + for (final s in [ + 'ints stay small', + 'seed: 4242', + 'case: ${f.caseId}', + 'generator: g1', + 'shrink limit: 300', + 'input', + '50', + f.replay, + ]) { + expect(f.report, contains(s)); + } + }); + + test('the generator version is whatever the property declares', () { + final r = runProperty( + name: 'v', + gen: G.intIn(0, 9), + body: (_) => throw StateError('x'), + genVersion: 'layout-gen/3', + config: const PropertyConfig(seed: 1)); + expect(r.failure!.report, contains('generator: layout-gen/3')); + }); + + test('a quote in the name is escaped in the replay command', () { + final f = _fails(G.intIn(0, 9), (_) => false, name: "it's a law"); + expect(f.replay, contains(r"--plain-name 'it'\''s a law'")); + }); + + test('an unknown test file is a visible placeholder, not omitted', () { + final f = _fails(G.intIn(0, 9), (_) => false); + expect(f.replay, contains('dart test ')); + }); + + test('the replay command reproduces the failure', () { + final gen = G.listOf(G.intIn(0, 99), maxLen: 25); + final f = _fails(gen, (l) => l.length < 20, seed: 11); + final again = + _fails(gen, (l) => l.length < 20, seed: 11, caseOnly: f.caseId); + expect(again.input, f.input); + expect(again.caseId, f.caseId); + }); + + test('is deterministic: the same run produces the same report', () { + final gen = G.pair(G.intIn(0, 500), G.doubleIn(0, 1)); + String report() => _fails(gen, (p) => p.$1 < 400).report; + expect(report(), report()); + }); + + test('a thrown TestFailure keeps its message', () { + final r = runProperty( + name: 'tf', + gen: G.intIn(0, 3), + body: (v) => expect(v, 99, reason: 'my reason'), + config: const PropertyConfig(seed: 1)); + expect(r.failure!.error, contains('my reason')); + }); + + test('a failing forced example is reported as forced, not shrunk', () { + final f = _fails(G.intIn(0, 1000), (v) => v != 777, examples: [3, 777]); + expect(f.kind, FailureKind.forcedExample); + expect(f.caseId, 'forced:1'); + expect(f.input, '777'); + expect(f.shrinkSteps, 0); + expect(f.replay, contains('PROPERTY_CASE=forced:1')); + }); + + test('a generator that throws is reported as a generator failure', () { + final r = runProperty( + name: 'g', + gen: _Boom(), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 3)); + expect(r.failure!.kind, FailureKind.generator); + expect(r.failure!.report, contains('boom')); + }); + }); + + group('shrinking ints', () { + test('finds the smallest failing int above zero', () { + expect(_fails(G.intIn(0, 1000), (v) => v < 50).input, '50'); + }); + + test('shrinks negatives toward zero', () { + expect(_fails(G.intIn(-100, 100), (v) => v > -7).input, '-7'); + }); + + test('shrinks toward the bound nearest zero when zero is out of range', + () { + expect(_fails(G.intIn(10, 100), (_) => false).input, '10'); + expect(_fails(G.intIn(-100, -10), (_) => false).input, '-10'); + }); + + test('an explicit origin is the shrink target', () { + expect(_fails(G.intIn(0, 100, origin: 60), (_) => false).input, '60'); + }); + + test('never leaves the generator domain', () { + final tried = []; + runProperty( + name: 'dom', + gen: G.intIn(13, 91), + body: (v) { + tried.add(v); + if (v > 20) throw StateError('x'); + }, + config: const PropertyConfig(seed: 5)); + expect(tried.every((v) => v >= 13 && v <= 91), isTrue); + }); + + test('a boolean shrinks to false', () { + expect(_fails(G.boolean(), (_) => false).input, 'false'); + }); + + test('elements shrink toward the earlier ones', () { + expect( + _fails(G.elements(['a', 'b', 'c', 'd']), (v) => v == 'a').input, 'b'); + }); + }); + + group('shrinking doubles', () { + test('reaches zero or the nearest bound when everything fails', () { + expect(_fails(G.doubleIn(-5, 5), (_) => false).input, '0.0'); + expect(_fails(G.doubleIn(5, 10), (_) => false).input, '5.0'); + }); + + test('lands just above a threshold, not far from it', () { + final f = _fails(G.doubleIn(0, 100), (v) => v < 3.5); + final v = double.parse(f.input); + expect(v, inInclusiveRange(3.5, 5)); + }); + + test('generation reaches the declared boundaries exactly', () { + final seen = _inputs( + G.doubleIn(-10, 10, boundaries: const [-10, 10, 0, 1e-9]), + cases: 200); + expect(seen, containsAll([-10.0, 10.0, 0.0, 1e-9])); + expect(seen.every((v) => v >= -10 && v <= 10), isTrue); + }); + + test('specials (NaN, infinities) appear when declared, never otherwise', + () { + final withSpecials = _inputs( + G.doubleIn(0, 1, specials: const [double.nan, double.infinity]), + cases: 200); + expect(withSpecials.any((v) => v.isNaN), isTrue); + expect(withSpecials.any((v) => v == double.infinity), isTrue); + final plain = _inputs(G.doubleIn(0, 1), cases: 200); + expect(plain.every((v) => v.isFinite && v >= 0 && v <= 1), isTrue); + }); + + test('a failing special shrinks to a simple finite value if that fails too', + () { + expect( + _fails(G.doubleIn(0, 1, specials: const [double.nan]), (_) => false) + .input, + '0.0'); + }); + + test('shrink candidates stay in range, differ from the value, are bounded', + () { + final g = G.doubleIn(-3, 100, boundaries: const [50]); + for (final v in [99.5, -2.25, 0.0, 3.0, 50.0, -3.0, 1e-12]) { + final c = g.shrink(v).toList(); + expect(c.length, lessThanOrEqualTo(40), reason: '$v'); + for (final x in c) { + expect(x >= -3 && x <= 100, isTrue, reason: '$v -> $x'); + expect(x, isNot(v), reason: '$v -> $x'); + } + } + expect(g.shrink(0.0), isEmpty, reason: 'the origin is already simplest'); + }); + + test('following the closest candidate always terminates', () { + final g = G.doubleIn(-3, 100); + var v = 87.123456; + var steps = 0; + while (true) { + final c = g.shrink(v).toList(); + if (c.isEmpty) break; + v = c.last; + expect(++steps, lessThan(200), reason: 'stuck at $v'); + } + expect(v, 0); + }); + + test('int shrink candidates: origin first, in range, never the value', () { + final g = G.intIn(-20, 40); + for (final v in [-20, -1, 0, 1, 7, 40]) { + final c = g.shrink(v).toList(); + if (v == 0) { + expect(c, isEmpty); + continue; + } + expect(c.first, 0); + expect(c.every((x) => x >= -20 && x <= 40 && x != v), isTrue); + expect(c.length, lessThanOrEqualTo(12)); + } + }); + }); + + group('shrinking lists, tuples, nullables', () { + test('a list shrinks to the shortest failing length of zeros', () { + final f = _fails(G.listOf(G.intIn(0, 9), maxLen: 20), (l) => l.length < 3); + expect(f.input, '[0, 0, 0]'); + }); + + test('a list keeps the one element that matters', () { + final f = _fails( + G.listOf(G.intIn(0, 20), maxLen: 20), (l) => !l.any((e) => e >= 5)); + expect(f.input, '[5]'); + }); + + test('a list never goes under its minimum or over its cap', () { + final lens = []; + runProperty>( + name: 'len', + gen: G.listOf(G.intIn(0, 9), minLen: 2, maxLen: 7), + body: (l) { + lens.add(l.length); + throw StateError('always'); + }, + config: const PropertyConfig(seed: 3)); + expect(lens.every((n) => n >= 2 && n <= 7), isTrue); + expect(_fails(G.listOf(G.intIn(0, 9), minLen: 2, maxLen: 7), (_) => false) + .input, '[0, 0]'); + }); + + test('the size cap holds across all generated cases', () { + final seen = _inputs(G.listOf(G.boolean(), maxLen: 60), cases: 200); + expect(seen.map((l) => l.length).reduce(math.max), 60, + reason: 'the cap itself is reached'); + expect(seen.every((l) => l.length <= 60), isTrue); + }); + + test('tuple components shrink TOGETHER when only a joint step still fails', + () { + // Fails only while a and b are within 3 of each other and a >= 5: moving + // ONE component far breaks the closeness, so only a joint step makes + // progress. + final f = _fails(G.pair(G.intIn(0, 1000), G.intIn(0, 1000)), + (p) => !((p.$1 - p.$2).abs() <= 3 && p.$1 >= 5), + cases: 2000, seed: 5); + final nums = RegExp(r'\d+').allMatches(f.input).map((m) => int.parse(m[0]!)); + final (a, b) = (nums.first, nums.last); + expect((a - b).abs(), lessThanOrEqualTo(3)); + expect(a, lessThanOrEqualTo(12), reason: f.input); + }); + + test('a triple and a quad shrink every component', () { + expect( + _fails(G.triple(G.intIn(0, 9), G.intIn(-9, 9), G.boolean()), + (_) => false) + .input, + '(0, 0, false)'); + expect( + _fails( + G.quad(G.intIn(0, 9), G.doubleIn(0, 1), G.boolean(), + G.elements(const ['x', 'y'])), + (_) => false) + .input, + '(0, 0.0, false, x)'); + }); + + test('a nullable shrinks to null first', () { + expect(_fails(G.nullable(G.intIn(5, 9)), (_) => false).input, 'null'); + expect( + _fails(G.nullable(G.intIn(5, 9), nullProbability: 0), + (v) => v == null || v < 7) + .input, + '7'); + }); + + test('a list and an int inside a tuple each shrink to their own minimum', + () { + final f = _fails( + G.pair(G.listOf(G.intIn(0, 9), minLen: 1, maxLen: 12), G.intIn(0, 11)), + (p) => !(p.$1.length >= 6 && p.$2 >= 4)); + expect(f.input, '([0, 0, 0, 0, 0, 0], 4)'); + }); + }); + + group('shrink bound', () { + test('the shrink limit caps the extra body runs', () { + var calls = 0; + final r = runProperty>( + name: 'limit', + gen: G.listOf(G.intIn(0, 1000), minLen: 40, maxLen: 50), + body: (_) { + calls++; + throw StateError('always'); + }, + config: const PropertyConfig(seed: 3, shrinkLimit: 5)); + expect(r.passed, isFalse); + expect(calls, lessThanOrEqualTo(1 + 5)); + expect(r.failure!.shrinkSteps, lessThanOrEqualTo(5)); + expect(r.failure!.report, contains('shrink limit: 5')); + }); + + test('with a limit of zero the original input is reported', () { + final f = _fails(G.intIn(500, 1000), (_) => false, shrinkLimit: 0); + expect(f.shrinkSteps, 0); + expect(int.parse(f.input), inInclusiveRange(500, 1000)); + }); + + test('the report shows the original input when shrinking changed it', () { + final f = _fails(G.intIn(0, 1000), (v) => v < 50, seed: 8); + expect(f.report, contains('original input')); + final z = _fails(G.intIn(0, 1000), (_) => false, examples: [0]); + expect(z.report, isNot(contains('original input'))); + }); + }); + + group('wall budget', () { + test('PROPERTY_BUDGET_MS is the hard limit over a property\'s looser default', + () { + // A law registers a 5 s in-test guard (headroom for a loaded parallel + // suite); the serial acceptance run sets PROPERTY_BUDGET_MS=2000 and + // that must be what fails a property, not the 5 s default. + final c = PropertyConfig.fromEnvironment( + const {'PROPERTY_BUDGET_MS': '2000'}, + defaultBudget: const Duration(seconds: 5), + ); + expect(c.budget, const Duration(milliseconds: 2000)); + var ticks = 0; + final r = runProperty( + name: 'serial acceptance', + gen: G.intIn(0, 9), + body: (_) {}, + config: c, + // 1 s per reading: over 2 s on the third, well inside the 5 s default. + elapsed: () => Duration(seconds: ticks++), + ); + expect(r.failure!.kind, FailureKind.budget); + expect(r.failure!.report, contains('2000 ms')); + }); + + test('a property over its budget fails, naming budget and cases run', () { + var ticks = 0; + final r = runProperty( + name: 'slow', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 200), + elapsed: () => Duration(milliseconds: 500 * ticks++), + ); + expect(r.failure!.kind, FailureKind.budget); + expect(r.casesRun, lessThan(200)); + expect(r.failure!.report, contains('2000 ms')); + expect(r.failure!.report, contains('budget')); + }); + + test('shrinking counts against the budget and reports the best so far', () { + // The clock reads past the budget after the 6th reading: the failure at + // case 0 is found, shrinking gets a few attempts, then stops. + var reads = 0; + final r = runProperty>( + name: 'slow shrink', + gen: G.listOf(G.intIn(0, 1000), minLen: 40, maxLen: 50), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1), + elapsed: () => Duration(milliseconds: 500 * reads++), + ); + final f = r.failure!; + expect(f.kind, FailureKind.counterexample); + expect(f.shrinkOverBudget, isTrue); + expect(f.report, contains('over budget (shrinking)')); + expect(f.shrinkSteps, lessThanOrEqualTo(6), + reason: 'unbudgeted, this shrink takes dozens of runs'); + // The best-so-far input is still a valid, failing input of the domain. + final len = RegExp(', ').allMatches(f.input).length + 1; + expect(len, inInclusiveRange(40, 50)); + }); + + test('shrinking inside the budget is not marked', () { + final r = runProperty( + name: 'quick shrink', + gen: G.intIn(500, 1000), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1), + elapsed: () => const Duration(milliseconds: 1), + ); + expect(r.failure!.shrinkOverBudget, isFalse); + expect(r.failure!.report, isNot(contains('over budget'))); + expect(r.failure!.input, '500'); + }); + + test('a single-case replay keeps shrinking whatever the clock says', () { + final r = runProperty( + name: 'replay shrink', + gen: G.intIn(500, 1000), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1, caseOnly: '0'), + elapsed: () => const Duration(hours: 1), + ); + expect(r.failure!.shrinkOverBudget, isFalse); + expect(r.failure!.input, '500'); + }); + + test('a property inside its budget passes', () { + final r = runProperty( + name: 'fast', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 20), + elapsed: () => const Duration(milliseconds: 5), + ); + expect(r.passed, isTrue); + }); + + test('a counterexample is reported before the budget', () { + final r = runProperty( + name: 'both', + gen: G.intIn(0, 9), + body: (_) => throw StateError('x'), + config: const PropertyConfig(seed: 1), + elapsed: () => const Duration(hours: 1), + ); + expect(r.failure!.kind, FailureKind.counterexample); + }); + + test('the default clock is a real stopwatch', () { + final r = runProperty( + name: 'spin', + gen: G.intIn(0, 9), + body: (_) { + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < 4) {} + }, + config: const PropertyConfig( + seed: 1, cases: 50, budget: Duration(milliseconds: 1)), + ); + expect(r.failure?.kind, FailureKind.budget); + }); + + test('a single-case replay is not held to the budget', () { + final r = runProperty( + name: 'replay', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, caseOnly: '3'), + elapsed: () => const Duration(hours: 1), + ); + expect(r.passed, isTrue); + }); + }); + + group('forAll', () { + var ran = 0; + forAll('self: forAll runs examples and generated cases', + G.intIn(0, 100), (v) { + ran++; + expect(v >= 0 && v <= 100, isTrue); + }, examples: const [0, 100], cases: 25); + + test('forAll ran its forced examples plus its cases', () { + expect(ran, anyOf(0, 27), + reason: '0 when this test alone is selected by name'); + }); + + forAll<(int, int)>('self: addition commutes', G.pair(G.intIn(-99, 99), G.intIn(-99, 99)), + (p) => expect(p.$1 + p.$2, p.$2 + p.$1)); + }); +} + +class _Boom extends Gen { + @override + int generate(Rng r, int size) => throw StateError('boom'); + @override + Iterable shrink(int v) => const []; +}