From 0725b6932e4ef60254d18ec76a21d930f885080b Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sun, 1 Nov 2026 09:30:42 -0700 Subject: [PATCH 1/4] RR correction and rhythm screening replay settled beats on each sync --- lib/onehz.dart | 7 +- .../onehz/clinical/irregular_diagnostics.dart | 258 ++ lib/src/onehz/clinical/irregular_rhythm.dart | 282 +- .../clinical/irregular_rhythm_state.dart | 427 +++ lib/src/onehz/clinical/irregular_window.dart | 65 + .../foundations/rr_correction_stream.dart | 822 +++++ lib/src/onehz/util.dart | 85 + .../irregular_diagnostics_stream_test.dart | 398 +++ .../irregular_screen_state_test.dart | 495 +++ test/onehz/incremental/rr_corrector_test.dart | 569 ++++ .../irregular_diagnostics_empty_test.dart | 54 + test/onehz/irregular_diagnostics_test.dart | 574 ++++ test/onehz/rr_artifact_cap_test.dart | 336 ++ test/onehz/rr_correction_oracle_test.dart | 322 ++ test/onehz/rr_correction_perf_test.dart | 48 + test/onehz/sliding_median_test.dart | 178 ++ test/onehz/support/correct_rr_reference.dart | 344 +++ test/onehz/support/rr_chunking.dart | 40 + test/onehz/support/rr_compare.dart | 44 + test/onehz/support/rr_runs.dart | 54 + test/onehz/support/rr_stream_driver.dart | 100 + test/onehz/support/rr_synth.dart | 151 + test/properties/rr_irregular_laws_test.dart | 2693 +++++++++++++++++ tool/incremental/bench_rr.dart | 83 + tool/incremental/driver.dart | 32 + tool/incremental/lag_stats.dart | 49 + tool/incremental/oracle_util.dart | 40 + tool/incremental/rr_stream.dart | 579 ++++ tool/incremental/synth.dart | 197 ++ 29 files changed, 9216 insertions(+), 110 deletions(-) create mode 100644 lib/src/onehz/clinical/irregular_diagnostics.dart create mode 100644 lib/src/onehz/clinical/irregular_rhythm_state.dart create mode 100644 lib/src/onehz/clinical/irregular_window.dart create mode 100644 lib/src/onehz/foundations/rr_correction_stream.dart create mode 100644 test/onehz/incremental/irregular_diagnostics_stream_test.dart create mode 100644 test/onehz/incremental/irregular_screen_state_test.dart create mode 100644 test/onehz/incremental/rr_corrector_test.dart create mode 100644 test/onehz/irregular_diagnostics_empty_test.dart create mode 100644 test/onehz/irregular_diagnostics_test.dart create mode 100644 test/onehz/rr_artifact_cap_test.dart create mode 100644 test/onehz/rr_correction_oracle_test.dart create mode 100644 test/onehz/rr_correction_perf_test.dart create mode 100644 test/onehz/sliding_median_test.dart create mode 100644 test/onehz/support/correct_rr_reference.dart create mode 100644 test/onehz/support/rr_chunking.dart create mode 100644 test/onehz/support/rr_compare.dart create mode 100644 test/onehz/support/rr_runs.dart create mode 100644 test/onehz/support/rr_stream_driver.dart create mode 100644 test/onehz/support/rr_synth.dart create mode 100644 test/properties/rr_irregular_laws_test.dart create mode 100644 tool/incremental/bench_rr.dart create mode 100644 tool/incremental/driver.dart create mode 100644 tool/incremental/lag_stats.dart create mode 100644 tool/incremental/oracle_util.dart create mode 100644 tool/incremental/rr_stream.dart create mode 100644 tool/incremental/synth.dart diff --git a/lib/onehz.dart b/lib/onehz.dart index ea5d9a40..845b7caf 100644 --- a/lib/onehz.dart +++ b/lib/onehz.dart @@ -18,7 +18,7 @@ export 'src/onehz/device.dart'; // Layer 0: foundations. export 'src/onehz/foundations/rr_correction.dart'; - +export 'src/onehz/foundations/rr_correction_stream.dart'; export 'src/onehz/foundations/baseline.dart'; export 'src/onehz/foundations/ewma_baselines.dart'; export 'src/onehz/foundations/fusion.dart'; @@ -34,8 +34,9 @@ export 'src/onehz/clinical/cosinor.dart'; export 'src/onehz/clinical/load_trimp.dart'; export 'src/onehz/clinical/stress_si.dart'; export 'src/onehz/clinical/irregular_rhythm.dart'; - - +export 'src/onehz/clinical/irregular_rhythm_state.dart'; +export 'src/onehz/clinical/irregular_diagnostics.dart' + hide diagnosticArtifactFraction; export 'src/onehz/clinical/cardiac_coherence.dart'; export 'src/onehz/clinical/vo2max.dart'; diff --git a/lib/src/onehz/clinical/irregular_diagnostics.dart b/lib/src/onehz/clinical/irregular_diagnostics.dart new file mode 100644 index 00000000..c2b91d2d --- /dev/null +++ b/lib/src/onehz/clinical/irregular_diagnostics.dart @@ -0,0 +1,258 @@ +// Internal rule for SD1/SD2, pNNx thresholds and sustained-window voting. +// Task Force 1996 supports successive-difference HRV components, not this +// classifier or its SD1/SD2 thresholds; see hrv_time.dart. +// Internal irregular-rhythm screen, ESTIMATE with low confidence. Batch and +// stream share irregularWindowVerdict(); centered moments reorder floating-point +// sums, while exact counters preserve window evidence. Shared adjacency excludes +// timestamp-gap pairs. Diagnostics account for evidence without adding a +// clinical estimator. +// CLINICAL — the evidence behind an irregular-rhythm screen verdict. +// +// What it is for: `irregularBeatScreen` computes the beat counts, the cleaning +// counts and the per-5-minute-window counts on its way to a flag, then throws +// them away. A reader who sees "flagged" or "not screened" cannot tell why. +// [IrregularDiagnostics] keeps them, for a screen that RAN and for one that +// ABSTAINED (an abstention carries the counts that caused it). +// +// Absent stays absent: an input the caller did not give is null, never 0. The +// batch call and the streaming `IrregularScreenState` must produce the same +// diagnostics for the same beats, however chunked or restored. + +import '../types.dart'; +import 'irregular_rhythm.dart'; + +/// What the RR corrector did before the screen saw the beats. Only the caller +/// has it (the screen sees the corrected NN), so it is handed in; batch passes +/// the fields of `RrCorrectionResult`, streaming those of `RrSnapshot`. +class RrCleaningCounts { + /// Beats fed to the corrector. + final int raw; + + /// Isolated artifacts replaced by the spline. + final int corrected; + + /// Beats dropped (part of a multi-beat run, or with no anchors to correct from). + final int dropped; + const RrCleaningCounts({ + required this.raw, + required this.corrected, + required this.dropped, + }); +} + +/// The artifact fraction to record as evidence: [fraction], or null when +/// [cleaning] says the corrector saw no beats (no denominator). +double? diagnosticArtifactFraction(double fraction, RrCleaningCounts? cleaning) => + cleaning != null && cleaning.raw == 0 ? null : fraction; + +/// Why a screen produced no value. Wire names (toJson): `too_few_beats`, +/// `artifact`, `no_successive_pairs`, `no_long_term_variability`. +enum IrregularAbstain { + tooFewBeats, + artifact, + noSuccessivePairs, + noLongTermVariability, +} + +/// The final window of the series. A series always ends mid-window, so the last +/// window is OPEN; it is counted, not silently dropped. Wire names: `none` +/// (no beats at all), `thin` (fewer than minWindowBeats, excluded from the +/// valid count), `unflagged` (valid, not flagged), `flagged` (valid, flagged). +enum IrregularOpenWindow { none, thin, unflagged, flagged } + +/// The thresholds the verdict was made against. +class IrregularThresholds { + final int minBeats; + final double maxArtifact; + final double sd1sd2Flag; + final double pnnThresholdMs; + final double pnnFlagPct; + final double windowMinutes; + final int minWindowBeats; + final double sustainedFraction; + const IrregularThresholds({ + required this.minBeats, + required this.maxArtifact, + required this.sd1sd2Flag, + required this.pnnThresholdMs, + required this.pnnFlagPct, + required this.windowMinutes, + required this.minWindowBeats, + required this.sustainedFraction, + }); +} + +/// Per-window counts. Totals INCLUDE the open window; [open] says what it was. +class IrregularWindowCounts { + /// Windows with at least one clean beat (a window opens on a beat). + final int total; + + /// Windows with at least minWindowBeats beats (the ones that vote). + final int valid; + + /// Valid windows where both SD1/SD2 and pNNx cleared their flag lines. + final int flagged; + + /// The final window: its beat count and what became of it. + final int openBeats; + final IrregularOpenWindow open; + const IrregularWindowCounts({ + required this.total, + required this.valid, + required this.flagged, + required this.openBeats, + required this.open, + }); + + /// flagged / valid, null (never 0 or NaN) when no window is valid. + double? get sustainedObserved => valid == 0 ? null : flagged / valid; +} + +class IrregularDiagnostics { + /// Null when the screen ran and produced a value. + final IrregularAbstain? abstain; + + /// Beats fed to the corrector / corrected / dropped. Null when the caller did + /// not hand [RrCleaningCounts] in (unknown, not zero). + final int? rrRaw, corrected, dropped; + + /// NN entries handed to the screen (NaN and out-of-range included) and the + /// ones inside [300, 2000] ms that it used (== `IrregularRhythm.nBeats`). + final int nnIn, nnKept; + + /// The artifact fraction handed in (0 when none was). Null when the corrector + /// was handed no beats ([RrCleaningCounts.raw] == 0): a share of nothing has + /// no value, and the `1 - cleanFraction` of an empty series (1.0) would read + /// as "every beat is an artifact". The verdict gate still reads the fraction + /// the caller passed; only this evidence field is absent. + final double? artifactFraction; + + /// Null when windows were not evaluated: no beat times (or a length that does + /// not match the beats), or a window config that fails closed. + final IrregularWindowCounts? windows; + final IrregularThresholds thresholds; + + const IrregularDiagnostics({ + required this.abstain, + required this.rrRaw, + required this.corrected, + required this.dropped, + required this.nnIn, + required this.nnKept, + required this.artifactFraction, + required this.windows, + required this.thresholds, + }); + + static const _abstainWire = { + IrregularAbstain.tooFewBeats: 'too_few_beats', + IrregularAbstain.artifact: 'artifact', + IrregularAbstain.noSuccessivePairs: 'no_successive_pairs', + IrregularAbstain.noLongTermVariability: 'no_long_term_variability', + }; + + /// Wire shape (version 1); see test/onehz/irregular_diagnostics_test.dart. + /// Doubles are written as they are held, so a JSON text round trip is exact. + Map toJson() => { + 'version': 1, + 'abstain': abstain == null ? null : _abstainWire[abstain], + 'beats': { + 'rr_raw': rrRaw, + 'nn_in': nnIn, + 'nn_kept': nnKept, + 'corrected': corrected, + 'dropped': dropped, + 'artifact_fraction': artifactFraction, + }, + 'windows': windows == null + ? null + : { + 'total': windows!.total, + 'valid': windows!.valid, + 'flagged': windows!.flagged, + 'sustained_observed': windows!.sustainedObserved, + 'open_beats': windows!.openBeats, + 'open': windows!.open.name, + }, + 'thresholds': { + 'min_beats': thresholds.minBeats, + 'max_artifact': thresholds.maxArtifact, + 'sd1sd2_flag': thresholds.sd1sd2Flag, + 'pnn_threshold_ms': thresholds.pnnThresholdMs, + 'pnn_flag_pct': thresholds.pnnFlagPct, + 'window_minutes': thresholds.windowMinutes, + 'min_window_beats': thresholds.minWindowBeats, + 'sustained_fraction': thresholds.sustainedFraction, + }, + }; + + /// Throws [FormatException] on a map of another version or a malformed one. + factory IrregularDiagnostics.fromJson(Map json) { + try { + return _restore(json); + } on FormatException { + rethrow; + } catch (e) { + throw FormatException('malformed IrregularDiagnostics: $e'); + } + } + + static IrregularDiagnostics _restore(Map j) { + if (j['version'] != 1) { + throw FormatException('unsupported IrregularDiagnostics version ${j['version']}'); + } + final b = j['beats'] as Map, th = j['thresholds'] as Map; + final w = j['windows'] as Map?; + final a = j['abstain'] as String?; + IrregularAbstain? abstain; + if (a != null) { + final hit = _abstainWire.entries.where((e) => e.value == a); + if (hit.isEmpty) throw FormatException('unknown abstain reason $a'); + abstain = hit.first.key; + } + double d(Map m, String k) => (m[k] as num).toDouble(); + return IrregularDiagnostics( + abstain: abstain, + rrRaw: b['rr_raw'] as int?, + corrected: b['corrected'] as int?, + dropped: b['dropped'] as int?, + nnIn: b['nn_in'] as int, + nnKept: b['nn_kept'] as int, + artifactFraction: + b['artifact_fraction'] == null ? null : d(b, 'artifact_fraction'), + windows: w == null + ? null + : IrregularWindowCounts( + total: w['total'] as int, + valid: w['valid'] as int, + flagged: w['flagged'] as int, + openBeats: w['open_beats'] as int, + open: IrregularOpenWindow.values.byName(w['open'] as String), + ), + thresholds: IrregularThresholds( + minBeats: th['min_beats'] as int, + maxArtifact: d(th, 'max_artifact'), + sd1sd2Flag: d(th, 'sd1sd2_flag'), + pnnThresholdMs: d(th, 'pnn_threshold_ms'), + pnnFlagPct: d(th, 'pnn_flag_pct'), + windowMinutes: d(th, 'window_minutes'), + minWindowBeats: th['min_window_beats'] as int, + sustainedFraction: d(th, 'sustained_fraction'), + ), + ); + } +} + +/// A screen verdict and the evidence behind it. +class IrregularScreenResult { + final Metric metric; + final IrregularDiagnostics diagnostics; + const IrregularScreenResult(this.metric, this.diagnostics); + + /// `metric.toJson((v) => v.toJson())` plus a `diagnostics` key: the envelope + /// edge persists. The existing keys are unchanged. + Map toJson() => { + ...metric.toJson((v) => v.toJson()), + 'diagnostics': diagnostics.toJson(), + }; +} diff --git a/lib/src/onehz/clinical/irregular_rhythm.dart b/lib/src/onehz/clinical/irregular_rhythm.dart index 2b0faf87..b4c9762e 100644 --- a/lib/src/onehz/clinical/irregular_rhythm.dart +++ b/lib/src/onehz/clinical/irregular_rhythm.dart @@ -1,3 +1,11 @@ +// Internal rule for SD1/SD2, pNNx thresholds and sustained-window voting. +// Task Force 1996 supports successive-difference HRV components, not this +// classifier or its SD1/SD2 thresholds; see hrv_time.dart. +// Internal irregular-rhythm screen, ESTIMATE with low confidence. Batch and +// stream share irregularWindowVerdict(); centered moments reorder floating-point +// sums, while exact counters preserve window evidence. Shared adjacency excludes +// timestamp-gap pairs. Diagnostics account for evidence without adding a +// clinical estimator. // CLINICAL — 24/7 irregular-rhythm SCREEN (NOT a diagnosis). // // A pulse-derived screen for sustained beat-to-beat irregularity over a long RR @@ -17,6 +25,8 @@ import 'dart:math' as math; import '../types.dart'; import '../util.dart'; +import 'irregular_diagnostics.dart'; +import 'irregular_window.dart'; class IrregularRhythm { final double sd1; // ms — short-term (beat-to-beat) scatter @@ -75,14 +85,59 @@ Metric irregularBeatScreen( double windowMinutes = 5, int minWindowBeats = 40, double sustainedFraction = 0.5, +}) => + irregularBeatScreenDetailed( + rrMs, + nnTimesMs: nnTimesMs, + artifactFraction: artifactFraction, + minBeats: minBeats, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + maxArtifact: maxArtifact, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction, + ).metric; + +/// [irregularBeatScreen] and the evidence behind its verdict: the beat counts, +/// the per-window counts (the final OPEN window included), the thresholds, and, +/// when the screen abstained, which gate stopped it. The Metric is exactly the +/// one [irregularBeatScreen] returns. [cleaning] is what the RR corrector did +/// upstream, when the caller knows (otherwise those counts are null, not 0). +/// Windows are counted whenever beat times are given, whether or not the +/// aggregate looked irregular and whether or not the screen abstained. +IrregularScreenResult irregularBeatScreenDetailed( + List rrMs, { + List? nnTimesMs, + double artifactFraction = 0.0, + int minBeats = irregularScreenMinBeats, + double sd1sd2Flag = 0.70, + double pnnThresholdMs = 70, + double pnnFlagPct = 30, + double maxArtifact = 0.30, + double windowMinutes = 5, + int minWindowBeats = 40, + double sustainedFraction = 0.5, + RrCleaningCounts? cleaning, }) { const inputs = ['rr_cleaned']; + final thresholds = IrregularThresholds( + minBeats: minBeats, + maxArtifact: maxArtifact, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction, + ); // The defensive [300, 2000] filter COMPACTS the series. Keep the mask too, so // successive differences below are taken only between beats that were both // kept AND adjacent in the input — otherwise every filtered beat manufactured // one spurious difference spanning the gap, which counted toward pNNx and // inflated sdsd/sd1, pushing both flag conditions toward a false "sustained - // irregularity" screen positive. + // irregularity" screen positive. (NaN fails both comparisons: not a beat.) final keep = [for (final v in rrMs) v >= 300 && v <= 2000]; final nn = [ for (var i = 0; i < rrMs.length; i++) @@ -117,33 +172,69 @@ Metric irregularBeatScreen( for (var i = 1; i < rrMs.length; i++) if (successive[i]) rrMs[i] - rrMs[i - 1] ]; + // Windows are built from the SAME clean beats as the aggregate (the [keep] + // mask), not the raw input — an artifact beat the aggregate correctly + // excludes must not be allowed back in to inflate one window's own ratio/pNN + // into a spurious per-window flag. Counted up front so an abstention can + // still report them; the verdict reads them only where it did before. + final windowsOk = irregularWindowConfigOk( + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction); + IrregularWindowCounts? windows; + if (hasTimes && windowsOk) { + final nnTimes = [ + for (var i = 0; i < rrMs.length; i++) + if (keep[i]) nnTimesMs[i] + ]; + windows = _countWindows( + nn, + nnTimes, + nnAdjacent, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + ); + } + + IrregularScreenResult abstain(IrregularAbstain why, String note) => + IrregularScreenResult( + Metric.absent( + tier: Tier.estimate, inputs_used: inputs, note: note), + IrregularDiagnostics( + abstain: why, + rrRaw: cleaning?.raw, + corrected: cleaning?.corrected, + dropped: cleaning?.dropped, + nnIn: rrMs.length, + nnKept: nn.length, + artifactFraction: diagnosticArtifactFraction(artifactFraction, cleaning), + windows: windows, + thresholds: thresholds, + ), + ); + // Gate on the contiguous pairs actually averaged, not the beat count: a // series fragmented by holes can hold minBeats beats yet only a handful of // successive pairs, and SD1/pNNx would then come from those few diffs. if (diffs.length < minBeats - 1) { - return const Metric.absent( - tier: Tier.estimate, - inputs_used: inputs, - note: 'too few clean beats for an irregular-rhythm screen', - ); + return abstain(IrregularAbstain.tooFewBeats, + 'too few clean beats for an irregular-rhythm screen'); } if (artifactFraction > maxArtifact) { - return Metric.absent( - tier: Tier.estimate, - inputs_used: inputs, - note: 'artifact fraction ${(artifactFraction * 100).round()}% > ' - '${(maxArtifact * 100).round()}% — screen suppressed on noisy RR', - ); + return abstain( + IrregularAbstain.artifact, + 'artifact fraction ${(artifactFraction * 100).round()}% > ' + '${(maxArtifact * 100).round()}% — screen suppressed on noisy RR'); } final sdsd = stddev(diffs); final sdnn = stddev(nn); if (sdsd == null || sdnn == null) { - return const Metric.absent( - tier: Tier.estimate, - inputs_used: inputs, - note: 'no successive clean beats to build a Poincare plot from', - ); + return abstain(IrregularAbstain.noSuccessivePairs, + 'no successive clean beats to build a Poincare plot from'); } final sd1 = sdsd / math.sqrt2; final v = 2 * sdnn * sdnn - sd1 * sd1; @@ -152,12 +243,10 @@ Metric irregularBeatScreen( // SD1/SD2 is undefined without long-term variability to divide by; emitting // ratio 0.0 with sd1 = sd2 = 0 published "perfectly regular" as a // measurement of a degenerate series. - return const Metric.absent( - tier: Tier.estimate, - inputs_used: inputs, - note: 'no long-term variability (SD2 = 0) — the SD1/SD2 ratio is ' - 'undefined, not "perfectly regular"', - ); + return abstain( + IrregularAbstain.noLongTermVariability, + 'no long-term variability (SD2 = 0) — the SD1/SD2 ratio is ' + 'undefined, not "perfectly regular"'); } final ratio = sd1 / sd2; @@ -180,57 +269,55 @@ Metric irregularBeatScreen( // that is what "sustained" is supposed to mean. Falls back to the old // whole-span verdict only when times weren't supplied (short/sleep-only // callers where the whole span already IS roughly one physiological state). - // Windows are built from the SAME clean beats as the aggregate above (the - // [keep] mask), not the raw input — an artifact beat the aggregate - // correctly excludes must not be allowed back in here to inflate one - // window's own ratio/pNN into a spurious per-window flag. - final nnTimes = hasTimes - ? [ - for (var i = 0; i < rrMs.length; i++) - if (keep[i]) nnTimesMs[i] - ] - : const []; + // A bad window config fails CLOSED (never sustained): a misconfigured caller + // must never manufacture a medical false positive. final flag = aggregateHigh && (!hasTimes || - _sustainedAcrossWindows( - nn, - nnTimes, - nnAdjacent, - sd1sd2Flag: sd1sd2Flag, - pnnThresholdMs: pnnThresholdMs, - pnnFlagPct: pnnFlagPct, - windowMinutes: windowMinutes, - minWindowBeats: minWindowBeats, - sustainedFraction: sustainedFraction, - )); + (windows != null && + windows.valid > 0 && + windows.flagged / windows.valid >= sustainedFraction)); // Confidence scales with beat count (~5000 beats ≈ a full strong night) AND // with the artifact fraction we were handed — it used to ignore it entirely, // so a barely-passing 29 %-artifact night published at the same confidence as // a clean one. final conf = (nn.length / 5000.0 * (1 - artifactFraction)).clamp(0.2, 0.9); - return Metric( - value: IrregularRhythm( - sd1: sd1, - sd2: sd2, - sd1sd2: ratio, - pnnPct: pnnPct, - nBeats: nn.length, - flag: flag, + return IrregularScreenResult( + Metric( + value: IrregularRhythm( + sd1: sd1, + sd2: sd2, + sd1sd2: ratio, + pnnPct: pnnPct, + nBeats: nn.length, + flag: flag, + ), + confidence: conf, + tier: Tier.estimate, + inputs_used: inputs, + note: 'irregular-rhythm SCREEN (not a diagnosis): Poincaré SD1/SD2 + pNN' + '${pnnThresholdMs.round()}. PRV not ECG — wrist pulse misses P-waves. ' + 'Discuss with a clinician only if you have symptoms.', + ), + IrregularDiagnostics( + abstain: null, + rrRaw: cleaning?.raw, + corrected: cleaning?.corrected, + dropped: cleaning?.dropped, + nnIn: rrMs.length, + nnKept: nn.length, + artifactFraction: diagnosticArtifactFraction(artifactFraction, cleaning), + windows: windows, + thresholds: thresholds, ), - confidence: conf, - tier: Tier.estimate, - inputs_used: inputs, - note: 'irregular-rhythm SCREEN (not a diagnosis): Poincaré SD1/SD2 + pNN' - '${pnnThresholdMs.round()}. PRV not ECG — wrist pulse misses P-waves. ' - 'Discuss with a clinician only if you have symptoms.', ); } -/// True when the SD1/SD2 + pNNx criteria independently hold in a real -/// fraction of short, roughly-stationary windows across [rrMs] — not just in -/// the one number the whole span blends into. [timesMs] must be the same -/// length as [rrMs] and index-aligned (elapsed ms per beat). -bool _sustainedAcrossWindows( +/// Counts the short windows across [rrMs] (clean beats only, with their +/// [timesMs] and [adjacent] flags, all the same length): how many exist, how +/// many are thick enough to vote, how many of those flag, and what the final +/// OPEN window was. The window rule and per-window verdict are the ones +/// `IrregularScreenState` uses (irregular_window.dart). +IrregularWindowCounts _countWindows( List rrMs, List timesMs, // Aligned to [rrMs]: whether each beat was truly adjacent (no dropped @@ -242,53 +329,29 @@ bool _sustainedAcrossWindows( required double pnnFlagPct, required double windowMinutes, required int minWindowBeats, - required double sustainedFraction, }) { - if (timesMs.length != rrMs.length || - adjacent.length != rrMs.length || - rrMs.length < 2) { - return false; - } - // Fail CLOSED (never sustained) on a bad config — a misconfigured caller - // must never manufacture a medical false positive. - if (!windowMinutes.isFinite || - windowMinutes <= 0 || - minWindowBeats < 2 || - !sustainedFraction.isFinite || - sustainedFraction < 0 || - sustainedFraction > 1) { - return false; - } final windowMs = windowMinutes * 60000; - var windowStart = timesMs.first; - var validWindows = 0; - var flaggedWindows = 0; + var total = 0, valid = 0, flagged = 0; + var openBeats = 0; + var open = IrregularOpenWindow.none; var bucket = []; var bucketAdjacent = []; + var windowStart = timesMs.isEmpty ? 0.0 : timesMs.first; void flush() { - // Mirror the aggregate's `keep[i] && keep[i-1]` guard: never diff - // across a beat that was dropped as an artifact in the original series, - // even though it's now a consecutive pair in this compacted bucket. - final diffs = [ - for (var i = 1; i < bucket.length; i++) - if (bucketAdjacent[i]) bucket[i] - bucket[i - 1] - ]; - // Valid on contiguous pairs, not beats (same reason as the aggregate). - if (diffs.length >= minWindowBeats - 1) { - validWindows++; - final sdsd = stddev(diffs); - final sdnn = stddev(bucket); - if (sdsd != null && sdnn != null) { - final sd1 = sdsd / math.sqrt2; - final v = 2 * sdnn * sdnn - sd1 * sd1; - final sd2 = v > 0 ? math.sqrt(v) : 0.0; - if (sd2 > 0) { - final ratio = sd1 / sd2; - final over = diffs.where((d) => d.abs() > pnnThresholdMs).length; - final pnn = 100.0 * over / diffs.length; - if (ratio >= sd1sd2Flag && pnn >= pnnFlagPct) flaggedWindows++; - } - } + if (bucket.isEmpty) return; + final verdict = irregularWindowVerdict(bucket, bucketAdjacent, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + minWindowBeats: minWindowBeats); + total++; + openBeats = bucket.length; + if (verdict == null) { + open = IrregularOpenWindow.thin; + } else { + valid++; + if (verdict) flagged++; + open = verdict ? IrregularOpenWindow.flagged : IrregularOpenWindow.unflagged; } bucket = []; bucketAdjacent = []; @@ -303,6 +366,11 @@ bool _sustainedAcrossWindows( bucketAdjacent.add(adjacent[i]); } flush(); - if (validWindows == 0) return false; - return flaggedWindows / validWindows >= sustainedFraction; + return IrregularWindowCounts( + total: total, + valid: valid, + flagged: flagged, + openBeats: openBeats, + open: open, + ); } diff --git a/lib/src/onehz/clinical/irregular_rhythm_state.dart b/lib/src/onehz/clinical/irregular_rhythm_state.dart new file mode 100644 index 00000000..27b3c765 --- /dev/null +++ b/lib/src/onehz/clinical/irregular_rhythm_state.dart @@ -0,0 +1,427 @@ +// Internal rule for SD1/SD2, pNNx thresholds and sustained-window voting. +// Task Force 1996 supports successive-difference HRV components, not this +// classifier or its SD1/SD2 thresholds; see hrv_time.dart. +// Internal irregular-rhythm screen, ESTIMATE with low confidence. Batch and +// stream share irregularWindowVerdict(); centered moments reorder floating-point +// sums, while exact counters preserve window evidence. Shared adjacency excludes +// timestamp-gap pairs. Diagnostics account for evidence without adding a +// clinical estimator. +// CLINICAL — incremental irregular-rhythm screen. +// +// [IrregularScreenState] is `irregularBeatScreen` as a fold over the corrected +// NN series that `RrCorrector` settles. Feed it each fold's settled NN with +// [IrregularScreenState.fold], and ask [IrregularScreenState.evaluate] with the +// corrector's provisional tail: the Metric equals +// `irregularBeatScreen(allNn, nnTimesMs: allTimes, ...)` over settled ++ tail. +// +// The aggregates (SD1/SD2 from the spread of successive differences and of the +// levels, pNNx) are running sums (Welford), equal to the batch's two-pass +// figures to ~1e-13. The "sustained across 5-minute windows" test is a COUNT of +// flagged / valid closed windows plus the one open window, which is evaluated +// on a copy at the end of the data, so it is exact. The per-window verdict is +// the batch's own (`irregularWindowVerdict`). +// +// [IrregularScreenState.evaluateDetailed] adds the evidence behind the verdict +// (`IrregularDiagnostics`): the same beat and window counts the batch call +// reports, including the final OPEN window, for a screen that ran and one that +// abstained. The checkpoint (version 2) carries two more counters for it, the +// NN entries seen and the windows closed; a version 1 checkpoint lacks them and +// is REFUSED rather than restored with counts it never kept. +// +// Version 3 adds the previous kept beat's time: two kept beats are successive +// only when no time hole lies between them (the batch's seam test), so the +// fold needs it. Older checkpoints are refused for the same reason. + +import 'dart:math' as math; + +import '../types.dart'; +import 'irregular_diagnostics.dart'; +import 'irregular_rhythm.dart'; +import 'irregular_window.dart'; + +class IrregularScreenState { + final double sd1sd2Flag, pnnThresholdMs, pnnFlagPct, windowMinutes; + final int minWindowBeats; + final double sustainedFraction; + + // ---- running state over the corrected NN list ---- + bool _prevKept = false; // the previous INPUT beat was inside [300, 2000] + double _prevV = 0; + // time of the previous kept beat; NaN when unknown (a NaN time is never + // within a seam, exactly as in the batch comparison) + double _prevT = double.nan; + int _nIn = 0; // NN entries folded, usable or not + int _nKept = 0; + // successive differences between adjacent kept beats + int _dN = 0; + double _dMean = 0, _dM2 = 0; + int _over = 0; + // levels of kept beats + int _lN = 0; + double _lMean = 0, _lM2 = 0; + // sustained-window bookkeeping + double? _winStart; + List _bk = []; + List _bkAdj = []; + int _total = 0, _valid = 0, _flagged = 0; + + IrregularScreenState({ + this.sd1sd2Flag = 0.70, + this.pnnThresholdMs = 70, + this.pnnFlagPct = 30, + this.windowMinutes = 5, + this.minWindowBeats = 40, + this.sustainedFraction = 0.5, + }); + + /// A broken window config fails CLOSED in the batch (never sustained). Here it + /// also means no windows are tracked, so a bad config cannot grow the state. + bool get _windowsOk => irregularWindowConfigOk( + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction); + + /// Restores a state from [toJson], parameters included. Throws + /// [FormatException] on a checkpoint of another type or version, or one that + /// is malformed. + factory IrregularScreenState.fromJson(Map json) { + try { + return _restore(json); + } on FormatException { + rethrow; + } catch (e) { + throw FormatException('malformed IrregularScreenState checkpoint: $e'); + } + } + + static IrregularScreenState _restore(Map j) { + if (j['type'] != 'IrregularScreenState') { + throw FormatException( + 'not an IrregularScreenState checkpoint: ${j['type']}'); + } + if (j['version'] != 3) { + throw FormatException( + 'unsupported IrregularScreenState version ${j['version']}'); + } + double d(String k) => (j[k] as num).toDouble(); + final s = IrregularScreenState( + sd1sd2Flag: d('sd1sd2Flag'), + pnnThresholdMs: d('pnnThresholdMs'), + pnnFlagPct: d('pnnFlagPct'), + windowMinutes: d('windowMinutes'), + minWindowBeats: j['minWindowBeats'] as int, + sustainedFraction: d('sustainedFraction'), + ); + s._prevKept = j['prevKept'] as bool; + s._prevV = d('prevV'); + // Present in every version 3 checkpoint; null only when unknown (NaN). + if (!j.containsKey('prevT')) throw const FormatException('missing prevT'); + s._prevT = (j['prevT'] as num?)?.toDouble() ?? double.nan; + s._nIn = j['nIn'] as int; + s._nKept = j['nKept'] as int; + s._dN = j['dN'] as int; + s._dMean = d('dMean'); + s._dM2 = d('dM2'); + s._over = j['over'] as int; + s._lN = j['lN'] as int; + s._lMean = d('lMean'); + s._lM2 = d('lM2'); + s._winStart = (j['winStart'] as num?)?.toDouble(); + s._bk = [for (final x in j['bk'] as List) (x as num).toDouble()]; + final bkAdj = j['bkAdj'] as List; + if (bkAdj.any((x) => x is! int || (x != 0 && x != 1))) { + throw const FormatException('invalid bkAdj'); + } + s._bkAdj = [for (final x in bkAdj) x == 1]; + s._total = j['total'] as int; + s._valid = j['valid'] as int; + s._flagged = j['flagged'] as int; + if (s._lN != s._nKept) throw const FormatException('invalid lN'); + if (s._dN > math.max(0, s._nKept - 1)) { + throw const FormatException('invalid dN'); + } + if (s._total > s._nKept - s._bk.length) { + throw const FormatException('invalid total'); + } + if (s._bk.isNotEmpty && s._winStart == null) { + throw const FormatException('invalid winStart'); + } + final ok = s._bk.length == s._bkAdj.length && + s._nKept >= 0 && + s._nIn >= s._nKept && + s._total >= s._valid && + s._dN >= 0 && + s._lN >= 0 && + s._over >= 0 && + s._over <= s._dN && + s._flagged >= 0 && + s._flagged <= s._valid; + if (!ok) { + throw const FormatException('inconsistent IrregularScreenState checkpoint'); + } + return s; + } + + /// Checkpoint: `{'version': 3, 'type': 'IrregularScreenState', ...}`, plain + /// JSON, bounded (running sums plus at most one open 5-minute window). + Map toJson() => { + 'version': 3, + 'type': 'IrregularScreenState', + 'sd1sd2Flag': sd1sd2Flag, + 'pnnThresholdMs': pnnThresholdMs, + 'pnnFlagPct': pnnFlagPct, + 'windowMinutes': windowMinutes, + 'minWindowBeats': minWindowBeats, + 'sustainedFraction': sustainedFraction, + 'prevKept': _prevKept, + 'prevV': _prevV, + 'prevT': _prevT.isFinite ? _prevT : null, + 'nIn': _nIn, + 'nKept': _nKept, + 'dN': _dN, + 'dMean': _dMean, + 'dM2': _dM2, + 'over': _over, + 'lN': _lN, + 'lMean': _lMean, + 'lM2': _lM2, + 'winStart': _winStart, + 'bk': _bk, + 'bkAdj': [for (final a in _bkAdj) a ? 1 : 0], + 'total': _total, + 'valid': _valid, + 'flagged': _flagged, + }; + + IrregularScreenState _copy() => IrregularScreenState( + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction, + ) + .._prevKept = _prevKept + .._prevV = _prevV + .._prevT = _prevT + .._nIn = _nIn + .._nKept = _nKept + .._dN = _dN + .._dMean = _dMean + .._dM2 = _dM2 + .._over = _over + .._lN = _lN + .._lMean = _lMean + .._lM2 = _lM2 + .._winStart = _winStart + .._bk = List.of(_bk) + .._bkAdj = List.of(_bkAdj) + .._total = _total + .._valid = _valid + .._flagged = _flagged; + + /// Closes the open window into the valid / flagged counts. + void _closeWindow() { + final verdict = irregularWindowVerdict(_bk, _bkAdj, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + minWindowBeats: minWindowBeats); + if (_bk.isNotEmpty) _total++; + if (verdict != null) { + _valid++; + if (verdict) _flagged++; + } + _bk = []; + _bkAdj = []; + } + + void _add(double v, double tm) { + _nIn++; + final kept = v >= 300 && v <= 2000; + if (kept) { + // Adjacent = the previous INPUT beat was kept too and no time hole lies + // between them: no difference is ever taken across a skipped beat or a + // dropout (the batch's `successive`). + final adjacent = _prevKept && tm - _prevT <= v + 0.5; + if (adjacent) { + final d = v - _prevV; + _dN++; + final dl = d - _dMean; + _dMean += dl / _dN; + _dM2 += dl * (d - _dMean); + if (d.abs() > pnnThresholdMs) _over++; + } + _lN++; + final dl = v - _lMean; + _lMean += dl / _lN; + _lM2 += dl * (v - _lMean); + _nKept++; + if (_windowsOk) { + final start = _winStart ??= tm; + if (tm - start >= windowMinutes * 60000) { + _closeWindow(); + _winStart = tm; + } + _bk.add(v); + _bkAdj.add(adjacent); + } + } + _prevKept = kept; + // Only read when the next beat is adjacent to a KEPT one; a NaN stored here + // would make the checkpoint unencodable. + if (kept) { + _prevV = v; + _prevT = tm; + } + } + + /// Appends settled NN ([nn], ms) and their beat times ([nnTimesMs], same + /// length). Beats outside [300, 2000] ms are skipped exactly as + /// `irregularBeatScreen` skips them, including its no-diff-across-a-gap rule. + void fold(List nn, List nnTimesMs) { + if (nn.length != nnTimesMs.length) { + throw ArgumentError('nn has ${nn.length} entries, nnTimesMs ' + '${nnTimesMs.length}'); + } + for (var i = 0; i < nn.length; i++) { + _add(nn[i], nnTimesMs[i]); + } + } + + /// The screen over everything folded plus the provisional ([tailNn], + /// [tailNnTimesMs]). Does not change the state. Absent (same note) exactly + /// where `irregularBeatScreen` is absent. + Metric evaluate( + List tailNn, + List tailNnTimesMs, { + double artifactFraction = 0.0, + int minBeats = irregularScreenMinBeats, + double maxArtifact = 0.30, + }) => + evaluateDetailed(tailNn, tailNnTimesMs, + artifactFraction: artifactFraction, + minBeats: minBeats, + maxArtifact: maxArtifact) + .metric; + + /// [evaluate] plus the evidence behind the verdict, equal to + /// `irregularBeatScreenDetailed(allNn, nnTimesMs: allTimes, ...)` over + /// settled ++ tail. [cleaning] is what the corrector did upstream (from its + /// snapshot), when known; otherwise those counts are null. Windows are + /// counted, the open one included, whether or not the screen abstained; they + /// are null when this state's window config fails closed. + IrregularScreenResult evaluateDetailed( + List tailNn, + List tailNnTimesMs, { + double artifactFraction = 0.0, + int minBeats = irregularScreenMinBeats, + double maxArtifact = 0.30, + RrCleaningCounts? cleaning, + }) { + const inputs = ['rr_cleaned']; + final c = _copy()..fold(tailNn, tailNnTimesMs); + final thresholds = IrregularThresholds( + minBeats: minBeats, + maxArtifact: maxArtifact, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction, + ); + // The open window closes at the end of the data, on the copy. + IrregularWindowCounts? windows; + if (c._windowsOk) { + final openBeats = c._bk.length; + final before = c._valid, beforeFlagged = c._flagged; + c._closeWindow(); + final open = openBeats == 0 + ? IrregularOpenWindow.none + : c._valid == before + ? IrregularOpenWindow.thin + : c._flagged > beforeFlagged + ? IrregularOpenWindow.flagged + : IrregularOpenWindow.unflagged; + windows = IrregularWindowCounts( + total: c._total, + valid: c._valid, + flagged: c._flagged, + openBeats: openBeats, + open: open, + ); + } + IrregularDiagnostics diag(IrregularAbstain? why) => IrregularDiagnostics( + abstain: why, + rrRaw: cleaning?.raw, + corrected: cleaning?.corrected, + dropped: cleaning?.dropped, + nnIn: c._nIn, + nnKept: c._nKept, + artifactFraction: diagnosticArtifactFraction(artifactFraction, cleaning), + windows: windows, + thresholds: thresholds, + ); + IrregularScreenResult abstain(IrregularAbstain why, String note) => + IrregularScreenResult( + Metric.absent( + tier: Tier.estimate, inputs_used: inputs, note: note), + diag(why), + ); + + // Gate on contiguous pairs, as the batch does. + if (c._dN < minBeats - 1) { + return abstain(IrregularAbstain.tooFewBeats, + 'too few clean beats for an irregular-rhythm screen'); + } + if (artifactFraction > maxArtifact) { + return abstain( + IrregularAbstain.artifact, + 'artifact fraction ${(artifactFraction * 100).round()}% > ' + '${(maxArtifact * 100).round()}% — screen suppressed on noisy RR'); + } + final sdsd = c._dN < 2 ? null : math.sqrt(c._dM2 / (c._dN - 1)); + final sdnn = c._lN < 2 ? null : math.sqrt(c._lM2 / (c._lN - 1)); + if (sdsd == null || sdnn == null) { + return abstain(IrregularAbstain.noSuccessivePairs, + 'no successive clean beats to build a Poincare plot from'); + } + final sd1 = sdsd / math.sqrt2; + final v = 2 * sdnn * sdnn - sd1 * sd1; + final sd2 = v > 0 ? math.sqrt(v) : 0.0; + if (sd2 <= 0) { + return abstain( + IrregularAbstain.noLongTermVariability, + 'no long-term variability (SD2 = 0) — the SD1/SD2 ratio is ' + 'undefined, not "perfectly regular"'); + } + final ratio = sd1 / sd2; + final pnnPct = c._dN == 0 ? 0.0 : 100.0 * c._over / c._dN; + final aggregateHigh = ratio >= sd1sd2Flag && pnnPct >= pnnFlagPct; + final flag = aggregateHigh && + windows != null && + c._valid != 0 && + c._flagged / c._valid >= sustainedFraction; + final conf = (c._nKept / 5000.0 * (1 - artifactFraction)).clamp(0.2, 0.9); + return IrregularScreenResult( + Metric( + value: IrregularRhythm( + sd1: sd1, + sd2: sd2, + sd1sd2: ratio, + pnnPct: pnnPct, + nBeats: c._nKept, + flag: flag, + ), + confidence: conf, + tier: Tier.estimate, + inputs_used: inputs, + note: 'irregular-rhythm SCREEN (not a diagnosis): Poincaré SD1/SD2 + pNN' + '${pnnThresholdMs.round()}. PRV not ECG — wrist pulse misses P-waves. ' + 'Discuss with a clinician only if you have symptoms.', + ), + diag(null), + ); + } +} diff --git a/lib/src/onehz/clinical/irregular_window.dart b/lib/src/onehz/clinical/irregular_window.dart new file mode 100644 index 00000000..407fe004 --- /dev/null +++ b/lib/src/onehz/clinical/irregular_window.dart @@ -0,0 +1,65 @@ +// Internal rule for SD1/SD2, pNNx thresholds and sustained-window voting. +// Task Force 1996 supports successive-difference HRV components, not this +// classifier or its SD1/SD2 thresholds; see hrv_time.dart. +// Internal irregular-rhythm screen, ESTIMATE with low confidence. Batch and +// stream share irregularWindowVerdict(); centered moments reorder floating-point +// sums, while exact counters preserve window evidence. Shared adjacency excludes +// timestamp-gap pairs. Diagnostics account for evidence without adding a +// clinical estimator. +// CLINICAL — the per-window pieces of the irregular-rhythm screen, shared by +// `irregularBeatScreen` (whole series) and `IrregularScreenState` (streaming) +// so the two cannot disagree about what a flagged window is. +// +// Not exported from `onehz.dart`. + +import 'dart:math' as math; +import '../util.dart'; + +/// Fail-CLOSED check on the sustained-window config: a misconfigured caller +/// must never manufacture a medical false positive. +bool irregularWindowConfigOk( + {required double windowMinutes, + required int minWindowBeats, + required double sustainedFraction}) { + return windowMinutes.isFinite && + windowMinutes > 0 && + minWindowBeats >= 2 && + sustainedFraction.isFinite && + sustainedFraction >= 0 && + sustainedFraction <= 1; +} + +/// One closed window of clean beats: null when it is too thin to count as a +/// window, otherwise whether SD1/SD2 and pNNx both clear their flag lines. +/// [adjacent] is aligned to [bucket]: whether each beat directly followed the +/// previous one in the ORIGINAL series, with no time hole between them (never +/// diff across a dropped beat or a dropout). +bool? irregularWindowVerdict( + List bucket, + List adjacent, { + required double sd1sd2Flag, + required double pnnThresholdMs, + required double pnnFlagPct, + required int minWindowBeats, +}) { + // Mirror the aggregate's `keep[i] && keep[i-1]` guard: never diff across a + // beat that was dropped as an artifact in the original series, even though + // it's now a consecutive pair in this compacted bucket. + final diffs = [ + for (var i = 1; i < bucket.length; i++) + if (adjacent[i]) bucket[i] - bucket[i - 1] + ]; + // Valid on contiguous pairs, not beats (same reason as the aggregate). + if (diffs.length < minWindowBeats - 1) return null; + final sdsd = stddev(diffs); + final sdnn = stddev(bucket); + if (sdsd == null || sdnn == null) return false; + final sd1 = sdsd / math.sqrt2; + final v = 2 * sdnn * sdnn - sd1 * sd1; + final sd2 = v > 0 ? math.sqrt(v) : 0.0; + if (sd2 <= 0) return false; + final ratio = sd1 / sd2; + final over = diffs.where((d) => d.abs() > pnnThresholdMs).length; + final pnn = 100.0 * over / diffs.length; + return ratio >= sd1sd2Flag && pnn >= pnnFlagPct; +} diff --git a/lib/src/onehz/foundations/rr_correction_stream.dart b/lib/src/onehz/foundations/rr_correction_stream.dart new file mode 100644 index 00000000..1e034eb5 --- /dev/null +++ b/lib/src/onehz/foundations/rr_correction_stream.dart @@ -0,0 +1,822 @@ +// Lipponen & Tarvainen 2019 classification with Peltola 2012 correction policy. +// The unpublished artifact-run cap is an internal rule, ESTIMATE with low +// confidence: +// over-cap runs become gaps and each side is re-corrected. Without a cap split +// the result equals uncapped correction. Shared kernel arithmetic and raw seam +// context preserve capped batch NN/classes; settled stream plus provisional tail +// equals capped batch. RrSandwich preserves source-beat absolute timestamps; +// uncapped arbitrarily long artifact runs have no exactness guarantee. +// FOUNDATION — streaming RR artifact correction. +// +// [RrCorrector] is `correctRr` (rr_correction.dart) turned into a fold: feed +// beats in chunks, get back the output that became FINAL with each chunk, and +// ask for the provisional tail on demand. At every prefix of the input, +// +// (all settled output so far) ++ (snapshot().tail*) == correctRr(prefix) +// +// bit for bit: nn, nnTimesMs, per-beat classes, counts, cleanFraction. Nothing +// is imputed here either: a beat whose neighbours are not there yet is not +// guessed at, it simply is not settled. +// +// Why a beat can be final long before the series ends: its class depends only +// on a centred window of `windowBeats ~/ 2` beats each side (twice that for +// the second threshold), one carried class, and — for the spline — the two +// nearest normal beats either side. Beats closer to the end of the data than +// that are PROVISIONAL, because `correctRr` clips its windows at the end. +// +// Derivation, with H = windowBeats ~/ 2 (45 at the default 91): +// +// beat clock t[i] prefix recurrence (current time, previous timestamp) +// dRR[i] rr[i] - rr[i-1] local +// th1, med QD of dRR / median of rr, over [i-H, i+H] needs rr up to i+H +// mRR from rr[i] and med[i] needs rr up to i+H +// th2 QD of mRR over [i-H, i+H] needs rr up to i+2H +// class first pass from the above, then the compensatory-pair rule, +// which reads only the previous FINAL class: one carried class. +// output an isolated artifact is spline-corrected from the two nearest +// normal beats each side (left pair carried, right pair needs +// classes up to the second normal after it); a run is dropped. +// +// So output settles 2H beats + (artifact run length) + 2 normals behind the +// newest beat. Beats nearer the end than that are provisional because +// `correctRr` clips its windows at the end of the data; [RrCorrector.snapshot] +// re-evaluates just that tail with the same clipping. +// +// With a cap, a future split also changes the windows to its left. The capped +// path keeps 4H + 2K + 4 beats provisional (248 at the defaults), then up to K +// more to avoid committing part of a short artifact run. Uncapped mode retains +// the original stage-by-stage latency described above. +// +// The arithmetic (window, threshold, class, spline) is shared with `correctRr` +// through rr_correction_kernel.dart. + +import 'dart:math' as math; + +import 'rr_correction.dart'; +import 'rr_correction_kernel.dart'; + +/// What [RrCorrector.fold] hands back: the output that became FINAL with that +/// fold, in order, ready to append to what the caller already holds. +class RrSettled { + /// Newly final cleaned NN intervals (ms), as `correctRr(...).nn` would hold. + final List nn; + + /// Beat times (ms) for [nn], as `correctRr(...).nnTimesMs` would hold. + final List nnTimes; + + /// Classes of the beats whose class became final with this fold, i.e. input + /// beats `[classifiedBeats before the fold, classifiedBeats after)`. A class + /// is final earlier than its output: a lone artifact waits for two normal + /// beats on its right before it can be spline-corrected. + final List classes; + + const RrSettled({ + required this.nn, + required this.nnTimes, + required this.classes, + }); +} + +/// The part of `correctRr(all beats so far)` that can still change, plus the +/// running totals. Recomputed on every [RrCorrector.snapshot]; never stored. +class RrSnapshot { + /// Beats folded so far. + final int n; + + /// Provisional NN / times that follow everything [RrCorrector.fold] has + /// settled so far. + final List tailNn; + final List tailNnTimes; + + /// Provisional classes of the beats `[classifiedBeats, n)`. + final List tailClasses; + + /// Totals as `correctRr(all beats so far)` reports them. + final int normalCount, droppedCount, correctedCount; + + /// Beats whose output is final / whose class is final. + final int settledBeats, classifiedBeats; + + const RrSnapshot({ + required this.n, + required this.tailNn, + required this.tailNnTimes, + required this.tailClasses, + required this.normalCount, + required this.droppedCount, + required this.correctedCount, + required this.settledBeats, + required this.classifiedBeats, + }); + + /// `correctRr`'s `cleanFraction`: 0 on an empty series, never NaN. + double get cleanFraction => n == 0 ? 0 : normalCount / n; +} + +/// Streaming Lipponen–Tarvainen correction. Same parameters as [correctRr]. +/// +/// Append-only: it cannot take beats back. Anything that changes data behind +/// the settled edge means starting a new corrector. +class RrCorrector { + final double alpha; + final int windowBeats; + final double minThresholdMs; + final double reanchorGapMs; + + /// Longest artifact run carried; a longer one is a gap. null = uncapped. + /// Same meaning as [correctRr]'s `maxArtifactRun`. + final int? maxArtifactRun; + final int _h; + final _CappedState? _capState; + + // ---- persisted state ---- + int _n = 0; // beats folded + bool? _wall; // timestamps given for this stream (null until the first beat) + double _origin = 0, _cur = 0, _prevTs = 0, _prevRr = 0; + int _off = 0; // global index of _rr[0] + final List _rr = [], _t = [], _d = []; + int _c1 = 0; // beats whose med / mRR / th1 are final + int _c2 = 0; // beats whose class is final + int _ce = 0; // beats whose output is final (handed out) + final List _fcls = []; // final classes for [_ce, _c2) + int _lastFinal = 0; // class index of beat _c2 - 1 + int _normalFinal = 0; + int _dropped = 0, _corrected = 0; + final List _lastNormals = []; // <= 2, oldest first + + // ---- derived, rebuilt on restore ---- + int _so = 0; // global index of _med[0] + final List _med = [], _mrr = [], _th1 = []; + final _wd = SortedMultiset(), _wr = SortedMultiset(), _wm = SortedMultiset(); + int _wPos = -1, _w2Pos = -1; + + RrCorrector({ + this.alpha = 5.2, + this.windowBeats = 91, + this.minThresholdMs = 100, + this.reanchorGapMs = 1000, + this.maxArtifactRun = kMaxArtifactRun, + }) : _h = windowBeats ~/ 2, + _capState = maxArtifactRun == null ? null : _CappedState() { + if (windowBeats < 1) throw ArgumentError.value(windowBeats, 'windowBeats'); + if (maxArtifactRun != null && maxArtifactRun! < 1) { + throw ArgumentError.value(maxArtifactRun, 'maxArtifactRun'); + } + } + + /// Restores a corrector from [toJson]. Throws [FormatException] on a + /// checkpoint of another type or version, or one that is malformed. + factory RrCorrector.fromJson(Map json) { + try { + return _restore(json); + } on FormatException { + rethrow; + } catch (e) { + throw FormatException('malformed RrCorrector checkpoint: $e'); + } + } + + static RrCorrector _restore(Map j) { + if (j['type'] != 'RrCorrector') { + throw FormatException('not an RrCorrector checkpoint: ${j['type']}'); + } + if (j['version'] != 2) { + throw FormatException('unsupported RrCorrector version ${j['version']}'); + } + List doubles(String k) => + [for (final x in j[k] as List) (x as num).toDouble()]; + final c = RrCorrector( + maxArtifactRun: j['maxArtifactRun'] as int?, + alpha: (j['alpha'] as num).toDouble(), + windowBeats: j['windowBeats'] as int, + minThresholdMs: (j['minThresholdMs'] as num).toDouble(), + reanchorGapMs: (j['reanchorGapMs'] as num).toDouble(), + ); + if (!j.containsKey('maxArtifactRun')) { + throw const FormatException('missing artifact cap'); + } + if (c._capState != null) { + c._capState.restore(j['capped'] as Map, c._h); + return c; + } + c._n = j['n'] as int; + c._wall = j['wall'] as bool?; + c._origin = (j['origin'] as num).toDouble(); + c._cur = (j['cur'] as num).toDouble(); + c._prevTs = (j['prevTs'] as num).toDouble(); + c._prevRr = (j['prevRr'] as num).toDouble(); + c._off = j['off'] as int; + c._rr.addAll(doubles('rr')); + c._t.addAll(doubles('t')); + c._d.addAll(doubles('d')); + c._c1 = j['c1'] as int; + c._c2 = j['c2'] as int; + c._ce = j['ce'] as int; + c._fcls.addAll((j['fcls'] as List).cast()); + c._lastFinal = j['lastFinal'] as int; + c._normalFinal = j['normalFinal'] as int; + c._dropped = j['dropped'] as int; + c._corrected = j['corrected'] as int; + c._lastNormals.addAll(doubles('lastNormals')); + if (c._n > 0 && c._wall == null) { + throw const FormatException('invalid wall'); + } + if (c._dropped < 0 || c._dropped > c._ce) { + throw const FormatException('invalid dropped'); + } + if (c._corrected < 0 || c._corrected > c._ce - c._dropped) { + throw const FormatException('invalid corrected'); + } + if (c._normalFinal < 0 || c._normalFinal > c._c2) { + throw const FormatException('invalid normalFinal'); + } + final buffered = c._n - c._off; + final ok = c._off >= 0 && + buffered >= 0 && + c._rr.length == buffered && + c._t.length == buffered && + c._d.length == buffered && + c._ce >= c._off && + c._ce <= c._c2 && + c._c2 <= c._c1 && + c._c1 <= c._n && + c._fcls.length == c._c2 - c._ce && + c._fcls.every((k) => k >= 0 && k < BeatClass.values.length) && + c._lastFinal >= 0 && + c._lastFinal < BeatClass.values.length && + c._lastNormals.length <= 2 && + c._off <= math.max(0, c._c2 - 2 * c._h - 1); + if (!ok) throw const FormatException('inconsistent RrCorrector checkpoint'); + c._rebuildStageArrays(); + return c; + } + + /// Checkpoint: `{'version': 2, 'type': 'RrCorrector', ...}`, plain JSON + /// (survives `jsonEncode`/`jsonDecode` bit-exactly), and bounded — it holds + /// the unsettled window, not the series. + Map toJson() { + if (_capState != null) { + return { + 'version': 2, 'type': 'RrCorrector', + 'maxArtifactRun': maxArtifactRun, 'alpha': alpha, + 'windowBeats': windowBeats, 'minThresholdMs': minThresholdMs, + 'reanchorGapMs': reanchorGapMs, 'capped': _capState.toJson(), + }; + } + return { + 'version': 2, + 'maxArtifactRun': maxArtifactRun, + 'type': 'RrCorrector', + 'alpha': alpha, + 'windowBeats': windowBeats, + 'minThresholdMs': minThresholdMs, + 'reanchorGapMs': reanchorGapMs, + 'n': _n, + 'wall': _wall, + 'origin': _origin, + 'cur': _cur, + 'prevTs': _prevTs, + 'prevRr': _prevRr, + 'off': _off, + 'rr': _rr, + 't': _t, + 'd': _d, + 'c1': _c1, + 'c2': _c2, + 'ce': _ce, + 'fcls': _fcls, + 'lastFinal': _lastFinal, + 'normalFinal': _normalFinal, + 'dropped': _dropped, + 'corrected': _corrected, + 'lastNormals': _lastNormals, + }; + } + + /// Beats whose output is final. + int get settledBeats => _capState?.edge ?? _ce; + + /// Appends beats. [tsMs] (same length as [rrMs]) is either given on EVERY + /// fold or on none; mixing throws [StateError]. A non-finite RR throws + /// [ArgumentError] and leaves the corrector unchanged. An empty chunk is a + /// no-op. + RrSettled fold(List rrMs, {List? tsMs}) { + if (_capState != null) return _capState.fold(this, rrMs, tsMs); + if (rrMs.isEmpty) { + return const RrSettled(nn: [], nnTimes: [], classes: []); + } + // Everything that can throw happens before the first mutation. + final wall = tsMs != null; + if (wall && tsMs.length != rrMs.length) { + throw ArgumentError('tsMs has ${tsMs.length} entries for ' + '${rrMs.length} beats'); + } + if (_wall != null && _wall != wall) { + throw StateError('timestamps must be given on every fold or on none'); + } + for (var k = 0; k < rrMs.length; k++) { + if (!rrMs[k].isFinite) throw ArgumentError('non-finite RR at index $k'); + if (wall && !tsMs[k].isFinite) { + throw ArgumentError('non-finite timestamp at index $k'); + } + } + _wall = wall; + for (var k = 0; k < rrMs.length; k++) { + final r = rrMs[k]; + final ts = wall ? tsMs[k] : 0.0; + if (_n == 0) { + _origin = wall ? ts - r : 0.0; + _cur = 0.0; + } + // Same clock as `correctRr`'s beat times: cumsum inside a contiguous run, + // re-anchored to the real timestamp at a dropout, never backwards. + final dropout = wall && _n > 0 && (ts - _prevTs) - r > reanchorGapMs; + final anchored = wall ? ts - _origin : 0.0; + _cur = (dropout && anchored > _cur) ? anchored : _cur + r; + if (wall) _prevTs = ts; + _rr.add(r); + _t.add(_cur); + _d.add(_n == 0 ? 0.0 : r - _prevRr); + _prevRr = r; + _n++; + } + final nn = [], times = []; + final classes = []; + _advance(classes); + _emitCommitted(nn, times); + _compact(); + return RrSettled(nn: nn, nnTimes: times, classes: classes); + } + + double _rrAt(int g) => _rr[g - _off]; + double _dAt(int g) => _d[g - _off]; + double _tAt(int g) => _t[g - _off]; + + void _advance(List classes) { + // Under 3 beats `correctRr` takes its short branch; nothing settles yet. + if (_n < 3) return; + while (_c1 + _h <= _n - 1) { + _stage1(_c1); + _c1++; + } + while (_c2 + 2 * _h <= _n - 1) { + classes.add(_stage2(_c2)); + _c2++; + } + } + + // th1 / med / mRR for beat g, whose window [max(0, g-h), g+h] is fully inside + // the data. + void _stage1(int g) { + final h = _h; + if (_wPos == g - 1 && _wPos >= 0) { + _wd.add(_dAt(g + h)); + _wr.add(_rrAt(g + h)); + if (g - 1 - h >= 0) { + _wd.remove(_dAt(g - 1 - h)); + _wr.remove(_rrAt(g - 1 - h)); + } + } else { + final lo = math.max(0, g - h); + _wd.resetFrom([for (var k = lo; k <= g + h; k++) _dAt(k)]); + _wr.resetFrom([for (var k = lo; k <= g + h; k++) _rrAt(k)]); + } + _wPos = g; + final m = medianExcluding(_wr.a, _rrAt(g)) ?? _rrAt(g); + _th1.add(thresholdOfSorted(_wd.a, alpha, minThresholdMs)); + _med.add(m); + _mrr.add(medianDeviation(_rrAt(g), m)); + } + + BeatClass _stage2(int g) { + final h = _h; + if (_w2Pos == g - 1 && _w2Pos >= 0) { + _wm.add(_mrr[g + h - _so]); + if (g - 1 - h >= 0) _wm.remove(_mrr[g - 1 - h - _so]); + } else { + final lo = math.max(0, g - h); + _wm.resetFrom([for (var k = lo; k <= g + h; k++) _mrr[k - _so]]); + } + _w2Pos = g; + final th2 = thresholdOfSorted(_wm.a, alpha, minThresholdMs); + final cls = _finalClass(g, _th1[g - _so], _med[g - _so], _mrr[g - _so], + th2, _lastFinal); + _fcls.add(cls.index); + _lastFinal = cls.index; + if (cls == BeatClass.normal) _normalFinal++; + return cls; + } + + /// Class of beat g given the class of g-1: first pass, then the + /// compensatory-pair rule. + BeatClass _finalClass(int g, double th1, double med, double mrr, double th2, + int prevClass) { + final rr = _rrAt(g), d = _dAt(g); + final cls = classifyBeat(rr, d, th1, med, mrr, th2); + if (g < 1 || cls != BeatClass.ectopic) return cls; + return isRecoveryBeat( + prevIsArtifact: prevClass != BeatClass.normal.index, + rr: rr, + dRR: d, + dRRPrev: _dAt(g - 1), + med: med) + ? BeatClass.normal + : cls; + } + + // Output over classes [ce, limit). Shared by the committed pass (limit = c2, + // isEnd false: an artifact whose fate is not decidable yet waits) and the + // provisional pass (limit = n, isEnd true: the end of the data resolves it + // exactly the way `correctRr` does). Returns the new ce. + int _emit({ + required int ce, + required int limit, + required bool isEnd, + required int Function(int g) clsAt, + required List lastNormals, + required void Function(double nn, double t) put, + required void Function(int runLen) drop, + required void Function() corrected, + }) { + const normal = 0; + while (ce < limit) { + if (clsAt(ce) == normal) { + put(_rrAt(ce), _tAt(ce)); + lastNormals.add(_rrAt(ce)); + if (lastNormals.length > 2) lastNormals.removeAt(0); + ce++; + continue; + } + var j = ce; + while (j < limit && clsAt(j) != normal) { + j++; + } + if (j == limit && !isEnd) break; + final runLen = j - ce; + if (runLen == 1) { + final right = []; + var k = j; + while (k < limit && right.length < 2) { + if (clsAt(k) == normal) right.add(_rrAt(k)); + k++; + } + if (right.length < 2 && k == limit && !isEnd) break; + final corr = splineMid(lastNormals, right); + if (corr != null) { + put(corr, _tAt(ce)); + corrected(); + } else { + drop(1); // no anchors: an honest drop; the clock already elapsed + } + } else { + drop(runLen); + } + ce = j; + } + return ce; + } + + void _emitCommitted(List nn, List times) { + final fo = _ce; + _ce = _emit( + ce: _ce, + limit: _c2, + isEnd: false, + clsAt: (g) => _fcls[g - fo], + lastNormals: _lastNormals, + put: (v, t) { + nn.add(v); + times.add(t); + }, + drop: (r) => _dropped += r, + corrected: () => _corrected++, + ); + final used = _ce - fo; + if (used > 0) _fcls.removeRange(0, used); + } + + /// Provisional tail and totals; does not change the corrector. + RrSnapshot snapshot() { + if (_capState != null) return _capState.snapshot(this); + final n = _n; + if (n == 0) { + return const RrSnapshot( + n: 0, + tailNn: [], + tailNnTimes: [], + tailClasses: [], + normalCount: 0, + droppedCount: 0, + correctedCount: 0, + settledBeats: 0, + classifiedBeats: 0); + } + if (n < 3) { + // `correctRr`'s short branch: plausible beats pass, the rest are + // long/short. Nothing is settled below 3 beats. + final nn = [], times = []; + final cl = []; + for (var g = 0; g < n; g++) { + final ok = _rrAt(g) >= 300 && _rrAt(g) <= 2000; + cl.add(ok ? BeatClass.normal : BeatClass.longShort); + if (ok) { + nn.add(_rrAt(g)); + times.add(_tAt(g)); + } + } + return RrSnapshot( + n: n, + tailNn: nn, + tailNnTimes: times, + tailClasses: cl, + normalCount: nn.length, + droppedCount: n - nn.length, + correctedCount: 0, + settledBeats: 0, + classifiedBeats: 0); + } + final h = _h; + final end = n - 1; + // Provisional stage 1 for beats [c1, n): windows clipped at the end. + final tMed = [], tMrr = [], tTh1 = []; + double mrrAt(int k) => k < _c1 ? _mrr[k - _so] : tMrr[k - _c1]; + for (var k = _c1; k < n; k++) { + final lo = math.max(0, k - h), hi = math.min(end, k + h); + final dd = [for (var q = lo; q <= hi; q++) _dAt(q)]..sort(); + final rs = [for (var q = lo; q <= hi; q++) _rrAt(q)]..sort(); + tTh1.add(thresholdOfSorted(dd, alpha, minThresholdMs)); + final m = medianExcluding(rs, _rrAt(k)) ?? _rrAt(k); + tMed.add(m); + tMrr.add(medianDeviation(_rrAt(k), m)); + } + // Provisional classes for [c2, n). + final tc = []; + var prev = _lastFinal; + var normalTent = 0; + for (var g = _c2; g < n; g++) { + final lo = math.max(0, g - h), hi = math.min(end, g + h); + final ms = [for (var q = lo; q <= hi; q++) mrrAt(q)]..sort(); + final th2 = thresholdOfSorted(ms, alpha, minThresholdMs); + final th1 = g < _c1 ? _th1[g - _so] : tTh1[g - _c1]; + final med = g < _c1 ? _med[g - _so] : tMed[g - _c1]; + final cls = _finalClass(g, th1, med, mrrAt(g), th2, prev); + tc.add(cls.index); + prev = cls.index; + if (cls == BeatClass.normal) normalTent++; + } + // Provisional output over [ce, n). + final fo = _ce; + final ln = List.of(_lastNormals); + final nn = [], times = []; + var dropped = _dropped, corrected = _corrected; + _emit( + ce: _ce, + limit: n, + isEnd: true, + clsAt: (g) => g < _c2 ? _fcls[g - fo] : tc[g - _c2], + lastNormals: ln, + put: (v, t) { + nn.add(v); + times.add(t); + }, + drop: (r) => dropped += r, + corrected: () => corrected++, + ); + return RrSnapshot( + n: n, + tailNn: nn, + tailNnTimes: times, + tailClasses: [for (final c in tc) BeatClass.values[c]], + normalCount: _normalFinal + normalTent, + droppedCount: dropped, + correctedCount: corrected, + settledBeats: _ce, + classifiedBeats: _c2, + ); + } + + /// Drops what no later beat can read: raw beats behind both the settled edge + /// and the oldest window a pending stage-2 beat still needs, and stage + /// arrays behind the oldest stage-2 window. + void _compact() { + final h = _h; + final newOff = math.max(0, math.min(_ce, _c2 - 2 * h - 1)); + if (newOff > _off) { + final drop = newOff - _off; + _rr.removeRange(0, drop); + _t.removeRange(0, drop); + _d.removeRange(0, drop); + _off = newOff; + } + final newSo = math.max(0, _c2 - h - 1); + if (newSo > _so && _med.isNotEmpty) { + final drop = math.min(newSo - _so, _med.length); + _med.removeRange(0, drop); + _mrr.removeRange(0, drop); + _th1.removeRange(0, drop); + _so += drop; + } + } + + /// The stage arrays are pure functions of the buffered raw beats; rebuild + /// them rather than persist them. + void _rebuildStageArrays() { + final h = _h; + _so = math.max(0, _c2 - h - 1); + _med.clear(); + _mrr.clear(); + _th1.clear(); + for (var g = _so; g < _c1; g++) { + final lo = math.max(0, g - h), hi = g + h; + final dd = [for (var q = lo; q <= hi; q++) _dAt(q)]..sort(); + final rs = [for (var q = lo; q <= hi; q++) _rrAt(q)]..sort(); + _th1.add(thresholdOfSorted(dd, alpha, minThresholdMs)); + final m = medianExcluding(rs, _rrAt(g)) ?? _rrAt(g); + _med.add(m); + _mrr.add(medianDeviation(_rrAt(g), m)); + } + _wPos = -1; + _w2Pos = -1; + } +} + +// The cap can cut a window that an uncapped detector has already read. Keep +// that decision provisional until both the run and its left influence have +// settled. No stage-1/2 arrays from across a cut are ever reused. +class _CappedState { + int n = 0, off = 0, edge = 0, segmentStart = 0; + int normal = 0, dropped = 0, corrected = 0; + bool? wall; + bool gap = false, hardGap = false; + double origin = 0, clock = 0; + BeatClass previous = BeatClass.normal; + final rr = [], ts = [], normals = []; + + RrCapWindow evaluate(RrCorrector c) => RrCapWindow.evaluate(rr, + ts: wall == true ? ts : null, + cap: c.maxArtifactRun!, + alpha: c.alpha, + windowBeats: c.windowBeats, + minThresholdMs: c.minThresholdMs, + reanchorGapMs: c.reanchorGapMs, + from: edge - off, + previous: previous, + leftNormals: normals, + leadingGap: gap, + leadingHardGap: hardGap, + segmentStart: math.max(0, segmentStart - off), + clock: clock, + origin: origin); + + RrSettled fold(RrCorrector c, List values, List? times) { + if (values.isEmpty) + return const RrSettled(nn: [], nnTimes: [], classes: []); + if (times != null && times.length != values.length) { + throw ArgumentError('RR and timestamp lengths differ'); + } + if (wall != null && wall != (times != null)) { + throw StateError('timestamps must be given on every fold or on none'); + } + for (var i = 0; i < values.length; i++) { + if (!values[i].isFinite || (times != null && !times[i].isFinite)) { + throw ArgumentError('non-finite beat at index $i'); + } + } + wall = times != null; + if (n == 0) origin = times == null ? 0 : times.first - values.first; + rr.addAll(values); + if (times != null) ts.addAll(times); + n += values.length; + final w = evaluate(c); + final nn = [], nt = [], classes = []; + // Two centred threshold windows, the cap decision and its backwards + // influence. Extra normal anchors settle isolated spline corrections. + var limit = math.max(edge, n - (4 * c._h + 2 * c.maxArtifactRun! + 4)); + // Do not split a short artifact run between folds: its final beat must + // never be mistaken for an isolated artifact at the next pending edge. + while (limit > edge && + w.classes[limit - off - 1] != BeatClass.normal && + !w.gaps[limit - off - 1]) { + limit--; + } + for (var i = edge - off; i < limit - off; i++) { + final cl = w.classes[i]; + classes.add(cl); + previous = cl; + if (cl == BeatClass.normal) { + normal++; + normals.add(rr[i]); + if (normals.length > 2) normals.removeAt(0); + } + if (w.gaps[i]) normals.clear(); + final v = w.values[i]; + if (v == null) { + dropped++; + } else { + nn.add(v); + nt.add(w.times[i]); + if (cl != BeatClass.normal) corrected++; + } + clock = w.times[i]; + if (gap && !w.gaps[i]) segmentStart = off + i; + gap = w.gaps[i]; + hardGap = w.hardGaps[i]; + } + edge = limit; + final nextOff = math.max(0, edge - 2 * c._h - 2); + final trim = nextOff - off; + if (trim > 0) { + rr.removeRange(0, trim); + if (wall == true) ts.removeRange(0, trim); + off = nextOff; + } + return RrSettled(nn: nn, nnTimes: nt, classes: classes); + } + + RrSnapshot snapshot(RrCorrector c) { + final w = evaluate(c); + final nn = [], nt = [], classes = []; + var nc = normal, dc = dropped, cc = corrected; + for (var i = edge - off; i < rr.length; i++) { + final cl = w.classes[i]; + classes.add(cl); + if (cl == BeatClass.normal) nc++; + final v = w.values[i]; + if (v == null) { + dc++; + } else { + nn.add(v); + nt.add(w.times[i]); + if (cl != BeatClass.normal) cc++; + } + } + return RrSnapshot( + n: n, + tailNn: nn, + tailNnTimes: nt, + tailClasses: classes, + normalCount: nc, + droppedCount: dc, + correctedCount: cc, + settledBeats: edge, + classifiedBeats: edge); + } + + Map toJson() => { + 'n': n, + 'off': off, + 'edge': edge, + 'segmentStart': segmentStart, + 'hardGap': hardGap, + 'normal': normal, + 'dropped': dropped, + 'corrected': corrected, + 'wall': wall, + 'gap': gap, + 'origin': origin, + 'clock': clock, + 'previous': previous.index, + 'rr': rr, + 'ts': ts, + 'normals': normals, + }; + + void restore(Map j, int half) { + n = j['n'] as int; + off = j['off'] as int; + edge = j['edge'] as int; + segmentStart = j['segmentStart'] as int; + hardGap = j['hardGap'] as bool; + normal = j['normal'] as int; + dropped = j['dropped'] as int; + corrected = j['corrected'] as int; + wall = j['wall'] as bool?; + gap = j['gap'] as bool; + origin = (j['origin'] as num).toDouble(); + clock = (j['clock'] as num).toDouble(); + previous = BeatClass.values[j['previous'] as int]; + List read(String key) => + [for (final v in j[key] as List) (v as num).toDouble()]; + rr.addAll(read('rr')); + ts.addAll(read('ts')); + normals.addAll(read('normals')); + if (segmentStart < 0 || + segmentStart > edge || + (hardGap && !gap) || + off != math.max(0, edge - 2 * half - 2) || + edge < off || + n < edge || + rr.length != n - off || + (gap && (previous == BeatClass.normal || normals.isNotEmpty)) || + (wall == true ? ts.length != rr.length : ts.isNotEmpty) || + (n > 0 && wall == null) || + normals.length > 2 || + normal < 0 || + dropped < 0 || + corrected < 0 || + normal + dropped + corrected != edge || + ![...rr, ...ts, ...normals, origin, clock].every((v) => v.isFinite)) { + throw const FormatException('inconsistent capped RR checkpoint'); + } + } +} diff --git a/lib/src/onehz/util.dart b/lib/src/onehz/util.dart index c847b5d2..db24f938 100644 --- a/lib/src/onehz/util.dart +++ b/lib/src/onehz/util.dart @@ -64,6 +64,85 @@ double? percentile(List values, double p) => /// Median (linear-interpolated), or null if empty. double? median(List xs) => percentile(xs, 50); +/// Linear-interpolated median of every clipped, centred [halfWin] window. +/// +/// `out[j]` is exactly +/// `median(xs.sublist(max(0, j - halfWin), min(xs.length - 1, j + halfWin) + 1))`. +/// In particular, ordering is [double.compareTo] ordering (including NaN last), +/// and an even-sized window uses [percentileSorted]'s interpolation. The input +/// is not changed. +/// +/// One sorted window is grown, then updated by binary insertion/removal; it +/// therefore avoids a fresh window allocation and sort at every position. +// Hyndman, R. J., Fan, Y. (1996). Sample quantiles in statistical packages. +// The American Statistician 50(4):361-365. +// percentileSorted implements type 7: zero-based rank = (n - 1) * p / 100. +// These clipped-window medians use that same adjacent-rank interpolation. +// Exact reformulation of clipped-window medians: insertion/removal preserves +// List.sort total order; percentileSorted retains the same ranks and +// interpolation. +List slidingMedians(List xs, int halfWin) { + if (xs.isEmpty) return []; + if (halfWin < 0) { + throw ArgumentError.value(halfWin, 'halfWin', 'must be non-negative'); + } + + final n = xs.length; + var lo = 0; + var hi = math.min(n, halfWin + 1); // exclusive + final sorted = []; + for (var i = lo; i < hi; i++) { + _insertSorted(sorted, xs[i]); + } + final out = List.filled(n, 0); + for (var j = 0; j < n; j++) { + out[j] = percentileSorted(sorted, 50)!; + if (j + 1 == n) break; + + final nextLo = math.max(0, j + 1 - halfWin); + final nextHi = math.min(n, j + 1 + halfWin + 1); // exclusive + while (lo < nextLo) { + _removeSorted(sorted, xs[lo]); + lo++; + } + while (hi < nextHi) { + _insertSorted(sorted, xs[hi]); + hi++; + } + } + return out; +} + +void _insertSorted(List sorted, double value) { + var lo = 0; + var hi = sorted.length; + while (lo < hi) { + final mid = lo + (hi - lo) ~/ 2; + if (sorted[mid].compareTo(value) < 0) { + lo = mid + 1; + } else { + hi = mid; + } + } + sorted.insert(lo, value); +} + +void _removeSorted(List sorted, double value) { + var lo = 0; + var hi = sorted.length; + while (lo < hi) { + final mid = lo + (hi - lo) ~/ 2; + if (sorted[mid].compareTo(value) < 0) { + lo = mid + 1; + } else { + hi = mid; + } + } + // [value] is known to be in the active window. A compare-equal duplicate is + // interchangeable for percentile arithmetic, including NaN and signed zero. + sorted.removeAt(lo); +} + /// Median absolute deviation, scaled to be a consistent estimator of σ for /// normal data (× 1.4826). Returns null if empty. NOTE: on quantized data the /// raw MAD can be 0 — callers must guard division (see [robustZ]). @@ -326,6 +405,12 @@ class LombScargle { /// /// This operates on NATIVE sample times — no resampling — which is exactly why /// it's the correct PSD for unevenly-sampled beat-time RR. +/// +/// PRECISION: every trig argument is built from `t - t.first`, never the raw +/// [t]. On epoch seconds (~1.8e9) the argument `2π·f·t` reaches ~1e10 rad, where +/// a double is only good to ~1e-6 rad, which cost up to ~5e-6 relative error in +/// the PSD. The spectrum is time-shift invariant, so the shift changes nothing +/// but that error (≈1e-13 after it). LombScargle? lombScargle(List t, List y, List freqsHz) { final n = t.length; if (n < 4 || y.length != n || freqsHz.isEmpty) return null; diff --git a/test/onehz/incremental/irregular_diagnostics_stream_test.dart b/test/onehz/incremental/irregular_diagnostics_stream_test.dart new file mode 100644 index 00000000..7839b2dd --- /dev/null +++ b/test/onehz/incremental/irregular_diagnostics_stream_test.dart @@ -0,0 +1,398 @@ +// Contract: the screen's DIAGNOSTICS are the same whether the beats arrive in +// one batch call or in chunks through `RrCorrector` + `IrregularScreenState`, +// across save/restore (design 04 "PRV diagnostics", item b). +// +// Oracle = `irregularBeatScreenDetailed` over `correctRr(prefix)`, with the +// corrector's own counts. At every cut, the streaming side is +// corrector.snapshot() -> state.evaluateDetailed(tail, cleaning: counts) +// and its diagnostics must equal the batch's JSON text, byte for byte: beat +// counts, corrected / dropped, artifact fraction, window counts, the final OPEN +// window, thresholds. Cuts are placed to split a 5-minute window, an isolated +// artifact (a spline correction that cannot settle until two normals follow it) +// and a multi-beat noise run (a drop). +// +// RED until implemented: `evaluateDetailed` / `irregularBeatScreenDetailed` +// are stubs that throw UnimplementedError. +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/rr_chunking.dart'; + +String _json(Object? o) => jsonEncode(o); + +/// A day of beats with known artifacts: calm sinus with AF bursts, isolated +/// early beats (+ compensatory pause) at [isolated], a 6-beat noise run at each +/// of [runs], and one 40 s dropout. Stamps are end-of-beat epoch ms. +({List rr, List ts, List isolated, List runs}) _day( + int seed, + {int beats = 9000}) { + final r = math.Random(seed); + final rr = [], ts = []; + final isolated = [], runs = []; + var clock = 1.76e12; // epoch ms + var af = false; + var left = 0; + var droppedOut = false; + void emit(double v) { + clock += v; + rr.add(v); + ts.add(clock.floorToDouble()); + } + + while (rr.length < beats) { + if (--left <= 0) { + af = r.nextDouble() < 0.4; + left = 300 + r.nextInt(900); + } + final i = rr.length; + if (!droppedOut && i >= 4000) { + droppedOut = true; + clock += 40000; // a sensor dropout + } + if (i > 100 && i % 523 == 7) { + isolated.add(i); + emit(380); // early beat + emit(1450); // compensatory pause + continue; + } + if (i > 100 && i % 811 == 400) { + runs.add(i); + for (var k = 0; k < 6; k++) { + emit(250 + r.nextInt(1800).toDouble()); + } + continue; + } + emit(af + ? (420 + r.nextInt(700)).toDouble() + : (850 + 40 * math.sin(i / 70) + 12 * (r.nextDouble() - .5)) + .roundToDouble()); + } + return (rr: rr, ts: ts, isolated: isolated, runs: runs); +} + +IrregularScreenResult _batch(List rr, List ts, int n) { + final c = correctRr(rr.sublist(0, n), rrTsMs: ts.sublist(0, n)); + return irregularBeatScreenDetailed( + c.nn, + nnTimesMs: c.nnTimesMs, + artifactFraction: (1.0 - c.cleanFraction).clamp(0.0, 1.0), + cleaning: RrCleaningCounts( + raw: n, corrected: c.correctedCount, dropped: c.droppedCount), + ); +} + +IrregularScreenResult _stream(RrCorrector c, IrregularScreenState st) { + final s = c.snapshot(); + return st.evaluateDetailed( + s.tailNn, + s.tailNnTimes, + artifactFraction: (1.0 - s.cleanFraction).clamp(0.0, 1.0), + cleaning: RrCleaningCounts( + raw: s.n, corrected: s.correctedCount, dropped: s.droppedCount), + ); +} + +void _same(IrregularScreenResult got, IrregularScreenResult want, String why) { + expect(_json(got.diagnostics.toJson()), _json(want.diagnostics.toJson()), + reason: 'diagnostics $why'); + expect(got.metric.present, want.metric.present, reason: 'present $why'); + expect(got.metric.note, want.metric.note, reason: 'note $why'); + if (want.metric.present) { + expect(got.metric.value!.flag, want.metric.value!.flag, reason: 'flag $why'); + expect(got.metric.value!.nBeats, want.metric.value!.nBeats, + reason: 'nBeats $why'); + } +} + +/// Cut points: random chunking plus neighbourhoods of every artifact and of +/// every 5-minute beat-clock edge. +List _cuts( + ({List rr, List ts, List isolated, List runs}) d, + int seed) { + final n = d.rr.length; + final cuts = {...randomCuts(math.Random(seed), n, sizes: [90, 500, 1500])}; + for (final k in d.isolated) { + for (var o = -1; o <= 4; o++) { + cuts.add(k + o); + } + } + for (final m in d.runs) { + for (var o = 0; o <= 9; o++) { + cuts.add(m + o); + } + } + var next = d.ts.first + 300000; + for (var i = 0; i < n; i++) { + if (d.ts[i] >= next) { + cuts..add(i - 1)..add(i)..add(i + 1); + next += 300000; + } + } + return [ + for (final c in cuts) + if (c > 0 && c <= n) c + ]..sort(); +} + +void main() { + group('corrector + screen state == batch, at every cut', () { + for (final restart in [false, true]) { + test('save/restore between chunks: $restart', () { + final d = _day(restart ? 2 : 1); + final c0 = correctRr(d.rr, rrTsMs: d.ts); + expect(c0.correctedCount, greaterThan(5), reason: 'fixture corrects'); + expect(c0.droppedCount, greaterThan(10), reason: 'fixture drops'); + + var corr = RrCorrector(); + var st = IrregularScreenState(); + var at = 0; + final opens = {}; + var flaggedSeen = 0, presentSeen = 0, abstainedSeen = 0; + for (final cut in _cuts(d, 4)) { + final settled = + corr.fold(d.rr.sublist(at, cut), tsMs: d.ts.sublist(at, cut)); + st.fold(settled.nn, settled.nnTimes); + at = cut; + if (restart) { + corr = RrCorrector.fromJson(jsonDecode(jsonEncode(corr.toJson())) + as Map); + st = IrregularScreenState.fromJson( + jsonDecode(jsonEncode(st.toJson())) as Map); + } + final got = _stream(corr, st); + _same(got, _batch(d.rr, d.ts, cut), 'cut=$cut restart=$restart'); + final w = got.diagnostics.windows; + if (w != null) opens.add(w.open); + got.metric.present ? presentSeen++ : abstainedSeen++; + if (got.diagnostics.windows != null && + got.diagnostics.windows!.flagged > 0) { + flaggedSeen++; + } + } + expect(presentSeen, greaterThan(20), reason: 'the screen really ran'); + expect(abstainedSeen, greaterThan(0), reason: 'and abstained early on'); + expect(flaggedSeen, greaterThan(5), reason: 'windows really flagged'); + expect(opens, containsAll([IrregularOpenWindow.thin]), + reason: 'cuts landed in a window that had not yet filled'); + expect( + opens.contains(IrregularOpenWindow.unflagged) || + opens.contains(IrregularOpenWindow.flagged), + isTrue, + reason: 'and in valid open windows'); + final last = _stream(corr, st); + expect(last.diagnostics.rrRaw, d.rr.length); + expect(last.diagnostics.corrected, c0.correctedCount); + expect(last.diagnostics.dropped, c0.droppedCount); + }); + } + + test('evaluateDetailed does not change the state', () { + final d = _day(3, beats: 3000); + final corr = RrCorrector(); + final st = IrregularScreenState(); + final settled = corr.fold(d.rr, tsMs: d.ts); + st.fold(settled.nn, settled.nnTimes); + final before = jsonEncode(st.toJson()); + final a = _stream(corr, st); + final b = _stream(corr, st); + expect(jsonEncode(st.toJson()), before); + expect(_json(a.diagnostics.toJson()), _json(b.diagnostics.toJson())); + }); + + test('one fold of the whole day == a hundred small ones', () { + final d = _day(5, beats: 4000); + final one = RrCorrector(), many = RrCorrector(); + final s1 = IrregularScreenState(), s2 = IrregularScreenState(); + final r1 = one.fold(d.rr, tsMs: d.ts); + s1.fold(r1.nn, r1.nnTimes); + for (var i = 0; i < d.rr.length; i += 40) { + final to = math.min(d.rr.length, i + 40); + final r = many.fold(d.rr.sublist(i, to), tsMs: d.ts.sublist(i, to)); + s2.fold(r.nn, r.nnTimes); + } + _same(_stream(many, s2), _stream(one, s1), 'chunking'); + _same(_stream(one, s1), _batch(d.rr, d.ts, d.rr.length), 'vs batch'); + }); + }); + + group('folding a corrected NN series directly (no corrector)', () { + ({List nn, List t}) fixture(int seed, {double salt = 0.03}) { + final r = math.Random(seed); + final nn = [], t = []; + var clock = 0.0; + var af = false; + var left = 0; + while (clock < 4 * 3600 * 1000.0) { + if (--left <= 0) { + af = r.nextDouble() < 0.5; + left = 200 + r.nextInt(1500); + } + var v = af + ? (420 + r.nextInt(700)).toDouble() + : (850 + 40 * math.sin(clock / 9000)).roundToDouble(); + final u = r.nextDouble(); + if (u < salt / 3) { + v = 250; + } else if (u < 2 * salt / 3) { + v = 2500; + } else if (u < salt) { + v = double.nan; + } + clock += v.isFinite ? v : 800; + nn.add(v); + t.add(clock); + } + return (nn: nn, t: t); + } + + IrregularScreenResult batch(List nn, List t, int n, + {double af = 0, + int minBeats = irregularScreenMinBeats, + double maxArtifact = 0.30}) => + irregularBeatScreenDetailed(nn.sublist(0, n), + nnTimesMs: t.sublist(0, n), + artifactFraction: af, + minBeats: minBeats, + maxArtifact: maxArtifact); + + for (final restart in [false, true]) { + test('out-of-range and NaN beats are counted in nnIn, not nnKept; ' + 'restore: $restart', () { + final x = fixture(8); + var st = IrregularScreenState(); + var at = 0; + var sawGap = false; + for (final cut + in randomCuts(math.Random(9), x.nn.length, sizes: [7, 90, 700, 3000])) { + st.fold(x.nn.sublist(at, cut), x.t.sublist(at, cut)); + at = cut; + if (restart) { + st = IrregularScreenState.fromJson( + jsonDecode(jsonEncode(st.toJson())) as Map); + } + for (final af in [0.0, 0.31]) { + final got = st.evaluateDetailed(const [], const [], artifactFraction: af); + _same(got, batch(x.nn, x.t, cut, af: af), 'cut=$cut af=$af'); + if (got.diagnostics.nnIn > got.diagnostics.nnKept) sawGap = true; + } + } + expect(sawGap, isTrue, reason: 'fixture has beats the filter removes'); + }); + } + + test('every prefix 0..700, one beat at a time: abstention reasons and ' + 'counts match the batch the whole way', () { + final x = fixture(12, salt: 0.0); + final st = IrregularScreenState(); + final reasons = {}; + for (var i = 0; i < 700; i++) { + st.fold([x.nn[i]], [x.t[i]]); + final got = st.evaluateDetailed(const [], const []); + _same(got, batch(x.nn, x.t, i + 1), 'n=${i + 1}'); + reasons.add(got.diagnostics.abstain); + } + expect(reasons, containsAll([IrregularAbstain.tooFewBeats, null]), + reason: 'thin first, then the screen runs'); + }); + + test('a smaller beat floor is a threshold the diagnostics report', () { + final x = fixture(13, salt: 0.0); + final st = IrregularScreenState() + ..fold(x.nn.sublist(0, 300), x.t.sublist(0, 300)); + final got = + st.evaluateDetailed(const [], const [], minBeats: 100, maxArtifact: 0.2); + expect(got.diagnostics.thresholds.minBeats, 100); + expect(got.diagnostics.thresholds.maxArtifact, 0.2); + _same(got, batch(x.nn, x.t, 300, minBeats: 100, maxArtifact: 0.2), + 'minBeats 100'); + }); + + test('tail: evaluateDetailed(tail) == batch(settled ++ tail), state ' + 'untouched', () { + final x = fixture(31); + final st = IrregularScreenState(); + const settled = 2500; + st.fold(x.nn.sublist(0, settled), x.t.sublist(0, settled)); + final before = jsonEncode(st.toJson()); + for (final tail in [0, 1, 50, 399]) { + final to = settled + tail; + _same( + st.evaluateDetailed(x.nn.sublist(settled, to), x.t.sublist(settled, to)), + batch(x.nn, x.t, to), + 'tail=$tail'); + expect(jsonEncode(st.toJson()), before); + } + }); + }); + + group('old checkpoints', () { + test('a version 1 checkpoint (no input-beat or window-total counters) is ' + 'REFUSED, never restored with counts it did not keep', () { + final st = IrregularScreenState() + ..fold([for (var i = 0; i < 80; i++) 800.0 + i % 9], [ + for (var i = 0; i < 80; i++) 800.0 * (i + 1) + ]); + final v1 = jsonDecode(jsonEncode(st.toJson())) as Map + ..['version'] = 1 + ..remove('nIn') + ..remove('total'); + expect(() => IrregularScreenState.fromJson(v1), throwsFormatException); + // The current format still restores. + final now = IrregularScreenState.fromJson( + jsonDecode(jsonEncode(st.toJson())) as Map); + expect(now.evaluateDetailed(const [], const []).diagnostics.nnIn, 80); + expect(st.toJson()['version'], 3); + }); + + test('a version 2 checkpoint missing a counter is refused too', () { + final j = jsonDecode(jsonEncode(IrregularScreenState().toJson())) + as Map + ..remove('nIn'); + expect(() => IrregularScreenState.fromJson(j), throwsFormatException); + }); + }); + + group('absent input stays absent', () { + test('a fresh state: absent, zero counts, nothing invented', () { + final got = IrregularScreenState().evaluateDetailed(const [], const []); + _same(got, irregularBeatScreenDetailed(const [], nnTimesMs: const []), + 'empty'); + expect(got.metric.present, isFalse); + expect(got.diagnostics.nnIn, 0); + expect(got.diagnostics.windows!.total, 0); + expect(got.diagnostics.windows!.sustainedObserved, isNull); + expect(got.diagnostics.rrRaw, isNull, + reason: 'no cleaning counts handed in: unknown, not 0'); + }); + + test('only unusable beats folded: counted as input, none kept, no window', + () { + final st = IrregularScreenState() + ..fold([double.nan, 100, 5000, double.nan], [1, 2, 3, 4]); + final got = st.evaluateDetailed(const [], const []); + expect(got.diagnostics.nnIn, 4); + expect(got.diagnostics.nnKept, 0); + expect(got.diagnostics.windows!.total, 0); + expect(got.diagnostics.windows!.open, IrregularOpenWindow.none); + expect(got.metric.present, isFalse); + }); + + test('a window config that fails closed: windows null, same as the batch', + () { + final r = math.Random(4); + final nn = [for (var i = 0; i < 1200; i++) 400.0 + 100 * r.nextInt(13)]; + var clock = 0.0; + final t = [for (final v in nn) clock += v]; + final st = IrregularScreenState(windowMinutes: 0)..fold(nn, t); + final got = st.evaluateDetailed(const [], const []); + expect(got.diagnostics.windows, isNull); + _same( + got, + irregularBeatScreenDetailed(nn, nnTimesMs: t, windowMinutes: 0), + 'windowMinutes 0'); + }); + }); +} diff --git a/test/onehz/incremental/irregular_screen_state_test.dart b/test/onehz/incremental/irregular_screen_state_test.dart new file mode 100644 index 00000000..6b118f4b --- /dev/null +++ b/test/onehz/incremental/irregular_screen_state_test.dart @@ -0,0 +1,495 @@ +// Contract for `IrregularScreenState` (clinical/irregular_rhythm_state). +// +// Oracle = `irregularBeatScreen` on the same NN series. The state folds the NN +// that `RrCorrector` settles and is evaluated with the corrector's provisional +// tail; the Metric must match the batch call on settled ++ tail at every prefix +// and under any chunking / save-restore: +// * present, note, tier, inputs, flag, nBeats, pNN%: identical; +// * SD1 / SD2 / ratio / confidence: equal to 1e-9 relative (the state uses a +// running variance, the batch a two-pass one); +// * ABSENT exactly where the batch is absent, with the same note — a thin, +// noisy, flat or gappy series never gets a made-up verdict. +// +// RED until the state is implemented: every test fails with UnimplementedError +// from the stub. +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/correct_rr_reference.dart'; +import '../support/rr_chunking.dart'; +import '../support/rr_synth.dart'; + +void _close(double got, double want, String why, {double rel = 1e-9}) { + final tol = rel * math.max(1.0, want.abs()); + expect((got - want).abs(), lessThanOrEqualTo(tol), + reason: '$why got=$got want=$want'); +} + +void _expectSameScreen(Metric got, + Metric want, String why) { + expect(got.present, want.present, reason: 'present $why'); + expect(got.note, want.note, reason: 'note $why'); + expect(got.tier, want.tier, reason: 'tier $why'); + expect(got.inputs_used, want.inputs_used, reason: 'inputs $why'); + if (!want.present) { + expect(got.value, isNull, reason: 'absent => no value $why'); + expect(got.confidence, 0, reason: 'absent => confidence 0 $why'); + return; + } + final g = got.value!, w = want.value!; + expect(g.flag, w.flag, reason: 'flag $why'); + expect(g.nBeats, w.nBeats, reason: 'nBeats $why'); + expect(g.pnnPct, w.pnnPct, reason: 'pnn $why'); + _close(g.sd1, w.sd1, 'sd1 $why'); + _close(g.sd2, w.sd2, 'sd2 $why'); + _close(g.sd1sd2, w.sd1sd2, 'sd1sd2 $why'); + _close(got.confidence, want.confidence, 'confidence $why', rel: 1e-12); +} + +/// AF-like bursts (irregularly irregular RR) in [afShare] of 5..30 min blocks, +/// otherwise sinus with slow drift. Optionally salted with out-of-range beats. +({List nn, List t}) _fixture(int seed, double afShare, + {double hours = 8, double outOfRange = 0}) { + final r = math.Random(seed); + final nn = [], t = []; + var clock = 0.0; + var af = false; + var left = 0; + while (clock < hours * 3600 * 1000.0) { + if (--left <= 0) { + af = r.nextDouble() < afShare; + left = 300 + r.nextInt(2000); + } + var v = af + ? (420 + r.nextInt(700)).toDouble() + : (850 + 40 * math.sin(clock / 9000) + 25 * (r.nextDouble() - .5)) + .roundToDouble(); + if (r.nextDouble() < outOfRange) v = r.nextBool() ? 2500 : 250; + clock += v; + nn.add(v); + t.add(clock); + } + return (nn: nn, t: t); +} + +Metric _batch(List nn, List t, int n, + {double af = 0.0, + int minBeats = irregularScreenMinBeats, + double maxArtifact = 0.30, + double sd1sd2Flag = 0.70, + double pnnThresholdMs = 70, + double pnnFlagPct = 30, + double windowMinutes = 5, + int minWindowBeats = 40, + double sustainedFraction = 0.5}) => + irregularBeatScreen(nn.sublist(0, n), + nnTimesMs: t.sublist(0, n), + artifactFraction: af, + minBeats: minBeats, + maxArtifact: maxArtifact, + sd1sd2Flag: sd1sd2Flag, + pnnThresholdMs: pnnThresholdMs, + pnnFlagPct: pnnFlagPct, + windowMinutes: windowMinutes, + minWindowBeats: minWindowBeats, + sustainedFraction: sustainedFraction); + +void main() { + group('folding a corrected NN series directly', () { + for (final share in [0.0, 0.35, 0.75, 1.0]) { + for (final restart in [false, true]) { + test('af share=$share, save/restore between chunks: $restart', () { + final x = _fixture((share * 100).round() + 3, share); + var st = IrregularScreenState(); + var at = 0; + var flags = 0, checks = 0; + final cuts = randomCuts(math.Random(5), x.nn.length, + sizes: [90, 500, 4000]); + for (final cut in cuts) { + st.fold(x.nn.sublist(at, cut), x.t.sublist(at, cut)); + at = cut; + if (restart) { + st = IrregularScreenState.fromJson(jsonRoundTrip(st.toJson())); + } + // 0.31 is over the 0.30 suppression line: must be absent. + for (final af in [0.0, 0.31]) { + final want = _batch(x.nn, x.t, cut, af: af); + final got = + st.evaluate(const [], const [], artifactFraction: af); + _expectSameScreen(got, want, 'cut=$cut af=$af'); + if (want.present) { + checks++; + if (want.value!.flag) flags++; + } + } + } + expect(checks, greaterThan(5), reason: 'screen was really evaluated'); + // ignore: avoid_print + print('irregular share=$share: checks=$checks flagged=$flags'); + if (share >= 0.75) { + expect(flags, greaterThan(0), reason: 'a flagged day is flagged'); + } + if (share == 0.0) expect(flags, 0); + }); + } + } + + test('out-of-range beats are skipped without bridging the gap', () { + // 2% of beats are 250 / 2500 ms. The batch never takes a difference + // ACROSS a skipped beat; the state must not either. + final x = _fixture(77, 0.5, hours: 4, outOfRange: 0.02); + var st = IrregularScreenState(); + var at = 0; + for (final cut in randomCuts(math.Random(6), x.nn.length, + sizes: [1, 7, 90, 700])) { + st.fold(x.nn.sublist(at, cut), x.t.sublist(at, cut)); + at = cut; + if (cut % 3 == 0) { + st = IrregularScreenState.fromJson(jsonRoundTrip(st.toJson())); + } + _expectSameScreen(st.evaluate(const [], const []), + _batch(x.nn, x.t, cut), 'cut=$cut'); + } + }); + + test('a window edge hit EXACTLY decides the verdict (>= closes the window)', + () { + // Window 1: 300 s of AF-like beats whose last interval lands the next + // beat on windowStart + 300000 EXACTLY. Window 2: calm, same trick. + // Window 3: calm, exactly minWindowBeats (40) beats, so it only counts + // as a window when the edge beat OPENS it. Closing on > instead of >= + // moves that beat into window 2, window 3 falls to 39 beats and stops + // counting, and the sustained fraction jumps from 1/3 to 1/2 (flag). + final r = math.Random(3); + final nn = [], t = []; + var clock = 0.0; + void add(double v) { + clock += v; + nn.add(v); + t.add(clock); + } + + void landOn(double target, double Function() next) { + while (target - clock > 2200) { + add(next()); + } + var rest = target - clock; + if (rest < 300) { + clock -= nn.removeLast(); + t.removeLast(); + rest = target - clock; + } + if (rest > 2000) { + add(rest / 2); + add(rest / 2); + } else { + add(rest); + } + } + + add(1000); // the first beat opens window 1 at t = 1000 + landOn(301000, () => 400.0 + 100 * r.nextInt(13)); // AF-like + double calm() => 800.0 + 10 * (r.nextInt(5) - 2); + landOn(601000, calm); + for (var i = 0; i < 39; i++) { + add(calm()); + } + expect(t.where((x) => x == 301000 || x == 601000).length, 2); + final want = _batch(nn, t, nn.length, minBeats: 500); + expect(want.present, isTrue, reason: 'sanity: crafted day is evaluable'); + expect(want.value!.flag, isFalse, reason: 'sanity: 1 of 3 windows flags'); + final st = IrregularScreenState(); + st.fold(nn, t); + _expectSameScreen(st.evaluate(const [], const []), want, 'edge day'); + // and split anywhere, with a restore, including right at the edge beats + for (final cut in [ + nn.length ~/ 2, + t.indexOf(301000), + t.indexOf(601000), + t.indexOf(601000) + 1 + ]) { + var a = IrregularScreenState() + ..fold(nn.sublist(0, cut), t.sublist(0, cut)); + a = IrregularScreenState.fromJson(jsonRoundTrip(a.toJson())); + a.fold(nn.sublist(cut), t.sublist(cut)); + _expectSameScreen(a.evaluate(const [], const []), want, 'cut=$cut'); + } + }); + + test('provisional tail: evaluate(tail) == batch(settled ++ tail), and ' + 'leaves the state untouched', () { + final x = _fixture(31, 0.75, hours: 3); + final st = IrregularScreenState(); + final settled = x.nn.length - 400; + st.fold(x.nn.sublist(0, settled), x.t.sublist(0, settled)); + final before = jsonEncode(st.toJson()); + for (final tailLen in [0, 1, 50, 399, 400]) { + final to = settled + tailLen; + final got = st.evaluate(x.nn.sublist(settled, to), x.t.sublist(settled, to)); + _expectSameScreen(got, _batch(x.nn, x.t, to), 'tail=$tailLen'); + expect(jsonEncode(st.toJson()), before, + reason: 'evaluate must not fold the tail into the state'); + } + // Fold the tail for real afterwards: still the batch answer. + st.fold(x.nn.sublist(settled), x.t.sublist(settled)); + _expectSameScreen(st.evaluate(const [], const []), + _batch(x.nn, x.t, x.nn.length), 'tail folded'); + }); + }); + + group('absent where the batch is absent (never fabricated)', () { + test('every prefix 0..700, one beat at a time: thin, then present', () { + final x = _fixture(12, 0.6, hours: 1); + final st = IrregularScreenState(); + var present = 0, absent = 0; + for (var i = 0; i < 700; i++) { + st.fold([x.nn[i]], [x.t[i]]); + final want = _batch(x.nn, x.t, i + 1); + final got = st.evaluate(const [], const []); + _expectSameScreen(got, want, 'n=${i + 1}'); + want.present ? present++ : absent++; + } + expect(absent, greaterThan(400), reason: 'thin prefixes were exercised'); + expect(present, greaterThan(100), reason: 'and then it speaks'); + }); + + test('a smaller beat floor and every prefix 0..120', () { + final x = _fixture(13, 1.0, hours: 1); + final st = IrregularScreenState(); + for (var i = 0; i < 120; i++) { + st.fold([x.nn[i]], [x.t[i]]); + _expectSameScreen( + st.evaluate(const [], const [], minBeats: 20), + _batch(x.nn, x.t, i + 1, minBeats: 20), + 'minBeats=20 n=${i + 1}'); + } + }); + + test('artifact fraction at / over the line, absent with the same note', () { + final x = _fixture(14, 0.5, hours: 2); + final st = IrregularScreenState()..fold(x.nn, x.t); + for (final af in [0.0, 0.29, 0.30, 0.3000001, 0.31, 0.5, 1.0]) { + _expectSameScreen(st.evaluate(const [], const [], artifactFraction: af), + _batch(x.nn, x.t, x.nn.length, af: af), 'af=$af'); + } + for (final cap in [0.1, 0.5]) { + _expectSameScreen( + st.evaluate(const [], const [], + artifactFraction: 0.2, maxArtifact: cap), + _batch(x.nn, x.t, x.nn.length, af: 0.2, maxArtifact: cap), + 'maxArtifact=$cap'); + } + }); + + test('flat series: SD2 = 0 is "undefined", not "perfectly regular"', () { + final nn = List.filled(900, 800); + final t = [for (var i = 1; i <= 900; i++) i * 800.0]; + final got = (IrregularScreenState()..fold(nn, t)) + .evaluate(const [], const []); + final want = _batch(nn, t, 900); + expect(want.present, isFalse, reason: 'the oracle abstains'); + _expectSameScreen(got, want, 'flat'); + }); + + test('no two adjacent clean beats: nothing to build a Poincare plot from', + () { + // every other beat out of range: 300 clean beats, zero adjacent pairs. + final nn = [for (var i = 0; i < 600; i++) i.isEven ? 800.0 : 250.0]; + final t = [for (var i = 1; i <= 600; i++) i * 800.0]; + final got = (IrregularScreenState()..fold(nn, t)) + .evaluate(const [], const [], minBeats: 5); + final want = _batch(nn, t, 600, minBeats: 5); + expect(want.present, isFalse); + _expectSameScreen(got, want, 'no adjacent pairs'); + }); + + test('empty state', () { + _expectSameScreen(IrregularScreenState().evaluate(const [], const []), + _batch(const [], const [], 0), 'empty'); + }); + }); + + group('parameters', () { + test('non-default thresholds and windows, with restore', () { + final x = _fixture(21, 0.4, hours: 3); + final params = ( + sd1sd2Flag: 0.5, + pnnThresholdMs: 50.0, + pnnFlagPct: 20.0, + windowMinutes: 2.0, + minWindowBeats: 20, + sustainedFraction: 0.3 + ); + var st = IrregularScreenState( + sd1sd2Flag: params.sd1sd2Flag, + pnnThresholdMs: params.pnnThresholdMs, + pnnFlagPct: params.pnnFlagPct, + windowMinutes: params.windowMinutes, + minWindowBeats: params.minWindowBeats, + sustainedFraction: params.sustainedFraction); + var at = 0; + for (final cut in randomCuts(math.Random(8), x.nn.length, + sizes: [100, 900, 3000])) { + st.fold(x.nn.sublist(at, cut), x.t.sublist(at, cut)); + at = cut; + st = IrregularScreenState.fromJson(jsonRoundTrip(st.toJson())); + _expectSameScreen( + st.evaluate(const [], const []), + _batch(x.nn, x.t, cut, + sd1sd2Flag: params.sd1sd2Flag, + pnnThresholdMs: params.pnnThresholdMs, + pnnFlagPct: params.pnnFlagPct, + windowMinutes: params.windowMinutes, + minWindowBeats: params.minWindowBeats, + sustainedFraction: params.sustainedFraction), + 'cut=$cut'); + } + // The parameters travel in the checkpoint. + expect(st.sd1sd2Flag, params.sd1sd2Flag); + expect(st.pnnThresholdMs, params.pnnThresholdMs); + expect(st.pnnFlagPct, params.pnnFlagPct); + expect(st.windowMinutes, params.windowMinutes); + expect(st.minWindowBeats, params.minWindowBeats); + expect(st.sustainedFraction, params.sustainedFraction); + }); + + test('a bad window config fails CLOSED: never a flag', () { + // AF all day, so the aggregate verdict is high; only the sustained-window + // check can hold the flag back, and a broken config must hold it back. + final x = _fixture(22, 1.0, hours: 2); + final base = _batch(x.nn, x.t, x.nn.length); + expect(base.present && base.value!.flag, isTrue, + reason: 'sanity: the default config flags this day'); + for (final bad in [ + IrregularScreenState(windowMinutes: 0), + IrregularScreenState(windowMinutes: -5), + IrregularScreenState(windowMinutes: double.nan), + IrregularScreenState(minWindowBeats: 1), + IrregularScreenState(sustainedFraction: 1.5), + IrregularScreenState(sustainedFraction: -0.1), + IrregularScreenState(sustainedFraction: double.nan), + ]) { + bad.fold(x.nn, x.t); + final want = _batch(x.nn, x.t, x.nn.length, + windowMinutes: bad.windowMinutes, + minWindowBeats: bad.minWindowBeats, + sustainedFraction: bad.sustainedFraction); + final got = bad.evaluate(const [], const []); + expect(want.present && !want.value!.flag, isTrue, + reason: 'oracle fails closed'); + _expectSameScreen(got, want, + 'window=${bad.windowMinutes} minBeats=${bad.minWindowBeats} ' + 'sustained=${bad.sustainedFraction}'); + } + }); + }); + + group('behind RrCorrector (the way the pipeline will use it)', () { + test('6 h dirty day, 15-minute passes, both states restored every pass', + () { + final s = synthRr(const SynthConfig( + seed: 5, + hours: 6, + ectopicPerMin: 0.8, + missedPerMin: 0.3, + extraPerMin: 0.3, + noiseRunPerMin: 0.1, + gapPerHour: 4)); + final cuts = timeCuts(s.ts, 900); + var corrector = RrCorrector(); + var st = IrregularScreenState(); + var from = 0, present = 0, maxBytes = 0; + for (var k = 0; k < cuts.length; k++) { + final settled = corrector.fold(s.rr.sublist(from, cuts[k]), + tsMs: s.ts.sublist(from, cuts[k])); + from = cuts[k]; + st.fold(settled.nn, settled.nnTimes); + corrector = RrCorrector.fromJson(jsonRoundTrip(corrector.toJson())); + st = IrregularScreenState.fromJson(jsonRoundTrip(st.toJson())); + maxBytes = math.max(maxBytes, jsonEncode(st.toJson()).length); + if (k % 4 != 0 && k != cuts.length - 1) continue; + final snap = corrector.snapshot(); + final ref = correctRrReference(s.rr.sublist(0, cuts[k]), + rrTsMs: s.ts.sublist(0, cuts[k])); + final af = (1.0 - ref.cleanFraction).clamp(0.0, 1.0).toDouble(); + final want = irregularBeatScreen(ref.nn, + nnTimesMs: ref.nnTimesMs, artifactFraction: af); + final got = st.evaluate(snap.tailNn, snap.tailNnTimes, + artifactFraction: 1.0 - snap.cleanFraction); + _expectSameScreen(got, want, 'pass $k n=${cuts[k]}'); + if (want.present) present++; + } + expect(present, greaterThan(3), reason: 'the screen was really run'); + expect(maxBytes, lessThan(16 * 1024), reason: 'checkpoint chars'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('23 h day: final screen equals the batch screen on the whole day', () { + final s = realShapedDay(); + final cuts = timeCuts(s.ts, 900); + final corrector = RrCorrector(); + final st = IrregularScreenState(); + var from = 0; + for (final to in cuts) { + final settled = + corrector.fold(s.rr.sublist(from, to), tsMs: s.ts.sublist(from, to)); + from = to; + st.fold(settled.nn, settled.nnTimes); + } + final snap = corrector.snapshot(); + final ref = correctRrReference(s.rr, rrTsMs: s.ts); + final af = (1.0 - ref.cleanFraction).clamp(0.0, 1.0).toDouble(); + final want = irregularBeatScreen(ref.nn, + nnTimesMs: ref.nnTimesMs, artifactFraction: af); + expect(want.present, isTrue, reason: 'sanity: a day is enough data'); + _expectSameScreen( + st.evaluate(snap.tailNn, snap.tailNnTimes, + artifactFraction: 1.0 - snap.cleanFraction), + want, + 'day'); + }, timeout: const Timeout(Duration(minutes: 10))); + }); + + group('contract', () { + test('checkpoint is versioned and typed like the other states', () { + final json = IrregularScreenState().toJson(); + expect(json['version'], 3); + expect(json['type'], 'IrregularScreenState'); + expect(() => IrregularScreenState.fromJson({...json, 'version': 2}), + throwsFormatException); + expect(() => IrregularScreenState.fromJson({...json, 'type': 'Other'}), + throwsFormatException); + expect(() => IrregularScreenState.fromJson({}), throwsFormatException); + }); + + test('checkpoint is bounded: running sums plus one open window', () { + final x = _fixture(41, 0.5, hours: 8); + expect(x.nn.length, greaterThan(20000)); + final st = IrregularScreenState(); + var at = 0, maxBytes = 0; + for (final cut in randomCuts(math.Random(2), x.nn.length, + sizes: [50, 500, 2000])) { + st.fold(x.nn.sublist(at, cut), x.t.sublist(at, cut)); + at = cut; + maxBytes = math.max(maxBytes, jsonEncode(st.toJson()).length); + } + // ~1.4 KB measured in the prototype; the series would be ~250 KB. + expect(maxBytes, lessThan(16 * 1024), reason: 'max checkpoint chars'); + }); + + test('save/restore is invisible: restored state folds on identically', () { + final x = _fixture(42, 0.5, hours: 3); + final cut = x.nn.length ~/ 2; + final a = IrregularScreenState()..fold(x.nn.sublist(0, cut), x.t.sublist(0, cut)); + final b = IrregularScreenState.fromJson(jsonRoundTrip(a.toJson())); + expect(jsonEncode(b.toJson()), jsonEncode(a.toJson())); + a.fold(x.nn.sublist(cut), x.t.sublist(cut)); + b.fold(x.nn.sublist(cut), x.t.sublist(cut)); + expect(jsonEncode(b.toJson()), jsonEncode(a.toJson())); + _expectSameScreen(b.evaluate(const [], const []), + a.evaluate(const [], const []), 'restored vs original'); + }); + }); +} diff --git a/test/onehz/incremental/rr_corrector_test.dart b/test/onehz/incremental/rr_corrector_test.dart new file mode 100644 index 00000000..ac4a1f67 --- /dev/null +++ b/test/onehz/incremental/rr_corrector_test.dart @@ -0,0 +1,569 @@ +// Contract for the streaming `RrCorrector` (foundations/rr_correction_stream). +// +// Oracle = the frozen copy of today's `correctRr` (support/correct_rr_reference +// .dart). At every prefix of the input, for any chunking and with the corrector +// saved to JSON and restored between chunks: +// +// settled output so far ++ snapshot().tail == correctRr(prefix) +// +// bit for bit (nn, nnTimes, per-beat classes, counts, cleanFraction). Settled +// output is final (a prefix of what the oracle says about the whole series), +// the settle horizon is bounded, and the checkpoint stays small. +// +// RED until RrCorrector is implemented: every test below fails with +// UnimplementedError from the stub. +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/correct_rr_reference.dart'; +import '../support/rr_compare.dart'; +import '../support/rr_stream_driver.dart'; +import '../support/rr_synth.dart'; + +/// Series used by several groups. Cached: generating is cheap, the oracle runs +/// are not. +final _dirty = SynthConfig( + seed: 23, + hours: 0.4, + ectopicPerMin: 2, + missedPerMin: 1, + extraPerMin: 1, + noiseRunPerMin: 0.5, + gapPerHour: 12); + +void main() { + group('every prefix', () { + for (final seed in [3, 4]) { + test('1 beat at a time, n = 0..260, seed $seed', () { + final s = synthRr(SynthConfig( + seed: seed, + hours: 0.12, + ectopicPerMin: 2, + missedPerMin: 1, + extraPerMin: 1, + noiseRunPerMin: 0.8, + gapPerHour: 20)); + final n = math.min(260, s.length); + final run = RrStreamRun(); + run.expectMatchesOracle(s.rr, s.ts, 0, why: 'n=0'); + for (var i = 0; i < n; i++) { + run.fold([s.rr[i]], [s.ts[i]]); + run.expectMatchesOracle(s.rr, s.ts, i + 1, why: 'seed=$seed n=${i + 1}'); + } + }); + } + + test('with a JSON save/restore between EVERY beat (n = 0..230)', () { + final s = synthRr(const SynthConfig( + seed: 11, + hours: 0.1, + ectopicPerMin: 2, + noiseRunPerMin: 0.6, + gapPerHour: 10)); + final n = math.min(230, s.length); + final run = RrStreamRun(); + run.restart(); // a corrector restored from an EMPTY checkpoint + run.expectMatchesOracle(s.rr, s.ts, 0, why: 'empty restore'); + for (var i = 0; i < n; i++) { + run.fold([s.rr[i]], [s.ts[i]]); + run.restart(); + run.expectMatchesOracle(s.rr, s.ts, i + 1, why: 'n=${i + 1}'); + } + }); + + test('1-beat folds over 4000 dirty beats: every alignment of a fold edge ' + 'and an artefact\'s right-anchor search', () { + final s = synthRr(const SynthConfig( + seed: 51, + hours: 1.2, + ectopicPerMin: 3, + missedPerMin: 2, + extraPerMin: 2, + noiseRunPerMin: 1, + gapPerHour: 8)); + final n = math.min(4000, s.length); + final full = correctRrReference(s.rr.sublist(0, n), + rrTsMs: s.ts.sublist(0, n)); + final run = RrStreamRun(); + for (var i = 0; i < n; i++) { + run.fold([s.rr[i]], [s.ts[i]]); + if (i % 997 == 0) run.restart(); + if (i % 50 == 0) run.expectSettledIsPrefixOf(full, 'i=$i'); + } + run.expectMatchesOracle(s.rr, s.ts, n, why: 'final'); + }); + }); + + group('random chunking', () { + final cfgs = { + 'clean': const SynthConfig( + seed: 21, + hours: 0.4, + ectopicPerMin: 0, + missedPerMin: 0, + extraPerMin: 0, + noiseRunPerMin: 0, + gapPerHour: 0), + 'default': const SynthConfig(seed: 22, hours: 0.5), + 'dirty': _dirty, + 'backwards-ts': const SynthConfig( + seed: 24, hours: 0.4, backwardsPerHour: 30, gapPerHour: 6), + }; + for (final e in cfgs.entries) { + for (final restart in [false, true]) { + test('${e.key}, save/restore between chunks: $restart', () { + final s = synthRr(e.value); + final full = correctRrReference(s.rr, rrTsMs: s.ts); + final r = math.Random(e.value.seed * 7 + (restart ? 1 : 0)); + final cuts = randomCuts(r, s.length); + final run = RrStreamRun(); + var from = 0; + for (var k = 0; k < cuts.length; k++) { + final to = cuts[k]; + run.fold(s.rr.sublist(from, to), s.ts.sublist(from, to)); + from = to; + if (restart && r.nextBool()) run.restart(); + run.expectSettledIsPrefixOf(full, '${e.key} cut=$to'); + if (k % 13 == 0 || k == cuts.length - 1) { + run.expectMatchesOracle(s.rr, s.ts, to, + why: '${e.key} cut=$to'); + } + } + }); + } + } + + test('chunking does not matter: 1 chunk, 7 chunks and 1-beat chunks end ' + 'identical', () { + final s = synthRr(const SynthConfig(seed: 61, hours: 0.3)); + final out = []; + for (final size in [s.length, 400, 1]) { + final run = RrStreamRun(); + for (var at = 0; at < s.length; at += size) { + final to = math.min(s.length, at + size); + run.fold(s.rr.sublist(at, to), s.ts.sublist(at, to)); + } + final snap = run.c.snapshot(); + out.add(jsonEncode([ + [...run.nn, ...snap.tailNn], + [...run.nnTimes, ...snap.tailNnTimes], + snap.normalCount, + snap.droppedCount, + snap.correctedCount, + ])); + } + expect(out[1], out[0]); + expect(out[2], out[0]); + }); + }); + + group('timestamps', () { + test('sub-second timestamps', () { + final s = synthRr(const SynthConfig(seed: 33, hours: 0.4, gapPerHour: 6)); + final r = math.Random(12); + final ts = [for (final t in s.ts) t + r.nextInt(1000) + r.nextDouble()]; + final run = RrStreamRun(); + var from = 0; + final cuts = randomCuts(r, s.length); + for (var k = 0; k < cuts.length; k++) { + run.fold(s.rr.sublist(from, cuts[k]), ts.sublist(from, cuts[k])); + from = cuts[k]; + if (k % 9 == 0) run.restart(); + if (k % 9 == 0 || k == cuts.length - 1) { + run.expectMatchesOracle(s.rr, ts, cuts[k], why: 'cut=${cuts[k]}'); + } + } + }); + + test('no timestamps at all (the clock is the RR cumsum)', () { + final s = synthRr(const SynthConfig(seed: 31, hours: 0.4)); + final run = RrStreamRun(); + final r = math.Random(5); + var from = 0; + final cuts = randomCuts(r, s.length); + for (var k = 0; k < cuts.length; k++) { + run.fold(s.rr.sublist(from, cuts[k]), null); + from = cuts[k]; + if (k % 4 == 0) run.restart(); + if (k % 7 == 0 || k == cuts.length - 1) { + run.expectMatchesOracle(s.rr, null, cuts[k], why: 'cut=${cuts[k]}'); + } + } + }); + + test('dropouts: the clock re-anchors across a hole, also across a fold ' + 'boundary and a restore', () { + // 300 beats, a 40 s hole before beat 150, a fold edge exactly there. + final rr = [for (var i = 0; i < 300; i++) 800.0 + (i % 5) * 3]; + var t = 1.7e12; + final ts = [ + for (var i = 0; i < 300; i++) (t += rr[i] + (i == 150 ? 40000 : 0)) + ]; + for (final edge in [149, 150, 151]) { + final run = RrStreamRun(); + run.fold(rr.sublist(0, edge), ts.sublist(0, edge)); + run.restart(); + run.fold(rr.sublist(edge), ts.sublist(edge)); + run.expectMatchesOracle(rr, ts, 300, why: 'edge=$edge'); + } + }); + }); + + group('parameters', () { + for (final p in [ + (alpha: 5.2, win: 31, floor: 100.0, reanchor: 1000.0), + (alpha: 4.0, win: 61, floor: 100.0, reanchor: 1000.0), + (alpha: 5.2, win: 121, floor: 100.0, reanchor: 1000.0), + (alpha: 6.5, win: 91, floor: 30.0, reanchor: 400.0), + (alpha: 5.2, win: 4, floor: 0.0, reanchor: 1000.0), + (alpha: 5.2, win: 90, floor: 100.0, reanchor: 5000.0), + ]) { + test('alpha=${p.alpha} win=${p.win} floor=${p.floor} ' + 'reanchor=${p.reanchor}, with restore', () { + final s = synthRr(const SynthConfig( + seed: 41, hours: 0.4, ectopicPerMin: 1, gapPerHour: 20)); + final run = RrStreamRun.params( + alpha: p.alpha, + win: p.win, + floor: p.floor, + reanchor: p.reanchor); + final r = math.Random(9); + var from = 0; + final cuts = randomCuts(r, s.length); + for (var k = 0; k < cuts.length; k++) { + run.fold(s.rr.sublist(from, cuts[k]), s.ts.sublist(from, cuts[k])); + from = cuts[k]; + if (r.nextBool()) run.restart(); + if (k % 9 == 0 || k == cuts.length - 1) { + run.expectMatchesOracle(s.rr, s.ts, cuts[k], + alpha: p.alpha, + win: p.win, + floor: p.floor, + reanchor: p.reanchor, + why: 'cut=${cuts[k]}'); + } + } + // The parameters travel in the checkpoint. + expect(run.c.alpha, p.alpha); + expect(run.c.windowBeats, p.win); + expect(run.c.minThresholdMs, p.floor); + expect(run.c.reanchorGapMs, p.reanchor); + }); + } + }); + + group('degenerate inputs', () { + void drive(List rr, List ts, String why) { + final run = RrStreamRun(); + final r = math.Random(3); + var from = 0; + for (final to in randomCuts(r, rr.length, sizes: [1, 2, 3, 50, 200])) { + run.fold(rr.sublist(from, to), ts.sublist(from, to)); + from = to; + if (r.nextInt(3) == 0) run.restart(); + run.expectMatchesOracle(rr, ts, to, why: '$why to=$to'); + } + } + + test('constant RR (QD = 0, the floor governs, massive ties)', () { + drive(List.filled(500, 800), + [for (var i = 0; i < 500; i++) 1.7e12 + i * 800.0], 'const'); + }); + + test('constant with one outlier at start / middle / end', () { + for (final at in [0, 1, 250, 498, 499]) { + final rr = List.filled(500, 800)..[at] = 1700; + drive(rr, [for (var i = 0; i < 500; i++) i * 800.0], 'outlier@$at'); + } + }); + + test('artefact runs longer than the window, and alternating artefacts', () { + final r = math.Random(5); + final rr = [ + for (var i = 0; i < 700; i++) + (i >= 150 && i < 290) + ? 250 + r.nextInt(2000).toDouble() + : (i >= 400 && i < 500 && i.isEven) + ? 2200 + : 800 + r.nextInt(40).toDouble() + ]; + var t = 0.0; + drive(rr, [for (final v in rr) (t += v)], 'longrun'); + }); + + test('artefact at beat 0 (no left anchors) and at the tail (no right)', () { + final rr = [ + 2500, + 800, + 810, + 790, + 805, + for (var i = 0; i < 300; i++) 800 + (i % 7) * 3.0, + 2500 + ]; + drive(rr, [for (var i = 0; i < rr.length; i++) i * 800.0], 'edges'); + }); + + test('all artefacts: nothing is ever invented', () { + final rr = List.filled(400, 2500); + drive(rr, [for (var i = 0; i < 400; i++) 1.7e12 + i * 2500.0], 'all'); + }); + + test('n = 1, 2, 3 and just past (the < 3 beat branch hands over)', () { + final rr = [800, 810, 790, 805, 3000, 800]; + final ts = [for (var i = 0; i < 6; i++) 1.7e12 + i * 800.0]; + final run = RrStreamRun(); + for (var i = 0; i < 6; i++) { + run.fold([rr[i]], [ts[i]]); + run.expectMatchesOracle(rr, ts, i + 1, why: 'n=${i + 1}'); + } + final out = RrStreamRun() + ..fold([250, 2500], [1.7e12, 1.7e12 + 2500]); + out.expectMatchesOracle([250, 2500], [1.7e12, 1.7e12 + 2500], 2, + why: 'two implausible beats'); + }); + }); + + group('settle horizon and state size', () { + test('capped settled edge includes the cap decision and its left influence', + () { + // Dirty series: only check the bound where the oracle's FINAL classes of + // the 5 beats just before the 90-beat look-ahead are all normal. Then no + // artefact run is in flight at the classified edge and no lone artefact + // is still waiting for its two right-hand anchors. + final s = synthRr(const SynthConfig( + seed: 23, + hours: 1.0, + ectopicPerMin: 0.5, + missedPerMin: 0.2, + extraPerMin: 0.2, + noiseRunPerMin: 0.1, + gapPerHour: 6)); + final full = correctRrReference(s.rr, rrTsMs: s.ts); + expect(full.classes.where((c) => c != BeatClass.normal).length, + greaterThan(20), + reason: 'sanity: the series really has artefacts'); + final run = RrStreamRun(); + final r = math.Random(17); + var checked = 0, maxLag = 0, from = 0; + final cuts = randomCuts(r, s.length, sizes: [1, 3, 25, 90, 400]); + var lastSettled = 0; + for (final to in cuts) { + run.fold(s.rr.sublist(from, to), s.ts.sublist(from, to)); + from = to; + final settled = run.c.settledBeats; + expect(settled, greaterThanOrEqualTo(lastSettled), + reason: 'settledBeats never goes backwards'); + expect(settled, lessThanOrEqualTo(to)); + lastSettled = settled; + if (to < 130) continue; + final quiet = [ + for (var k = to - 95; k < to - 90; k++) full.classes[k] + ].every((c) => c == BeatClass.normal); + if (quiet) { + checked++; + maxLag = math.max(maxLag, to - settled); + expect(to - settled, lessThanOrEqualTo(280), + reason: 'to=$to settled=$settled'); + } + } + expect(checked, greaterThan(20), reason: 'the bound was really checked'); + // ignore: avoid_print + print('settle lag: checked=$checked max=$maxLag beats'); + }); + + test('a clean series settles right up to the look-ahead', () { + final r = math.Random(71); + var t = 1.7e12; + final rr = [ + for (var i = 0; i < 1500; i++) + (800 + 40 * math.sin(i / 9) + r.nextInt(21) - 10).roundToDouble() + ]; + final ts = [for (final v in rr) (t += v)]; + final full = correctRrReference(rr, rrTsMs: ts); + // The comparison below only holds if the series really is all normal. + expect(full.classes.every((c) => c == BeatClass.normal), isTrue); + final run = RrStreamRun(); + for (var at = 0; at < rr.length; at += 37) { + final to = math.min(rr.length, at + 37); + run.fold(rr.sublist(at, to), ts.sublist(at, to)); + if (to >= 91) { + expect(to - run.c.settledBeats, lessThanOrEqualTo(248), + reason: 'to=$to'); + } + } + }); + + test('checkpoint is bounded: it holds the window, not the series', () { + final s = synthRr(const SynthConfig(seed: 81, hours: 6)); + expect(s.length, greaterThan(20000)); + final run = RrStreamRun(); + var maxBytes = 0; + var from = 0; + for (final to in timeCuts(s.ts, 900)) { + run.fold(s.rr.sublist(from, to), s.ts.sublist(from, to)); + from = to; + maxBytes = math.max(maxBytes, jsonEncode(run.c.toJson()).length); + } + // ~4.6 KB measured in the prototype; the series would be ~300 KB. + expect(maxBytes, lessThan(32 * 1024), reason: 'max checkpoint chars'); + // ignore: avoid_print + print('checkpoint max: $maxBytes chars over ${s.length} beats'); + }); + }); + + group('contract', () { + test('snapshot() changes nothing: twice is equal, folding on is unaffected', + () { + final s = synthRr(const SynthConfig(seed: 91, hours: 0.2)); + final a = RrStreamRun(), b = RrStreamRun(); + final cut = s.length ~/ 2; + a.fold(s.rr.sublist(0, cut), s.ts.sublist(0, cut)); + b.fold(s.rr.sublist(0, cut), s.ts.sublist(0, cut)); + final before = jsonEncode(a.c.toJson()); + final one = a.c.snapshot(), two = a.c.snapshot(); + expect(jsonEncode(a.c.toJson()), before, reason: 'checkpoint unchanged'); + expectBitIdentical(one.tailNn, two.tailNn, 'tail twice'); + // b never snapshots; both then fold the rest and must agree. + a.fold(s.rr.sublist(cut), s.ts.sublist(cut)); + b.fold(s.rr.sublist(cut), s.ts.sublist(cut)); + expectBitIdentical(a.nn, b.nn, 'settled nn with/without snapshot'); + expect(jsonEncode(a.c.toJson()), jsonEncode(b.c.toJson())); + }); + + test('toJson survives jsonEncode/jsonDecode: restored == original', () { + final s = synthRr(_dirty); + final cut = s.length ~/ 2; + final a = RrStreamRun(); + a.fold(s.rr.sublist(0, cut), s.ts.sublist(0, cut)); + final restored = RrCorrector.fromJson(jsonRoundTrip(a.c.toJson())); + expect(jsonEncode(restored.toJson()), jsonEncode(a.c.toJson())); + expect(restored.settledBeats, a.c.settledBeats); + final x = a.c.fold(s.rr.sublist(cut), tsMs: s.ts.sublist(cut)); + final y = restored.fold(s.rr.sublist(cut), tsMs: s.ts.sublist(cut)); + expectBitIdentical(y.nn, x.nn, 'nn after restore'); + expectBitIdentical(y.nnTimes, x.nnTimes, 'times after restore'); + expectSameClasses(y.classes, x.classes, 'classes after restore'); + }); + + test('checkpoint is versioned and typed like the other states', () { + final json = RrCorrector().toJson(); + expect(json['version'], 2); + expect(json['type'], 'RrCorrector'); + expect(() => RrCorrector.fromJson({...json, 'version': 1}), + throwsFormatException); + expect(() => RrCorrector.fromJson({...json, 'type': 'Other'}), + throwsFormatException); + expect(() => RrCorrector.fromJson({}), throwsFormatException); + expect(() => RrCorrector.fromJson({'version': 1, 'type': 'RrCorrector'}), + throwsFormatException); + }); + + test('an empty chunk is a no-op', () { + final s = synthRr(const SynthConfig(seed: 92, hours: 0.1)); + final run = RrStreamRun(); + run.fold(s.rr.sublist(0, 300), s.ts.sublist(0, 300)); + final before = jsonEncode(run.c.toJson()); + final out = run.c.fold(const [], tsMs: const []); + expect(out.nn, isEmpty); + expect(out.nnTimes, isEmpty); + expect(out.classes, isEmpty); + expect(jsonEncode(run.c.toJson()), before); + }); + + test('a fresh corrector reports an empty series, never NaN', () { + final snap = RrCorrector().snapshot(); + expect(snap.n, 0); + expect(snap.tailNn, isEmpty); + expect(snap.tailNnTimes, isEmpty); + expect(snap.tailClasses, isEmpty); + expect(snap.cleanFraction, 0); + expect(snap.droppedCount, 0); + expect(snap.correctedCount, 0); + expect(RrCorrector().settledBeats, 0); + }); + + test('a non-finite RR is refused and leaves the corrector unchanged', () { + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + final run = RrStreamRun(); + run.fold([800, 810, 790], [1.7e12, 1.7e12 + 810, 1.7e12 + 1600]); + final before = jsonEncode(run.c.toJson()); + // Good beats BEFORE the bad one in the same chunk must not stick. + expect( + () => run.c.fold([805, 800, bad, 810], + tsMs: [1.7e12 + 2400, 1.7e12 + 3200, 1.7e12 + 4000, 1.7e12 + 4800]), + throwsArgumentError, + reason: '$bad'); + expect(jsonEncode(run.c.toJson()), before, reason: 'unchanged $bad'); + } + }); + + test('timestamps are given on every fold or on none', () { + final a = RrCorrector()..fold([800, 810], tsMs: [1.7e12, 1.7e12 + 810]); + final before = jsonEncode(a.toJson()); + expect(() => a.fold([790, 805]), throwsStateError); + expect(jsonEncode(a.toJson()), before, reason: 'unchanged by the throw'); + final b = RrCorrector()..fold([800, 810]); + expect(() => b.fold([790, 805], tsMs: [1.0, 2.0]), throwsStateError); + }); + }); + + group('whole nights and days', () { + test('real 8.9 h WHOOP 4 night, 15-minute passes, restore every pass', () { + final s = realNightRr()!; + final full = correctRrReference(s.rr, rrTsMs: s.ts); + final cuts = timeCuts(s.ts, 900); + final run = RrStreamRun(); + var from = 0, maxLag = 0; + for (var k = 0; k < cuts.length; k++) { + run.fold(s.rr.sublist(from, cuts[k]), s.ts.sublist(from, cuts[k])); + from = cuts[k]; + run.restart(); + run.expectSettledIsPrefixOf(full, 'pass $k'); + maxLag = math.max(maxLag, cuts[k] - run.c.settledBeats); + if (k == 7 || k == 20) { + run.expectMatchesOracle(s.rr, s.ts, cuts[k], why: 'pass $k'); + } + } + run.expectMatchesOracle(s.rr, s.ts, s.length, why: 'final'); + // Final cross-check against the whole-series oracle, field by field. + final snap = run.c.snapshot(); + expectBitIdentical([...run.nn, ...snap.tailNn], full.nn, 'night nn'); + expect(sameBits(snap.cleanFraction, full.cleanFraction), isTrue); + expect(run.restarts, cuts.length); + // Real nights have artefact runs; the horizon stays within a few hundred. + expect(maxLag, lessThan(400), reason: 'max unsettled beats'); + // ignore: avoid_print + print('real night: ${s.length} beats, ${cuts.length} passes, ' + 'max unsettled=$maxLag'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('23 h day (~96k beats), 15-minute passes, restore every pass', () { + final s = realShapedDay(); + final full = correctRrReference(s.rr, rrTsMs: s.ts); + final cuts = timeCuts(s.ts, 900); + final run = RrStreamRun(); + var from = 0, maxBytes = 0, maxLag = 0; + for (var k = 0; k < cuts.length; k++) { + run.fold(s.rr.sublist(from, cuts[k]), s.ts.sublist(from, cuts[k])); + from = cuts[k]; + run.restart(); + run.expectSettledIsPrefixOf(full, 'pass $k'); + maxBytes = math.max(maxBytes, jsonEncode(run.c.toJson()).length); + maxLag = math.max(maxLag, cuts[k] - run.c.settledBeats); + if (k == 3 || k == 20) { + run.expectMatchesOracle(s.rr, s.ts, cuts[k], why: 'pass $k'); + } + } + run.expectMatchesOracle(s.rr, s.ts, s.length, why: 'final'); + expect(maxBytes, lessThan(32 * 1024), reason: 'checkpoint chars'); + expect(maxLag, lessThan(400), reason: 'max unsettled beats'); + // ignore: avoid_print + print('day: ${s.length} beats, ${cuts.length} passes, ' + 'max checkpoint=$maxBytes chars, max unsettled=$maxLag'); + }, timeout: const Timeout(Duration(minutes: 10))); + }); +} diff --git a/test/onehz/irregular_diagnostics_empty_test.dart b/test/onehz/irregular_diagnostics_empty_test.dart new file mode 100644 index 00000000..6034bef9 --- /dev/null +++ b/test/onehz/irregular_diagnostics_empty_test.dart @@ -0,0 +1,54 @@ +// The screen's artifact fraction is a share of the beats the corrector saw. When +// it saw none there is no denominator: the diagnostics carry null (absent), never +// the 1.0 a caller computes from `1 - cleanFraction` of an empty series, which +// would read as "100% of beats are artifacts" beside zero beats. Verdicts are +// unchanged: the fraction handed in still drives the artifact gate. +import 'dart:convert'; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +const _none = RrCleaningCounts(raw: 0, corrected: 0, dropped: 0); + +void main() { + test('batch: zero beats in -> no artifact fraction, same absent verdict', () { + final r = irregularBeatScreenDetailed(const [], + artifactFraction: 1.0, cleaning: _none); + expect(r.diagnostics.artifactFraction, isNull); + expect((r.diagnostics.toJson()['beats'] as Map)['artifact_fraction'], isNull); + expect(r.diagnostics.rrRaw, 0, reason: 'the raw count stays a real zero'); + expect(r.metric.present, isFalse); + expect(jsonEncode(r.metric.toJson((v) => v.toJson())), + jsonEncode(irregularBeatScreen(const [], artifactFraction: 1.0) + .toJson((v) => v.toJson()))); + }); + + test('batch: the fraction is kept whenever the corrector saw beats, or the ' + 'caller gave no counts (unchanged)', () { + const seen = RrCleaningCounts(raw: 100, corrected: 5, dropped: 2); + expect( + irregularBeatScreenDetailed(const [], artifactFraction: 0.07, cleaning: seen) + .diagnostics + .artifactFraction, + 0.07); + expect(irregularBeatScreenDetailed(const []).diagnostics.artifactFraction, 0.0); + }); + + test('streaming: zero beats in -> no artifact fraction', () { + final r = IrregularScreenState().evaluateDetailed(const [], const [], + artifactFraction: 1.0, cleaning: _none); + expect(r.diagnostics.artifactFraction, isNull); + expect(r.metric.present, isFalse); + }); + + test('the wire round-trips an absent fraction', () { + final d = irregularBeatScreenDetailed(const [], + artifactFraction: 1.0, cleaning: _none) + .diagnostics; + final text = jsonEncode(d.toJson()); + final back = IrregularDiagnostics.fromJson( + (jsonDecode(text) as Map).cast()); + expect(back.artifactFraction, isNull); + expect(jsonEncode(back.toJson()), text); + }); +} diff --git a/test/onehz/irregular_diagnostics_test.dart b/test/onehz/irregular_diagnostics_test.dart new file mode 100644 index 00000000..138ae257 --- /dev/null +++ b/test/onehz/irregular_diagnostics_test.dart @@ -0,0 +1,574 @@ +// Contract for the irregular-rhythm screen's DIAGNOSTICS (batch side). +// +// Today `irregularBeatScreen` returns SD1/SD2/ratio/pNN/n_beats/flag and throws +// away the evidence: how many beats went in, how many survived the [300, 2000] +// filter, how the corrector treated them, how many 5-minute windows existed, +// how many voted, how many flagged, what the final OPEN window was, and what +// fraction of windows flagged against what the rule needs. An abstention also +// carried only a note string. The contract: +// +// a. `irregularBeatScreenDetailed` returns the SAME Metric as +// `irregularBeatScreen` plus an `IrregularDiagnostics`, present when the +// screen ran AND when it abstained (the abstention names its cause + counts); +// c. absent input stays absent: no beats => absent metric, zero counts, no +// fabricated windows; an unknown count is null, never 0; NaN is not a beat; +// d. toJson round-trips, existing JSON keys and verdicts are unchanged. +// +// (b, batch == resumed, is test/onehz/incremental/irregular_diagnostics_stream_test.dart.) +// +// RED until implemented: the stub throws UnimplementedError. +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +// ---- fixtures ------------------------------------------------------------- + +/// Calm, slowly drifting sinus: valid windows, none flagged. +List _calm(int n) => + [for (var i = 0; i < n; i++) (1000 + 15 * math.sin(i / 8)).roundToDouble()]; + +/// Irregularly irregular: every window of it flags. +List _af(int n, int seed) { + final r = math.Random(seed); + return [ + for (var i = 0; i < n; i++) + (800.0 + (r.nextBool() ? 250 : -50) + r.nextInt(120)).roundToDouble() + ]; +} + +List _cumsum(List rr) { + var t = 0.0; + return [for (final v in rr) t += v]; +} + +/// Three-window day crafted so every window edge is hand-known (the same trick +/// as the screen-state test): window 1 opens at the first beat (t = 1000) and +/// is AF-like; the beat that lands EXACTLY 300 s after a window's first beat +/// opens the next one. Window 2 is calm. The final window holds [lastBeats] +/// beats in total (the edge beat that opens it included) of [lastMode] +/// ('calm' | 'af') and is OPEN when the data ends. +({List nn, List t}) _threeWindowDay( + int lastBeats, String lastMode) { + final r = math.Random(3); + final nn = [], t = []; + var clock = 0.0; + void add(double v) { + clock += v; + nn.add(v); + t.add(clock); + } + + void landOn(double target, double Function() next) { + while (target - clock > 2200) { + add(next()); + } + var rest = target - clock; + if (rest < 300) { + clock -= nn.removeLast(); + t.removeLast(); + rest = target - clock; + } + if (rest > 2000) { + add(rest / 2); + add(rest / 2); + } else { + add(rest); + } + } + + double afLike() => 400.0 + 100 * r.nextInt(13); + double calm() => 800.0 + 10 * (r.nextInt(5) - 2); + add(1000); + landOn(301000, afLike); + landOn(601000, calm); + for (var i = 1; i < lastBeats; i++) { + add(lastMode == 'af' ? afLike() : calm()); + } + expect(t.where((x) => x == 301000 || x == 601000).length, 2, + reason: 'fixture: both window edges landed exactly'); + return (nn: nn, t: t); +} + +/// Independent window oracle: the same bucketing rule written out, each window +/// judged by calling the PUBLIC screen on its beats (whole-span verdict, no +/// times), so it shares no code with the window helpers it checks. Only for +/// series with no out-of-range beats (no gaps inside a bucket). +({int total, int valid, int flagged, int openBeats, IrregularOpenWindow open}) + _refWindows(List nn, List t, + {double windowMin = 5, int minWindowBeats = 40}) { + var total = 0, valid = 0, flagged = 0; + var bucket = []; + IrregularOpenWindow last = IrregularOpenWindow.none; + void close() { + if (bucket.isEmpty) return; + total++; + if (bucket.length < minWindowBeats) { + last = IrregularOpenWindow.thin; + } else { + valid++; + final m = irregularBeatScreen(bucket, minBeats: 2); + final f = m.present && m.value!.flag; + if (f) flagged++; + last = f ? IrregularOpenWindow.flagged : IrregularOpenWindow.unflagged; + } + } + + var start = t.isEmpty ? 0.0 : t.first; + var openBeats = 0; + for (var i = 0; i < nn.length; i++) { + if (t[i] - start >= windowMin * 60000) { + close(); + bucket = []; + start = t[i]; + } + bucket.add(nn[i]); + } + openBeats = bucket.length; + close(); + return ( + total: total, + valid: valid, + flagged: flagged, + openBeats: openBeats, + open: last, + ); +} + +String _json(Object? o) => jsonEncode(o); + +void main() { + group('present screen: the evidence behind a verdict', () { + test('beat counts: raw in, kept after the [300,2000] filter, NaN is not a ' + 'beat', () { + final rr = _af(1200, 1); + rr[10] = 250; // short + rr[11] = 2500; // long + rr[12] = double.nan; + rr[500] = 299.999; + final res = irregularBeatScreenDetailed(rr, + nnTimesMs: _cumsum(rr.map((v) => v.isFinite ? v : 800.0).toList())); + final d = res.diagnostics; + expect(d.nnIn, 1200); + expect(d.nnKept, 1200 - 4, reason: '250, 2500, NaN and 299.999 are out'); + expect(res.metric.present, isTrue); + expect(d.nnKept, res.metric.value!.nBeats, + reason: 'diagnostics and value agree on the beats analysed'); + expect(d.abstain, isNull); + expect(d.artifactFraction, 0.0); + }); + + test('the verdict is exactly irregularBeatScreen\'s, to the last digit', () { + for (final rr in [_calm(1200), _af(1200, 7), _af(900, 8)]) { + final t = _cumsum(rr); + final plain = irregularBeatScreen(rr, nnTimesMs: t); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: t); + expect(_json(res.metric.toJson((v) => v.toJson())), + _json(plain.toJson((v) => v.toJson()))); + } + }); + + test('hand-known windows: AF-like, calm, then an OPEN window of exactly ' + 'minWindowBeats (40) calm beats is VALID and not flagged', () { + final x = _threeWindowDay(40, 'calm'); + final res = irregularBeatScreenDetailed(x.nn, nnTimesMs: x.t); + expect(res.metric.present, isTrue, reason: 'sanity: evaluable day'); + final w = res.diagnostics.windows!; + expect(w.total, 3); + expect(w.valid, 3); + expect(w.flagged, 1); + expect(w.openBeats, 40); + expect(w.open, IrregularOpenWindow.unflagged); + expect(w.sustainedObserved, closeTo(1 / 3, 1e-12)); + expect(res.diagnostics.thresholds.sustainedFraction, 0.5, + reason: 'required vs observed: 0.5 needed, 1/3 seen'); + expect(res.metric.value!.flag, isFalse); + }); + + test('hand-known windows: an open window of 39 beats is EXCLUDED (thin) but ' + 'still counted in total', () { + final x = _threeWindowDay(39, 'calm'); + final w = + irregularBeatScreenDetailed(x.nn, nnTimesMs: x.t).diagnostics.windows!; + expect(w.total, 3); + expect(w.valid, 2, reason: 'the 39-beat tail does not vote'); + expect(w.flagged, 1); + expect(w.openBeats, 39); + expect(w.open, IrregularOpenWindow.thin); + expect(w.sustainedObserved, 0.5); + }); + + test('hand-known windows: an open window of AF-like beats is VALID and ' + 'FLAGGED', () { + final x = _threeWindowDay(60, 'af'); + final res = irregularBeatScreenDetailed(x.nn, nnTimesMs: x.t); + final w = res.diagnostics.windows!; + expect(w.total, 3); + expect(w.valid, 3); + expect(w.flagged, 2); + expect(w.openBeats, 60); + expect(w.open, IrregularOpenWindow.flagged); + expect(w.sustainedObserved, closeTo(2 / 3, 1e-12)); + }); + + test('window counts equal an independent oracle across seeds', () { + for (final seed in [1, 2, 3, 4, 5, 6]) { + final r = math.Random(seed); + // Blocks of calm and AF of random length, so windows straddle modes. + final rr = []; + while (rr.length < 3000) { + rr.addAll(r.nextBool() ? _calm(100 + r.nextInt(500)) : _af(100 + r.nextInt(500), seed)); + } + final t = _cumsum(rr); + final ref = _refWindows(rr, t); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: t); + final w = res.diagnostics.windows!; + expect(w.total, ref.total, reason: 'seed $seed'); + expect(w.valid, ref.valid, reason: 'seed $seed'); + expect(w.flagged, ref.flagged, reason: 'seed $seed'); + expect(w.openBeats, ref.openBeats, reason: 'seed $seed'); + expect(w.open, ref.open, reason: 'seed $seed'); + expect(w.valid, lessThanOrEqualTo(w.total)); + expect(w.flagged, lessThanOrEqualTo(w.valid)); + if (res.metric.present && res.metric.value!.flag) { + expect(w.sustainedObserved, greaterThanOrEqualTo(0.5), + reason: 'a flag implies the sustained rule was met'); + } + } + }); + + test('windows are counted even when the aggregate is not high (they explain ' + 'a NOT flagged too)', () { + final rr = _calm(1500); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: _cumsum(rr)); + final w = res.diagnostics.windows!; + expect(res.metric.value!.flag, isFalse); + expect(w.total, greaterThan(1)); + expect(w.valid, greaterThan(1)); + expect(w.flagged, 0); + expect(w.sustainedObserved, 0.0, reason: 'valid windows, none flagged: 0, a fact'); + }); + + test('cleaning counts pass through; without them they are null, not 0', () { + final rr = _af(1200, 3); + final t = _cumsum(rr); + final withCounts = irregularBeatScreenDetailed(rr, + nnTimesMs: t, + artifactFraction: 0.07, + cleaning: const RrCleaningCounts(raw: 1290, corrected: 41, dropped: 49)); + expect(withCounts.diagnostics.rrRaw, 1290); + expect(withCounts.diagnostics.corrected, 41); + expect(withCounts.diagnostics.dropped, 49); + expect(withCounts.diagnostics.artifactFraction, 0.07); + final without = irregularBeatScreenDetailed(rr, nnTimesMs: t); + expect(without.diagnostics.rrRaw, isNull); + expect(without.diagnostics.corrected, isNull); + expect(without.diagnostics.dropped, isNull); + }); + + test('real corrector output: counts equal correctRr\'s own', () { + final r = math.Random(11); + final raw = [ + for (var i = 0; i < 2500; i++) + (i % 97 == 50 + ? 420 // isolated early beat + : (i % 211 >= 100 && i % 211 < 106) + ? 250 + r.nextInt(1800) // a noise run + : 900 + 30 * math.sin(i / 7) + r.nextInt(20)) + .toDouble() + ]; + final c = correctRr(raw); + expect(c.correctedCount, greaterThan(0), reason: 'fixture corrects'); + expect(c.droppedCount, greaterThan(0), reason: 'fixture drops'); + final res = irregularBeatScreenDetailed(c.nn, + nnTimesMs: c.nnTimesMs, + artifactFraction: (1 - c.cleanFraction).clamp(0.0, 1.0), + cleaning: RrCleaningCounts( + raw: raw.length, + corrected: c.correctedCount, + dropped: c.droppedCount)); + expect(res.diagnostics.rrRaw, raw.length); + expect(res.diagnostics.corrected, c.correctedCount); + expect(res.diagnostics.dropped, c.droppedCount); + expect(res.diagnostics.nnIn, c.nn.length); + }); + }); + + group('abstention carries the counts that caused it', () { + test('too few beats', () { + final rr = _calm(212); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: _cumsum(rr)); + expect(res.metric.present, isFalse); + final d = res.diagnostics; + expect(d.abstain, IrregularAbstain.tooFewBeats); + expect(d.nnIn, 212); + expect(d.nnKept, 212); + expect(d.thresholds.minBeats, irregularScreenMinBeats); + expect(res.metric.note, contains('too few clean beats'), + reason: 'the existing note is unchanged'); + }); + + test('too few beats because the filter removed them (kept < minBeats <= in)', + () { + final rr = [ + for (var i = 0; i < 700; i++) i % 2 == 0 ? 1000.0 + i % 40 : 250.0 + ]; // 350 kept + final d = irregularBeatScreenDetailed(rr).diagnostics; + expect(d.abstain, IrregularAbstain.tooFewBeats); + expect(d.nnIn, 700); + expect(d.nnKept, 350); + }); + + test('artifact fraction over the line', () { + final rr = _calm(1200); + final res = irregularBeatScreenDetailed(rr, + nnTimesMs: _cumsum(rr), + artifactFraction: 0.45, + cleaning: const RrCleaningCounts(raw: 2200, corrected: 100, dropped: 900)); + expect(res.metric.present, isFalse); + final d = res.diagnostics; + expect(d.abstain, IrregularAbstain.artifact); + expect(d.artifactFraction, 0.45); + expect(d.thresholds.maxArtifact, 0.30); + expect(d.nnKept, 1200); + expect(d.dropped, 900); + expect(d.corrected, 100); + }); + + // The gate counts successive pairs (upstream, merged 2026-10-18), so with + // any minBeats >= 2 an all-isolated series is "too few"; minBeats 1 is the + // one setting that still reaches the no-pairs abstention. + test('no successive clean pairs (every kept beat is isolated)', () { + final rr = [ + for (var i = 0; i < 1000; i++) i % 2 == 0 ? 900.0 + i % 50 : 250.0 + ]; + expect(irregularBeatScreenDetailed(rr, minBeats: 100).diagnostics.abstain, + IrregularAbstain.tooFewBeats); + final res = irregularBeatScreenDetailed(rr, minBeats: 1); + expect(res.metric.present, isFalse); + expect(res.diagnostics.abstain, IrregularAbstain.noSuccessivePairs); + expect(res.diagnostics.nnIn, 1000); + expect(res.diagnostics.nnKept, 500); + }); + + test('no long-term variability (SD2 = 0), not "perfectly regular"', () { + final rr = [for (var i = 0; i < 1200; i++) 1000.0]; + final res = irregularBeatScreenDetailed(rr, nnTimesMs: _cumsum(rr)); + expect(res.metric.present, isFalse); + expect(res.diagnostics.abstain, IrregularAbstain.noLongTermVariability); + expect(res.diagnostics.nnKept, 1200); + }); + + test('an abstained screen still reports its windows (they were there)', () { + final x = _threeWindowDay(60, 'af'); + final res = irregularBeatScreenDetailed(x.nn, + nnTimesMs: x.t, artifactFraction: 0.5); + expect(res.metric.present, isFalse); + expect(res.diagnostics.abstain, IrregularAbstain.artifact); + expect(res.diagnostics.windows!.total, 3); + expect(res.diagnostics.windows!.flagged, 2); + }); + }); + + group('absent input stays absent', () { + test('no beats: absent metric, zero counts, no fabricated windows', () { + final res = irregularBeatScreenDetailed(const [], nnTimesMs: const []); + expect(res.metric.present, isFalse); + expect(res.metric.confidence, 0); + final d = res.diagnostics; + expect(d.abstain, IrregularAbstain.tooFewBeats); + expect(d.nnIn, 0); + expect(d.nnKept, 0); + expect(d.rrRaw, isNull, reason: 'no cleaning info given: unknown, not 0'); + final w = d.windows!; + expect((w.total, w.valid, w.flagged, w.openBeats), (0, 0, 0, 0)); + expect(w.open, IrregularOpenWindow.none); + expect(w.sustainedObserved, isNull, reason: 'no valid window: null, not 0 or NaN'); + }); + + test('no beat times: windows are NOT evaluated (null), not zero windows', () { + final res = irregularBeatScreenDetailed(_af(1200, 2)); + expect(res.diagnostics.windows, isNull); + expect(res.metric.present, isTrue); + }); + + test('times of the wrong length: windows null, whole-span verdict as before', + () { + final rr = _af(1200, 2); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: _cumsum(rr).sublist(1)); + expect(res.diagnostics.windows, isNull); + expect(_json(res.metric.toJson((v) => v.toJson())), + _json(irregularBeatScreen(rr, nnTimesMs: _cumsum(rr).sublist(1)).toJson((v) => v.toJson()))); + }); + + test('a window config that fails closed: windows null, never flagged', () { + final rr = _af(1200, 2); + final res = irregularBeatScreenDetailed(rr, + nnTimesMs: _cumsum(rr), windowMinutes: 0); + expect(res.diagnostics.windows, isNull); + expect(res.metric.value!.flag, isFalse); + }); + }); + + group('JSON', () { + List variety() { + final x = _threeWindowDay(40, 'calm'); + final af = _af(1200, 5); + return [ + irregularBeatScreenDetailed(x.nn, + nnTimesMs: x.t, + cleaning: const RrCleaningCounts(raw: 900, corrected: 3, dropped: 4)), + irregularBeatScreenDetailed(af, nnTimesMs: _cumsum(af)), + irregularBeatScreenDetailed(af), // windows null, cleaning null + irregularBeatScreenDetailed(_calm(100)), // tooFewBeats + irregularBeatScreenDetailed(_calm(1200), artifactFraction: 0.5), + irregularBeatScreenDetailed([for (var i = 0; i < 1200; i++) 1000.0]), + irregularBeatScreenDetailed(const []), + ]; + } + + test('every shape survives real JSON text and comes back identical', () { + for (final r in variety()) { + final text = jsonEncode(r.diagnostics.toJson()); + final back = + IrregularDiagnostics.fromJson(jsonDecode(text) as Map); + expect(jsonEncode(back.toJson()), text); + expect(back.abstain, r.diagnostics.abstain); + expect(back.nnIn, r.diagnostics.nnIn); + expect(back.nnKept, r.diagnostics.nnKept); + expect(back.rrRaw, r.diagnostics.rrRaw); + expect(back.windows?.total, r.diagnostics.windows?.total); + expect(back.windows?.open, r.diagnostics.windows?.open); + } + }); + + test('the wire shape (what edge persists and Nerd stats reads)', () { + final x = _threeWindowDay(40, 'calm'); + final j = irregularBeatScreenDetailed(x.nn, + nnTimesMs: x.t, + artifactFraction: 0.1, + cleaning: const RrCleaningCounts(raw: 900, corrected: 3, dropped: 4)) + .diagnostics + .toJson(); + expect(j['version'], 1); + expect(j['abstain'], isNull); + final b = j['beats'] as Map; + expect(b.keys.toSet(), { + 'rr_raw', 'nn_in', 'nn_kept', 'corrected', 'dropped', 'artifact_fraction' + }); + expect(b['rr_raw'], 900); + expect(b['corrected'], 3); + expect(b['dropped'], 4); + expect(b['nn_in'], x.nn.length); + expect(b['artifact_fraction'], 0.1); + final w = j['windows'] as Map; + expect(w.keys.toSet(), { + 'total', 'valid', 'flagged', 'sustained_observed', 'open_beats', 'open' + }); + expect(w['total'], 3); + expect(w['open'], 'unflagged'); + expect(w['open_beats'], 40); + final th = j['thresholds'] as Map; + expect(th.keys.toSet(), { + 'min_beats', 'max_artifact', 'sd1sd2_flag', 'pnn_threshold_ms', + 'pnn_flag_pct', 'window_minutes', 'min_window_beats', 'sustained_fraction' + }); + expect(th['min_beats'], 500); + expect(th['sustained_fraction'], 0.5); + }); + + test('abstain wire names; unknown counts and windows are JSON null', () { + expect( + irregularBeatScreenDetailed(_calm(100)).diagnostics.toJson()['abstain'], + 'too_few_beats'); + expect( + irregularBeatScreenDetailed(_calm(1200), artifactFraction: 0.5) + .diagnostics.toJson()['abstain'], + 'artifact'); + final j = irregularBeatScreenDetailed(_calm(100)).diagnostics.toJson(); + expect(j['windows'], isNull); + expect((j['beats'] as Map)['rr_raw'], isNull); + expect((j['beats'] as Map)['corrected'], isNull); + expect((j['beats'] as Map)['dropped'], isNull); + expect( + irregularBeatScreenDetailed([for (var i = 0; i < 1200; i++) 1000.0]) + .diagnostics.toJson()['abstain'], + 'no_long_term_variability'); + expect( + irregularBeatScreenDetailed([ + for (var i = 0; i < 1000; i++) i % 2 == 0 ? 900.0 + i % 50 : 250.0 + ], minBeats: 1).diagnostics.toJson()['abstain'], + 'no_successive_pairs'); + }); + + test('fromJson refuses another version and malformed maps', () { + final j = irregularBeatScreenDetailed(_calm(1200)).diagnostics.toJson(); + expect(() => IrregularDiagnostics.fromJson({...j, 'version': 2}), + throwsFormatException); + expect(() => IrregularDiagnostics.fromJson({...j, 'beats': 'x'}), + throwsFormatException); + expect(() => IrregularDiagnostics.fromJson({...j, 'abstain': 'because'}), + throwsFormatException); + expect(() => IrregularDiagnostics.fromJson(const {}), throwsFormatException); + }); + + test('IrregularScreenResult.toJson = the old envelope + a diagnostics key', + () { + final rr = _af(1200, 9); + final t = _cumsum(rr); + final res = irregularBeatScreenDetailed(rr, nnTimesMs: t); + final env = res.toJson(); + final old = irregularBeatScreen(rr, nnTimesMs: t).toJson((v) => v.toJson()); + expect(env.keys.toSet(), {...old.keys, 'diagnostics'}); + expect(_json({...env}..remove('diagnostics')), _json(old), + reason: 'every pre-existing key and value is untouched'); + expect(_json(env['diagnostics']), _json(res.diagnostics.toJson())); + // Absent envelope too. + final ab = irregularBeatScreenDetailed(_calm(50)).toJson(); + expect(ab['value'], '—'); + expect(ab['confidence'], 0); + expect(ab['diagnostics'], isA()); + }); + }); + + group('regression: nothing the screen already says changes', () { + test('IrregularRhythm.toJson keeps exactly its six keys (green today)', () { + final j = irregularBeatScreen(_af(1200, 1)).value!.toJson(); + expect(j.keys.toSet(), + {'sd1_ms', 'sd2_ms', 'sd1_sd2', 'pnn_pct', 'n_beats', 'flag'}); + }); + + test('same flag / sd1 / sd2 / pnn / nBeats / confidence / note on the ' + 'existing test inputs, and on salted ones', () { + final r = math.Random(7); + final salted = _af(2000, 4); + salted[5] = 250; + salted[900] = 2200; + final cases = <({List rr, List? t, double af})>[ + (rr: [for (var i = 0; i < 1200; i++) 1000 + 15 * math.sin(i / 8)], t: null, af: 0.0), + (rr: [for (var i = 0; i < 1200; i++) 800.0 + (r.nextBool() ? 250 : -50) + r.nextInt(120)], t: null, af: 0.0), + (rr: [for (var i = 0; i < 100; i++) 1000.0], t: null, af: 0.0), + (rr: [for (var i = 0; i < 1200; i++) 1000.0], t: null, af: 0.4), + (rr: salted, t: null, af: 0.1), + ]; + for (final c in cases) { + final t = c.t ?? _cumsum(c.rr); + for (final times in [null, t]) { + final plain = irregularBeatScreen(c.rr, nnTimesMs: times, artifactFraction: c.af); + final det = irregularBeatScreenDetailed(c.rr, nnTimesMs: times, artifactFraction: c.af); + expect(det.metric.present, plain.present); + expect(det.metric.note, plain.note); + expect(det.metric.confidence, plain.confidence); + expect(det.metric.tier, plain.tier); + expect(det.metric.inputs_used, plain.inputs_used); + if (plain.present) { + final a = det.metric.value!, b = plain.value!; + expect((a.flag, a.nBeats, a.pnnPct, a.sd1, a.sd2, a.sd1sd2), + (b.flag, b.nBeats, b.pnnPct, b.sd1, b.sd2, b.sd1sd2)); + } + } + } + }); + }); +} diff --git a/test/onehz/rr_artifact_cap_test.dart b/test/onehz/rr_artifact_cap_test.dart new file mode 100644 index 00000000..773473c5 --- /dev/null +++ b/test/onehz/rr_artifact_cap_test.dart @@ -0,0 +1,336 @@ +// RED: RR artifact-run cap (combined bump, chunk A1). +// +// Contract: +// * `kMaxArtifactRun == 32` is the default of `correctRr(maxArtifactRun:)` and +// `RrCorrector(maxArtifactRun:)`; null means uncapped (today's behaviour). +// * Every artifact run <= K => output bit-identical to today's (the frozen +// reference in support/correct_rr_reference.dart): nn, nnTimesMs, classes, +// counts, cleanFraction. +// * A run LONGER than K splits the series: the output is +// correctRr(before) ++ correctRr(after), times kept absolute (the clock +// keeps running through the run), and the run's beats are ABSENT (a gap, +// not a correction). Each side is corrected as a series of its own, so its +// windows do not see the run. +// * The capped streaming `RrCorrector` equals the capped batch at every prefix +// (settled ++ snapshot tail), for any chunking and with JSON save/restore. +// * Its state is bounded: a checkpoint does not grow with an unbounded run. +// +// The long-run series are built so the split is unambiguous: clean integer RR, +// runs of 3000 ms beats (> 2000 is a hard artifact whatever the windows say), +// timestamps = exact cumulative sums, so every clock value is an integer and the +// comparison can be bit-exact. +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/property.dart'; +import 'support/correct_rr_reference.dart'; +import 'support/rr_chunking.dart'; +import 'support/rr_compare.dart'; +import 'support/rr_runs.dart'; +import 'support/rr_synth.dart'; + +const _t0 = rrRunsT0; + +int _longestRun(List classes) { + var best = 0, cur = 0; + for (final c in classes) { + cur = c == BeatClass.normal ? 0 : cur + 1; + best = math.max(best, cur); + } + return best; +} + +/// What the cap promises for [runs] (each longer than K): the pieces between +/// the runs corrected on their own, their times moved onto the one clock. +void _expectSplit(RrCorrectionResult got, RunSeries s, List<(int, int)> runs, String why) { + final n = s.rr.length; + final bounds = <(int, int)>[]; + var from = 0; + for (final (a, b) in runs) { + bounds.add((from, a)); + from = b; + } + bounds.add((from, n)); + final nn = [], times = []; + for (final (a, b) in bounds) { + final piece = correctRr(s.rr.sublist(a, b), + rrTsMs: s.ts.sublist(a, b), maxArtifactRun: null); + final shift = (s.ts[a] - s.rr[a]) - (s.ts[0] - s.rr[0]); + nn.addAll(piece.nn); + times.addAll([for (final t in piece.nnTimesMs) t + shift]); + expectSameClasses(got.classes.sublist(a, b), piece.classes, + '$why: classes of piece [$a, $b)'); + } + for (final (a, b) in runs) { + // the run's own beats sit between the pieces: artifacts, absent from nn + expect(got.classes.sublist(a, b).every((c) => c != BeatClass.normal), isTrue, + reason: '$why: run [$a, $b) is all artifact'); + } + expect(got.classes.length, n, reason: '$why: one class per input beat'); + expectBitIdentical(got.nn, nn, '$why nn'); + expectBitIdentical(got.nnTimesMs, times, '$why nnTimesMs'); +} + +void main() { + test('kMaxArtifactRun is 32 (2x the longest run on 18 days of real data)', () { + expect(kMaxArtifactRun, 32); + }); + + group('runs <= K: bit-identical to today', () { + forAll('synthetic series whose longest artifact run fits', G.intIn(1, 100000), + (seed) { + final s = synthRr(SynthConfig( + seed: seed, + hours: 0.12, + ectopicPerMin: 1.5, + missedPerMin: 0.8, + extraPerMin: 0.8, + noiseRunPerMin: 0.6, + gapPerHour: 10)); + final want = correctRrReference(s.rr, rrTsMs: s.ts); + final longest = _longestRun(want.classes); + if (longest > kMaxArtifactRun) return; // another property's business + expectSameCorrection(correctRr(s.rr, rrTsMs: s.ts), want, 'default cap'); + expectSameCorrection( + correctRr(s.rr, rrTsMs: s.ts, maxArtifactRun: kMaxArtifactRun), + want, + 'K = 32'); + expectSameCorrection(correctRr(s.rr, rrTsMs: s.ts, maxArtifactRun: null), + want, 'uncapped'); + // The cap is a ceiling: K equal to the longest run changes nothing. + expectSameCorrection( + correctRr(s.rr, rrTsMs: s.ts, maxArtifactRun: math.max(1, longest)), + want, + 'K = longest run ($longest)'); + }, cases: 30, budget: const Duration(seconds: 60)); + + test('the checked-in real night and a real-shaped day', () { + final night = realNightRr()!; + final day = realShapedDay(); + for (final (name, d) in [('night', night), ('day', day)]) { + final want = correctRrReference(d.rr, rrTsMs: d.ts); + expect(_longestRun(want.classes), lessThanOrEqualTo(kMaxArtifactRun), + reason: '$name: precondition, no run over the cap'); + expectSameCorrection(correctRr(d.rr, rrTsMs: d.ts), want, name); + } + }); + + test('a run of exactly K is still corrected the old way (dropped, no split)', () { + final (s, runs) = withRuns(5, [300, 300], [kMaxArtifactRun], nearRun: true); + expect(runs.single.$2 - runs.single.$1, kMaxArtifactRun); + expectSameCorrection(correctRr(s.rr, rrTsMs: s.ts), + correctRrReference(s.rr, rrTsMs: s.ts), 'run == K'); + }); + }); + + group('a run longer than K is a gap', () { + test('K + 1 splits the series', () { + final (s, runs) = withRuns(6, [300, 300], [kMaxArtifactRun + 1]); + _expectSplit(correctRr(s.rr, rrTsMs: s.ts), s, runs, 'K+1'); + }); + + test('one 40-beat run, clean sides', () { + final (s, runs) = withRuns(7, [360, 360], [40]); + _expectSplit(correctRr(s.rr, rrTsMs: s.ts), s, runs, '40-run'); + }); + + test('isolated artifacts 20 beats either side of the run are judged without ' + 'the run in their window', () { + final (s, runs) = withRuns(8, [360, 360], [40], nearRun: true); + _expectSplit(correctRr(s.rr, rrTsMs: s.ts), s, runs, 'near-run artifacts'); + }); + + for (final seed in [11, 12, 13, 14, 15, 16]) { + test('two long runs, random lengths and sides (seed $seed)', () { + final r = Rng(seed); + final (s, runs) = withRuns( + seed, + [r.intIn(150, 400), r.intIn(150, 400), r.intIn(150, 400)], + [r.intIn(33, 90), r.intIn(33, 90)], + nearRun: r.nextBool()); + _expectSplit(correctRr(s.rr, rrTsMs: s.ts), s, runs, 'seed $seed'); + }); + } + + test('an explicit smaller K splits at that K', () { + final (s, runs) = withRuns(9, [300, 300], [6]); + _expectSplit(correctRr(s.rr, rrTsMs: s.ts, maxArtifactRun: 5), s, runs, 'K=5'); + // ...and the same run is under the default cap: not a split. + expectSameCorrection(correctRr(s.rr, rrTsMs: s.ts), + correctRrReference(s.rr, rrTsMs: s.ts), 'K=32 keeps the 6-run'); + }); + + test('uncapped (null) still drops the run and keeps one series', () { + final (s, _) = withRuns(10, [300, 300], [40]); + expectSameCorrection(correctRr(s.rr, rrTsMs: s.ts, maxArtifactRun: null), + correctRrReference(s.rr, rrTsMs: s.ts), 'null'); + }); + }); + + group('streaming RrCorrector with the cap', () { + void expectPrefix(RrCorrector c, List nn, List times, + List classes, RunSeries s, int n, int k, String why) { + final snap = c.snapshot(); + final want = correctRr(s.rr.sublist(0, n), + rrTsMs: s.ts.sublist(0, n), maxArtifactRun: k); + expectBitIdentical([...nn, ...snap.tailNn], want.nn, '$why nn'); + expectBitIdentical([...times, ...snap.tailNnTimes], want.nnTimesMs, '$why t'); + expectSameClasses([...classes, ...snap.tailClasses], want.classes, why); + expect(snap.droppedCount, want.droppedCount, reason: '$why dropped'); + expect(snap.correctedCount, want.correctedCount, reason: '$why corrected'); + expect(sameBits(snap.cleanFraction, want.cleanFraction), isTrue, + reason: '$why cleanFraction'); + } + + for (final k in [32, 5]) { + test('one beat at a time, every prefix, K = $k, a 40-beat run in the middle', () { + final (s, _) = withRuns(21, [110, 110], [40], nearRun: true); + final c = RrCorrector(maxArtifactRun: k); + final nn = [], times = []; + final classes = []; + for (var i = 0; i < s.rr.length; i++) { + final out = c.fold([s.rr[i]], tsMs: [s.ts[i]]); + nn.addAll(out.nn); + times.addAll(out.nnTimes); + classes.addAll(out.classes); + expectPrefix(c, nn, times, classes, s, i + 1, k, 'K=$k n=${i + 1}'); + } + }); + } + + for (final seed in [31, 32, 33]) { + test('random chunks with a JSON save/restore between them (seed $seed)', () { + final r = Rng(seed); + final (s, _) = withRuns( + seed, [r.intIn(120, 250), r.intIn(120, 250)], [r.intIn(33, 70)], + nearRun: true); + var c = RrCorrector(); + final nn = [], times = []; + final classes = []; + var at = 0; + for (final cut in randomCuts(math.Random(seed), s.rr.length)) { + final out = c.fold(s.rr.sublist(at, cut), tsMs: s.ts.sublist(at, cut)); + nn.addAll(out.nn); + times.addAll(out.nnTimes); + classes.addAll(out.classes); + at = cut; + c = RrCorrector.fromJson(jsonRoundTrip(c.toJson())); + expect(c.maxArtifactRun, kMaxArtifactRun, reason: 'cap survives restore'); + expectPrefix(c, nn, times, classes, s, at, kMaxArtifactRun, 'seed $seed n=$at'); + } + }); + } + + test('settling through a gap retains the new segment boundary', () { + final (s, _) = withRuns(51, [300, 700], [100], nearRun: true); + var c = RrCorrector(); + final nn = [], times = []; + final classes = []; + for (var i = 0; i < s.rr.length; i++) { + final out = c.fold([s.rr[i]], tsMs: [s.ts[i]]); + nn.addAll(out.nn); + times.addAll(out.nnTimes); + classes.addAll(out.classes); + if (i % 19 == 0 || i == s.rr.length - 1) { + c = RrCorrector.fromJson(jsonRoundTrip(c.toJson())); + expectPrefix(c, nn, times, classes, s, i + 1, kMaxArtifactRun, + 'across a settled gap at $i'); + } + } + }); + + test('maxArtifactRun survives the checkpoint (a value, the default, null)', () { + for (final k in [5, kMaxArtifactRun, null]) { + final c = RrCorrector(maxArtifactRun: k) + ..fold([for (var i = 0; i < 50; i++) 800.0 + i % 7], + tsMs: [for (var i = 0; i < 50; i++) _t0 + 800.0 * i]); + final back = RrCorrector.fromJson(jsonRoundTrip(c.toJson())); + expect(back.maxArtifactRun, k, reason: 'K = $k'); + } + expect(RrCorrector().maxArtifactRun, kMaxArtifactRun); + }); + + test('the checkpoint is bounded: 10,000 artifact beats do not grow it', () { + final r = Rng(41); + final clean = cleanRr(r, 300); + final c = RrCorrector(); + var t = _t0; + c.fold(clean, tsMs: [for (final v in clean) t += v]); + final before = jsonEncode(c.toJson()).length; + var worst = before; + for (var fed = 0; fed < 10000; fed += 100) { + final chunk = List.filled(100, 3000.0); + c.fold(chunk, tsMs: [for (final v in chunk) t += v]); + worst = math.max(worst, jsonEncode(c.toJson()).length); + } + expect(worst, lessThan(60000), reason: 'checkpoint chars during the run'); + expect(worst, lessThan(before * 3 + 2000), + reason: 'does not scale with the run (started at $before)'); + // and it resumes: after the run the series is a new segment, and the + // stream still equals the capped batch over everything it was fed. + final tail = cleanRr(r, 300); + final out = c.fold(tail, tsMs: [for (final v in tail) t += v]); + final rrAll = [...clean, ...List.filled(10000, 3000.0), ...tail]; + var u = _t0; + final all = RunSeries(rrAll, [for (final v in rrAll) u += v]); + final want = correctRr(rrAll, rrTsMs: all.ts); + final snap = c.snapshot(); + expect(snap.n, rrAll.length); + expect(out.nn.length, greaterThan(0), reason: 'the tail settles'); + expect(snap.droppedCount, want.droppedCount); + expect(snap.correctedCount, want.correctedCount); + expect(sameBits(snap.cleanFraction, want.cleanFraction), isTrue); + }); + }); + group('nnSourceIndex: which input beat each NN came from', () { + void expectSources(List rr, List ts, String why, + {int? cap = kMaxArtifactRun}) { + final got = correctRr(rr, rrTsMs: ts, maxArtifactRun: cap); + final src = got.nnSourceIndex; + expect(src.length, got.nn.length, reason: '$why length'); + for (var k = 0; k < src.length; k++) { + expect(src[k], inInclusiveRange(0, rr.length - 1), reason: '$why range'); + if (k > 0) expect(src[k], greaterThan(src[k - 1]), reason: '$why order'); + if (got.classes[src[k]] == BeatClass.normal) { + expect(sameBits(got.nn[k], rr[src[k]]), isTrue, reason: '$why value'); + } + } + // every beat that is not a source was dropped + expect(rr.length - src.length, got.droppedCount, reason: '$why dropped'); + } + + test('synthetic nights (corrected, dropped, gap beats)', () { + for (final seed in [3, 23, 77]) { + final d = synthRr(SynthConfig( + seed: seed, hours: 0.2, ectopicPerMin: 2, missedPerMin: 1, + extraPerMin: 1, noiseRunPerMin: 0.5, gapPerHour: 12)); + expectSources(d.rr, d.ts, 'seed $seed'); + expectSources(d.rr, d.ts, 'seed $seed uncapped', cap: null); + } + }); + + test('over the cap, tiny inputs, and empty', () { + final (s, runs) = withRuns(8, [120, 120, 120], [33, 45]); + expectSources(s.rr, s.ts, 'runs'); + final src = correctRr(s.rr, rrTsMs: s.ts).nnSourceIndex; + for (final (a, b) in runs) { + expect(src.where((i) => i >= a && i < b), isEmpty); + } + for (final n in [0, 1, 2, 3, 5]) { + expectSources(s.rr.sublist(0, n), s.ts.sublist(0, n), 'n=$n'); + } + }); + + test('a spline-corrected beat is its own source', () { + final rr = cleanRr(Rng(5), 200); + rr[100] = rr[100] * 0.5; + final ts = rrSeries(rr).ts; + final got = correctRr(rr, rrTsMs: ts); + expect(got.correctedCount, 1); + expect(got.nnSourceIndex, contains(100)); + }); + }); +} diff --git a/test/onehz/rr_correction_oracle_test.dart b/test/onehz/rr_correction_oracle_test.dart new file mode 100644 index 00000000..6fd13601 --- /dev/null +++ b/test/onehz/rr_correction_oracle_test.dart @@ -0,0 +1,322 @@ +// Guard for the `correctRr` speed rewrite: the production function must stay +// BIT-IDENTICAL to a frozen copy of the implementation it replaces +// (support/correct_rr_reference.dart) — nn, nnTimesMs, per-beat classes, counts +// and cleanFraction — on synthetic, real-shaped and hostile inputs. +// +// This passes against today's implementation by construction (it IS the frozen +// copy). It is the net under the rewrite, not a failing test. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import 'support/correct_rr_reference.dart'; +import 'support/rr_compare.dart'; +import 'support/rr_synth.dart'; + +void _check(List rr, List? ts, String why, + {double alpha = 5.2, + int win = 91, + double floor = 100, + double reanchor = 1000}) { + final want = correctRrReference(rr, + rrTsMs: ts, + alpha: alpha, + windowBeats: win, + minThresholdMs: floor, + reanchorGapMs: reanchor); + final got = correctRr(rr, + rrTsMs: ts, + alpha: alpha, + windowBeats: win, + minThresholdMs: floor, + reanchorGapMs: reanchor); + expectSameCorrection(got, want, why); + // The input must not be modified. + expect(got.classes.length, rr.length, reason: '$why classes per input beat'); +} + +/// Timestamps for [rr] in one of several shapes. `null` = none supplied. +List? _timestamps(math.Random r, List rr, int mode) { + final n = rr.length; + var t = 1.7e12; + switch (mode) { + case 0: + return null; + case 1: // contiguous, whole-second like rec_ts*1000 + return [ + for (final v in rr) ((t += v) / 1000).floorToDouble() * 1000.0 + ]; + case 2: // dropouts of 3..600 s + return [ + for (final v in rr) + ((t += v + (r.nextDouble() < 0.01 ? 3000 + r.nextInt(600000) : 0)) / + 1000) + .floorToDouble() * + 1000.0 + ]; + case 3: // sub-second stamps + return [for (final v in rr) (t += v) + r.nextDouble() * 40]; + case 4: // counter-reset style: some stamps step backwards + return [ + for (final v in rr) + (t += v) - (r.nextDouble() < 0.02 ? 1000.0 * (1 + r.nextInt(3)) : 0) + ]; + case 5: // length mismatch: the oracle ignores the stamps entirely + return List.filled(math.max(0, n - 1), 1.7e12); + default: // every beat on the same stamp + return List.filled(n, 1.7e12); + } +} + +/// A hostile-ish RR series: drifting base, then bursts of every artefact shape. +List _randomRr(math.Random r, int n) { + final base = 500 + r.nextInt(700).toDouble(); + final rate = r.nextDouble() * 0.12; + final rr = []; + var drift = 0.0; + while (rr.length < n) { + drift += (r.nextDouble() - .5) * 8; + drift = drift.clamp(-200.0, 200.0); + final v = base + drift + (r.nextDouble() - .5) * 60; + if (r.nextDouble() < rate) { + switch (r.nextInt(7)) { + case 0: // ectopic + compensatory pause + rr..add((v * .6).roundToDouble())..add((v * 1.4).roundToDouble()); + case 1: // missed beat + rr.add((v * 2).roundToDouble()); + case 2: // extra beat + rr..add((v * .45).roundToDouble())..add((v * .55).roundToDouble()); + case 3: // multi-beat garbage run + for (var k = 3 + r.nextInt(12); k > 0; k--) { + rr.add(200 + r.nextInt(2500).toDouble()); + } + case 4: // saturated / zero / negative / absurd values + rr.add([0.0, -50.0, 2400.0, 5000.0, 1e6][r.nextInt(5)]); + case 5: // non-integer ms + rr.add(v + r.nextDouble()); + default: + rr.add(v.roundToDouble()); + } + } else { + rr.add(v.roundToDouble()); + } + } + return rr.sublist(0, n); +} + +void main() { + group('edge cases', () { + test('empty', () { + _check(const [], null, 'empty no ts'); + _check(const [], const [], 'empty with ts'); + }); + + test('fewer than the 3 beats the dispersion estimate needs', () { + final cases = >[ + [800], + [250], + [2500], + [300], + [2000], + [2001], + [800, 810], + [800, 250], + [2500, 2500], + [299, 300], + ]; + for (final rr in cases) { + _check(rr, null, 'short $rr no ts'); + _check(rr, [for (var i = 0; i < rr.length; i++) 1.7e12 + i * 800.0], + 'short $rr ts'); + } + }); + + test('every length around the window sizes, clean and dirty', () { + final r = math.Random(5); + final lengths = { + for (var n = 3; n <= 16; n++) n, + for (final c in [31, 45, 46, 47, 90, 91, 92, 93, 135, 136, 181, 182, 183]) + for (var d = -1; d <= 1; d++) c + d, + }; + for (final n in lengths) { + final rr = _randomRr(r, n); + _check(rr, null, 'n=$n no ts'); + _check(rr, _timestamps(r, rr, 2), 'n=$n ts'); + } + }); + + test('all artefacts', () { + final r = math.Random(6); + for (final n in [3, 4, 10, 100, 400]) { + _check(List.filled(n, 2500.0), null, 'all long n=$n'); + _check(List.filled(n, 250.0), null, 'all short n=$n'); + final rr = [for (var i = 0; i < n; i++) 250 + r.nextInt(2250).toDouble()]; + _check(rr, null, 'random garbage n=$n'); + _check(rr, _timestamps(r, rr, 1), 'random garbage ts n=$n'); + // alternating normal / artefact: no run is ever longer than 1 + final alt = [for (var i = 0; i < n; i++) i.isEven ? 800.0 : 2500.0]; + _check(alt, null, 'alternating n=$n'); + } + }); + + test('constant RR (QD = 0: the floor governs, massive ties)', () { + final rr = List.filled(500, 800); + _check(rr, null, 'constant'); + for (final at in [0, 1, 2, 250, 497, 498, 499]) { + final o = List.of(rr)..[at] = 1700; + _check(o, null, 'constant + outlier@$at'); + _check(o, [for (var i = 0; i < 500; i++) 1.7e12 + i * 800.0], + 'constant + outlier@$at ts'); + } + // two-valued series: dRR is mostly 0 with exact ties everywhere + final two = [for (var i = 0; i < 400; i++) i % 3 == 0 ? 810.0 : 800.0]; + _check(two, null, 'two-valued'); + }); + + test('artefact runs longer than the window, and at the edges', () { + final r = math.Random(7); + final rr = [ + for (var i = 0; i < 700; i++) + (i >= 150 && i < 290) + ? 250 + r.nextInt(2000).toDouble() + : (i >= 400 && i < 500 && i.isEven) + ? 2200 + : 800 + r.nextInt(40).toDouble() + ]; + _check(rr, null, 'long runs'); + _check(rr, _timestamps(r, rr, 1), 'long runs ts'); + final edges = [ + 2500, + 800, + 810, + 790, + 805, + for (var i = 0; i < 300; i++) 800 + (i % 7) * 3.0, + 2500 + ]; + _check(edges, null, 'artefacts at both ends'); + }); + + test('gaps, re-anchoring and timestamp shapes', () { + final r = math.Random(8); + final rr = _randomRr(r, 900); + for (var mode = 0; mode <= 6; mode++) { + _check(rr, _timestamps(r, rr, mode), 'ts mode $mode'); + } + // a dropout exactly at, just under and just over the re-anchor slack + for (final extra in [999.0, 1000.0, 1001.0]) { + var t = 1.7e12; + final ts = [ + for (var i = 0; i < 300; i++) + t += 800 + (i == 150 ? extra : 0) + ]; + _check(List.filled(300, 800.0), ts, 'slack extra=$extra'); + } + }); + + test('rrTsMs present vs absent changes only the clock', () { + final r = math.Random(9); + final rr = _randomRr(r, 600); + final ts = _timestamps(r, rr, 2)!; + final a = correctRr(rr, rrTsMs: ts); + final b = correctRr(rr); + expectSameClasses(a.classes, b.classes, 'classes ignore the clock'); + expectBitIdentical(a.nn, b.nn, 'nn ignores the clock'); + }); + + test('parameter grid', () { + final r = math.Random(10); + final rr = _randomRr(r, 450); + final ts = _timestamps(r, rr, 2); + for (final alpha in [5.2, 2.0, 8.0]) { + for (final win in [1, 2, 3, 4, 31, 61, 90, 91, 92, 121, 1000]) { + for (final floor in [100.0, 0.0, 250.0]) { + _check(rr, ts, 'a=$alpha w=$win f=$floor', + alpha: alpha, win: win, floor: floor); + } + } + } + for (final reanchor in [0.0, 400.0, 5000.0]) { + _check(rr, ts, 'reanchor=$reanchor', reanchor: reanchor); + } + }); + }); + + group('randomized', () { + // 90 independent series: random length, artefact mix, timestamp shape and + // parameters. Seeds are fixed so a failure names a reproducible case. + for (var seed = 0; seed < 90; seed++) { + test('seed $seed', () { + final r = math.Random(1000 + seed); + final n = [ + r.nextInt(12), + r.nextInt(200), + r.nextInt(900), + r.nextInt(2500) + ][r.nextInt(4)]; + final rr = _randomRr(r, n); + final ts = _timestamps(r, rr, r.nextInt(7)); + final custom = r.nextInt(4) == 0; + _check(rr, ts, 'seed=$seed n=$n', + alpha: custom ? [3.0, 4.5, 6.5][r.nextInt(3)] : 5.2, + win: custom ? [5, 31, 61, 121][r.nextInt(4)] : 91, + floor: custom ? [0.0, 50.0, 150.0][r.nextInt(3)] : 100, + reanchor: custom ? [0.0, 500.0, 3000.0][r.nextInt(3)] : 1000); + }); + } + }); + + group('real-shaped', () { + test('generator configs (clean, default, dirty, backwards-ts, noisy)', () { + final cfgs = [ + const SynthConfig( + seed: 21, + hours: 0.5, + ectopicPerMin: 0, + missedPerMin: 0, + extraPerMin: 0, + noiseRunPerMin: 0, + gapPerHour: 0), + const SynthConfig(seed: 22, hours: 0.6), + const SynthConfig( + seed: 23, + hours: 0.5, + ectopicPerMin: 2, + missedPerMin: 1, + extraPerMin: 1, + noiseRunPerMin: 0.5, + gapPerHour: 12), + const SynthConfig( + seed: 24, hours: 0.5, backwardsPerHour: 30, gapPerHour: 6), + const SynthConfig( + seed: 25, hours: 0.4, noiseRunPerMin: 2, gapPerHour: 30), + ]; + for (final c in cfgs) { + final s = synthRr(c); + _check(s.rr, s.ts, 'synth seed=${c.seed} ts'); + _check(s.rr, null, 'synth seed=${c.seed} no ts'); + } + }); + + test('real 8.9 h WHOOP 4 night: full, with and without timestamps', () { + final s = realNightRr(); + expect(s, isNotNull, reason: 'real night fixture missing'); + _check(s!.rr, s.ts, 'real night ts'); + _check(s.rr, null, 'real night no ts'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('real night prefixes (every window alignment)', () { + final s = realNightRr()!; + for (final n in [1, 2, 3, 4, 45, 46, 90, 91, 92, 181, 1000, 4999]) { + _check(s.rr.sublist(0, n), s.ts.sublist(0, n), 'real prefix n=$n'); + } + }, timeout: const Timeout(Duration(minutes: 5))); + + test('a whole 23 h day (~96k beats)', () { + final s = realShapedDay(); + expect(s.length, inInclusiveRange(90000, 100000)); + _check(s.rr, s.ts, 'day'); + }, timeout: const Timeout(Duration(minutes: 10))); + }); +} diff --git a/test/onehz/rr_correction_perf_test.dart b/test/onehz/rr_correction_perf_test.dart new file mode 100644 index 00000000..2aed0186 --- /dev/null +++ b/test/onehz/rr_correction_perf_test.dart @@ -0,0 +1,48 @@ +// Performance guard for `correctRr`. +// +// Today every beat sorts five 91-element windows, so one call over a 23 h day +// (~96k beats) takes ~18 s on a loaded desktop and the app pays it on every +// derive pass. Sliding sorted windows give the same output in ~0.56 s (see +// test/onehz/rr_correction_oracle_test.dart for the bit-identical guard). +// +// The budgets are ~5x the rewrite's measured time, so a loaded CI host does not +// flake, and far below today's implementation, so this FAILS until the rewrite +// lands. Timing is wall clock around ONE call, JIT warm-up included. +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import 'support/rr_synth.dart'; + +const _dayBudget = Duration(seconds: 3); // rewrite: 0.56 s; today: ~18 s +const _nightBudget = Duration(seconds: 1); // rewrite: 0.19 s; today: ~7 s + +void main() { + test('23 h day (~96k beats) corrects within ${_dayBudget.inSeconds} s', () { + // 5 h synthetic + the real 8.9 h night + 9 h synthetic = 96,712 beats. + final s = realShapedDay(); + expect(s.length, inInclusiveRange(90000, 100000)); + final sw = Stopwatch()..start(); + final out = correctRr(s.rr, rrTsMs: s.ts); + sw.stop(); + // ignore: avoid_print + print('correctRr: ${s.length} beats in ${sw.elapsedMilliseconds} ms'); + // Not a stub result: the call really processed the series. + expect(out.classes.length, s.length); + expect(sw.elapsed, lessThan(_dayBudget), + reason: '${sw.elapsedMilliseconds} ms for ${s.length} beats'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('real 8.9 h night (~32k beats) corrects within ' + '${_nightBudget.inSeconds} s', () { + final s = realNightRr(); + expect(s, isNotNull, reason: 'real night fixture missing'); + final sw = Stopwatch()..start(); + final out = correctRr(s!.rr, rrTsMs: s.ts); + sw.stop(); + // ignore: avoid_print + print('correctRr: ${s.length} beats in ${sw.elapsedMilliseconds} ms'); + expect(out.classes.length, s.length); + expect(sw.elapsed, lessThan(_nightBudget), + reason: '${sw.elapsedMilliseconds} ms for ${s.length} beats'); + }, timeout: const Timeout(Duration(minutes: 5))); +} diff --git a/test/onehz/sliding_median_test.dart b/test/onehz/sliding_median_test.dart new file mode 100644 index 00000000..947bc2b6 --- /dev/null +++ b/test/onehz/sliding_median_test.dart @@ -0,0 +1,178 @@ +// slidingMedians (round 15, phase 0) - RED until the implementation lands. +// +// Contract (r15-phase0.md section 1): for every j, +// out[j] == median(xs.sublist(max(0, j - halfWin), min(n - 1, j + halfWin) + 1)) +// bit for bit, where `median` is util.dart's (copy + List.sort default order, +// NaN last, linear-interpolated percentile at 50). +// +// Assumed signature: List slidingMedians(List xs, int halfWin) +// exported from package:openstrap_analytics/onehz.dart (via util.dart). + +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +/// The definition, written independently of the implementation. +List naiveSlidingMedians(List xs, int halfWin) { + final n = xs.length; + return [ + for (var j = 0; j < n; j++) + median(xs.sublist( + math.max(0, j - halfWin), math.min(n - 1, j + halfWin) + 1))! + ]; +} + +bool sameD(double a, double b) { + if (a.isNaN || b.isNaN) return a.isNaN && b.isNaN; + return a == b && a.isNegative == b.isNegative; +} + +void expectBitIdentical(List xs, int halfWin) { + final got = slidingMedians(xs, halfWin); + final want = naiveSlidingMedians(xs, halfWin); + expect(got.length, want.length, reason: 'length xs=$xs h=$halfWin'); + for (var j = 0; j < want.length; j++) { + if (!sameD(got[j], want[j])) { + fail('j=$j h=$halfWin xs=$xs: got ${got[j]}, want ${want[j]}'); + } + } +} + +void main() { + const specials = [ + double.nan, + double.infinity, + double.negativeInfinity, + 0.0, + -0.0, + ]; + + test('empty input and zero window', () { + expect(slidingMedians(const [], 0), isEmpty); + expect(slidingMedians(const [], 5), isEmpty); + expectBitIdentical([3.5], 0); + expectBitIdentical([3.5], 7); + final xs = [4.0, 1.0, 9.0, 2.0]; + expect(slidingMedians(xs, 0), xs); // halfWin 0: each window is one value + }); + + test('does not mutate or alias its input', () { + final xs = [5.0, 1.0, 4.0, 2.0, 3.0]; + final copy = List.of(xs); + final out = slidingMedians(xs, 2); + expect(xs, copy); + expect(identical(out, xs), isFalse); + }); + + test('exhaustive small: n 0..20, halfWin 0..12, distinct values', () { + final r = math.Random(1); + for (var n = 0; n <= 20; n++) { + for (var h = 0; h <= 12; h++) { + expectBitIdentical([for (var i = 0; i < n; i++) r.nextDouble() * 100 - 50], h); + } + } + }); + + test('exhaustive small: heavy ties', () { + final r = math.Random(2); + for (var n = 0; n <= 20; n++) { + for (var h = 0; h <= 12; h++) { + for (var rep = 0; rep < 3; rep++) { + expectBitIdentical([for (var i = 0; i < n; i++) r.nextInt(3).toDouble()], h); + } + } + } + }); + + test('exhaustive small: NaN, +-inf, +-0.0 mixed with ties', () { + final r = math.Random(3); + for (var n = 0; n <= 20; n++) { + for (var h = 0; h <= 12; h++) { + for (var rep = 0; rep < 4; rep++) { + final xs = [ + for (var i = 0; i < n; i++) + r.nextInt(3) == 0 + ? specials[r.nextInt(specials.length)] + : r.nextInt(4).toDouble() - 1, + ]; + expectBitIdentical(xs, h); + } + } + } + }); + + test('all-special and all-equal series', () { + for (final v in specials) { + for (var n = 1; n <= 9; n++) { + for (var h = 0; h <= 5; h++) { + expectBitIdentical(List.filled(n, v), h); + } + } + } + for (final pair in [ + [double.nan, double.infinity], + [double.negativeInfinity, double.infinity], + [double.negativeInfinity, double.nan], + [-0.0, 0.0], + ]) { + for (var n = 2; n <= 12; n++) { + for (var h = 0; h <= 6; h++) { + expectBitIdentical([for (var i = 0; i < n; i++) pair[i % 2]], h); + } + } + } + }); + + test('infinities make even-window interpolation NaN exactly as the definition', + () { + // inf - inf and (-inf) + inf in the interpolation: the result must match + // the naive percentile arithmetic, not a "smarter" average. + expectBitIdentical([double.infinity, double.infinity], 1); + expectBitIdentical([double.negativeInfinity, double.infinity], 1); + expectBitIdentical([double.negativeInfinity, double.negativeInfinity, 1.0], 1); + expectBitIdentical([1.0, double.nan, 2.0, double.nan], 1); + }); + + test('halfWin >= n and very large halfWin', () { + final r = math.Random(4); + for (final n in [1, 2, 5, 17, 40]) { + final xs = [for (var i = 0; i < n; i++) r.nextDouble()]; + for (final h in [n - 1, n, n + 1, 3 * n, 1 << 30]) { + expectBitIdentical(xs, h); + } + } + }); + + test('monotone and reversed runs (window slides over sorted data)', () { + for (final n in [10, 33]) { + for (final h in [1, 3, 8]) { + expectBitIdentical([for (var i = 0; i < n; i++) i.toDouble()], h); + expectBitIdentical([for (var i = 0; i < n; i++) (n - i).toDouble()], h); + } + } + }); + + test('production-sized: CVHR window (halfWin 65) over 3000 s', () { + final r = math.Random(5); + final xs = [ + for (var i = 0; i < 3000; i++) + 900 + 60 * math.sin(i / 9.0) + (r.nextDouble() - 0.5) * 30 + ]; + expectBitIdentical(xs, 65); + // quantised values => many ties + expectBitIdentical([for (final x in xs) x.roundToDouble()], 65); + }); + + test('production-sized with sprinkled NaN and inf', () { + final r = math.Random(6); + final xs = [ + for (var i = 0; i < 1500; i++) + r.nextInt(40) == 0 + ? specials[r.nextInt(specials.length)] + : (r.nextDouble() * 20).roundToDouble() + ]; + expectBitIdentical(xs, 65); + expectBitIdentical(xs, 165); + }); +} diff --git a/test/onehz/support/correct_rr_reference.dart b/test/onehz/support/correct_rr_reference.dart new file mode 100644 index 00000000..999f68db --- /dev/null +++ b/test/onehz/support/correct_rr_reference.dart @@ -0,0 +1,344 @@ +// FROZEN REFERENCE: a verbatim copy of `correctRr` as it stood at analytics +// a19f439 (before the sliding-window speed rewrite), with its helpers and the +// two `util.dart` functions it calls (`percentile`, `median`) copied in too, so +// nothing the rewrite touches can change what this file computes. +// +// It is the oracle for the production `correctRr` and for the streaming +// `RrCorrector`: same input => bit-identical nn, nnTimesMs, classes, counts and +// cleanFraction. DO NOT "fix" or speed this file up; if the production +// behaviour is meant to change, that is a kAlgoVersion-level decision and this +// copy is replaced deliberately, in its own commit. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart' + show BeatClass, RrCorrectionResult; + +double? _percentileSortedRef(List sorted, double p) { + if (sorted.isEmpty) return null; + if (sorted.length == 1) return sorted[0]; + final rank = (p / 100) * (sorted.length - 1); + final lo = rank.floor(); + final hi = rank.ceil(); + if (lo == hi) return sorted[lo]; + final frac = rank - lo; + return sorted[lo] + (sorted[hi] - sorted[lo]) * frac; +} + +double? _percentileRef(List values, double p) => + _percentileSortedRef([...values]..sort(), p); + +double? _medianRef(List xs) => _percentileRef(xs, 50); + +/// Lipponen–Tarvainen RR artifact correction. +/// +/// [rrMs] the raw RR series (ms). [alpha] scales the time-varying threshold +/// (paper default 5.2 on the QD estimate). [windowBeats] sliding window for the +/// local dispersion estimate. +/// +/// [rrTsMs] the beat timestamps that came with [rrMs] (same length, sorted, +/// epoch ms). PASS THEM. Without them the output clock is Σrr, which splices +/// every sensor dropout out of the record: the intervals the band never +/// reported are not in [rrMs] at all, so no amount of book-keeping inside this +/// function can recover them. Measured on 13 real nights, the cumsum span was +/// 0.13–0.87 of the true rec_ts span, and the Lomb-Scargle spectrum built on it +/// moved LF/HF across the sympatho-vagal line (0.65 → 1.53 on one MG night). +/// With them, the clock cumsums RR *inside* a contiguous run — the RR values +/// are the only sub-second information that exists, since `rr_ts_ms` is +/// `rec_ts*1000` and therefore whole-second — and re-anchors to the real +/// timestamp whenever the wall step exceeds the interval by more than +/// [reanchorGapMs], i.e. at a dropout. +RrCorrectionResult correctRrReference( + List rrMs, { + List? rrTsMs, + double alpha = 5.2, + int windowBeats = 91, + double minThresholdMs = 100, + double reanchorGapMs = 1000, +}) { + final n = rrMs.length; + // Beat-end time for EVERY input beat, kept or not. One array, built once, so + // every branch below reads a clock instead of advancing its own. + final tAll = _beatTimesRef(rrMs, rrTsMs, reanchorGapMs); + if (n == 0) { + return const RrCorrectionResult( + nn: [], + nnTimesMs: [], + classes: [], + cleanFraction: 0, + droppedCount: 0, + correctedCount: 0, + ); + } + if (n < 3) { + // Too short to estimate dispersion: pass through physiologically-plausible + // beats only, classify the rest as long/short. No fabrication. + final classes = [ + for (final rr in rrMs) + (rr >= 300 && rr <= 2000) ? BeatClass.normal : BeatClass.longShort + ]; + final nn = []; + final times = []; + for (var i = 0; i < n; i++) { + if (classes[i] == BeatClass.normal) { + nn.add(rrMs[i]); + times.add(tAll[i]); + } + } + return RrCorrectionResult( + nn: nn, + nnTimesMs: times, + classes: classes, + cleanFraction: nn.length / n, + droppedCount: n - nn.length, + correctedCount: 0, + ); + } + + // dRR[i] = rr[i] - rr[i-1]. + final dRR = List.filled(n, 0); + for (var i = 1; i < n; i++) { + dRR[i] = rrMs[i] - rrMs[i - 1]; + } + + // Time-varying threshold from a sliding quartile-deviation (QD) of dRR. + // th1 ~ dispersion of dRR (short artifacts), th2 ~ dispersion of medianed RR. + final th1 = _slidingThresholdRef(dRR, windowBeats, alpha, minThresholdMs); + + // medRR: rr minus local median (for missed/extra long-range tests). + final med = _slidingMedianRef(rrMs, windowBeats); + final mRR = List.generate(n, (i) { + final d = rrMs[i] - med[i]; + return d < 0 ? d * 2 : d; // paper asymmetry weight + }); + final th2 = _slidingThresholdRef(mRR, windowBeats, alpha, minThresholdMs); + + final classes = List.filled(n, BeatClass.normal); + for (var i = 0; i < n; i++) { + final hardLong = rrMs[i] > 2000; + final hardShort = rrMs[i] < 300; + final bigJump = dRR[i].abs() > th1[i]; + final bigDev = mRR[i].abs() > th2[i]; + if (hardLong || (bigDev && mRR[i] > 0)) { + // Long interval: likely a MISSED beat (interval ~ multiple of normal). + classes[i] = (med[i] > 0 && rrMs[i] > 1.5 * med[i]) + ? BeatClass.missed + : BeatClass.longShort; + } else if (hardShort || (bigDev && mRR[i] < 0)) { + // Short interval: likely an EXTRA (spurious) beat. + classes[i] = (med[i] > 0 && rrMs[i] < 0.6 * med[i]) + ? BeatClass.extra + : BeatClass.longShort; + } else if (bigJump) { + classes[i] = BeatClass.ectopic; + } + } + + // Compensatory-pair reconciliation. A single ectopic/extra beat shows up as + // TWO successive dRR spikes of opposite sign (the bad beat, then the + // recovery). If beat i was flagged ONLY by the dRR jump (ectopic) but its + // value is physiologically normal and close to the local median, while its + // predecessor was a short/extra or long/missed event of opposite-sign dRR, + // then i is just the recovery — demote it to normal so the event stays a + // single isolated artifact rather than a spurious 2-beat run. + for (var k = 1; k < n; k++) { + if (classes[k] != BeatClass.ectopic) continue; + final prevBad = classes[k - 1] == BeatClass.extra || + classes[k - 1] == BeatClass.missed || + classes[k - 1] == BeatClass.ectopic || + classes[k - 1] == BeatClass.longShort; + if (!prevBad) continue; + final oppositeSign = dRR[k] * dRR[k - 1] < 0; + final valueNormal = med[k] > 0 && + rrMs[k] >= 300 && + rrMs[k] <= 2000 && + (rrMs[k] - med[k]).abs() <= 0.2 * med[k]; + if (oppositeSign && valueNormal) { + classes[k] = BeatClass.normal; + } + } + + // Build cleaned NN. Isolated single artifact -> cubic-spline (Catmull-Rom on + // the 4 surrounding NORMAL beats). A run of ≥2 consecutive artifacts -> drop. + final isArtifact = [for (final c in classes) c != BeatClass.normal]; + final nn = []; + final times = []; + var dropped = 0; + var corrected = 0; + var i = 0; + while (i < n) { + if (!isArtifact[i]) { + nn.add(rrMs[i]); + times.add(tAll[i]); + i++; + continue; + } + // Measure run length. + var j = i; + while (j < n && isArtifact[j]) { + j++; + } + final runLen = j - i; + if (runLen == 1) { + // Isolated -> spline-correct from surrounding normals. + final corr = _splineCorrectRef(rrMs, isArtifact, i); + if (corr != null) { + // The beat keeps its REAL time, not the interpolated one. `corr` is our + // best guess at what the NN *should* have been (a missed beat splits one + // ~2000 ms interval into two ~1000 ms ones), but the 2000 ms still + // elapsed. Advancing by `corr` deleted ~1 s of record per corrected beat + // — 1.6 % of a night at one missed beat in sixty. + nn.add(corr); + times.add(tAll[i]); + corrected++; + } else { + dropped++; // no anchors -> honest drop; the clock already elapsed + } + } else { + // Multi-beat run: NEVER interpolate. The clock is not touched here at all + // — `tAll` already carries the run's elapsed time, so the two sides of a + // dropped run are not spliced together. Not doing this compacted + // `nnTimesMs` (299.0 s of real record emitted as a 294.0 s span) and + // inflated everything keyed off it: cvhr_per_hour's analyzedHours, the + // Lomb-Scargle axis, spanSec — on exactly the noisy nights that needed + // the correction most. + dropped += runLen; + } + i = j; + } + + final normalCount = classes.where((c) => c == BeatClass.normal).length; + return RrCorrectionResult( + nn: nn, + nnTimesMs: times, + classes: classes, + cleanFraction: normalCount / n, + droppedCount: dropped, + correctedCount: corrected, + ); +} + +/// Beat-end time (ms, relative to the start of beat 0) for every input beat. +/// +/// RR intervals tile the timeline, so the clock advances for EVERY interval, +/// kept or dropped — but only for intervals the band actually REPORTED. A +/// sensor dropout is not an interval at all: it is absent from [rrMs], and +/// summing across it splices the two sides of the hole together. [rrTsMs] is +/// the only witness that the hole existed, so when the wall step between two +/// consecutive reported beats exceeds that beat's own interval by more than +/// [reanchorGapMs], the clock is RE-ANCHORED to the real timestamp instead of +/// accumulated. Inside a contiguous run the RR cumsum is kept, because +/// `rr_ts_ms = rec_ts*1000` is whole-second and the RR values are the only +/// sub-second information in the record. +/// +/// The slack has to be ≥ the max plausible RR (2,400 ms saturation) minus a +/// normal beat, or ordinary quantisation would re-anchor constantly; 1,000 ms +/// is the audit's figure and is ~1 whole quantisation step. +List _beatTimesRef( + List rrMs, List? rrTsMs, double reanchorGapMs) { + final n = rrMs.length; + final t = List.filled(n, 0); + if (n == 0) return t; + final wall = rrTsMs != null && rrTsMs.length == n; + final origin = wall ? rrTsMs[0] - rrMs[0] : 0.0; + var cur = 0.0; + for (var i = 0; i < n; i++) { + // The test is LOCAL — this beat's wall step against this beat's own + // interval — so a run's accumulated cumsum-vs-wall drift never triggers it. + // + // That drift is real and NOT resolved here. Inside contiguous runs of ≥200 + // beats, Σrr / rec_ts-span measures 0.9631 on gen4 against 0.9988 (W5) and + // 1.0013 (MG). Which clock is wrong is not decidable from the exports: the + // band's `hr` field is computed from the same beat detector as `rr_ms`, so + // it is not independent evidence, and a low rate of beats the band never + // reports produces the same deficit with a perfectly correct clock. So + // there is deliberately no `rrClockScale` — applying 0.963 to gen4's RR + // values would be actively wrong under that second explanation. The + // consequence to carry: **gen4 and gen5/MG nights are not comparable on any + // frequency-domain or per-hour quantity** until it is resolved. + final dropout = + wall && i > 0 && (rrTsMs[i] - rrTsMs[i - 1]) - rrMs[i] > reanchorGapMs; + final anchored = wall ? rrTsMs[i] - origin : 0.0; + // `> cur` keeps the clock monotonic even if timestamps arrive out of order + // or the cumsum has run ahead of the wall — a backwards jump is never taken. + cur = (dropout && anchored > cur) ? anchored : cur + rrMs[i]; + t[i] = cur; + } + return t; +} + +/// Time-varying threshold: alpha × QD of the SIGNED series x in a sliding +/// window, where QD = (Q3 − Q1)/2 (Lipponen & Tarvainen 2019, eq. for th1/th2: +/// "th = α · quartile deviation of dRR over the 91-beat window", α = 5.2). +/// +/// The quartile deviation MUST be taken on the SIGNED series. Taking it on |x| +/// folds the symmetric ±dRR distribution onto one side, collapsing QD by ~an +/// order of magnitude; the threshold then sinks to [floor] and the detector +/// degenerates into a fixed 100 ms cut-off that flags ordinary RSA as ectopy. +List _slidingThresholdRef( + List x, int win, double alpha, double floor) { + final n = x.length; + final out = List.filled(n, 0); + final half = win ~/ 2; + for (var i = 0; i < n; i++) { + final lo = math.max(0, i - half); + final hi = math.min(n - 1, i + half); + final seg = []; + for (var k = lo; k <= hi; k++) { + seg.add(x[k]); + } + final q1 = _percentileRef(seg, 25) ?? 0; + final q3 = _percentileRef(seg, 75) ?? 0; + final qd = (q3 - q1) / 2; + // Floor keeps a gross outlier detectable on (near-)quantized clean data + // where the QD genuinely collapses to 0 (constant RR). On any series with + // real beat-to-beat variability α·QD dominates the floor, so the floor + // never governs a physiological signal. + out[i] = math.max(alpha * qd, floor); + } + return out; +} + +List _slidingMedianRef(List x, int win) { + final n = x.length; + final out = List.filled(n, 0); + final half = win ~/ 2; + for (var i = 0; i < n; i++) { + final lo = math.max(0, i - half); + final hi = math.min(n - 1, i + half); + final seg = []; + for (var k = lo; k <= hi; k++) { + if (k == i) continue; + seg.add(x[k]); + } + out[i] = _medianRef(seg) ?? x[i]; + } + return out; +} + +/// Catmull-Rom cubic interpolation at the artifact index using the nearest two +/// NORMAL beats on each side. Returns null if anchors are unavailable. +double? _splineCorrectRef(List rr, List isArtifact, int idx) { + final left = []; + for (var k = idx - 1; k >= 0 && left.length < 2; k--) { + if (!isArtifact[k]) left.insert(0, rr[k]); + } + final right = []; + for (var k = idx + 1; k < rr.length && right.length < 2; k++) { + if (!isArtifact[k]) right.add(rr[k]); + } + if (left.isEmpty || right.isEmpty) return null; + final p1 = left.last; + final p2 = right.first; + final p0 = left.length >= 2 ? left.first : p1; + final p3 = right.length >= 2 ? right.last : p2; + // Catmull-Rom at t=0.5 between p1 and p2. + const t = 0.5; + final t2 = t * t; + final t3 = t2 * t; + final v = 0.5 * + ((2 * p1) + + (-p0 + p2) * t + + (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 + + (-p0 + 3 * p1 - 3 * p2 + p3) * t3); + return v; +} diff --git a/test/onehz/support/rr_chunking.dart b/test/onehz/support/rr_chunking.dart new file mode 100644 index 00000000..83c2eba1 --- /dev/null +++ b/test/onehz/support/rr_chunking.dart @@ -0,0 +1,40 @@ +// Shared helpers for the RR correction / streaming oracle tests. +import 'dart:convert'; +import 'dart:math' as math; + +/// JSON round trip through real text (what a SQLite blob would hold). +Map jsonRoundTrip(Map m) => + jsonDecode(jsonEncode(m)) as Map; + +/// Random chunk boundaries over [0, n): sizes drawn from [sizes] (a mix of 1, +/// tiny, one-window, and huge) so every alignment against the 91/181-beat +/// windows is hit. +List randomCuts(math.Random r, int n, + {List sizes = const [1, 2, 3, 7, 45, 46, 90, 91, 92, 180, 181, 500, 1500]}) { + final cuts = []; + var at = 0; + while (at < n) { + final s = sizes[r.nextInt(sizes.length)]; + at = math.min(n, at + 1 + r.nextInt(s)); + cuts.add(at); + } + return cuts; +} + +/// Cuts that fall every [sec] seconds of the beat clock: what a 15-minute +/// derive pass sees. [tsMs] sorted-ish epoch ms. +List timeCuts(List tsMs, double sec) { + final cuts = []; + if (tsMs.isEmpty) return cuts; + var next = tsMs.first + sec * 1000; + for (var i = 0; i < tsMs.length; i++) { + if (tsMs[i] >= next) { + cuts.add(i); + while (next <= tsMs[i]) { + next += sec * 1000; + } + } + } + cuts.add(tsMs.length); + return cuts; +} diff --git a/test/onehz/support/rr_compare.dart b/test/onehz/support/rr_compare.dart new file mode 100644 index 00000000..a2f772e4 --- /dev/null +++ b/test/onehz/support/rr_compare.dart @@ -0,0 +1,44 @@ +// Bit-for-bit comparison helpers for the RR correction oracle tests. +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +/// `compareTo` is 0 only for the same double bit pattern in every case that +/// matters here (it tells -0.0 from 0.0 and treats NaN as equal to NaN), which +/// `==` does not. +bool sameBits(double a, double b) => a.compareTo(b) == 0; + +/// Fails with the FIRST differing index instead of dumping two 96k-element +/// lists. +void expectBitIdentical(List got, List want, String why) { + if (got.length != want.length) { + fail('$why: length ${got.length} != ${want.length}'); + } + for (var i = 0; i < got.length; i++) { + if (!sameBits(got[i], want[i])) { + fail('$why: index $i got ${got[i]} want ${want[i]}'); + } + } +} + +void expectSameClasses(List got, List want, String why) { + if (got.length != want.length) { + fail('$why: classes length ${got.length} != ${want.length}'); + } + for (var i = 0; i < got.length; i++) { + if (got[i] != want[i]) { + fail('$why: class[$i] got ${got[i]} want ${want[i]}'); + } + } +} + +/// Everything a [RrCorrectionResult] carries. +void expectSameCorrection( + RrCorrectionResult got, RrCorrectionResult want, String why) { + expectBitIdentical(got.nn, want.nn, '$why nn'); + expectBitIdentical(got.nnTimesMs, want.nnTimesMs, '$why nnTimesMs'); + expectSameClasses(got.classes, want.classes, why); + expect(sameBits(got.cleanFraction, want.cleanFraction), isTrue, + reason: '$why cleanFraction ${got.cleanFraction} vs ${want.cleanFraction}'); + expect(got.droppedCount, want.droppedCount, reason: '$why droppedCount'); + expect(got.correctedCount, want.correctedCount, reason: '$why correctedCount'); +} diff --git a/test/onehz/support/rr_runs.dart b/test/onehz/support/rr_runs.dart new file mode 100644 index 00000000..86370dc1 --- /dev/null +++ b/test/onehz/support/rr_runs.dart @@ -0,0 +1,54 @@ +// Test support: clean RR series with long artifact runs, for the cap tests. +// Integer RR, timestamps = exact running sums (every clock value is an integer, +// so comparisons can be bit-exact). Seeded (property.dart's Rng). +import 'dart:math' as math; + +import '../../support/property.dart' show Rng; + +const rrRunsT0 = 1.7e12; + +class RunSeries { + final List rr, ts; + RunSeries(this.rr, this.ts); +} + +/// Clean integer RR (RSA-like wobble), timestamps the exact running sum. +List cleanRr(Rng r, int n) => [ + for (var i = 0; i < n; i++) + (850 + 40 * math.sin(i * 0.7) + 25 * math.sin(i * 0.13) + r.intIn(-15, 15)) + .roundToDouble() + ]; + +RunSeries rrSeries(List rr) { + var t = rrRunsT0; + return RunSeries(rr, [for (final v in rr) t += v]); +} + +/// Segments of [segLens] clean beats separated by runs of 3000 ms beats +/// ([runLens]); returns the series and each run's [start, end). With [nearRun], +/// one short (extra-like) beat sits 20 beats before and 20 after every run. +(RunSeries, List<(int, int)>) withRuns(int seed, List segLens, List runLens, + {bool nearRun = false}) { + final r = Rng(seed); + final rr = []; + final runs = <(int, int)>[]; + for (var k = 0; k < segLens.length; k++) { + final seg = cleanRr(r, segLens[k]); + if (nearRun) { + if (k > 0) seg[20] = (seg[20] * 0.55).roundToDouble(); + if (k < runLens.length) seg[seg.length - 20] = (seg[seg.length - 20] * 0.55).roundToDouble(); + } + rr.addAll(seg); + if (k < runLens.length) { + final a = rr.length; + // The last two values rise (2500, 3000), so the dRR spike on the normal + // beat that follows has the opposite sign and the compensatory-pair rule + // takes that beat back as normal: the run is exactly the junk beats. + rr.addAll(List.filled(runLens[k], 3000.0)); + rr[rr.length - 2] = 2500.0; + runs.add((a, rr.length)); + } + } + return (rrSeries(rr), runs); +} + diff --git a/test/onehz/support/rr_stream_driver.dart b/test/onehz/support/rr_stream_driver.dart new file mode 100644 index 00000000..6c8d07b9 --- /dev/null +++ b/test/onehz/support/rr_stream_driver.dart @@ -0,0 +1,100 @@ +// Driver + oracle comparison for the streaming RrCorrector tests. +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import 'correct_rr_reference.dart'; +import 'rr_chunking.dart'; +import 'rr_compare.dart'; + +export 'rr_chunking.dart'; + +/// Accumulates everything a corrector has SETTLED, the way the app will hold it +/// between derive passes, and can save/restore the corrector at any boundary. +class RrStreamRun { + RrCorrector c; + final nn = []; + final nnTimes = []; + final classes = []; + int folded = 0; + int restarts = 0; + + RrStreamRun([RrCorrector? corrector]) : c = corrector ?? RrCorrector(); + + RrStreamRun.params( + {double alpha = 5.2, + int win = 91, + double floor = 100, + double reanchor = 1000}) + : c = RrCorrector( + alpha: alpha, + windowBeats: win, + minThresholdMs: floor, + reanchorGapMs: reanchor); + + RrSettled fold(List rr, List? ts) { + final s = c.fold(rr, tsMs: ts); + nn.addAll(s.nn); + nnTimes.addAll(s.nnTimes); + classes.addAll(s.classes); + folded += rr.length; + return s; + } + + /// Save to text and restore into a NEW corrector. + void restart() { + c = RrCorrector.fromJson(jsonRoundTrip(c.toJson())); + restarts++; + } + + /// `settled ++ tail` over the first [n] beats equals the frozen `correctRr` + /// on exactly those beats: nn, times, per-beat classes, counts, cleanFraction. + void expectMatchesOracle(List rr, List? ts, int n, + {double alpha = 5.2, + int win = 91, + double floor = 100, + double reanchor = 1000, + String why = ''}) { + expect(folded, n, reason: 'driver folded $why'); + final want = correctRrReference(rr.sublist(0, n), + rrTsMs: ts?.sublist(0, n), + alpha: alpha, + windowBeats: win, + minThresholdMs: floor, + reanchorGapMs: reanchor); + final snap = c.snapshot(); + expect(snap.n, n, reason: 'snapshot.n $why'); + expect(snap.classifiedBeats, classes.length, + reason: 'classifiedBeats == classes handed out $why'); + expect(snap.settledBeats, c.settledBeats, reason: 'settledBeats $why'); + expect(snap.settledBeats, lessThanOrEqualTo(snap.classifiedBeats), + reason: 'output cannot settle before its class $why'); + expect(snap.tailClasses.length, n - snap.classifiedBeats, + reason: 'tailClasses covers [classifiedBeats, n) $why'); + expectBitIdentical([...nn, ...snap.tailNn], want.nn, '$why nn'); + expectBitIdentical( + [...nnTimes, ...snap.tailNnTimes], want.nnTimesMs, '$why nnTimes'); + expectSameClasses( + [...classes, ...snap.tailClasses], want.classes, '$why classes'); + expect(snap.normalCount, + want.classes.where((k) => k == BeatClass.normal).length, + reason: 'normalCount $why'); + expect(snap.droppedCount, want.droppedCount, reason: 'dropped $why'); + expect(snap.correctedCount, want.correctedCount, reason: 'corrected $why'); + expect(sameBits(snap.cleanFraction, want.cleanFraction), isTrue, + reason: 'cleanFraction ${snap.cleanFraction} vs ${want.cleanFraction} ' + '$why'); + } + + /// Settled output is FINAL: it is a prefix of what the oracle says about the + /// WHOLE series, not just about the prefix seen so far. [full] is the oracle + /// run once over everything. + void expectSettledIsPrefixOf(RrCorrectionResult full, String why) { + expect(nn.length, lessThanOrEqualTo(full.nn.length), reason: '$why nn len'); + expectBitIdentical(nn, full.nn.sublist(0, nn.length), '$why settled nn'); + expectBitIdentical( + nnTimes, full.nnTimesMs.sublist(0, nnTimes.length), '$why settled t'); + expect(classes.length, lessThanOrEqualTo(full.classes.length)); + expectSameClasses( + classes, full.classes.sublist(0, classes.length), '$why settled cls'); + } +} diff --git a/test/onehz/support/rr_synth.dart b/test/onehz/support/rr_synth.dart new file mode 100644 index 00000000..ed0598d0 --- /dev/null +++ b/test/onehz/support/rr_synth.dart @@ -0,0 +1,151 @@ +// Test support: real-shaped RR generators and the real-night fixture loader. +// Deterministic (seeded), no wall clock. (Copied from the research prototypes in +// tool/incremental/synth.dart; the accelerometer generator stayed there because +// the day-curve states it feeds live in edge, not here.) +// +// "Real-shaped" means what the app really feeds correctRr: +// * integer-millisecond RR, end-of-beat timestamps quantised to WHOLE seconds +// (rr_ts_ms = rec_ts * 1000, several beats can share one stamp), +// * ectopic pairs, missed beats (2x interval), extra beats (half interval), +// multi-beat noise runs, sensor dropouts (no beats for seconds..minutes), +// * optional counter-reset style timestamp steps backwards. +import 'dart:io'; +import 'dart:math' as math; + +class RrData { + final List rr; + final List ts; // epoch-ish ms, whole seconds + RrData(this.rr, this.ts); + int get length => rr.length; +} + +class SynthConfig { + final int seed; + final double hours; + final double startSec; + final double ectopicPerMin; // premature beat + compensatory pause + final double missedPerMin; + final double extraPerMin; + final double noiseRunPerMin; // run of 3..12 garbage beats + final double gapPerHour; // dropouts of 5..900 s + final double backwardsPerHour; // ts steps back 1..3 s (counter reset-ish) + final double nightFraction; // fraction of the span that is "asleep" + const SynthConfig({ + this.seed = 1, + this.hours = 24, + this.startSec = 1.7e9, + this.ectopicPerMin = 0.3, + this.missedPerMin = 0.1, + this.extraPerMin = 0.1, + this.noiseRunPerMin = 0.05, + this.gapPerHour = 1.5, + this.backwardsPerHour = 0.0, + this.nightFraction = 0.33, + }); +} + +/// Physiologically-flavoured RR: slow HR drift + RSA (0.25 Hz) + LF (0.1 Hz) +/// + white jitter, integer ms, with injected artefacts. +RrData synthRr(SynthConfig c) { + final rnd = math.Random(c.seed); + final rr = [], ts = []; + final endT = c.hours * 3600.0; + var t = 0.0; // seconds since start (beat clock) + var phase = rnd.nextDouble() * 6.28; + final nightStart = endT * 0.05; + final nightEnd = nightStart + endT * c.nightFraction; + void emit(double interval) { + final v = interval.roundToDouble(); + t += v / 1000.0; + var stamp = (c.startSec + t).floorToDouble() * 1000.0; + rr.add(v); + ts.add(stamp); + } + + while (t < endT) { + final asleep = t >= nightStart && t < nightEnd; + // Base HR: sleep ~52 bpm, day 65..120 with activity bouts. + final bout = (math.sin(t / 1900.0 + c.seed) + 1) / 2; // 0..1 + final bpm = asleep ? 52 + 3 * math.sin(t / 5400) : 66 + 55 * bout * bout; + var base = 60000.0 / bpm; + final rsaAmp = asleep ? 45.0 : 22.0 * (1 - 0.7 * bout); + final rsaHz = asleep ? 0.24 : 0.27; + final v = base + + rsaAmp * math.sin(2 * math.pi * rsaHz * t + phase) + + 28 * math.sin(2 * math.pi * 0.1 * t) + + 8 * (rnd.nextDouble() - .5) * 2; + final beatMin = v / 60000.0; // minutes per beat → per-beat probabilities + final u = rnd.nextDouble(); + var p = c.noiseRunPerMin * beatMin; + if (u < p) { + final len = 3 + rnd.nextInt(10); + for (var k = 0; k < len; k++) { + emit(250 + rnd.nextInt(2100).toDouble()); + } + continue; + } + p += c.ectopicPerMin * beatMin; + if (u < p) { + emit(v * (0.55 + 0.15 * rnd.nextDouble())); + emit(v * (1.3 + 0.2 * rnd.nextDouble())); + continue; + } + p += c.missedPerMin * beatMin; + if (u < p) { + emit(v * (1.9 + 0.2 * rnd.nextDouble())); + continue; + } + p += c.extraPerMin * beatMin; + if (u < p) { + final f = 0.35 + 0.3 * rnd.nextDouble(); + emit(v * f); + emit(v * (1 - f)); + continue; + } + p += c.gapPerHour * beatMin / 60.0; + if (u < p) { + final gap = 5 + rnd.nextInt(896); + t += gap.toDouble(); + continue; + } + p += c.backwardsPerHour * beatMin / 60.0; + if (u < p && ts.isNotEmpty) { + emit(v); + ts[ts.length - 1] = ts.last - 1000.0 * (1 + rnd.nextInt(3)); + continue; + } + emit(v); + } + return RrData(rr, ts); +} + +/// The checked-in anonymised 8.9 h WHOOP-4 night (rebased to 0 s). Returns +/// null if the fixture is missing (callers decide whether to fail). +RrData? realNightRr({double startSec = 1.7e9}) { + final f = File('test/onehz/fixtures/real_night_2026_07_rr.csv'); + if (!f.existsSync()) return null; + final rr = [], ts = []; + for (final line in f.readAsLinesSync().skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(','); + ts.add(startSec * 1000 + double.parse(p[0])); + rr.add(double.parse(p[1])); + } + return RrData(rr, ts); +} + +/// A whole 24 h-ish day: synthetic morning + REAL night + synthetic evening. +/// The synthetic parts carry the artefact load; the night is as recorded. +RrData realShapedDay({int seed = 7, double startSec = 1.7e9}) { + final real = realNightRr(startSec: startSec)!; + final nightSpanMs = real.ts.last - real.ts.first; + final morning = synthRr(SynthConfig( + seed: seed, hours: 5, startSec: startSec - 5 * 3600, nightFraction: 0)); + final evening = synthRr(SynthConfig( + seed: seed + 1, + hours: 9, + startSec: startSec + nightSpanMs / 1000 + 120, + nightFraction: 0)); + return RrData([...morning.rr, ...real.rr, ...evening.rr], + [...morning.ts, ...real.ts, ...evening.ts]); +} diff --git a/test/properties/rr_irregular_laws_test.dart b/test/properties/rr_irregular_laws_test.dart new file mode 100644 index 00000000..240ac7d4 --- /dev/null +++ b/test/properties/rr_irregular_laws_test.dart @@ -0,0 +1,2693 @@ +// LAWS of the streaming RR cleaning and the irregular-rhythm screen (design 05, +// pilot cluster C2A, analytics side): `RrCorrector` (foundations/ +// rr_correction_stream.dart), `IrregularScreenState` (clinical/ +// irregular_rhythm_state.dart) and the diagnostics they report +// (`IrregularDiagnostics`, clinical/irregular_diagnostics.dart), alone and +// integrated the way the day pass uses them (corrector settles NN, the screen +// folds it, the corrector's provisional tail and counts are handed to +// `evaluateDetailed`). +// +// L1 chunk invariance: any VALID chunking of the input (empty chunks and a +// save/restore through real JSON text between chunks included) gives the +// same checkpoint TEXT, the same settled output and the same snapshot as +// one fold of the whole input. This is NOT a monoid claim (neither module +// has a `combine`); it is a law of the append transition. The documented +// refusals (mixing timestamped and timestamp-free folds, a non-finite RR +// or timestamp, a length mismatch) are asserted as refusals that leave +// the state unchanged. +// L1b streamed vs batch: at every checked seam `settled ++ snapshot` is +// `correctRr` of the prefix bit for bit (the FROZEN reference copy for +// short series, the production batch beyond the reference's budget), and +// the screen is the batch screen: counts, flags, abstentions and the +// diagnostics text exact; SD1 / SD2 / ratio within 1e-9 relative and +// confidence within 1e-12 (running sums against a two-pass batch). The +// window counts are checked against a window partition written HERE, +// independently of `irregularWindowVerdict` (which both sides share). +// L2 checkpoint round trip: `toJson` -> real JSON text -> `fromJson` -> +// `toJson` is the same text, and the restored object reads what the live +// one reads on an explicit projection and goes on identically. +// L3 a checkpoint of another type or version, a truncated, padded or +// inconsistent one, hostile counts, parts that disagree: refused whole +// with a FormatException (never another error type, never a half-built +// object). A mutation the reader cannot tell from a legal checkpoint +// (one number changed) may be accepted, but then the object holds exactly +// what was written: nothing is silently normalised. +// L4 identity: an empty chunk leaves a checkpoint unchanged (the corrector +// records NOTHING on a first empty fold, not even whether timestamps are +// used), `evaluate*` never changes the state it reads, and the other +// way round a refused fold leaves the state unchanged. +// L5 conservation, integrated, provisional NN included: nn_in == rr_raw - +// dropped; normal + corrected + dropped == beats; the windows cover the +// kept NN (independent partition); flagged <= valid <= total <= kept; an +// invalid window config gives NO window evidence and cannot flag; the +// artifact fraction is a share of the beats the corrector saw, or ABSENT +// (null, never 1.0) when it saw none (0fc5768). +// W the diagnostics wire (what edge persists): round trip through JSON +// text, refusal of another version / unknown reason / malformed map, and +// the "absent, not zero" rules for the cleaning counts. +// +// The laws describe the code at analytics 0fc57682, with upstream's screen +// rules since (f026a4e merge): a pair across a time hole is not successive, the +// beat gate and window validity count successive pairs (minBeats - 1), and the +// screen checkpoint is version 3 (it carries the previous beat's time). A failing law is first +// checked against the module's contract; only a violation of the intended +// contract is a bug, and a wrong oracle is fixed here, never in lib/. +// +// A series is a RECIPE (flavour, beats, seed, ...) expanded deterministically, +// so a failing input shrinks to a few readable integers, not to thousands of +// doubles. Beats are capped at 3,000 per series and the chunking at 6 split +// points. No clock is read. +// +// Replay a failure with the command in its report, e.g. +// PROPERTY_SEED= PROPERTY_CASE= TZ=UTC dart test \ +// test/properties/rr_irregular_laws_test.dart --plain-name '' + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../onehz/support/correct_rr_reference.dart'; +import '../onehz/support/rr_compare.dart'; +import '../support/fold_law_support.dart'; +import '../support/law_registry.dart'; +import '../support/property.dart'; + +final _laws = LawSet(); + +// ── the recipe of a beat series ───────────────────────────────────────────── + +const _flavours = [ + 'clean', // regular beats, a little jitter + 'mixed', // ectopic pairs, missed / extra beats, noise runs, dropouts + 'flagged', // irregularly irregular (AF-like) throughout + 'gappy', // a dropout every 90 beats + 'backwards', // a beat time that steps back now and then + 'artefact heavy', // many artefacts + one run of 120 (longer than the window) + 'flat', // constant RR: QD = 0, massive ties + 'outliers', // flat, with an outlier at the start, middle and end + 'edge gaps', // dropouts of exactly, one under and one over each re-anchor gap + 'hard limits', // 300 and 2000 exactly, then a too-long and a too-short beat +]; +const int _nFlavours = 10; + +/// 2025-10-09 08:53:20 UTC in ms: a whole second, as production stamps are. +const double _t0 = 1760000000000; + +/// Corrector parameters: (alpha, windowBeats, minThresholdMs, reanchorGapMs). +/// Index 0 is the default. They travel in the checkpoint. +const List<(double, int, double, double)> _corrCfgs = [ + (5.2, 91, 100, 1000), + (5.2, 31, 100, 1000), + (4.0, 61, 100, 1000), + (6.5, 91, 30, 400), + (5.2, 4, 0, 1000), + (5.2, 90, 100, 5000), + (5.2, 1, 100, 1000), // a window of one beat: no look-ahead at all + (5.2, 2, 100, 1000), + (5.2, 3, 30, 400), +]; + +RrCorrector _newCorrector(int cfg) { + final c = _corrCfgs[cfg]; + return RrCorrector( + maxArtifactRun: null, // This harness pins the uncapped pipeline layout. + alpha: c.$1, windowBeats: c.$2, minThresholdMs: c.$3, reanchorGapMs: c.$4); +} + +class _Series { + _Series(this.rr, this.ts); + final List rr, ts; + int get n => rr.length; +} + +final Map<(int, int, int), _Series> _seriesCache = {}; + +/// The beats of a recipe (cached: expansion is not the cost under test). +_Series _series(int flavour, int n, int seed) => + _seriesCache.putIfAbsent((flavour, n, seed), () { + if (_seriesCache.length > 400) _seriesCache.clear(); + return _expandSeries(flavour, n, seed); + }); + +_Series _expandSeries(int flavour, int n, int seed) { + final r = Rng(seed * 7919 + flavour * 131 + 11); + final rr = [], ts = []; + var clock = _t0; + void emit(double v) { + final e = v.roundToDouble(); + clock += e; + rr.add(e); + // End-of-beat time quantised to whole seconds (rec_ts * 1000). + ts.add(flavour == 8 ? clock : (clock / 1000).floorToDouble() * 1000); + } + + var longRun = false; + while (rr.length < n) { + final i = rr.length; + final v = 800 + 40 * math.sin(i / 70) + 12 * (r.nextDouble() - .5); + switch (flavour) { + case 1: + case 5: + final u = r.nextDouble(); + final heavy = flavour == 5; + if (heavy && !longRun && i >= n ~/ 3 && n >= 240) { + longRun = true; + for (var k = 0; k < 120; k++) { + emit(250 + r.nextInt(2100).toDouble()); + } + } else if (u < (heavy ? .08 : .02) && i > 3) { + emit(v * .6); + emit(v * 1.4); + } else if (u < (heavy ? .10 : .03)) { + emit(v * 1.95); + } else if (u < (heavy ? .12 : .04)) { + emit(v * .5); + emit(v * .5); + } else if (u < (heavy ? .14 : .046)) { + for (var k = 3 + r.nextInt(10); k > 0; k--) { + emit(250 + r.nextInt(2100).toDouble()); + } + } else if (!heavy && u < .051) { + clock += 5000 + r.nextInt(300000); + emit(v); + } else { + emit(v); + } + case 2: + emit(420 + r.nextInt(700).toDouble()); + case 3: + if (i % 90 == 89) clock += 5000 + r.nextInt(115000); + emit(v); + case 4: + emit(v); + if (i % 97 == 96) ts[ts.length - 1] -= 1000.0 * (1 + r.nextInt(3)); + case 6: + emit(800); + case 7: + emit(i == 0 || i == n ~/ 2 || i == n - 1 ? 1300 : 800); + case 9: + // The edges of the plausible range, exactly, then one beat past each: + // 300 and 2000 are plausible, 2500 and 250 are not. + emit(i < 4 ? const [300.0, 2000.0, 2500.0, 250.0][i] : v); + case 8: + // The wall-clock step exceeds the beat by exactly the re-anchor gap of + // one of the configs (400, 1000, 5000 ms), one ms under it, one over. + emit(v); + if (i % 12 == 11) { + const gaps = [400.0, 399.0, 401.0, 1000.0, 999.0, 1001.0, 5000.0, 4999.0, 5001.0]; + clock += gaps[(i ~/ 12) % gaps.length]; + ts[ts.length - 1] = clock; // (exact milliseconds, not whole seconds) + } + default: + emit(v); + } + } + if (rr.length > n) { + rr.removeRange(n, rr.length); + ts.removeRange(n, ts.length); + } + return _Series(rr, ts); +} + +// ── the recipe of a chunking ──────────────────────────────────────────────── + +/// (flavour, beats, seed, mode: 0 timestamps given, 1 none). +typedef _Rec = (int, int, int, int); + +/// (splits, cut seed, restart mask, corrector config index). +typedef _Plan = (int, int, int, int); +typedef _Case = (_Rec, _Plan); + +class _FlavourGen extends Gen { + @override + int generate(Rng r, int size) => r.nextInt(_nFlavours); + @override + Iterable shrink(int v) => G.intIn(0, _nFlavours - 1).shrink(v); +} + +/// Beat counts: the shapes that matter (empty, under 3, the window edges 45 / +/// 91 and their neighbours), then growing with the case index up to [maxN]. +class _NGen extends Gen { + _NGen(this.maxN); + final int maxN; + static const _pool = [ + 0, 1, 2, 3, 4, 5, 10, 44, 45, 46, 47, 89, 90, 91, 92, 93, 94, 135, 136, // + 137, 181, 182, 183, 272, 273, + ]; + @override + int generate(Rng r, int size) { + final p = r.nextDouble(); + if (p < .04) return 0; + if (p < .29) { + final ok = [for (final x in _pool) if (x <= maxN) x]; + return ok[r.nextInt(ok.length)]; + } + if (p < .40) return r.intIn(math.min(500, maxN), maxN); + final cap = math.max(120, maxN * size ~/ 100); + if (p < .62) return r.intIn(0, math.min(cap, 400)); + return r.intIn(0, math.min(cap, maxN)); + } + + @override + Iterable shrink(int v) => G.intIn(0, maxN).shrink(v); +} + +class _CutSeedGen extends Gen { + @override + int generate(Rng r, int size) => r.nextInt(1 << 20); + @override + Iterable shrink(int v) => G.intIn(0, 1 << 20).shrink(v); +} + +Gen<_Case> _caseGen(int maxN) => G.pair( + G.quad(_FlavourGen(), _NGen(maxN), G.intIn(0, 1 << 16), G.elements(const [0, 0, 0, 1])), + G.quad(G.intIn(0, 6), _CutSeedGen(), G.intIn(0, 127), + G.elements(const [0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8])), + ); + +const _windowEdges = [0, 1, 2, 3, 44, 45, 46, 90, 91, 92, 93, 136, 137, 182, 183]; + +/// Chunk boundaries, half of the cuts on the edges of the corrector's windows +/// (the beats where classes and output settle). +List _bounds(int n, int splits, int cutSeed) => + foldBounds(n, splits, cutSeed, special: _windowEdges); + +// ── folding ───────────────────────────────────────────────────────────────── + +String _text(RrCorrector c) => jsonEncode(c.toJson()); + +RrCorrector _restore(RrCorrector c) => RrCorrector.fromJson( + (jsonDecode(jsonEncode(c.toJson())) as Map).cast()); + +List? _tsSlice(_Series s, bool wall, int a, int b) => + wall ? s.ts.sublist(a, b) : null; + +/// What a driven corrector handed out. +class _Run { + RrCorrector c; + final nn = [], times = []; + final classes = []; + _Run(this.c); +} + +void _foldInto(_Run run, _Series s, bool wall, int a, int b) { + final out = run.c.fold(s.rr.sublist(a, b), tsMs: _tsSlice(s, wall, a, b)); + run.nn.addAll(out.nn); + run.times.addAll(out.nnTimes); + run.classes.addAll(out.classes); +} + +/// Folds [s] in the chunks [bounds]; after chunk k the corrector is restored +/// through JSON text when bit k of [restart] is set. [seam] sees every seam +/// after the chunk (and the restore) with the end of the chunk. +_Run _drive(_Series s, bool wall, int cfg, List bounds, int restart, + {void Function(_Run run, int k, int at)? seam}) { + final run = _Run(_newCorrector(cfg)); + for (var k = 0; k + 1 < bounds.length; k++) { + _foldInto(run, s, wall, bounds[k], bounds[k + 1]); + if ((restart >> (k % 7)) & 1 == 1) run.c = _restore(run.c); + seam?.call(run, k, bounds[k + 1]); + } + return run; +} + +void _sameSnapshot(RrSnapshot a, RrSnapshot b, String why) { + expect(a.n, b.n, reason: 'n $why'); + expectBitIdentical(a.tailNn, b.tailNn, '$why tailNn'); + expectBitIdentical(a.tailNnTimes, b.tailNnTimes, '$why tailNnTimes'); + expectSameClasses(a.tailClasses, b.tailClasses, '$why tailClasses'); + expect(a.normalCount, b.normalCount, reason: 'normal $why'); + expect(a.droppedCount, b.droppedCount, reason: 'dropped $why'); + expect(a.correctedCount, b.correctedCount, reason: 'corrected $why'); + expect(a.settledBeats, b.settledBeats, reason: 'settled $why'); + expect(a.classifiedBeats, b.classifiedBeats, reason: 'classified $why'); + expect(sameBits(a.cleanFraction, b.cleanFraction), isTrue, + reason: 'cleanFraction $why'); +} + +/// Case-level facts from the recipe alone (no fold), for the reach observers. +void _observeRr(_Case c, void Function(String) bump) { + final (flavour, n, _, mode) = c.$1; + final (splits, cutSeed, restart, cfg) = c.$2; + bump('flavour: ${_flavours[flavour]}'); + if (n == 0) bump('beats: none'); + if (n > 0 && n < 3) bump('beats: under 3 (the short branch)'); + if (n >= 3 && n < 91) bump('beats: inside the window (3 to 90)'); + if (n >= 91) bump('beats: past the window (91 or more)'); + if (n >= 1000) bump('beats: 1000 or more'); + if (mode == 1) bump('no timestamps'); + if (cfg != 0) bump('non-default config'); + final b = _bounds(n, splits, cutSeed); + if (b.length >= 4) bump('chunks: three or more'); + if (b.length >= 6) bump('chunks: five or more'); + if (hasEmptyChunk(b)) bump('an empty chunk'); + if (restart != 0 && b.length > 2) bump('a restore between chunks'); + if (b.any((x) => x == 91 || x == 92 || x == 45 || x == 46)) { + bump('a cut on a window edge (45 / 46 / 91 / 92)'); + } +} + +const Map _rrShares = { + 'flavour: clean': .03, + 'flavour: mixed': .03, + 'flavour: flagged': .03, + 'flavour: gappy': .03, + 'flavour: backwards': .03, + 'flavour: artefact heavy': .03, + 'flavour: flat': .03, + 'flavour: outliers': .03, + 'flavour: edge gaps': .02, + 'flavour: hard limits': .02, + 'beats: none': .01, + 'beats: under 3 (the short branch)': .015, + 'beats: inside the window (3 to 90)': .1, + 'beats: past the window (91 or more)': .4, + 'no timestamps': .08, + 'non-default config': .2, + 'chunks: three or more': .4, + 'chunks: five or more': .15, + 'an empty chunk': .05, + 'a restore between chunks': .3, + 'a cut on a window edge (45 / 46 / 91 / 92)': .1, +}; + +// ── forced scenarios ──────────────────────────────────────────────────────── + +_Case _c(int flavour, int n, + {int seed = 5, + int mode = 0, + int splits = 3, + int cutSeed = 1, + int restart = 0, + int cfg = 0}) => + ((flavour, n, seed, mode), (splits, cutSeed, restart, cfg)); + +/// Thin, flat, artefact runs, boundaries (the window edges, exactly 3 beats and +/// exactly one window), timestamp-free, every config. Every law that takes a +/// recipe meets these first. +final List<_Case> _forced = [ + _c(0, 0), // no beats at all + _c(0, 1), // one beat + _c(0, 2, restart: 127), // two: the < 3 branch + _c(0, 3, restart: 127), // exactly three + _c(9, 1), // exactly 300 ms: plausible + _c(9, 2, restart: 127), // 300 and 2000 exactly + _c(9, 3, restart: 127), // and a too-long beat: the first main-path verdict + _c(9, 4, restart: 127), // and a too-short one + _c(9, 120, splits: 4, cutSeed: 28, restart: 127), + _c(1, 5, splits: 6, restart: 127), + _c(0, 44, splits: 4, cutSeed: 3), // under the half window + _c(0, 46, splits: 5, cutSeed: 4), // the first beat whose window is complete + _c(0, 90, splits: 3, cutSeed: 5), // one short of the first settled class + _c(0, 91, splits: 3, cutSeed: 6), // exactly the first settled class + _c(0, 92, splits: 3, cutSeed: 7), + _c(6, 200, splits: 4, cutSeed: 8, restart: 127), // flat: QD = 0 + _c(7, 150, splits: 4, cutSeed: 9), // outlier at start / middle / end + _c(7, 3), // the outlier is the first and the last beat + _c(5, 400, splits: 6, cutSeed: 10, restart: 85), // artefacts + a 120 run + _c(4, 300, splits: 5, cutSeed: 11, restart: 127), // a beat time steps back + _c(3, 300, splits: 5, cutSeed: 12), // dropouts + _c(1, 350, splits: 6, cutSeed: 13, mode: 1), // no timestamps + _c(2, 400, splits: 6, cutSeed: 14, restart: 127), // irregularly irregular + _c(1, 300, splits: 6, cutSeed: 15, cfg: 1), + _c(1, 300, splits: 6, cutSeed: 16, cfg: 2, restart: 127), + _c(1, 300, splits: 6, cutSeed: 17, cfg: 3, mode: 1), + _c(1, 300, splits: 6, cutSeed: 18, cfg: 4, restart: 127), // window of 4 beats + _c(5, 300, splits: 6, cutSeed: 19, cfg: 5), // an even window of 90 + _c(8, 300, splits: 5, cutSeed: 22, restart: 127), // dropouts exactly at the gap + _c(8, 300, splits: 5, cutSeed: 23, cfg: 3, restart: 85), // gap 400 + _c(8, 300, splits: 5, cutSeed: 24, cfg: 5, restart: 127), // gap 5000 + _c(1, 200, splits: 6, cutSeed: 25, cfg: 6, restart: 127), // a window of one beat + _c(5, 200, splits: 6, cutSeed: 26, cfg: 7), // two beats + _c(1, 200, splits: 6, cutSeed: 27, cfg: 8, restart: 127), // three beats + _c(1, 1200, splits: 6, cutSeed: 20, restart: 127), // past 1000 beats + _c(1, 3000, splits: 6, cutSeed: 21, restart: 127), // the cap +]; + +/// A day with settled beats, artefacts and several windows: the base the +/// mutated-checkpoint law cuts its mutations from. +final _Case _l3Base = _forced.firstWhere( + (c) => c.$1.$1 == 1 && c.$1.$2 >= 300 && c.$1.$4 == 0 && c.$2.$4 == 0); + +// ── RC: the corrector ─────────────────────────────────────────────────────── + +String _tag(_Case c) { + final ((f, n, seed, mode), (splits, cutSeed, restart, cfg)) = c; + return '${_flavours[f]} n=$n seed=$seed ts=${mode == 0} splits=$splits ' + 'cut=$cutSeed restart=$restart cfg=$cfg'; +} + +void _l1Corrector(_Case c) { + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final (splits, cutSeed, restart, cfg) = plan; + final s = _series(flavour, n, seed); + final wall = mode == 0; + final tag = _tag(c); + final whole = _drive(s, wall, cfg, [0, n], 0); + final bounds = _bounds(n, splits, cutSeed); + final got = _drive(s, wall, cfg, bounds, restart); + expect(_text(got.c), _text(whole.c), + reason: '$tag bounds=$bounds: the checkpoint text is the same'); + expectBitIdentical(got.nn, whole.nn, '$tag settled nn'); + expectBitIdentical(got.times, whole.times, '$tag settled times'); + expectSameClasses(got.classes, whole.classes, '$tag settled classes'); + _sameSnapshot(got.c.snapshot(), whole.c.snapshot(), tag); +} + + +/// Largest series the frozen reference is run on in a property (it is the +/// expensive side: about 0.14 ms a beat). Beyond it the production batch is +/// the oracle: weaker (it shares the kernel with the stream), but pinned to the +/// reference by rr_correction_oracle_test.dart. +const int _refCap = 300; + +RrCorrectionResult _oracle(_Series s, bool wall, int cfg, int at) { + final p = _corrCfgs[cfg]; + final rr = s.rr.sublist(0, at), ts = wall ? s.ts.sublist(0, at) : null; + return at <= _refCap + ? correctRrReference(rr, + rrTsMs: ts, + alpha: p.$1, + windowBeats: p.$2, + minThresholdMs: p.$3, + reanchorGapMs: p.$4) + : correctRr(rr, + rrTsMs: ts, + alpha: p.$1, + windowBeats: p.$2, + minThresholdMs: p.$3, + reanchorGapMs: p.$4); +} + +void _expectOracleAt(_Run run, _Series s, bool wall, int cfg, int at, String why) { + final want = _oracle(s, wall, cfg, at); + if (at <= _refCap) { + // The production batch is the oracle beyond the reference's budget; hold it + // to the reference wherever both run, so that swap stays justified. + final p = _corrCfgs[cfg]; + expectSameCorrection( + correctRr(s.rr.sublist(0, at), + rrTsMs: wall ? s.ts.sublist(0, at) : null, + alpha: p.$1, + windowBeats: p.$2, + minThresholdMs: p.$3, + reanchorGapMs: p.$4), + want, + '$why: production batch vs frozen reference'); + } + final snap = run.c.snapshot(); + expect(snap.n, at, reason: 'snapshot.n $why'); + expect(snap.classifiedBeats, run.classes.length, + reason: 'classifiedBeats == classes handed out $why'); + expect(snap.settledBeats, run.c.settledBeats, reason: 'settledBeats $why'); + expect(snap.settledBeats, lessThanOrEqualTo(snap.classifiedBeats), + reason: 'output cannot settle before its class $why'); + expectBitIdentical([...run.nn, ...snap.tailNn], want.nn, '$why nn'); + expectBitIdentical( + [...run.times, ...snap.tailNnTimes], want.nnTimesMs, '$why times'); + expectSameClasses([...run.classes, ...snap.tailClasses], want.classes, why); + expect(snap.normalCount, + want.classes.where((k) => k == BeatClass.normal).length, + reason: 'normalCount $why'); + expect(snap.droppedCount, want.droppedCount, reason: 'dropped $why'); + expect(snap.correctedCount, want.correctedCount, reason: 'corrected $why'); + expect(sameBits(snap.cleanFraction, want.cleanFraction), isTrue, + reason: 'cleanFraction ${snap.cleanFraction} vs ${want.cleanFraction} $why'); +} + +void _l1bCorrector(_Case c) { + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final (splits, cutSeed, restart, cfg) = plan; + final s = _series(flavour, n, seed); + final wall = mode == 0; + final tag = _tag(c); + final bounds = _bounds(n, splits, cutSeed); + final picks = pickSeams(bounds.length - 1); + _drive(s, wall, cfg, bounds, restart, seam: (run, k, at) { + if (picks.contains(k)) { + _expectOracleAt(run, s, wall, cfg, at, '$tag seam $k at $at'); + } + }); +} + +/// What a reader can see of a corrector, as text: its parameters, its settled +/// edge and its snapshot (every field). +String _projectCorrector(RrCorrector c) { + final s = c.snapshot(); + return jsonEncode([ + c.alpha, + c.windowBeats, + c.minThresholdMs, + c.reanchorGapMs, + c.settledBeats, + s.n, + s.tailNn, + s.tailNnTimes, + [for (final k in s.tailClasses) k.index], + s.normalCount, + s.droppedCount, + s.correctedCount, + s.settledBeats, + s.classifiedBeats, + s.cleanFraction, + ]); +} + +void _l2Corrector(_Case c) { + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final (splits, cutSeed, _, cfg) = plan; + final s = _series(flavour, n, seed); + final wall = mode == 0; + final tag = _tag(c); + final bounds = _bounds(n, splits, cutSeed); + // Fold the first part, copy the state through JSON text, fold the rest into + // both: the copy reads the same and goes on identically. + final at = bounds.length > 2 ? bounds[1] : n ~/ 2; + final live = _Run(_newCorrector(cfg)); + _foldInto(live, s, wall, 0, at); + final text = _text(live.c); + final map = (jsonDecode(text) as Map).cast(); + final back = _Run(RrCorrector.fromJson(map)); + expect(_text(back.c), text, reason: '$tag: write(read(b)) == b'); + expect(_projectCorrector(back.c), _projectCorrector(live.c), + reason: '$tag: the restored one reads what the live one reads'); + // The restored object owns its state: the map it was read from is not shared. + for (final k in ['rr', 't', 'd', 'fcls', 'lastNormals']) { + (map[k] as List).clear(); + } + expect(_text(back.c), text, reason: '$tag: not aliased to the map'); + final ownSettled = live.nn.length; + _foldInto(live, s, wall, at, n); + _foldInto(back, s, wall, at, n); + expectBitIdentical(back.nn, live.nn.sublist(ownSettled), '$tag nn after restore'); + expectBitIdentical(back.times, live.times.sublist(ownSettled), '$tag times after restore'); + expectSameClasses(back.classes, live.classes.sublist(live.classes.length - back.classes.length), tag); + expect(_text(back.c), _text(live.c), reason: '$tag: same checkpoint after the rest'); + expect(_projectCorrector(back.c), _projectCorrector(live.c), reason: tag); +} + +// ── L3: mutated checkpoints ───────────────────────────────────────────────── + +/// Applies mutation [m] to a copy of [src] (a decoded checkpoint). Returns the +/// mutated map and what it may do. The "must refuse" cases are the ones that +/// make the checkpoint contradict itself or its type; they are chosen from what +/// a reachable state can never look like, not from the reader's code. +(Map, Must) _mutateCorrector(Map src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + final keys = j.keys.toList(); + List l(String k) => j[k] as List; + switch (kind % 11) { + case 0: + const versions = [0, 1, 3, -1, 99, '2', null, true, 1 << 40, 'v2']; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse); + case 1: + const types = ['IrregularScreenState', 'Other', null, '', 'rrcorrector', 1]; + j['type'] = types[a % types.length]; + return (j, Must.refuse); + case 2: + // A key that every real checkpoint has, gone ('wall' may legally be null). + final k = keys.where((k) => k != 'wall').toList()[a % (keys.length - 1)]; + j.remove(k); + return (j, Must.refuse); + case 3: + // The wrong kind of value under a key. + final k = keys.where((k) => k != 'wall').toList()[a % (keys.length - 1)]; + const junk = ['x', {}, [[]]]; + j[k] = junk[b % junk.length]; + return (j, Must.refuse); + case 4: + // The buffered arrays disagree with each other or with n - off. + final k = const ['rr', 't', 'd'][a % 3]; + final arr = l(k); + switch (b % 3) { + case 0: + if (arr.isEmpty) { + arr.add(1.0); + } else { + arr.removeLast(); + } + case 1: + arr.add(arr.isEmpty ? 1.0 : arr.last); + default: + arr.clear(); + if (j['n'] == j['off']) arr.add(1.0); + } + return (j, Must.refuse); + case 5: + // The counters contradict their own order off <= ce <= c2 <= c1 <= n. + switch (a % 6) { + case 0: + j['ce'] = (j['c2'] as int) + 1; + case 1: + j['c2'] = (j['c1'] as int) + 1; + case 2: + j['c1'] = (j['n'] as int) + 1; + case 3: + j['off'] = -1; + case 4: + j['n'] = (j['n'] as int) + 1 + b % 3; + default: + j['n'] = (j['n'] as int) - 1 - b % 3; + } + return (j, Must.refuse); + case 6: + // The class buffer: wrong length, or a class that is not one. + final f = l('fcls'); + switch (a % 3) { + case 0: + f.add(0); + case 1: + if (f.isEmpty) { + f.add(5); + } else { + f[b % f.length] = const [5, -1, 99, 1 << 40][b % 4]; + } + default: + if (f.isEmpty) { + f.add(0); + } else { + f.removeLast(); + } + } + return (j, Must.refuse); + case 7: + // The carried state: more than two normals, a class that is not one. + if (a.isEven) { + l('lastNormals') + ..add(800.0) + ..add(801.0) + ..add(802.0); + } else { + j['lastFinal'] = const [5, -1, 99, 1 << 40][b % 4]; + } + return (j, Must.refuse); + case 8: + // Hostile sizes: the checkpoint says it holds far more than it does. + const huge = [1 << 40, 1 << 62, 0x7fffffffffffffff, -1]; + j[const ['n', 'off', 'c1', 'c2', 'ce'][a % 5]] = huge[b % huge.length]; + return (j, Must.refuse); + case 9: + // One number changed: the reader cannot tell it from a legal checkpoint. + switch (a % 9) { + case 0: + if (l('rr').isNotEmpty) l('rr')[b % l('rr').length] = (l('rr')[b % l('rr').length] as num) + 1.0; + case 1: + if (l('t').isNotEmpty) l('t')[b % l('t').length] = (l('t')[b % l('t').length] as num) + 1.0; + case 2: + if (l('d').isNotEmpty) l('d')[b % l('d').length] = (l('d')[b % l('d').length] as num) + 1.0; + case 3: + j['alpha'] = (j['alpha'] as num) + 1.0; + case 4: + j['minThresholdMs'] = (j['minThresholdMs'] as num) + 1.0; + case 5: + j['reanchorGapMs'] = (j['reanchorGapMs'] as num) + 1.0; + case 6: + j['dropped'] = (j['dropped'] as int) + 1; + case 7: + j['normalFinal'] = (j['normalFinal'] as int) + 1; + default: + j['wall'] = !(j['wall'] as bool? ?? false); + } + return (j, Must.faithful); + default: + // A key this version does not know. + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + return (j, Must.ignored); + } +} + +String _checkpointOutcome(Map j) { + try { + final c = RrCorrector.fromJson((deepCopy(j) as Map).cast()); + return 'accepted:${jsonEncode(c.toJson())}'; + } on FormatException { + return 'refused'; + } catch (e) { + return 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } +} + +final Gen _mutGen = mutGen(11); + +void _l3Corrector((_Case, Mut) arg) { + final (c, m) = arg; + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final cfg = plan.$4; + final s = _series(flavour, n, seed); + final tag = '${_tag(c)} mutation=${m.$1 % 11}(${m.$2},${m.$3})'; + final run = _drive(s, mode == 0, cfg, [0, n], 0); + final src = (jsonDecode(_text(run.c)) as Map).cast(); + final (bad, must) = _mutateCorrector(src, m); + expectMutationOutcome(_checkpointOutcome(bad), must, jsonEncode(bad), + jsonEncode(src), tag); +} + +// ── L4 / refusals ─────────────────────────────────────────────────────────── + +void _l4Corrector(_Case c) { + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final (_, _, _, cfg) = plan; + final s = _series(flavour, n, seed); + final wall = mode == 0; + final tag = _tag(c); + // Identity: an empty chunk, with or without timestamps, at the start (before + // anything records the mode), in the middle and at the end. + final fresh = _text(_newCorrector(cfg)); + for (final withTs in [true, false]) { + final f = _newCorrector(cfg); + final out = f.fold(const [], tsMs: withTs ? const [] : null); + expect(out.nn, isEmpty, reason: tag); + expect(out.nnTimes, isEmpty, reason: tag); + expect(out.classes, isEmpty, reason: tag); + expect(_text(f), fresh, + reason: '$tag: an empty first fold records nothing (not even the mode)'); + } + final cut = n ~/ 2; + final run = _Run(_newCorrector(cfg)); + // An empty fold before the first beat must not decide whether timestamps are + // used: the series still folds in either mode and reads as if it never ran. + run.c.fold(const [], tsMs: wall ? null : const []); + _foldInto(run, s, wall, 0, cut); + final before = _text(run.c); + for (final withTs in [true, false]) { + final out = run.c.fold(const [], tsMs: withTs ? const [] : null); + expect(out.nn.length + out.nnTimes.length + out.classes.length, 0, reason: tag); + expect(_text(run.c), before, reason: '$tag: an empty chunk is the identity'); + } + // snapshot() reads and changes nothing. + final one = _projectCorrector(run.c); + expect(_text(run.c), before, reason: '$tag: snapshot() is read-only'); + expect(_projectCorrector(run.c), one, reason: '$tag: twice is equal'); + // Refusals leave the state exactly as it was. + if (cut > 0) { + final good = s.rr.sublist(cut, math.min(n, cut + 3)); + final goodTs = s.ts.sublist(cut, math.min(n, cut + 3)); + if (good.isNotEmpty) { + // The other mode. + expect(() => run.c.fold(good, tsMs: wall ? null : goodTs), throwsStateError, + reason: '$tag: timestamps on every fold or on none'); + expect(_text(run.c), before, reason: '$tag: unchanged by the mixed fold'); + // A length mismatch (timestamps on). + if (wall) { + expect(() => run.c.fold(good, tsMs: goodTs.sublist(0, goodTs.length - 1)), + throwsArgumentError, reason: tag); + expect(_text(run.c), before, reason: '$tag: unchanged by the mismatch'); + } + // A non-finite value AFTER good beats in the same chunk must not stick. + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + final rr = [...good, bad], ts = [...goodTs, goodTs.last + 1000]; + expect(() => run.c.fold(rr, tsMs: wall ? ts : null), throwsArgumentError, + reason: '$tag: non-finite RR $bad'); + expect(_text(run.c), before, reason: '$tag: unchanged by $bad'); + if (wall) { + final badTs = [...goodTs, bad]; + expect(() => run.c.fold([...good, 800], tsMs: badTs), throwsArgumentError, + reason: '$tag: non-finite timestamp $bad'); + expect(_text(run.c), before, reason: '$tag: unchanged by timestamp $bad'); + } + } + } + } +} + +// ── L5: conservation ──────────────────────────────────────────────────────── + +void _checkConserved(RrSnapshot snap, _Run run, int cfg, String why) { + final n = snap.n; + // The class of a beat is final once the window two thresholds deep (2h beats + // each side) is in the data (header of rr_correction_stream.dart); under 3 + // beats nothing is. + final h = _corrCfgs[cfg].$2 ~/ 2; + expect(snap.classifiedBeats, n < 3 ? 0 : math.max(0, n - 2 * h), + reason: 'classes settle exactly 2h beats behind the newest $why'); + final nnTotal = run.nn.length + snap.tailNn.length; + expect(nnTotal, n - snap.droppedCount, + reason: 'nn_in == rr_raw - dropped $why'); + expect(snap.normalCount + snap.correctedCount + snap.droppedCount, n, + reason: 'every beat is kept, corrected or dropped, once $why'); + expect(snap.droppedCount >= 0 && snap.correctedCount >= 0, isTrue, reason: why); + expect(run.classes.length + snap.tailClasses.length, n, + reason: 'a class for every beat $why'); + final normals = [...run.classes, ...snap.tailClasses] + .where((k) => k == BeatClass.normal) + .length; + expect(snap.normalCount, normals, reason: 'normal count is the class count $why'); + expect(snap.settledBeats <= snap.classifiedBeats && snap.classifiedBeats <= n, + isTrue, + reason: 'settled <= classified <= folded $why'); + expect(snap.cleanFraction, n == 0 ? 0.0 : snap.normalCount / n, + reason: 'cleanFraction is a share of the beats, 0 on none, never NaN $why'); + expect(snap.cleanFraction >= 0 && snap.cleanFraction <= 1, isTrue, reason: why); + expect(run.nn.length, run.times.length, reason: why); + expect(snap.tailNn.length, snap.tailNnTimes.length, reason: why); + final all = [...run.times, ...snap.tailNnTimes]; + for (var i = 1; i < all.length; i++) { + if (all[i] < all[i - 1]) { + fail('beat times never run backwards, whatever the stamps do: ' + '${all[i - 1]} then ${all[i]} at $i $why'); + } + } +} + +void _l5Corrector(_Case c) { + final (rec, plan) = c; + final (flavour, n, seed, mode) = rec; + final (splits, cutSeed, restart, cfg) = plan; + final s = _series(flavour, n, seed); + final tag = _tag(c); + final bounds = _bounds(n, splits, cutSeed); + _drive(s, mode == 0, cfg, bounds, restart, seam: (run, k, at) { + _checkConserved(run.c.snapshot(), run, cfg, '$tag seam $k at $at'); + }); +} + +// ── SC: the screen state, folding a corrected NN series directly ──────────── + +const _nnFlavours = [ + 'sinus', // regular, slow drift + 'af', // irregularly irregular throughout + 'blocks', // blocks of AF and sinus, 5 to 30 minutes + 'flat', // constant + 'two point', // alternating 800 / 1000 + 'sinus + gaps', // sinus with dropouts (time jumps) + 'af heavy', // AF in most blocks + 'range edges', // 300 and 2000 exactly (kept), 299 and 2001 (not), then calm beats +]; +const int _nNnFlavours = 8; + +/// (out-of-range share, NaN share) of the entries. +const List<(double, double)> _salts = [(0, 0), (.02, 0), (.2, .02), (0, .02)]; + +class _Nn { + _Nn(this.nn, this.t); + final List nn, t; + int get n => nn.length; +} + +final Map<(int, int, int, int), _Nn> _nnCache = {}; + +_Nn _nnSeries(int flavour, int n, int seed, int salt) => + _nnCache.putIfAbsent((flavour, n, seed, salt), () { + if (_nnCache.length > 400) _nnCache.clear(); + return _expandNn(flavour, n, seed, salt); + }); + +_Nn _expandNn(int flavour, int n, int seed, int salt) { + final r = Rng(seed * 104729 + flavour * 31 + salt + 3); + final (outShare, nanShare) = _salts[salt]; + final nn = [], t = []; + var clock = 0.0; + var af = false; + var left = 0; + for (var i = 0; i < n; i++) { + if (--left <= 0) { + af = switch (flavour) { + 1 => true, + 2 => r.nextBool(.4), + 6 => r.nextBool(.8), + _ => false, + }; + left = 300 + r.nextInt(2000); + } + var v = switch (flavour) { + 3 => 800.0, + 4 => i.isEven ? 800.0 : 1000.0, + 7 => const [300.0, 2000.0, 299.0, 2001.0, 800.0, 1000.0, 850.0][i % 7], + _ => af + ? (420 + r.nextInt(700)).toDouble() + : (850 + 40 * math.sin(i / 90) + 25 * (r.nextDouble() - .5)).roundToDouble(), + }; + var step = v; + final u = r.nextDouble(); + if (u < nanShare) { + v = double.nan; + step = 800; + } else if (u < nanShare + outShare) { + v = r.nextBool() ? 2500 : 250; + step = v; + } + if (flavour == 5 && i % 211 == 210) clock += 40000 + r.nextInt(300000); + clock += step; + nn.add(v); + t.add(clock); + } + return _Nn(nn, t); +} + +/// (flavour, beats, seed, salt). +typedef _NnRec = (int, int, int, int); + +/// Screen parameters: (sd1sd2Flag, pnnThresholdMs, pnnFlagPct, windowMinutes, +/// minWindowBeats, sustainedFraction). The last five are invalid window +/// configs: the batch fails CLOSED on them and the state keeps no windows. +const List<(double, double, double, double, int, double)> _scCfgs = [ + (.70, 70, 30, 5, 40, .5), + (.70, 70, 30, 1, 10, .5), + (.50, 50, 20, 2, 2, 0.0), + (.70, 70, 30, 5, 40, 1.0), + (.70, 70, 30, 0, 40, .5), // invalid: no window length + (.70, 70, 30, 5, 1, .5), // invalid: windows of fewer than 2 beats + (.70, 70, 30, 5, 40, 1.5), // invalid: a fraction above 1 + (.70, 70, 30, -5, 40, -.1), // invalid: two at once + (.90, 100, 60, 10, 100, .25), +]; +bool _scCfgValid(int i) { + final c = _scCfgs[i]; + return c.$4 > 0 && c.$5 >= 2 && c.$6 >= 0 && c.$6 <= 1; +} + +IrregularScreenState _newState(int cfg) { + final c = _scCfgs[cfg]; + return IrregularScreenState( + sd1sd2Flag: c.$1, + pnnThresholdMs: c.$2, + pnnFlagPct: c.$3, + windowMinutes: c.$4, + minWindowBeats: c.$5, + sustainedFraction: c.$6); +} + +/// What `evaluateDetailed` is asked: (minBeats, maxArtifact, artifactFraction, +/// cleaning: 0 none, 1 raw=beats+3, 2 raw=0). +const List<(int, double, double, int)> _evals = [ + (500, .30, 0.0, 1), + (10, .30, 0.0, 1), + (40, .30, .07, 0), + (100, .05, .31, 1), + (2, 1.0, 1.0, 1), + (20, .30, .29, 0), + (1, .30, 0.0, 2), + (30, .30, .30, 1), // the artifact gate is strict: exactly the maximum passes +]; + +RrCleaningCounts? _cleaningFor(int kind, int beats) => switch (kind) { + 0 => null, + 1 => RrCleaningCounts(raw: beats + 3, corrected: 2, dropped: 3), + _ => const RrCleaningCounts(raw: 0, corrected: 0, dropped: 0), + }; + +IrregularScreenResult _evaluate(IrregularScreenState st, int ev, + {List tailNn = const [], List tailT = const [], int beats = 0}) { + final e = _evals[ev]; + return st.evaluateDetailed(tailNn, tailT, + artifactFraction: e.$3, + minBeats: e.$1, + maxArtifact: e.$2, + cleaning: _cleaningFor(e.$4, beats)); +} + +IrregularScreenResult _batchScreenOf(List nn, List t, int cfg, int ev) { + final c = _scCfgs[cfg]; + final e = _evals[ev]; + return irregularBeatScreenDetailed(nn, + nnTimesMs: t, + artifactFraction: e.$3, + minBeats: e.$1, + maxArtifact: e.$2, + sd1sd2Flag: c.$1, + pnnThresholdMs: c.$2, + pnnFlagPct: c.$3, + windowMinutes: c.$4, + minWindowBeats: c.$5, + sustainedFraction: c.$6, + cleaning: _cleaningFor(e.$4, nn.length)); +} + +String _stext(IrregularScreenState s) => jsonEncode(s.toJson()); + +IrregularScreenState _srestore(IrregularScreenState s) => + IrregularScreenState.fromJson( + (jsonDecode(jsonEncode(s.toJson())) as Map).cast()); + +void _close(double got, double want, String why, {double rel = 1e-9}) { + final tol = rel * math.max(1.0, want.abs()); + if (!((got - want).abs() <= tol)) { + fail('$why: got $got want $want (tolerance $tol)'); + } +} + +/// Whether input beat [i] and the one before it form a successive pair, from +/// the documented rule alone: both kept (300..2000 ms), neighbours in the input +/// and, when beat times are given, no time hole between them (the elapsed time +/// is at most the interval, +0.5 ms of rounding; a NaN time is never within). +bool _successive(List nn, List? t, int i) { + bool kept(double v) => v >= 300 && v <= 2000; + return i > 0 && + kept(nn[i]) && + kept(nn[i - 1]) && + (t == null || t[i] - t[i - 1] <= nn[i] + 0.5); +} + +/// The kept beats of [nn] (300..2000 ms) and the successive differences taken +/// only between two successive beats (see [_successive]), from the documented +/// rule alone. +({List kept, List diffs}) _keptAndDiffs( + List nn, List? t) { + bool kept(double v) => v >= 300 && v <= 2000; + return ( + kept: [for (final v in nn) if (kept(v)) v], + diffs: [ + for (var i = 1; i < nn.length; i++) + if (_successive(nn, t, i)) nn[i] - nn[i - 1] + ], + ); +} + +/// The successive pairs the screen gates on (its minimum is minBeats - 1). +int _pairsOf(List nn, List? t) => _keptAndDiffs(nn, t).diffs.length; + +/// Whether SD2 is numerically zero for [nn]: 2*SDNN^2 - SD1^2 is within 1e-9 of +/// its own scale of nothing (an exactly two-valued alternating series is the +/// plain case). There the sign of float noise decides "no long-term +/// variability" versus a tiny SD2 and a ratio of 1e8, and running sums and a +/// two-pass batch legitimately disagree. Computed independently, from the +/// documented rule. +bool _sd2Degenerate(List nn, List? t) { + final k = _keptAndDiffs(nn, t); + final sdnn = _sd(k.kept), sdsd = _sd(k.diffs); + if (sdnn == null || sdsd == null || sdnn == 0) return false; + final v = 2 * sdnn * sdnn - sdsd * sdsd / 2; + return v.abs() <= 1e-9 * 2 * sdnn * sdnn; +} + +/// A reported SD2 is "numerically zero" when sd2^2 is within the same 1e-9 of +/// the scale 2*SDNN^2 (= sd1^2 + sd2^2) that [_sd2Degenerate] uses. The bound +/// is the float noise of the cancellation 2*SDNN^2 - SD1^2: two sums of squares +/// of the same magnitude differ by a few ulps (about 1e-15 relative, times a +/// factor that grows with the beat count, well under 1e-12 at the 3,000 beats +/// the generators cap at), so 1e-9 leaves a margin of more than 1e3 and +/// still rejects any SD2 that is a real fraction of SD1. +void _expectNumericallyZeroSd2(IrregularRhythm r, String why) { + expect(r.sd2.isFinite && r.sd2 >= 0, isTrue, reason: 'sd2 finite, >= 0 $why'); + expect(r.sd2 * r.sd2 <= 1e-9 * (r.sd1 * r.sd1 + r.sd2 * r.sd2), isTrue, + reason: 'a reported SD2 where the other side abstains is numerically ' + 'zero (sd1 ${r.sd1}, sd2 ${r.sd2}) $why'); +} + +/// Counts, flags, abstentions, notes and the diagnostics text exact; SD1 / SD2 / +/// ratio within 1e-9 relative, confidence within 1e-12 (running sums against a +/// two-pass batch). +/// +/// Pass [nn] (the beats the screen saw) to allow the one ambiguity of a +/// numerically zero SD2 (see [_sd2Degenerate]): the two sides may disagree on +/// whether SD2 is zero. That exempts ONLY the SD2 presence and what hangs on +/// it: SD2 and the ratio (both sides, when both report: each only has to be +/// numerically zero), and, when exactly one side reports, the note and the +/// flag, because the abstaining side has nothing to compare them with. +/// Everything else is still held: tier, inputs, every diagnostics field but the +/// abstention reason, SD1 (against an independent value when only one side +/// reports), the beat count, pNN and confidence (each against an independent +/// value when only one side reports, so [pnnThresholdMs] and the screen's +/// [artifactFraction] are passed along with [nn]). When +/// exactly one side reports, the other must abstain with +/// `noLongTermVariability` and the reported SD2 must be numerically zero. +void _sameScreen(IrregularScreenResult got, IrregularScreenResult want, String why, + {List? nn, + List? t, + double? pnnThresholdMs, + double? artifactFraction}) { + assert(nn == null || (pnnThresholdMs != null && artifactFraction != null), + 'the independent values need the pNN threshold and the artifact fraction'); + final g = got.metric, w = want.metric; + final degenerate = nn != null && _sd2Degenerate(nn, t); + final ambiguous = degenerate && g.present != w.present; + String diagText(IrregularScreenResult r, {bool noAbstain = false}) { + final j = r.diagnostics.toJson(); + if (noAbstain) j['abstain'] = null; + return jsonEncode(j); + } + + expect(g.tier, w.tier, reason: 'tier $why'); + expect(g.inputs_used, w.inputs_used, reason: 'inputs $why'); + expect(diagText(got, noAbstain: ambiguous), diagText(want, noAbstain: ambiguous), + reason: 'diagnostics$why${ambiguous ? " (but the abstention reason)" : ""}'); + if (ambiguous) { + final rep = g.present ? got : want, abs = g.present ? want : got; + expect(abs.diagnostics.abstain, IrregularAbstain.noLongTermVariability, + reason: 'the only disagreement allowed at a numerically zero SD2 $why'); + expect(rep.diagnostics.abstain, isNull, reason: 'a reported screen did not abstain $why'); + expect(abs.metric.value, isNull, reason: 'absent => no value $why'); + expect(abs.metric.confidence, 0, reason: 'absent => confidence 0 $why'); + final v = rep.metric.value!; + _expectNumericallyZeroSd2(v, why); + final k = _keptAndDiffs(nn, t); + expect(v.nBeats, k.kept.length, reason: 'nBeats $why'); + expect(v.nBeats, rep.diagnostics.nnKept, reason: 'nBeats is the kept count $why'); + _close(v.sd1, _sd(k.diffs)! / math.sqrt2, 'sd1 against the independent value $why'); + // pNN and confidence do not depend on SD2: held to their documented values. + final over = k.diffs.where((d) => d.abs() > pnnThresholdMs!).length; + expect(v.pnnPct, k.diffs.isEmpty ? 0.0 : 100.0 * over / k.diffs.length, + reason: 'pnn against the independent value $why'); + _close(rep.metric.confidence, + (k.kept.length / 5000.0 * (1 - artifactFraction!)).clamp(0.2, 0.9), + 'confidence against the independent value $why', + rel: 1e-12); + return; + } + expect(g.present, w.present, reason: 'present $why'); + expect(g.note, w.note, reason: 'note $why'); + if (!w.present) { + expect(g.value, isNull, reason: 'absent => no value $why'); + expect(g.confidence, 0, reason: 'absent => confidence 0 $why'); + return; + } + final a = g.value!, b = w.value!; + expect(a.flag, b.flag, reason: 'flag $why'); + expect(a.nBeats, b.nBeats, reason: 'nBeats $why'); + expect(a.pnnPct, b.pnnPct, reason: 'pnn $why'); + _close(a.sd1, b.sd1, 'sd1 $why'); + if (degenerate) { + // Both report: SD2 and the ratio are float noise around zero on both sides. + _expectNumericallyZeroSd2(a, why); + _expectNumericallyZeroSd2(b, why); + } else { + _close(a.sd2, b.sd2, 'sd2 $why'); + _close(a.sd1sd2, b.sd1sd2, 'sd1sd2 $why'); + } + _close(g.confidence, w.confidence, 'confidence $why', rel: 1e-12); +} + +// The independent window oracle: written from the documented rule, not from +// `irregularWindowVerdict` (which the batch and the stream share). + +double? _sd(List x) { + if (x.length < 2) return null; + final m = x.reduce((a, b) => a + b) / x.length; + var ss = 0.0; + for (final v in x) { + ss += (v - m) * (v - m); + } + return math.sqrt(ss / (x.length - 1)); +} + +/// Verdict of one closed window of kept beats: null when it holds fewer than +/// [minBeats] - 1 successive differences (validity counts pairs, not beats); +/// else whether SD1/SD2 and pNNx both clear their flag lines. A difference is +/// taken only between two successive beats (never across a skipped beat or a +/// time hole). +bool? _refVerdict(List v, List adj, double flag, double pnnMs, + double pnnPct, int minBeats) { + final d = [ + for (var i = 1; i < v.length; i++) + if (adj[i]) v[i] - v[i - 1] + ]; + if (d.length < minBeats - 1) return null; + final sdsd = _sd(d), sdnn = _sd(v); + if (sdsd == null || sdnn == null) return false; + final sd1 = sdsd / math.sqrt2; + final q = 2 * sdnn * sdnn - sd1 * sd1; + final sd2 = q > 0 ? math.sqrt(q) : 0.0; + if (sd2 <= 0) return false; + final over = d.where((x) => x.abs() > pnnMs).length; + return sd1 / sd2 >= flag && 100.0 * over / d.length >= pnnPct; +} + +typedef _RefWindows = ({ + int total, + int valid, + int flagged, + int openBeats, + IrregularOpenWindow open, + int kept, + int nIn +}); + +/// The windows of [nn] / [t] (the whole input, kept or not) under screen +/// config [cfg]: a window opens on its first kept beat and closes when a kept +/// beat is a full window later; the sizes cover the kept beats exactly once. +_RefWindows _refWindows(List nn, List t, int cfg) { + final c = _scCfgs[cfg]; + final windowMs = c.$4 * 60000; + final sizes = []; + final verdicts = []; + var vals = [], adj = []; + double? start; + var kept = 0; + void close() { + if (vals.isEmpty) return; + sizes.add(vals.length); + verdicts.add(_refVerdict(vals, adj, c.$1, c.$2, c.$3, c.$5)); + vals = []; + adj = []; + } + + for (var i = 0; i < nn.length; i++) { + final isKept = nn[i] >= 300 && nn[i] <= 2000; + if (isKept) { + kept++; + start ??= t[i]; + if (t[i] - start >= windowMs) { + close(); + start = t[i]; + } + vals.add(nn[i]); + adj.add(_successive(nn, t, i)); + } + } + close(); + expect(sizes.fold(0, (a, b) => a + b), kept, + reason: 'the windows cover the kept NN exactly once'); + final valid = verdicts.where((v) => v != null).length; + return ( + total: sizes.length, + valid: valid, + flagged: verdicts.where((v) => v == true).length, + openBeats: sizes.isEmpty ? 0 : sizes.last, + open: sizes.isEmpty + ? IrregularOpenWindow.none + : verdicts.last == null + ? IrregularOpenWindow.thin + : verdicts.last! + ? IrregularOpenWindow.flagged + : IrregularOpenWindow.unflagged, + kept: kept, + nIn: nn.length, + ); +} + +/// The diagnostics' beat and window evidence against the independent oracle. +void _expectEvidence(IrregularScreenResult res, List nn, List t, + int cfg, String why) { + final dg = res.diagnostics; + final ref = _refWindows(nn, t, cfg); + expect(dg.nnIn, ref.nIn, reason: 'nn_in is every entry handed in $why'); + expect(dg.nnKept, ref.kept, reason: 'nn_kept is the in-range ones $why'); + final w = dg.windows; + if (!_scCfgValid(cfg)) { + expect(w, isNull, reason: 'an invalid config gives no window evidence $why'); + if (res.metric.present) { + expect(res.metric.value!.flag, isFalse, reason: 'and cannot flag $why'); + } + return; + } + expect(w, isNotNull, reason: 'a valid config counts windows $why'); + expect(w!.total, ref.total, reason: 'windows total $why'); + expect(w.valid, ref.valid, reason: 'windows valid $why'); + expect(w.flagged, ref.flagged, reason: 'windows flagged $why'); + expect(w.openBeats, ref.openBeats, reason: 'open window beats $why'); + expect(w.open, ref.open, reason: 'open window label $why'); + expect(w.flagged >= 0 && w.flagged <= w.valid && w.valid <= w.total, isTrue, + reason: 'flagged <= valid <= total $why'); + expect(w.total <= dg.nnKept, isTrue, reason: 'total <= kept $why'); + expect(w.total == 0, dg.nnKept == 0, reason: 'no windows <=> no kept NN $why'); + expect(w.valid * _scCfgs[cfg].$5 <= dg.nnKept, isTrue, + reason: 'valid windows are full $why'); + expect(w.sustainedObserved, w.valid == 0 ? isNull : w.flagged / w.valid, + reason: 'flagged / valid, absent (not 0) when no window is valid $why'); + final m = res.metric; + if (m.present && m.value!.flag) { + expect(w.valid > 0 && w.flagged >= _scCfgs[cfg].$6 * w.valid, isTrue, + reason: 'a flag is backed by enough flagged windows $why'); + } + expect(m.present, dg.abstain == null, reason: 'present <=> not abstained $why'); +} + +typedef _NnCase = ((int, int, int, int), (int, int, int, int)); + +/// ((flavour, beats, seed, salt), (cfg, splits + cut seed, restart mask, +/// evaluation)). +Gen<_NnCase> _nnCaseGen(int maxN) => G.pair( + G.quad(G.intIn(0, _nNnFlavours - 1), _NGen(maxN), G.intIn(0, 1 << 16), + G.elements(const [0, 0, 1, 2, 3])), + G.quad(G.intIn(0, _scCfgs.length - 1), G.intIn(0, 6 * (1 << 20) + (1 << 20)), + G.intIn(0, 127), G.intIn(0, _evals.length - 1)), + ); + +String _nnTag(_NnCase c) { + final ((f, n, seed, salt), (cfg, sc, restart, ev)) = c; + return '${_nnFlavours[f]} n=$n seed=$seed salt=$salt cfg=$cfg ' + 'splits/cut=${sc ~/ (1 << 20)}/${sc % (1 << 20)} restart=$restart eval=$ev'; +} + +List _nnBounds(_NnCase c) { + final n = c.$1.$2; + final sc = c.$2.$2; + return _bounds(n, sc ~/ (1 << 20), sc % (1 << 20)); +} + +void _observeNn(_NnCase c, void Function(String) bump) { + final ((f, n, _, salt), (cfg, _, restart, ev)) = c; + bump('flavour: ${_nnFlavours[f]}'); + if (n == 0) bump('beats: none'); + if (n >= 1 && n < 40) bump('beats: under a thin window'); + if (n >= 500) bump('beats: 500 or more'); + if (n >= 1500) bump('beats: 1500 or more'); + if (salt >= 1) bump('out-of-range entries'); + if (salt >= 2) bump('NaN entries'); + bump(_scCfgValid(cfg) ? 'config: valid' : 'config: invalid'); + if (cfg != 0) bump('config: not the default'); + final b = _nnBounds(c); + if (b.length >= 4) bump('chunks: three or more'); + if (hasEmptyChunk(b)) bump('an empty chunk'); + if (restart != 0 && b.length > 2) bump('a restore between chunks'); + bump('evaluation: $ev'); +} + +const Map _nnShares = { + 'flavour: sinus': .03, + 'flavour: af': .03, + 'flavour: blocks': .03, + 'flavour: flat': .03, + 'flavour: two point': .03, + 'flavour: sinus + gaps': .03, + 'flavour: af heavy': .03, + 'flavour: range edges': .03, + 'beats: under a thin window': .05, + 'beats: 500 or more': .08, + 'out-of-range entries': .3, + 'NaN entries': .2, + 'config: valid': .4, + 'config: invalid': .15, + 'config: not the default': .4, + 'chunks: three or more': .4, + 'an empty chunk': .05, + 'a restore between chunks': .3, + 'evaluation: 0': .03, + 'evaluation: 1': .03, + 'evaluation: 2': .03, + 'evaluation: 3': .03, + 'evaluation: 4': .03, + 'evaluation: 5': .03, + 'evaluation: 6': .03, + 'evaluation: 7': .03, +}; + +_NnCase _n(int flavour, int n, + {int seed = 5, + int salt = 0, + int cfg = 0, + int splits = 3, + int cut = 1, + int restart = 0, + int ev = 1}) => + ((flavour, n, seed, salt), (cfg, splits * (1 << 20) + cut, restart, ev)); + +final List<_NnCase> _nnForced = [ + _n(0, 0, ev: 0), // nothing folded + _n(0, 0, ev: 6), // nothing folded, zero beats seen by the corrector + _n(0, 1), + _n(0, 2), + _n(0, 39, splits: 4, cfg: 1, ev: 1), // one beat short of a thin window of 40 + _n(0, 40, splits: 4, cfg: 0, ev: 1), // exactly the minimum window + _n(0, 41, splits: 4, cfg: 0, ev: 1), + _n(0, 2, cfg: 2, ev: 4), // minimum window of 2 beats, minimum beats 2 + _n(0, 3, cfg: 2, ev: 4), // exactly two successive differences + _n(1, 3, cfg: 2, ev: 4), + _n(1, 4, cfg: 2, ev: 4), + _n(7, 200, cfg: 1, ev: 1, splits: 5, restart: 127), // the edges of the kept range + _n(7, 7, cfg: 2, ev: 4, splits: 3), // exactly one period + _n(3, 300, cfg: 0, ev: 1, splits: 5, restart: 127), // flat: SD2 = 0 + _n(4, 300, cfg: 2, ev: 1, splits: 5, restart: 127), // two point + _n(1, 900, cfg: 0, ev: 1, splits: 6, cut: 3, restart: 127), // AF: flags + _n(6, 1500, cfg: 1, ev: 1, splits: 6, cut: 4, restart: 127), // AF heavy, short windows + _n(2, 1200, cfg: 0, ev: 0, splits: 6, cut: 5), // blocks, default minimum 500 + _n(0, 800, salt: 2, cfg: 3, ev: 5, splits: 6, cut: 6, restart: 85), // out-of-range + NaN + _n(1, 400, salt: 3, cfg: 2, ev: 7, splits: 4, cut: 7, restart: 127), // NaN, artifact exactly at the maximum + _n(5, 900, cfg: 1, ev: 3, splits: 6, cut: 8), // gaps; artifact fraction over the maximum + _n(1, 600, cfg: 4, ev: 1, splits: 3, cut: 9), // invalid: no window length + _n(1, 600, cfg: 5, ev: 1, splits: 3, cut: 10), // invalid: windows of 1 beat + _n(1, 600, cfg: 6, ev: 1, splits: 3, cut: 11), // invalid: fraction 1.5 + _n(1, 600, cfg: 7, ev: 1, splits: 3, cut: 12), // invalid: two at once + _n(2, 700, cfg: 8, ev: 2, splits: 5, cut: 13), // wide thresholds + _n(0, 3000, cfg: 0, ev: 0, splits: 6, cut: 14, restart: 127), // the cap, default minimum + _n(1, 3000, cfg: 1, ev: 1, splits: 6, cut: 15, restart: 127), +]; + +/// Days with an open window, completed windows and counts worth contradicting: +/// the base the mutated-checkpoint law cuts its mutations from. +final List<_NnCase> _nnL3 = _nnForced + .where((c) => c.$1.$2 >= 600 && _scCfgValid(c.$2.$1)) + .take(3) + .toList(); + +/// Window edge hit EXACTLY: a kept beat a full window after the window's first +/// closes it (>=). Hand-built, so it does not depend on a generator reaching it. +({List nn, List t}) _edgeSeries() { + final nn = [], t = []; + // 60 beats at 5000 ms: the 61st lands exactly 300000 ms after the first. + for (var i = 0; i < 62; i++) { + nn.add(800.0 + (i % 7) * 11); + t.add(10000.0 + i * 5000); + } + return (nn: nn, t: t); +} + +void _foldChunks(IrregularScreenState Function() make, _Nn s, List bounds, + int restart, void Function(IrregularScreenState st, int k, int at)? seam, + {IrregularScreenState? into}) { + var st = into ?? make(); + for (var k = 0; k + 1 < bounds.length; k++) { + st.fold(s.nn.sublist(bounds[k], bounds[k + 1]), s.t.sublist(bounds[k], bounds[k + 1])); + if ((restart >> (k % 7)) & 1 == 1) st = _srestore(st); + seam?.call(st, k, bounds[k + 1]); + } +} + +void _scL1(_NnCase c) { + final ((f, n, seed, salt), (cfg, _, restart, ev)) = c; + final s = _nnSeries(f, n, seed, salt); + final tag = _nnTag(c); + final bounds = _nnBounds(c); + final whole = _newState(cfg)..fold(s.nn, s.t); + late IrregularScreenState got; + var st = _newState(cfg); + for (var k = 0; k + 1 < bounds.length; k++) { + st.fold(s.nn.sublist(bounds[k], bounds[k + 1]), s.t.sublist(bounds[k], bounds[k + 1])); + if ((restart >> (k % 7)) & 1 == 1) st = _srestore(st); + } + got = st; + expect(_stext(got), _stext(whole), reason: '$tag bounds=$bounds: same checkpoint text'); + for (var e = 0; e < _evals.length; e++) { + expect(jsonEncode(_evaluate(got, e, beats: n).toJson()), + jsonEncode(_evaluate(whole, e, beats: n).toJson()), + reason: '$tag: evaluation $e reads the same'); + } +} + +/// An evaluation with a provisional tail equals the state after folding it. +void _scL1b(_NnCase c) { + final ((f, n, seed, salt), (cfg, _, restart, ev)) = c; + final s = _nnSeries(f, n, seed, salt); + final tag = _nnTag(c); + final bounds = _nnBounds(c); + final picks = pickSeams(bounds.length - 1); + _foldChunks(() => _newState(cfg), s, bounds, restart, (st, k, at) { + if (!picks.contains(k)) return; + // The tail is what a corrector would still hold provisional: some of the + // beats after the seam. + final tailEnd = math.min(n, at + (at * 7 + k * 13) % 97); + final tailNn = s.nn.sublist(at, tailEnd), tailT = s.t.sublist(at, tailEnd); + final before = _stext(st); + final got = _evaluate(st, ev, tailNn: tailNn, tailT: tailT, beats: tailEnd); + expect(_stext(st), before, reason: '$tag: evaluate reads, never writes'); + final want = _batchScreenOf(s.nn.sublist(0, tailEnd), s.t.sublist(0, tailEnd), cfg, ev); + final why = '$tag seam $k at $at tail to $tailEnd'; + _sameScreen(got, want, why, + nn: s.nn.sublist(0, tailEnd), + t: s.t.sublist(0, tailEnd), + pnnThresholdMs: _scCfgs[cfg].$2, + artifactFraction: _evals[ev].$3); + _expectEvidence(got, s.nn.sublist(0, tailEnd), s.t.sublist(0, tailEnd), cfg, why); + }); +} + +String _projectScreen(IrregularScreenState st, int beats) => jsonEncode([ + for (var e = 0; e < _evals.length; e++) _evaluate(st, e, beats: beats).toJson(), + _evaluate(st, 0, tailNn: const [800, 810, 790], tailT: const [1e12, 1e12 + 810, 1e12 + 1600], beats: beats).toJson(), + ]); + +void _scL2(_NnCase c) { + final ((f, n, seed, salt), (cfg, _, restart, _)) = c; + final s = _nnSeries(f, n, seed, salt); + final tag = _nnTag(c); + final bounds = _nnBounds(c); + final at = bounds.length > 2 ? bounds[1] : n ~/ 2; + final live = _newState(cfg)..fold(s.nn.sublist(0, at), s.t.sublist(0, at)); + final text = _stext(live); + final map = (jsonDecode(text) as Map).cast(); + final back = IrregularScreenState.fromJson(map); + expect(_stext(back), text, reason: '$tag: write(read(b)) == b'); + expect(_projectScreen(back, at), _projectScreen(live, at), + reason: '$tag: the restored one reads what the live one reads'); + // Its parameters travel with it: a restored state is not re-owned by whoever + // reads it (the thresholds it reports are the ones it was built with). + final c0 = _scCfgs[cfg]; + expect(back.windowMinutes, c0.$4, reason: tag); + expect(back.minWindowBeats, c0.$5, reason: tag); + expect(back.sustainedFraction, c0.$6, reason: tag); + expect(back.sd1sd2Flag, c0.$1, reason: tag); + // Not aliased to the map it came from. + (map['bk'] as List).clear(); + (map['bkAdj'] as List).clear(); + expect(_stext(back), text, reason: '$tag: not aliased to the map'); + live.fold(s.nn.sublist(at), s.t.sublist(at)); + back.fold(s.nn.sublist(at), s.t.sublist(at)); + expect(_stext(back), _stext(live), reason: '$tag: same checkpoint after the rest'); + expect(_projectScreen(back, n), _projectScreen(live, n), reason: tag); +} + +(Map, Must) _mutateScreen(Map src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + final keys = j.keys.toList(); + List l(String k) => j[k] as List; + switch (kind % 11) { + case 0: + // Version 1 (before the diagnostics counters) and version 2 (before the + // previous beat's time) are REFUSED, not restored with state they never + // kept; so is everything else but 3. + const versions = [0, 1, 2, -1, 99, '3', null, true, 1 << 40]; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse); + case 1: + const types = ['RrCorrector', 'Other', null, '', 'irregularscreenstate', 2]; + j['type'] = types[a % types.length]; + return (j, Must.refuse); + case 2: + final k = keys.where((k) => k != 'winStart').toList()[a % (keys.length - 1)]; + j.remove(k); + return (j, Must.refuse); + case 3: + final k = keys.where((k) => k != 'winStart').toList()[a % (keys.length - 1)]; + const junk = ['x', {}, [[]]]; + j[k] = junk[b % junk.length]; + return (j, Must.refuse); + case 4: + // The open window's beats and their neighbour flags disagree. + if (b.isEven) { + l('bkAdj').add(1); + } else if (l('bk').isEmpty) { + l('bk').add(800.0); + } else { + l('bk').removeLast(); + } + return (j, Must.refuse); + case 5: + // Counts that contradict each other: more kept than seen, more valid + // windows than windows, more flagged than valid, more over than diffs. + switch (a % 5) { + case 0: + j['nKept'] = (j['nIn'] as int) + 1; + case 1: + j['total'] = (j['valid'] as int) - 1; + case 2: + j['flagged'] = (j['valid'] as int) + 1; + case 3: + j['over'] = (j['dN'] as int) + 1; + default: + j['nIn'] = (j['nKept'] as int) - 1; + } + return (j, Must.refuse); + case 6: + // Negative counts. + j[const ['nIn', 'nKept', 'dN', 'lN', 'over', 'flagged'][a % 6]] = -1 - b % 5; + return (j, Must.refuse); + case 7: + // The neighbour flags are not flags. (Integers other than 0 / 1 are the + // reader's one leniency, see the skipped finding below.) + const notFlags = ['x', null, 1.5, true, []]; + if (l('bkAdj').isEmpty) { + l('bkAdj').add(notFlags[a % notFlags.length]); + l('bk').add(800.0); + } else { + l('bkAdj')[b % l('bkAdj').length] = notFlags[a % notFlags.length]; + } + return (j, Must.refuse); + case 8: + // Hostile sizes, and parts that disagree in ways the reader does not + // cross-check (the skipped FINDING below): refused, or exactly what was + // written. + const huge = [1 << 40, 1 << 62, 0x7fffffffffffffff]; + switch (a % 8) { + case 0: + j['nIn'] = huge[b % huge.length]; + case 1: + j['nKept'] = (j['nIn'] as int); + j['lN'] = (j['lN'] as int) + 1; // levels seen != kept beats + case 2: + j['dN'] = huge[b % huge.length]; // more differences than beats + case 3: + j['lN'] = huge[b % huge.length]; + case 4: + j['total'] = (j['nKept'] as int) + 1 + b % 5; // more windows than beats + case 5: + j['valid'] = huge[b % huge.length]; + j['total'] = huge[b % huge.length]; + case 6: + j['over'] = (j['dN'] as int); // every difference over the threshold + default: + l('bk').addAll(List.filled(1 + b % 3, 800.0)); + l('bkAdj').addAll(List.filled(1 + b % 3, 1)); + } + return (j, Must.faithful); + case 9: + // One number changed: the reader cannot tell it from a legal checkpoint. + switch (a % 9) { + case 0: + j['dMean'] = (j['dMean'] as num) + 1.0; + case 1: + j['lM2'] = (j['lM2'] as num) + 1.0; + case 2: + j['prevV'] = (j['prevV'] as num) + 1.0; + case 3: + j['prevKept'] = !(j['prevKept'] as bool); + case 4: + j['pnnFlagPct'] = (j['pnnFlagPct'] as num) + 1.0; + case 5: + j['sd1sd2Flag'] = (j['sd1sd2Flag'] as num) + 0.1; + case 6: + j['winStart'] = ((j['winStart'] as num?) ?? 0) + 1.0; + case 7: + j['prevT'] = ((j['prevT'] as num?) ?? 0) + 1.0; + default: + if (l('bk').isNotEmpty) l('bk')[b % l('bk').length] = (l('bk')[b % l('bk').length] as num) + 1.0; + } + return (j, Must.faithful); + default: + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + return (j, Must.ignored); + } +} + +String _screenCheckpointOutcome(Map j) { + try { + final s = IrregularScreenState.fromJson((deepCopy(j) as Map).cast()); + return 'accepted:${jsonEncode(s.toJson())}'; + } on FormatException { + return 'refused'; + } catch (e) { + return 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } +} + +void _scL3((_NnCase, Mut) arg) { + final (c, m) = arg; + final ((f, n, seed, salt), (cfg, _, _, _)) = c; + final tag = '${_nnTag(c)} mutation=${m.$1 % 11}(${m.$2},${m.$3})'; + final s = _nnSeries(f, n, seed, salt); + final st = _newState(cfg)..fold(s.nn, s.t); + final src = (jsonDecode(_stext(st)) as Map).cast(); + final (bad, must) = _mutateScreen(src, m); + expectMutationOutcome(_screenCheckpointOutcome(bad), must, jsonEncode(bad), + jsonEncode(src), tag); +} + +void _scL4(_NnCase c) { + final ((f, n, seed, salt), (cfg, _, _, ev)) = c; + final s = _nnSeries(f, n, seed, salt); + final tag = _nnTag(c); + final fresh = _stext(_newState(cfg)); + // An empty fold on a fresh state records nothing. + final empty = _newState(cfg)..fold(const [], const []); + expect(_stext(empty), fresh, reason: '$tag: an empty first fold records nothing'); + final cut = n ~/ 2; + final st = _newState(cfg)..fold(s.nn.sublist(0, cut), s.t.sublist(0, cut)); + final before = _stext(st); + st.fold(const [], const []); + expect(_stext(st), before, reason: '$tag: an empty fold is the identity'); + // evaluate reads and never writes, with or without a tail, twice equal. + final one = _evaluate(st, ev, beats: cut); + expect(_stext(st), before, reason: '$tag: evaluate is read-only'); + final tail = s.nn.sublist(cut, math.min(n, cut + 20)); + final tailT = s.t.sublist(cut, math.min(n, cut + 20)); + _evaluate(st, ev, tailNn: tail, tailT: tailT, beats: cut + tail.length); + expect(_stext(st), before, reason: '$tag: and with a tail'); + expect(jsonEncode(_evaluate(st, ev, beats: cut).toJson()), jsonEncode(one.toJson()), + reason: '$tag: twice is equal'); + // Refusal: lengths that disagree change nothing. + expect(() => st.fold([800, 810], const [1.0]), throwsArgumentError, reason: tag); + expect(() => st.fold(const [], [1.0]), throwsArgumentError, reason: tag); + expect(_stext(st), before, reason: '$tag: a refused fold leaves the state unchanged'); +} + +void _scL5(_NnCase c) { + final ((f, n, seed, salt), (cfg, _, restart, ev)) = c; + final s = _nnSeries(f, n, seed, salt); + final tag = _nnTag(c); + final bounds = _nnBounds(c); + _foldChunks(() => _newState(cfg), s, bounds, restart, (st, k, at) { + final res = _evaluate(st, ev, beats: at); + final why = '$tag seam $k at $at'; + final dg = res.diagnostics; + final inRange = [for (final v in s.nn.sublist(0, at)) if (v >= 300 && v <= 2000) v]; + expect(dg.nnIn, at, reason: 'nn_in is every entry folded $why'); + expect(dg.nnKept, inRange.length, reason: 'nn_kept is the in-range ones $why'); + expect(dg.nnKept <= dg.nnIn, isTrue, reason: why); + _expectEvidence(res, s.nn.sublist(0, at), s.t.sublist(0, at), cfg, why); + final m = res.metric; + // The beat gate counts successive pairs (minBeats - 1 of them), not beats: + // too few pairs abstains, and nothing past that gate does. + final pairs = _pairsOf(s.nn.sublist(0, at), s.t.sublist(0, at)); + final enoughPairs = pairs >= dg.thresholds.minBeats - 1; + expect(dg.abstain == IrregularAbstain.tooFewBeats, !enoughPairs, + reason: 'too few beats <=> fewer than minBeats - 1 pairs ($pairs) $why'); + if (m.present) { + expect(m.value!.nBeats, dg.nnKept, reason: 'nBeats == nn_kept $why'); + expect(dg.nnKept >= dg.thresholds.minBeats, isTrue, reason: 'min beats $why'); + } + if (dg.abstain == IrregularAbstain.artifact) { + expect(enoughPairs, isTrue, reason: why); + expect(dg.nnKept >= dg.thresholds.minBeats, isTrue, reason: why); + expect(_evals[ev].$3 > dg.thresholds.maxArtifact, isTrue, reason: why); + } + }); +} + +// ── IP: the corrector and the screen together, as the day pass runs them ───── + +/// (a beat series and its chunking, screen config index, evaluation index). +typedef _IpCase = (_Case, int, int); + +Gen<_IpCase> _ipGen(int maxN) => G.triple(_caseGen(maxN), + G.intIn(0, _scCfgs.length - 1), G.intIn(0, _evals.length - 1)); + +String _ipTag(_IpCase c) => '${_tag(c.$1)} screen=${c.$2} eval=${c.$3}'; + +void _observeIp(_IpCase c, void Function(String) bump) { + _observeRr(c.$1, bump); + final (flavour, n, _, _) = c.$1.$1; + bump(_scCfgValid(c.$2) ? 'config: valid' : 'config: invalid'); + if (c.$2 != 0) bump('config: not the default'); + bump('evaluation: ${c.$3}'); + if (flavour == 2 && n >= 600) bump('a long irregularly irregular day'); +} + +const Map _ipShares = { + ..._rrShares, + 'config: valid': .4, + 'config: invalid': .1, + 'config: not the default': .4, + 'evaluation: 0': .02, + 'evaluation: 1': .02, + 'evaluation: 2': .02, + 'evaluation: 3': .02, + 'evaluation: 4': .02, + 'evaluation: 5': .02, + 'evaluation: 6': .02, + 'evaluation: 7': .02, + 'a long irregularly irregular day': .02, +}; + +/// What the day pass computes at a seam: the corrector's provisional tail and +/// counts handed to the screen state. +IrregularScreenResult _ipStream(RrCorrector corr, IrregularScreenState st, int ev) { + final snap = corr.snapshot(); + final e = _evals[ev]; + return st.evaluateDetailed(snap.tailNn, snap.tailNnTimes, + artifactFraction: (1.0 - snap.cleanFraction).clamp(0.0, 1.0), + minBeats: e.$1, + maxArtifact: e.$2, + cleaning: RrCleaningCounts( + raw: snap.n, corrected: snap.correctedCount, dropped: snap.droppedCount)); +} + +void _ipBody(_IpCase arg) { + final (c, scr, ev) = arg; + final ((flavour, n, seed, mode), (splits, cutSeed, restart, cfg)) = c; + final s = _series(flavour, n, seed); + final wall = mode == 0; + final tag = _ipTag(arg); + final bounds = _bounds(n, splits, cutSeed); + final picks = pickSeams(bounds.length - 1); + var corr = _newCorrector(cfg); + var st = _newState(scr); + for (var k = 0; k + 1 < bounds.length; k++) { + final settled = corr.fold(s.rr.sublist(bounds[k], bounds[k + 1]), + tsMs: _tsSlice(s, wall, bounds[k], bounds[k + 1])); + st.fold(settled.nn, settled.nnTimes); + if ((restart >> (k % 7)) & 1 == 1) { + corr = _restore(corr); + st = _srestore(st); + } + if (!picks.contains(k)) continue; + final at = bounds[k + 1]; + final why = '$tag seam $k at $at'; + final got = _ipStream(corr, st, ev); + final want = _oracle(s, wall, cfg, at); + final e = _evals[ev]; + final cs = _scCfgs[scr]; + final batch = irregularBeatScreenDetailed(want.nn, + nnTimesMs: want.nnTimesMs, + artifactFraction: (1.0 - want.cleanFraction).clamp(0.0, 1.0), + minBeats: e.$1, + maxArtifact: e.$2, + sd1sd2Flag: cs.$1, + pnnThresholdMs: cs.$2, + pnnFlagPct: cs.$3, + windowMinutes: cs.$4, + minWindowBeats: cs.$5, + sustainedFraction: cs.$6, + cleaning: RrCleaningCounts( + raw: at, corrected: want.correctedCount, dropped: want.droppedCount)); + _sameScreen(got, batch, why, + nn: want.nn, + t: want.nnTimesMs, + pnnThresholdMs: cs.$2, + artifactFraction: (1.0 - want.cleanFraction).clamp(0.0, 1.0)); + // L5, integrated: the corrector's counts are the screen's evidence. + final dg = got.diagnostics; + expect(dg.rrRaw, at, reason: 'rr_raw is every beat the corrector saw $why'); + expect(dg.dropped, want.droppedCount, reason: 'dropped $why'); + expect(dg.corrected, want.correctedCount, reason: 'corrected $why'); + expect(dg.nnIn, at - dg.dropped!, reason: 'nn_in == rr_raw - dropped $why'); + expect(dg.corrected! + dg.dropped! <= at, isTrue, reason: why); + expect(dg.nnKept <= dg.nnIn, isTrue, reason: 'kept <= in $why'); + if (at == 0) { + expect(dg.artifactFraction, isNull, + reason: 'a share of no beats is absent, not 1.0 $why'); + } else { + expect(dg.artifactFraction, (1.0 - want.cleanFraction).clamp(0.0, 1.0), + reason: 'artifact fraction $why'); + } + if (dg.abstain == IrregularAbstain.artifact) { + // The gate reads the fraction handed in. With nothing seen (and a pair + // gate of zero) that is 1 - cleanFraction of no beats, while the + // diagnostics report the share as absent, not 1.0 (0fc5768). + final handed = (1.0 - want.cleanFraction).clamp(0.0, 1.0); + expect(handed > dg.thresholds.maxArtifact, isTrue, reason: why); + if (at != 0) { + expect(dg.artifactFraction! > dg.thresholds.maxArtifact, isTrue, reason: why); + } + } + _expectEvidence(got, want.nn, want.nnTimesMs, scr, why); + } +} + +// ── W: the diagnostics wire ───────────────────────────────────────────────── + +const List _fractions = [0.0, .07, .30, .31, 1.0, .5]; + +/// (flavour, beats, seed) of a small NN series, screen config, evaluation, and +/// the artifact fraction index handed in. +typedef _WireCase = (_NnRec, int, int, int); + +Gen<_WireCase> _wireGen() => G.quad( + G.quad(G.intIn(0, _nNnFlavours - 1), _NGen(300), G.intIn(0, 1 << 12), + G.elements(const [0, 1, 2, 3])), + G.intIn(0, _scCfgs.length - 1), + G.intIn(0, _evals.length - 1), + G.intIn(0, _fractions.length - 1)); + +IrregularScreenResult _wireResult(_WireCase c) { + final ((f, n, seed, salt), cfg, ev, fr) = c; + final s = _nnSeries(f, n, seed, salt); + final e = _evals[ev]; + final cs = _scCfgs[cfg]; + return irregularBeatScreenDetailed(s.nn, + nnTimesMs: s.t, + artifactFraction: _fractions[fr], + minBeats: math.min(e.$1, 30), + maxArtifact: e.$2, + sd1sd2Flag: cs.$1, + pnnThresholdMs: cs.$2, + pnnFlagPct: cs.$3, + windowMinutes: cs.$4, + minWindowBeats: cs.$5, + sustainedFraction: cs.$6, + cleaning: _cleaningFor(e.$4, n)); +} + +/// Every field a reader of the diagnostics can see. +String _projectDiagnostics(IrregularDiagnostics d) { + final w = d.windows, t = d.thresholds; + return jsonEncode([ + d.abstain?.index, + d.rrRaw, + d.corrected, + d.dropped, + d.nnIn, + d.nnKept, + d.artifactFraction, + w == null + ? null + : [w.total, w.valid, w.flagged, w.openBeats, w.open.index, w.sustainedObserved], + [ + t.minBeats, + t.maxArtifact, + t.sd1sd2Flag, + t.pnnThresholdMs, + t.pnnFlagPct, + t.windowMinutes, + t.minWindowBeats, + t.sustainedFraction + ], + ]); +} + +void _wL2(_WireCase c) { + final res = _wireResult(c); + final text = jsonEncode(res.diagnostics.toJson()); + final back = IrregularDiagnostics.fromJson( + (jsonDecode(text) as Map).cast()); + expect(jsonEncode(back.toJson()), text, reason: 'write(read(b)) == b'); + expect(_projectDiagnostics(back), _projectDiagnostics(res.diagnostics), + reason: 'the restored evidence reads what the live one reads'); + // The envelope edge persists: the old metric keys, unchanged, and one more. + final env = res.toJson(); + final old = res.metric.toJson((v) => v.toJson()); + for (final k in old.keys) { + expect(jsonEncode(env[k]), jsonEncode(old[k]), reason: 'envelope key $k'); + } + expect(env.keys.toSet(), {...old.keys, 'diagnostics'}); + expect(jsonEncode(env['diagnostics']), text); + // Absent stays absent on the wire: an unknown count or window is JSON null. + final beats = (jsonDecode(text) as Map)['beats'] as Map; + expect(beats['rr_raw'], res.diagnostics.rrRaw); + expect(beats['artifact_fraction'], res.diagnostics.artifactFraction); + expect((jsonDecode(text) as Map)['windows'] == null, res.diagnostics.windows == null); +} + +(Map, Must) _mutateWire(Map src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + Map sec(String k) => j[k] as Map; + switch (kind % 8) { + case 0: + const versions = [0, 2, 3, -1, 99, '1', null, true, 1 << 40]; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse); + case 1: + const names = ['unknown', '', 'tooFewBeats', 'TOO_FEW_BEATS', 3, []]; + j['abstain'] = names[a % names.length]; + return (j, Must.refuse); + case 2: + j.remove(const ['beats', 'thresholds', 'version'][a % 3]); + return (j, Must.refuse); + case 3: + final k = const ['beats', 'thresholds'][a % 2]; + const junk = ['x', [], 1, null]; + j[k] = junk[b % junk.length]; + return (j, Must.refuse); + case 4: + // A field of the wrong kind inside a section. + final keys = [...sec('beats').keys.where((k) => k == 'nn_in' || k == 'nn_kept')]; + final tk = sec('thresholds').keys.toList(); + if (a.isEven) { + sec('beats')[keys[b % keys.length]] = const ['x', null, 1.5, []][a % 4]; + } else { + sec('thresholds')[tk[b % tk.length]] = const ['x', null, []][a % 3]; + } + return (j, Must.refuse); + case 5: + // The open window named something that is not one. + if (j['windows'] == null) { + j['windows'] = { + 'total': 1, + 'valid': 1, + 'flagged': 0, + 'sustained_observed': 0.0, + 'open_beats': 1, + 'open': 'bogus' + }; + } else { + sec('windows')['open'] = const ['bogus', '', 'FLAGGED', 3, null][a % 5]; + } + return (j, Must.refuse); + case 6: + // Numbers changed, or parts that disagree about the evidence: the wire + // is evidence, not a state, and carries no cross-check (the skipped + // FINDING below): refused, or exactly what was written. + switch (a % 6) { + case 0: + sec('beats')['nn_kept'] = (sec('beats')['nn_kept'] as int) + 1 + b % 5; + case 1: + sec('beats')['nn_in'] = (sec('beats')['nn_in'] as int) + 1; + case 2: + sec('thresholds')['max_artifact'] = (sec('thresholds')['max_artifact'] as num) + .1; + case 3: + sec('beats')['artifact_fraction'] = 1.0; + case 4: + if (j['windows'] != null) { + sec('windows')['flagged'] = (sec('windows')['valid'] as int) + 1 + b % 3; + } + default: + if (j['windows'] != null) sec('windows')['total'] = 0; + } + // `sustained_observed` is derived (flagged / valid) and written afresh; + // keep the mutated text consistent with that. + final w = j['windows'] as Map?; + if (w != null) { + w['sustained_observed'] = (w['valid'] as int) == 0 + ? null + : (w['flagged'] as int) / (w['valid'] as int); + } + return (j, Must.faithful); + default: + if (a % 4 == 3 && j['windows'] != null) { + // The derived figure is not read: whatever it says, the reader writes + // flagged / valid back. + sec('windows')['sustained_observed'] = 0.123456; + } else { + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + } + return (j, Must.ignored); + } +} + +String _wireOutcome(Map j) { + try { + final d = IrregularDiagnostics.fromJson( + (deepCopy(j) as Map).cast()); + return 'accepted:${jsonEncode(d.toJson())}'; + } on FormatException { + return 'refused'; + } catch (e) { + return 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } +} + +void _wL3((_WireCase, Mut) arg) { + final (c, m) = arg; + final tag = 'wire ${c.$1} cfg=${c.$2} eval=${c.$3} frac=${c.$4} ' + 'mutation=${m.$1 % 8}(${m.$2},${m.$3})'; + final src = (jsonDecode(jsonEncode(_wireResult(c).diagnostics.toJson())) as Map) + .cast(); + final (bad, must) = _mutateWire(src, m); + expectMutationOutcome(_wireOutcome(bad), must, jsonEncode(bad), jsonEncode(src), tag); +} + +/// The "absent, not zero" rules (0fc5768): cleaning counts are null when the +/// caller gave none; the artifact fraction is null when the corrector saw no +/// beats (a share of nothing; the 1.0 of `1 - cleanFraction` of an empty series +/// would read as every beat being an artifact), the fraction handed in +/// otherwise. The verdict gate still reads the fraction handed in. The batch +/// and the stream agree. +void _wAbsent(_WireCase c) { + final ((f, n, seed, salt), cfg, ev, fr) = c; + final s = _nnSeries(f, n, seed, salt); + final e = _evals[ev]; + final fraction = _fractions[fr]; + final cleaning = _cleaningFor(e.$4, n); + final tag = 'n=$n flavour=${_nnFlavours[f]} cleaning=${e.$4} fraction=$fraction'; + final batch = _batchScreenOf(s.nn, s.t, cfg, ev); + final st = _newState(cfg)..fold(s.nn, s.t); + final stream = st.evaluateDetailed(const [], const [], + artifactFraction: e.$3, minBeats: e.$1, maxArtifact: e.$2, cleaning: cleaning); + for (final r in [batch, stream]) { + final d = r.diagnostics; + expect(d.rrRaw, cleaning?.raw, reason: '$tag: rr_raw is absent without counts'); + expect(d.corrected, cleaning?.corrected, reason: tag); + expect(d.dropped, cleaning?.dropped, reason: tag); + final noBeats = cleaning != null && cleaning.raw == 0; + expect(d.artifactFraction, noBeats ? isNull : e.$3, + reason: '$tag: the artifact fraction of no beats is absent, not ${e.$3}'); + final wire = d.toJson()['beats'] as Map; + expect(wire['artifact_fraction'], d.artifactFraction, reason: tag); + // The gate still reads the fraction handed in. + // Past the pair gate (minBeats - 1 successive pairs), the artifact gate + // comes next. + if (_pairsOf(s.nn, s.t) >= e.$1 - 1 && e.$3 > e.$2) { + expect(d.abstain, IrregularAbstain.artifact, reason: '$tag: gate'); + } + if (d.abstain == IrregularAbstain.artifact) { + expect(e.$3 > e.$2, isTrue, reason: '$tag: gate'); + } + } + // And a screen that ran with no beats counted is still not 1.0. + // (fraction is also drawn directly: see the wire cases below.) + expect(fraction, _fractions[fr]); + final aj = jsonEncode(batch.diagnostics.toJson()), bj = jsonEncode(stream.diagnostics.toJson()); + if (n > 0 || cleaning == null) { + // Same beats in: the same evidence from both sides, abstention included + // (but at a numerically zero SD2, see _sd2Degenerate, the reason may differ). + String noReason(String t) => + jsonEncode((jsonDecode(t) as Map)..['abstain'] = null); + if (_sd2Degenerate(s.nn, s.t)) { + // Only the abstention reason may differ, and only between "reported" and + // "no long-term variability". + expect(noReason(bj), noReason(aj), reason: '$tag: batch and stream agree'); + final ba = batch.diagnostics.abstain, sa = stream.diagnostics.abstain; + if (ba != sa) { + expect({ba, sa}, {null, IrregularAbstain.noLongTermVariability}, + reason: '$tag: the only reason change allowed at a zero SD2'); + } + } else { + expect(bj, aj, reason: '$tag: batch and stream agree'); + } + } +} + +void main() { + group('RC the corrector', () { + _laws.law<_Case>( + 'L1 RrCorrector: any chunking, empty chunks and restores included, ' + 'gives the checkpoint text, settled output and snapshot of one fold', + _caseGen(3000), + _l1Corrector, + examples: _forced, + cases: 60, + reach: Reach<_Case>(_rrShares, _observeRr), + ); + _laws.law<_Case>( + 'L1b RrCorrector: at the seams, settled ++ snapshot is correctRr of the ' + 'prefix, bit for bit (frozen reference up to $_refCap beats)', + _caseGen(1500), + _l1bCorrector, + examples: [for (final c in _forced) if (c.$1.$2 <= 450) c], + cases: 8, + reach: Reach<_Case>({ + for (final e in _rrShares.entries) + if (!e.key.startsWith('flavour: ') && + !const { + 'beats: 1000 or more', + 'chunks: five or more', + 'beats: none', + 'beats: under 3 (the short branch)', + 'flavour: flagged', + 'beats: past the window (91 or more)', + }.contains(e.key)) + e.key: e.value + }, _observeRr), + ); + _laws.law<_Case>( + 'L2 RrCorrector: write(read(b)) == b, the restored one reads the same ' + 'and goes on identically', + _caseGen(1500), + _l2Corrector, + examples: _forced, + cases: 50, + reach: Reach<_Case>(_rrShares, _observeRr), + ); + _laws.law<(_Case, Mut)>( + 'L3 RrCorrector: a mutated checkpoint is refused whole or holds exactly ' + 'what was written', + G.pair(_caseGen(600), _mutGen), + _l3Corrector, + examples: [ + for (var i = 0; i < 11 * 4; i++) + (_forced[const [3, 8, 13, 16, 20, 17, 11][i % 7] + (i % 3 == 0 ? 0 : 0)], (i % 11, i ~/ 11, 7 + i)), + for (var k = 0; k < 11; k++) (_l3Base, (k, k, 0)), + for (var a = 0; a < 10; a++) (_l3Base, (0, a, 0)), + for (var a = 0; a < 5; a++) for (var b = 0; b < 4; b++) (_l3Base, (8, a, b)), + ], + cases: 200, + reach: Reach<(_Case, Mut)>({ + for (var k = 0; k < 11; k++) 'kind: $k': .02, + 'a state with settled beats': .3, + 'an empty state': .008, + }, (arg, bump) { + bump('kind: ${arg.$2.$1 % 11}'); + final n = arg.$1.$1.$2; + if (n >= 91) bump('a state with settled beats'); + if (n == 0) bump('an empty state'); + }), + ); + _laws.law<_Case>( + 'L4 RrCorrector: an empty chunk is the identity and records nothing; a ' + 'refused fold leaves the state unchanged; snapshot() reads only', + _caseGen(800), + _l4Corrector, + examples: _forced, + cases: 150, + reach: Reach<_Case>(_rrShares, _observeRr), + ); + _laws.law<_Case>( + 'L5 RrCorrector: beats are conserved (nn_in == rr_raw - dropped, normal + ' + 'corrected + dropped == beats), classes cover every beat, times never run ' + 'backwards', + _caseGen(3000), + _l5Corrector, + examples: _forced, + cases: 40, + reach: Reach<_Case>(_rrShares, _observeRr), + ); + }); + + group('SC the screen state', () { + final shares = _nnShares; + _laws.law<_NnCase>( + 'L1 IrregularScreenState: any chunking, empty chunks and restores ' + 'included, gives the checkpoint text and every evaluation of one fold', + _nnCaseGen(3000), + _scL1, + examples: _nnForced, + cases: 200, + reach: Reach<_NnCase>(shares, _observeNn), + ); + _laws.law<_NnCase>( + 'L1b IrregularScreenState: evaluate(tail) is the batch screen of settled ' + '++ tail; the window counts are an independent partition\'s', + _nnCaseGen(1500), + _scL1b, + examples: [for (final c in _nnForced) if (c.$1.$2 <= 1500) c], + cases: 150, + reach: Reach<_NnCase>({ + for (final e in shares.entries) + if (!const {'beats: 1500 or more', 'beats: 500 or more'}.contains(e.key)) + e.key: e.value + }, _observeNn), + ); + _laws.law<_NnCase>( + 'L2 IrregularScreenState: write(read(b)) == b, the restored one reads the ' + 'same, keeps its own parameters and goes on identically', + _nnCaseGen(1500), + _scL2, + examples: _nnForced, + cases: 200, + reach: Reach<_NnCase>(shares, _observeNn), + ); + _laws.law<(_NnCase, Mut)>( + 'L3 IrregularScreenState: a version 1 or otherwise mutated checkpoint is ' + 'refused whole or holds exactly what was written', + G.pair(_nnCaseGen(600), _mutGen), + _scL3, + examples: [ + for (var k = 0; k < 11; k++) + for (var a = 0; a < 3; a++) (_nnL3[a], (k, a + k, 3 * k + a)), + for (var a = 0; a < 9; a++) (_nnL3[0], (0, a, 0)), + // Every key but winStart removed in turn (prevT included: a version 3 + // checkpoint without it is refused, not read as an unknown time). + for (var a = 0; a < 25; a++) (_nnL3[0], (2, a, 0)), + (_nnL3[0], (9, 7, 0)), // prevT changed: held exactly + ], + cases: 200, + reach: Reach<(_NnCase, Mut)>({ + for (var k = 0; k < 11; k++) 'kind: $k': .02, + 'a state with an open window': .3, + }, (arg, bump) { + bump('kind: ${arg.$2.$1 % 11}'); + if (arg.$1.$1.$2 >= 40) bump('a state with an open window'); + }), + ); + _laws.law<_NnCase>( + 'L4 IrregularScreenState: an empty fold is the identity, evaluate never ' + 'writes, a refused fold leaves the state unchanged', + _nnCaseGen(800), + _scL4, + examples: _nnForced, + cases: 200, + reach: Reach<_NnCase>(shares, _observeNn), + ); + _laws.law<_NnCase>( + 'L5 IrregularScreenState: nn_in/nn_kept count the input, the windows ' + 'cover the kept NN, flagged <= valid <= total, an invalid config has no ' + 'windows and cannot flag', + _nnCaseGen(3000), + _scL5, + examples: _nnForced, + cases: 150, + reach: Reach<_NnCase>(shares, _observeNn), + ); + test('a kept beat a full window after the first closes the window (>=)', () { + final e = _edgeSeries(); + // 62 beats: the 61st sits exactly 300000 ms after the first. + final st = IrregularScreenState(minWindowBeats: 2)..fold(e.nn, e.t); + final res = st.evaluateDetailed(const [], const [], minBeats: 1); + expect(res.diagnostics.windows!.total, 2, reason: 'closed AT the edge'); + expect(res.diagnostics.windows!.openBeats, 2); + final ref = _refWindows(e.nn, e.t, 0); + expect(ref.total, 2); + // One millisecond short of the edge: still one window. + final t2 = [...e.t]..[60] -= 1; + final st2 = IrregularScreenState(minWindowBeats: 2)..fold(e.nn, t2); + expect(st2.evaluateDetailed(const [], const [], minBeats: 1).diagnostics.windows!.total, 2, + reason: 'beat 62 is past the edge anyway'); + final t3 = [...e.t.sublist(0, 61)]..[60] -= 1; + final st3 = IrregularScreenState(minWindowBeats: 2)..fold(e.nn.sublist(0, 61), t3); + expect(st3.evaluateDetailed(const [], const [], minBeats: 1).diagnostics.windows!.total, 1, + reason: 'one millisecond short of a full window: still open'); + }); + }); + + group('IP the corrector and the screen together', () { + _laws.law<_IpCase>( + 'L1b+L5 corrector -> screen: at the seams the diagnostics are the batch ' + 'diagnostics over the corrected prefix, and the beats are conserved', + _ipGen(1200), + _ipBody, + examples: [ + for (var i = 0; i < _forced.length; i++) + if (_forced[i].$1.$2 <= 450 && i % 2 == 0) (_forced[i], const [0, 1, 2, 3, 4, 8, 5, 6, 7][i % 9], const [0, 1, 2, 3, 4, 5, 6, 7][i % 8]), + // The flagged day under the default screen, and the same under each + // window config, at 500 beats and over. + (_c(2, 450, splits: 4, cutSeed: 31, restart: 127), 0, 1), + (_c(2, 450, splits: 4, cutSeed: 32, restart: 127), 1, 1), + (_c(2, 450, splits: 4, cutSeed: 33), 2, 1), + (_c(2, 450, splits: 4, cutSeed: 34), 3, 1), + (_c(2, 450, splits: 4, cutSeed: 35), 4, 1), + (_c(2, 450, splits: 4, cutSeed: 36), 5, 1), + (_c(2, 450, splits: 4, cutSeed: 37), 6, 1), + (_c(2, 450, splits: 4, cutSeed: 38), 7, 1), + (_c(2, 450, splits: 4, cutSeed: 39), 8, 1), + (_c(2, 600, splits: 3, cutSeed: 40), 0, 0), // default minimum of 500 + (_c(0, 0), 0, 6), // no beats, nothing seen by the corrector + (_c(0, 0, mode: 1), 1, 1), + ], + cases: 8, + reach: Reach<_IpCase>({ + for (final e in _ipShares.entries) + if (!e.key.startsWith('flavour: ') && + !const { + 'beats: 1000 or more', + 'chunks: five or more', + 'beats: none', + 'beats: under 3 (the short branch)', + 'flavour: flagged', + 'a long irregularly irregular day', + 'no timestamps', + 'config: invalid', + 'evaluation: 0', + 'evaluation: 1', + 'evaluation: 2', + 'evaluation: 3', + 'evaluation: 4', + 'evaluation: 5', + 'evaluation: 6', + 'evaluation: 7', + }.contains(e.key)) + e.key: e.value + }, _observeIp), + ); + }); + + group('W the diagnostics wire', () { + final reach = Reach<_WireCase>({ + 'abstained': .1, + 'ran': .05, + 'no cleaning counts': .1, + 'zero beats seen': .02, + 'windows present': .3, + 'windows absent (invalid config)': .05, + }, (c, bump) { + final ((f, n, _, _), cfg, ev, _) = c; + bump(n >= math.min(_evals[ev].$1, 30) && n >= 1 ? 'ran' : 'abstained'); + if (_evals[ev].$4 == 0) bump('no cleaning counts'); + if (_evals[ev].$4 == 2) bump('zero beats seen'); + bump(_scCfgValid(cfg) ? 'windows present' : 'windows absent (invalid config)'); + }); + final forcedWire = <_WireCase>[ + ((0, 0, 1, 0), 0, 6, 4), // zero beats, corrector saw none, fraction 1.0 + ((0, 0, 1, 0), 0, 0, 4), // zero beats, counts say 3 + ((0, 0, 1, 0), 0, 2, 4), // zero beats, no counts at all + ((1, 200, 1, 0), 0, 1, 1), + ((1, 300, 2, 0), 2, 4, 0), + ((2, 300, 3, 0), 1, 7, 2), // artifact fraction exactly at the maximum + ((2, 300, 3, 0), 1, 3, 3), // and just over it + ((3, 100, 1, 0), 0, 1, 0), // flat + ((4, 100, 1, 0), 2, 1, 0), // two point + ((0, 50, 1, 2), 4, 1, 0), // invalid window config + ((0, 50, 1, 0), 0, 1, 0), // exactly the minimum windows + ((5, 280, 1, 3), 3, 5, 1), + ]; + _laws.law<_WireCase>( + 'W-L2 IrregularDiagnostics: write(read(b)) == b through JSON text, the ' + 'restored evidence reads the same, the envelope keeps the old keys', + _wireGen(), + _wL2, + examples: forcedWire, + cases: 200, + reach: reach, + ); + _laws.law<(_WireCase, Mut)>( + 'W-L3 IrregularDiagnostics: another version, an unknown reason, a missing ' + 'or malformed section is refused whole; changed numbers are held exactly', + G.pair(_wireGen(), _mutGen), + _wL3, + examples: [ + for (var k = 0; k < 8; k++) + for (var i = 0; i < 3; i++) (forcedWire[3 + i], (k, 5 * k + i, 2 * k + i)), + for (var a = 0; a < 9; a++) (forcedWire[3], (0, a, 0)), + for (var a = 0; a < 6; a++) (forcedWire[3], (1, a, 0)), + for (var a = 0; a < 6; a++) (forcedWire[3], (6, a, 0)), + for (var a = 0; a < 5; a++) (forcedWire[9], (5, a, 0)), + ], + cases: 200, + reach: Reach<(_WireCase, Mut)>({ + for (var k = 0; k < 8; k++) 'kind: $k': .03, + }, (arg, bump) => bump('kind: ${arg.$2.$1 % 8}')), + ); + _laws.law<_WireCase>( + 'W-L5 diagnostics: cleaning counts are absent unless given, the artifact ' + 'fraction is absent when the corrector saw no beats and kept otherwise; ' + 'batch and stream agree (0fc5768)', + _wireGen(), + _wAbsent, + examples: forcedWire, + cases: 200, + reach: reach, + ); + }); + + group('RC accepted-checkpoint continuation', () { + test('a checkpoint the reader accepts with a nonzero first difference goes ' + 'on from it: the second beat can be the recovery of the first', () { + // A writer stores d[0] = 0, but the reader takes any finite difference. + // Resumed from d[0] = -200, beat 1 (+200 against beat 0) is a recovery + // from beat 0's jump, and is demoted to normal; with d[0] = 0 it is not. + // (If the reader is ever hardened to refuse a nonzero d[0], this becomes + // the refusal test.) + List classesAfter(double d0) { + final c = RrCorrector(windowBeats: 1, maxArtifactRun: null)..fold([600]); + final j = (jsonDecode(jsonEncode(c.toJson())) as Map).cast(); + (j['d'] as List)[0] = d0; + return RrCorrector.fromJson(j).fold([800, 800]).classes; + } + + expect(classesAfter(-200.0), + [BeatClass.ectopic, BeatClass.normal, BeatClass.normal]); + expect(classesAfter(0.0), isNot(classesAfter(-200.0)), + reason: 'the stored difference is what decides it'); + }); + }); + + group('the forced scenarios say what they claim', () { + test('every abstention reason, every open-window label, a flag and the ' + 'absent artifact fraction are met by hand-built series, and the stream ' + 'agrees with the batch on each', () { + // (nn, minBeats, artifact fraction, maxArtifact) -> the reason, known by hand. + List sinus(int n) => [for (var i = 0; i < n; i++) 800.0 + (i % 9) * 13]; + List af(int n) => + [for (var i = 0; i < n; i++) (i % 2 == 0 ? 450.0 : 1100.0) + (i % 5) * 7]; + // A [holeEvery] > 0 puts a 10 s dropout before every holeEvery-th beat. + List times(List nn, int holeEvery) { + var t = 0.0; + var i = 0; + return [ + for (final v in nn) + t += (v >= 300 && v <= 2000 ? v : 800) + + (holeEvery > 0 && i++ % holeEvery == holeEvery - 1 ? 10000 : 0) + ]; + } + + final cases = <(String, List, int, double, IrregularAbstain?, int)>[ + ('too few beats', sinus(30), 40, 0.0, IrregularAbstain.tooFewBeats, 0), + // 60 beats, but a dropout before every second one leaves 29 pairs: + // the gate counts successive pairs (39 needed), not beats. + ('enough beats, too few successive pairs', sinus(60), 40, 0.0, + IrregularAbstain.tooFewBeats, 2), + // Zero pairs clear a gate of minBeats - 1 = 0, and then there is no + // difference to build the Poincare plot from. + ('nothing at all', const [], 1, 0.0, IrregularAbstain.noSuccessivePairs, 0), + ('too noisy', sinus(60), 40, .31, IrregularAbstain.artifact, 0), + ('exactly at the artifact maximum', sinus(60), 40, .30, null, 0), + ('exactly the minimum beats', sinus(40), 40, 0.0, null, 0), + // Two kept beats that are not neighbours: no difference exists. + ('no successive pair', const [800, 5000, 810], 1, 0.0, + IrregularAbstain.noSuccessivePairs, 0), + // Two kept neighbours with a dropout between them: not successive. + ('no successive pair across a time hole', const [800, 810], 1, 0.0, + IrregularAbstain.noSuccessivePairs, 1), + ('no long-term variability', List.filled(60, 800), 40, 0.0, + IrregularAbstain.noLongTermVariability, 0), + ('irregularly irregular', af(200), 40, 0.0, null, 0), + ]; + final opens = {}; + var flagged = false; + for (final (name, nn, minBeats, fraction, reason, holeEvery) in cases) { + final t = times(nn, holeEvery); + final batch = irregularBeatScreenDetailed(nn, + nnTimesMs: t, minBeats: minBeats, artifactFraction: fraction); + final st = IrregularScreenState()..fold(nn, t); + final stream = st.evaluateDetailed(const [], const [], + minBeats: minBeats, artifactFraction: fraction); + expect(batch.diagnostics.abstain, reason, reason: '$name: batch'); + _sameScreen(stream, batch, name); + final w = batch.diagnostics.windows; + if (w != null) opens.add(w.open); + if (batch.metric.present && batch.metric.value!.flag) flagged = true; + } + expect(flagged, isTrue, reason: 'a flagged day'); + expect(opens, { + IrregularOpenWindow.none, + IrregularOpenWindow.thin, + IrregularOpenWindow.unflagged, + IrregularOpenWindow.flagged, + }, reason: 'every open-window label'); + }); + + test('the forced corrector days are thin, flat, artefact-heavy, boundary, ' + 'timestamp-free and past one window', () { + bool any(bool Function(_Case c, _Series s) f) => _forced.any((c) => + f(c, _series(c.$1.$1, c.$1.$2, c.$1.$3))); + for (final n in [0, 1, 2, 3, 44, 46, 90, 91, 92]) { + expect(any((c, s) => s.n == n), isTrue, reason: '$n beats'); + } + expect(any((c, s) => c.$1.$4 == 1), isTrue, reason: 'no timestamps'); + for (var cfg = 0; cfg < _corrCfgs.length; cfg++) { + expect(any((c, s) => c.$2.$4 == cfg), isTrue, reason: 'config $cfg'); + } + // Flat: QD = 0, everything ties. Outliers at the first and last beat. + expect(any((c, s) => _flavours[c.$1.$1] == 'flat' && s.rr.toSet().length == 1), + isTrue); + expect( + any((c, s) => + _flavours[c.$1.$1] == 'outliers' && + s.n >= 3 && + s.rr.first == 1300 && + s.rr.last == 1300), + isTrue); + // A run of artefacts longer than the window: dropped, never corrected. + expect( + any((c, s) => + _flavours[c.$1.$1] == 'artefact heavy' && + correctRr(s.rr, rrTsMs: s.ts).droppedCount >= 20), + isTrue, + reason: 'artefacts that are dropped, not corrected (a run of 120 garbage beats is longer than the corrector window)'); + // A beat time that steps back, a dropout. + expect( + any((c, s) { + if (_flavours[c.$1.$1] != 'backwards') return false; + for (var i = 1; i < s.n; i++) { + if (s.ts[i] < s.ts[i - 1]) return true; + } + return false; + }), + isTrue, + reason: 'beat times that step backwards'); + expect( + any((c, s) { + if (_flavours[c.$1.$1] != 'gappy') return false; + for (var i = 1; i < s.n; i++) { + if (s.ts[i] - s.ts[i - 1] > 5000) return true; + } + return false; + }), + isTrue, + reason: 'a dropout'); + }); + + test('the forced integrated days reach a flag, a thin open window, a valid ' + 'one and the abstentions', () { + final reasons = {}; + final opens = {}; + var flagged = false, noBeatsAbsent = false; + for (final c in _nnForced) { + final ((f, n, seed, salt), (cfg, _, _, ev)) = c; + final s = _nnSeries(f, n, seed, salt); + final res = _evaluate(_newState(cfg)..fold(s.nn, s.t), ev, beats: n); + reasons.add(res.diagnostics.abstain); + final w = res.diagnostics.windows; + if (w != null) opens.add(w.open); + if (res.metric.present && res.metric.value!.flag) flagged = true; + if (n == 0 && _evals[ev].$4 == 2 && res.diagnostics.artifactFraction == null) { + noBeatsAbsent = true; + } + } + expect(reasons, containsAll([ + null, + IrregularAbstain.tooFewBeats, + IrregularAbstain.artifact, + IrregularAbstain.noLongTermVariability, + ])); + expect(opens, containsAll([ + IrregularOpenWindow.none, + IrregularOpenWindow.thin, + IrregularOpenWindow.unflagged, + ])); + expect(flagged, isTrue, reason: 'a flagged screen among the forced days'); + expect(noBeatsAbsent, isTrue, reason: 'zero beats: the fraction is absent'); + }); + }); + + group('FINDINGS (skipped: suspected gaps against the documented contract)', () { + test('IrregularScreenState.fromJson bounds differences and closed windows', () { + final nn = [for (var i = 0; i < 300; i++) 800.0 + (i % 7) * 11]; + final t = [for (var i = 0; i < 300; i++) 10000.0 + i * 1000]; + final st = IrregularScreenState()..fold(nn, t); + Map fresh() => + (jsonDecode(jsonEncode(st.toJson())) as Map).cast(); + final good = fresh(); + expect((good['bk'] as List).length, 300); + expect(IrregularScreenState.fromJson(good).toJson(), good); + + for (final (field, value) in [('dN', 300), ('total', 1)]) { + final bad = fresh()..[field] = value; + expect(() => IrregularScreenState.fromJson(bad), throwsFormatException, + reason: '$field: $value'); + } + }); + + test('IrregularScreenState.fromJson refuses a checkpoint whose parts ' + 'contradict each other', () { + final nn = [for (var i = 0; i < 300; i++) 800.0 + (i % 7) * 11]; + final t = [for (var i = 0; i < 300; i++) 10000.0 + i * 1000]; + final st = IrregularScreenState()..fold(nn, t); + Map fresh() => + (jsonDecode(jsonEncode(st.toJson())) as Map).cast(); + final bad = )>{ + 'levels seen != kept beats': (j) => j['lN'] = 5, + 'more differences than beats': (j) => j['dN'] = 1 << 40, + 'more windows than beats': (j) => j['total'] = 1 << 40, + 'a neighbour flag that is not 0 / 1': (j) => (j['bkAdj'] as List)[0] = 2, + 'a negative neighbour flag': (j) => (j['bkAdj'] as List)[0] = -1, + 'an open window without a start': (j) => j['winStart'] = null, + }; + for (final e in bad.entries) { + final j = fresh(); + e.value(j); + expect(() => IrregularScreenState.fromJson(j), throwsFormatException, + reason: e.key); + } + }); + + test('RrCorrector.fromJson bounds counters by their pipeline stages', () { + final short = RrCorrector(maxArtifactRun: null)..fold([800.0, 800.0, 800.0]); + Map shortJson() => + (jsonDecode(jsonEncode(short.toJson())) as Map).cast(); + final shortCheckpoint = shortJson(); + expect(shortCheckpoint['n'], 3); + expect(shortCheckpoint['c2'], 0); + expect(shortCheckpoint['ce'], 0); + expect(RrCorrector.fromJson(shortCheckpoint).toJson(), shortCheckpoint); + for (final (field, value) in [ + ('normalFinal', 1), + ('dropped', 1), + ('corrected', 1), + ]) { + final bad = shortJson()..[field] = value; + expect(() => RrCorrector.fromJson(bad), throwsFormatException, + reason: '$field: $value'); + } + + final long = RrCorrector(maxArtifactRun: null)..fold([for (var i = 0; i < 300; i++) 800.0]); + final longCheckpoint = + (jsonDecode(jsonEncode(long.toJson())) as Map).cast(); + final c2 = longCheckpoint['c2'] as int; + final ce = longCheckpoint['ce'] as int; + expect(c2, greaterThan(0)); + expect(ce, greaterThan(0)); + expect(longCheckpoint['off'], greaterThan(0)); + expect(RrCorrector.fromJson(longCheckpoint).toJson(), longCheckpoint); + final tooManyOutcomes = { + ...longCheckpoint, + 'dropped': ce, + 'corrected': 1, + }; + expect(() => RrCorrector.fromJson(tooManyOutcomes), throwsFormatException, + reason: 'dropped + corrected exceeds ce'); + }); + + test('RrCorrector.fromJson refuses a checkpoint whose counters are ' + 'impossible', () { + final c = RrCorrector(maxArtifactRun: null) + ..fold([for (var i = 0; i < 300; i++) 800.0 + (i % 7) * 11], + tsMs: [for (var i = 0; i < 300; i++) 1.76e12 + i * 1000]); + Map fresh() => + (jsonDecode(jsonEncode(c.toJson())) as Map).cast(); + final bad = )>{ + 'beats folded but timestamp mode unknown': (j) => j['wall'] = null, + 'negative dropped count': (j) => j['dropped'] = -5, + 'more dropped than beats': (j) => j['dropped'] = 1 << 40, + 'negative corrected count': (j) => j['corrected'] = -1, + 'more normal beats than classified': (j) => j['normalFinal'] = 1 << 40, + }; + for (final e in bad.entries) { + final j = fresh(); + e.value(j); + expect(() => RrCorrector.fromJson(j), throwsFormatException, reason: e.key); + } + expect(() => RrCorrector.fromJson({...fresh(), 'alpha': -1.0}), returnsNormally); + }); + }); + + _laws.registerReachTest(); +} diff --git a/tool/incremental/bench_rr.dart b/tool/incremental/bench_rr.dart new file mode 100644 index 00000000..61ff9426 --- /dev/null +++ b/tool/incremental/bench_rr.dart @@ -0,0 +1,83 @@ +// Timing: oracle correctRr vs streaming RrCorrector on a 23 h real-shaped day. +// dart run tool/incremental/bench_rr.dart +import 'dart:convert'; +import 'package:openstrap_analytics/onehz.dart' + hide RrCorrector, RrSettled, RrSnapshot, IrregularScreenState; +import 'oracle_util.dart'; +import 'rr_stream.dart'; +import 'synth.dart'; + +double ms(Stopwatch s) => s.elapsedMicroseconds / 1000.0; + +void main() { + final day = realShapedDay(); + final n = day.length; + print('day beats=$n span h=${(day.ts.last - day.ts.first) / 3.6e6}'); + // warm + correctRr(day.rr.sublist(0, 4000), rrTsMs: day.ts.sublist(0, 4000)); + + final sw = Stopwatch()..start(); + final oracle = correctRr(day.rr, rrTsMs: day.ts); + print('oracle one-shot: ${ms(sw).toStringAsFixed(0)} ms ' + '(${(ms(sw) * 1000 / n).toStringAsFixed(1)} us/beat)'); + + // streaming, one fold of the whole day + sw..reset()..start(); + final one = RrCorrector(); + final s1 = one.fold(day.rr, tsMs: day.ts); + final snap1 = one.snapshot(); + print('stream one-shot fold+snapshot: ${ms(sw).toStringAsFixed(0)} ms ' + '(${(ms(sw) * 1000 / n).toStringAsFixed(1)} us/beat)'); + final same = [...s1.nn, ...snap1.tailNn]; + print(' identical nn: ${same.length == oracle.nn.length && _eq(same, oracle.nn)}'); + + // streaming, 15-minute folds, JSON restart every fold (worst case) + final cuts = timeCuts(day.ts, 900); + var c = RrCorrector(); + final foldMs = [], snapMs = [], jsonMs = []; + var bytes = 0, maxBytes = 0; + var from = 0; + final all = []; + for (final to in cuts) { + sw..reset()..start(); + final s = c.fold(day.rr.sublist(from, to), tsMs: day.ts.sublist(from, to)); + foldMs.add(ms(sw)); + all.addAll(s.nn); + sw..reset()..start(); + final snap = c.snapshot(); + snapMs.add(ms(sw)); + if (to == cuts.last) all.addAll(snap.tailNn); + sw..reset()..start(); + final txt = jsonEncode(c.toJson()); + c = RrCorrector.fromJson(jsonRoundTrip(jsonDecode(txt) as Map)); + jsonMs.add(ms(sw)); + bytes = txt.length; + if (bytes > maxBytes) maxBytes = bytes; + from = to; + } + print('stream 15-min folds: ${cuts.length} folds'); + print(' fold mean=${_mean(foldMs).toStringAsFixed(1)} ms max=${_max(foldMs).toStringAsFixed(1)} sum=${_sum(foldMs).toStringAsFixed(0)}'); + print(' snapshot mean=${_mean(snapMs).toStringAsFixed(2)} ms max=${_max(snapMs).toStringAsFixed(2)}'); + print(' json enc+dec+restore mean=${_mean(jsonMs).toStringAsFixed(2)} ms max=${_max(jsonMs).toStringAsFixed(2)} state bytes max=$maxBytes'); + print(' identical nn over 96 folds: ${_eq(all, oracle.nn)}'); + + // what the app does today: whole-day correctRr at pass k + for (final frac in [0.25, 0.5, 0.75, 1.0]) { + final m = (n * frac).floor(); + sw..reset()..start(); + correctRr(day.rr.sublist(0, m), rrTsMs: day.ts.sublist(0, m)); + print('oracle at ${(frac * 100).round()}% of day (n=$m): ${ms(sw).toStringAsFixed(0)} ms'); + } +} + +bool _eq(List a, List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +double _mean(List x) => x.isEmpty ? 0 : _sum(x) / x.length; +double _sum(List x) => x.fold(0.0, (a, b) => a + b); +double _max(List x) => x.fold(0.0, (a, b) => a > b ? a : b); diff --git a/tool/incremental/driver.dart b/tool/incremental/driver.dart new file mode 100644 index 00000000..4d3b552a --- /dev/null +++ b/tool/incremental/driver.dart @@ -0,0 +1,32 @@ +// Test/bench driver: RR stream -> RrCorrector -> per-fold callback with the +// SETTLED NN delta and the PROVISIONAL tail, so downstream states can be +// folded exactly the way the app would (settled in, tail evaluated on a copy). +import 'oracle_util.dart'; +import 'rr_stream.dart'; +import 'synth.dart'; + +class FoldStep { + final int fold; + final int n; // raw beats folded so far + final RrSettled settled; // newly settled NN + final RrSnapshot snap; // provisional tail + totals + FoldStep(this.fold, this.n, this.settled, this.snap); + double get artifactFraction => + (1.0 - snap.cleanFraction).clamp(0.0, 1.0).toDouble(); +} + +void driveRr(RrData s, List cuts, void Function(FoldStep) onFold, + {bool restart = false, int restartEvery = 1}) { + var c = RrCorrector(); + var from = 0; + for (var k = 0; k < cuts.length; k++) { + final to = cuts[k]; + final st = c.fold(s.rr.sublist(from, to), tsMs: s.ts.sublist(from, to)); + final snap = c.snapshot(); + onFold(FoldStep(k, to, st, snap)); + if (restart && k % restartEvery == 0) { + c = RrCorrector.fromJson(jsonRoundTrip(c.toJson())); + } + from = to; + } +} diff --git a/tool/incremental/lag_stats.dart b/tool/incremental/lag_stats.dart new file mode 100644 index 00000000..ebd0506f --- /dev/null +++ b/tool/incremental/lag_stats.dart @@ -0,0 +1,49 @@ +// Settle horizon in the wild: how many beats / seconds behind the newest beat +// is the last SETTLED output, distribution over 1-beat folds. +import 'package:openstrap_analytics/onehz.dart' + hide RrCorrector, RrSettled, RrSnapshot, IrregularScreenState; +import 'rr_stream.dart'; +import 'synth.dart'; + +void report(String name, RrData s) { + final c = RrCorrector(); + final lagBeats = [], lagSec = []; + var ce0 = 0; + for (var i = 0; i < s.length; i++) { + c.fold([s.rr[i]], tsMs: [s.ts[i]]); + final settled = c.settledBeats; + if (settled > ce0 || true) { + lagBeats.add(i + 1 - settled); + lagSec.add(settled == 0 ? 0 : (s.ts[i] - s.ts[settled - 1]) / 1000.0); + } + ce0 = settled; + } + lagBeats.sort(); + lagSec.sort(); + double q(List x, double p) => x[((x.length - 1) * p).round()].toDouble(); + print('$name: beats=${s.length} lag beats p50=${q(lagBeats, .5)} p95=${q(lagBeats, .95)} ' + 'p99=${q(lagBeats, .99)} max=${lagBeats.last} lag s p50=${q(lagSec, .5).toStringAsFixed(0)} ' + 'p99=${q(lagSec, .99).toStringAsFixed(0)} max=${lagSec.last.toStringAsFixed(0)} ' + 'state buffer max beats: ${c.bufferedBeats}'); +} + +void main() { + report('real night', realNightRr()!); + report('synthetic day (default artefacts)', synthRr(const SynthConfig(seed: 3, hours: 8))); + report('synthetic dirty', synthRr(const SynthConfig( + seed: 4, hours: 3, ectopicPerMin: 3, missedPerMin: 2, extraPerMin: 2, noiseRunPerMin: 1, gapPerHour: 8))); + // one-shot micro bench, 3 runs each, 32k beats + final n = realNightRr()!; + final sw = Stopwatch(); + final o = [], r = []; + for (var k = 0; k < 3; k++) { + sw..reset()..start(); + correctRr(n.rr, rrTsMs: n.ts); + o.add(sw.elapsedMilliseconds); + sw..reset()..start(); + final c = RrCorrector()..fold(n.rr, tsMs: n.ts); + c.snapshot(); + r.add(sw.elapsedMilliseconds); + } + print('32k-beat night one-shot ms: oracle $o streaming $r'); +} diff --git a/tool/incremental/oracle_util.dart b/tool/incremental/oracle_util.dart new file mode 100644 index 00000000..6f12de69 --- /dev/null +++ b/tool/incremental/oracle_util.dart @@ -0,0 +1,40 @@ +// Shared helpers for the incremental oracle tests. +import 'dart:convert'; +import 'dart:math' as math; + +/// JSON round trip through real text (what a SQLite blob would hold). +Map jsonRoundTrip(Map m) => + jsonDecode(jsonEncode(m)) as Map; + +/// Random chunk boundaries over [0, n): sizes drawn from [sizes] (a mix of 1, +/// tiny, one-window, and huge) so every alignment against the 91/181-beat +/// windows is hit. +List randomCuts(math.Random r, int n, + {List sizes = const [1, 2, 3, 7, 45, 46, 90, 91, 92, 180, 181, 500, 1500]}) { + final cuts = []; + var at = 0; + while (at < n) { + final s = sizes[r.nextInt(sizes.length)]; + at = math.min(n, at + 1 + r.nextInt(s)); + cuts.add(at); + } + return cuts; +} + +/// Cuts that fall every [sec] seconds of the beat clock: what a 15-minute +/// derive pass sees. [tsMs] sorted-ish epoch ms. +List timeCuts(List tsMs, double sec) { + final cuts = []; + if (tsMs.isEmpty) return cuts; + var next = tsMs.first + sec * 1000; + for (var i = 0; i < tsMs.length; i++) { + if (tsMs[i] >= next) { + cuts.add(i); + while (next <= tsMs[i]) { + next += sec * 1000; + } + } + } + cuts.add(tsMs.length); + return cuts; +} diff --git a/tool/incremental/rr_stream.dart b/tool/incremental/rr_stream.dart new file mode 100644 index 00000000..6499b3a1 --- /dev/null +++ b/tool/incremental/rr_stream.dart @@ -0,0 +1,579 @@ +// RESEARCH PROTOTYPE — streaming Lipponen-Tarvainen RR correction. +// +// Goal: fold NEW beats onto saved state and get, after every fold, EXACTLY +// what `correctRr` returns over the whole series so far (bit-identical nn, +// nnTimes, classes, counts, cleanFraction), while never re-reading the settled +// prefix. +// +// Why that is possible (derivation, with H = windowBeats ~/ 2 = 45): +// beat clock t[i] prefix recurrence (cur, prevTs, origin) +// dRR[i] rr[i]-rr[i-1] local +// th1[i] QD of dRR over [i-H, i+H] -> needs rr up to i+H +// med[i] median of rr[i-H..i+H] \ {i} -> needs rr up to i+H +// mRR[i] f(rr[i], med[i]) -> needs rr up to i+H +// th2[i] QD of mRR over [i-H, i+H] -> needs rr up to i+2H +// class0[i] f(rr,dRR,th1,th2,mRR,med) -> final once n-1 >= i+2H +// class[i] left-to-right compensatory-pair pass; reads only final +// class[i-1], so it folds with one carried class. +// output a run of artifacts is dropped, a single one is spline- +// interpolated from the 2 nearest NORMAL beats each side. The +// left anchors are carried (last two normals); the right anchors +// need classes up to the 2nd normal after the artifact. So the +// settle horizon is 2H beats + (artifact run length) + 2 normals. +// Windows are clipped at n-1, so the last 2H beats are PROVISIONAL: [snapshot] +// re-evaluates just that tail (directly, with the oracle's own clipping) and +// is exactly what correctRr would say at the current n. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart' + hide RrCorrector, RrSettled, RrSnapshot, IrregularScreenState; + +/// Provisional + counters view after a fold. `settled*` is NOT here: [fold] +/// returns the newly settled output; this is the part that can still change. +class RrSnapshot { + final int n; + final List tailNn; + final List tailNnTimes; + final int normalCount, droppedCount, correctedCount; + final int settledBeats, classifiedBeats; + final List? tailClasses; // classes of beats [classifiedBeats, n) + const RrSnapshot({ + required this.n, + required this.tailNn, + required this.tailNnTimes, + required this.normalCount, + required this.droppedCount, + required this.correctedCount, + required this.settledBeats, + required this.classifiedBeats, + required this.tailClasses, + }); + double get cleanFraction => n == 0 ? 0 : normalCount / n; +} + +class RrSettled { + final List nn = []; + final List nnTimes = []; +} + +/// Sorted multiset window with O(log w) search + O(w) memmove, w ~ 91. +class _Sorted { + final List a = []; + int _lb(double v) { + var lo = 0, hi = a.length; + while (lo < hi) { + final m = (lo + hi) >> 1; + if (a[m] < v) { + lo = m + 1; + } else { + hi = m; + } + } + return lo; + } + + void add(double v) => a.insert(_lb(v), v); + void remove(double v) => a.removeAt(_lb(v)); + void resetFrom(Iterable xs) { + a + ..clear() + ..addAll(xs) + ..sort(); + } +} + +class RrCorrector { + final double alpha; + final int windowBeats; + final double minThresholdMs; + final double reanchorGapMs; + final int _h; + + // ---- prefix state (the persisted tuple) ---- + int n = 0; // total beats folded + bool? _wall; // timestamps supplied for this stream + double _origin = 0, _cur = 0, _prevTs = 0, _prevRr = 0; + int _off = 0; // global index of _rr[0] + final List _rr = [], _t = [], _d = []; + int _so = 0; // global index of _med[0] (stage arrays) + final List _med = [], _mrr = [], _th1 = []; + int _c1 = 0; // beats with med/mRR/th1 final + int _c2 = 0; // beats with class final + int _ce = 0; // beats whose OUTPUT is final (emitted) + final List _fcls = []; // final classes for [_ce, _c2) + int _lastFinal = 0; // class index of beat _c2-1 + int _normalFinal = 0; + int _dropped = 0, _corrected = 0; + final List _lastNormals = []; // <= 2, oldest first + + // ---- derived caches (not persisted) ---- + final _Sorted _wd = _Sorted(), _wr = _Sorted(), _wm = _Sorted(); + int _wPos = -1, _w2Pos = -1; + + RrCorrector({ + this.alpha = 5.2, + this.windowBeats = 91, + this.minThresholdMs = 100, + this.reanchorGapMs = 1000, + }) : _h = windowBeats ~/ 2 { + if (windowBeats < 3) throw ArgumentError('windowBeats >= 3'); + } + + /// Test hook: called once per beat when its class becomes final. + void Function(int g, int cls)? debugClassSink; + + int get settledBeats => _ce; + int get classifiedBeats => _c2; + int get bufferedBeats => n - _off; + + // ------------------------------------------------------------------ fold + /// Append beats. Returns the output that became FINAL with this fold. + RrSettled fold(List rrMs, {List? tsMs}) { + final out = RrSettled(); + if (rrMs.isEmpty) return out; + final wall = tsMs != null && tsMs.length == rrMs.length; + _wall ??= wall; + if (_wall != wall) { + throw StateError('timestamps must be supplied for every fold or none'); + } + for (var k = 0; k < rrMs.length; k++) { + final r = rrMs[k]; + if (!r.isFinite) throw ArgumentError('non-finite rr at $k'); + final i = n; + double cur; + final ts = wall ? tsMs[k] : 0.0; + if (i == 0) { + _origin = wall ? ts - r : 0.0; + _cur = 0.0; + } + final dropout = wall && i > 0 && (ts - _prevTs) - r > reanchorGapMs; + final anchored = wall ? ts - _origin : 0.0; + cur = (dropout && anchored > _cur) ? anchored : _cur + r; + _cur = cur; + if (wall) _prevTs = ts; + _rr.add(r); + _t.add(cur); + _d.add(i == 0 ? 0.0 : r - _prevRr); + _prevRr = r; + n++; + } + _advance(out); + _compact(); + return out; + } + + double _rrAt(int g) => _rr[g - _off]; + double _dAt(int g) => _d[g - _off]; + double _tAt(int g) => _t[g - _off]; + + void _advance(RrSettled out) { + if (n < 3) return; + final h = _h; + while (_c1 + h <= n - 1) { + _stage1(_c1); + _c1++; + } + while (_c2 + 2 * h <= n - 1) { + _stage2(_c2); + _c2++; + } + _emitCommitted(out); + } + + // th1/med/mRR for beat g; windows [max(0,g-h), g+h] fully inside the data. + void _stage1(int g) { + final h = _h; + if (_wPos == g - 1 && _wPos >= 0) { + _wd.add(_dAt(g + h)); + _wr.add(_rrAt(g + h)); + if (g - 1 - h >= 0) { + _wd.remove(_dAt(g - 1 - h)); + _wr.remove(_rrAt(g - 1 - h)); + } + } else { + final lo = math.max(0, g - h); + _wd.resetFrom([for (var k = lo; k <= g + h; k++) _dAt(k)]); + _wr.resetFrom([for (var k = lo; k <= g + h; k++) _rrAt(k)]); + } + _wPos = g; + final th1 = _thOf(_wd.a); + final m = _medExcl(_wr.a, _rrAt(g)) ?? _rrAt(g); + final d = _rrAt(g) - m; + _med.add(m); + _mrr.add(d < 0 ? d * 2 : d); + _th1.add(th1); + } + + void _stage2(int g) { + final h = _h; + if (_w2Pos == g - 1 && _w2Pos >= 0) { + _wm.add(_mrr[g + h - _so]); + if (g - 1 - h >= 0) _wm.remove(_mrr[g - 1 - h - _so]); + } else { + final lo = math.max(0, g - h); + _wm.resetFrom([for (var k = lo; k <= g + h; k++) _mrr[k - _so]]); + } + _w2Pos = g; + final th2 = _thOf(_wm.a); + var cls = _class0(_rrAt(g), _dAt(g), _th1[g - _so], _med[g - _so], + _mrr[g - _so], th2); + cls = _reconcile(g, cls, _lastFinal, _rrAt(g), _dAt(g), + g > 0 ? _dAt(g - 1) : 0, _med[g - _so]); + _fcls.add(cls.index); + debugClassSink?.call(g, cls.index); + _lastFinal = cls.index; + if (cls == BeatClass.normal) _normalFinal++; + } + + double _thOf(List sorted) { + final q1 = percentileSorted(sorted, 25) ?? 0; + final q3 = percentileSorted(sorted, 75) ?? 0; + final qd = (q3 - q1) / 2; + return math.max(alpha * qd, minThresholdMs); + } + + /// median of [sorted] with ONE occurrence of [v] removed — what + /// `_slidingMedian` computes (it skips k == i), without re-sorting. + static double? _medExcl(List sorted, double v) { + final len = sorted.length - 1; + if (len <= 0) return null; + var lo = 0, hi = sorted.length; + while (lo < hi) { + final m = (lo + hi) >> 1; + if (sorted[m] < v) { + lo = m + 1; + } else { + hi = m; + } + } + final p = lo; + double at(int j) => j < p ? sorted[j] : sorted[j + 1]; + if (len == 1) return at(0); + final rank = (50.0 / 100) * (len - 1); + final l = rank.floor(), u = rank.ceil(); + if (l == u) return at(l); + final frac = rank - l; + return at(l) + (at(u) - at(l)) * frac; + } + + BeatClass _class0( + double rr, double d, double th1, double med, double mrr, double th2) { + final hardLong = rr > 2000; + final hardShort = rr < 300; + final bigJump = d.abs() > th1; + final bigDev = mrr.abs() > th2; + if (hardLong || (bigDev && mrr > 0)) { + return (med > 0 && rr > 1.5 * med) ? BeatClass.missed : BeatClass.longShort; + } else if (hardShort || (bigDev && mrr < 0)) { + return (med > 0 && rr < 0.6 * med) ? BeatClass.extra : BeatClass.longShort; + } else if (bigJump) { + return BeatClass.ectopic; + } + return BeatClass.normal; + } + + BeatClass _reconcile(int g, BeatClass cls, int prevCls, double rr, double d, + double dPrev, double med) { + if (g < 1 || cls != BeatClass.ectopic) return cls; + final prevBad = prevCls != BeatClass.normal.index; + if (!prevBad) return cls; + final opposite = d * dPrev < 0; + final valueNormal = + med > 0 && rr >= 300 && rr <= 2000 && (rr - med).abs() <= 0.2 * med; + return (opposite && valueNormal) ? BeatClass.normal : cls; + } + + static double? _spline(List left, List right) { + if (left.isEmpty || right.isEmpty) return null; + final p1 = left.last; + final p2 = right.first; + final p0 = left.length >= 2 ? left.first : p1; + final p3 = right.length >= 2 ? right.last : p2; + const t = 0.5; + final t2 = t * t; + final t3 = t2 * t; + return 0.5 * + ((2 * p1) + + (-p0 + p2) * t + + (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 + + (-p0 + 3 * p1 - 3 * p2 + p3) * t3); + } + + // Emission over classes [ce, limit). Shared by the committed pass (limit = + // c2, isEnd=false: undecidable artifacts wait) and the provisional pass + // (limit = n, isEnd=true: end of data resolves them exactly like the batch). + int _emit({ + required int ce, + required int limit, + required bool isEnd, + required int Function(int g) clsAt, + required List lastNormals, + required void Function(double nn, double t) put, + required void Function(int runLen) drop, + required void Function() corrected, + }) { + const normal = 0; + while (ce < limit) { + if (clsAt(ce) == normal) { + put(_rrAt(ce), _tAt(ce)); + lastNormals.add(_rrAt(ce)); + if (lastNormals.length > 2) lastNormals.removeAt(0); + ce++; + continue; + } + var j = ce; + while (j < limit && clsAt(j) != normal) { + j++; + } + if (j == limit && !isEnd) break; + final runLen = j - ce; + if (runLen == 1) { + final right = []; + var k = j; + while (k < limit && right.length < 2) { + if (clsAt(k) == normal) right.add(_rrAt(k)); + k++; + } + if (right.length < 2 && k == limit && !isEnd) break; + final corr = _spline(lastNormals, right); + if (corr != null) { + put(corr, _tAt(ce)); + corrected(); + } else { + drop(1); + } + } else { + drop(runLen); + } + ce = j; + } + return ce; + } + + void _emitCommitted(RrSettled out) { + final fo = _ce; + _ce = _emit( + ce: _ce, + limit: _c2, + isEnd: false, + clsAt: (g) => _fcls[g - fo], + lastNormals: _lastNormals, + put: (v, t) { + out.nn.add(v); + out.nnTimes.add(t); + }, + drop: (r) => _dropped += r, + corrected: () => _corrected++, + ); + // forget classes that are now emitted + final used = _ce - fo; + if (used > 0) _fcls.removeRange(0, used); + } + + // ------------------------------------------------------------- snapshot + /// The part of `correctRr(all beats so far)` that is not settled yet, plus + /// the totals. Settled output (from [fold]) ++ tail == the oracle, exactly. + RrSnapshot snapshot({bool withClasses = false}) { + if (n == 0) { + return const RrSnapshot( + n: 0, + tailNn: [], + tailNnTimes: [], + normalCount: 0, + droppedCount: 0, + correctedCount: 0, + settledBeats: 0, + classifiedBeats: 0, + tailClasses: []); + } + if (n < 3) { + // oracle's n<3 branch (nothing is ever settled below 3 beats) + final nn = [], times = []; + final cl = []; + for (var g = 0; g < n; g++) { + final ok = _rrAt(g) >= 300 && _rrAt(g) <= 2000; + cl.add(ok ? BeatClass.normal : BeatClass.longShort); + if (ok) { + nn.add(_rrAt(g)); + times.add(_tAt(g)); + } + } + return RrSnapshot( + n: n, + tailNn: nn, + tailNnTimes: times, + normalCount: nn.length, + droppedCount: n - nn.length, + correctedCount: 0, + settledBeats: 0, + classifiedBeats: 0, + tailClasses: withClasses ? cl : null); + } + final h = _h; + final end = n - 1; + // ---- provisional stage 1 for beats [c1, n): windows clipped at `end` + final tMed = [], tMrr = [], tTh1 = []; + double mrrAt(int k) => + k < _c1 ? _mrr[k - _so] : tMrr[k - _c1]; + for (var k = _c1; k < n; k++) { + final lo = math.max(0, k - h), hi = math.min(end, k + h); + final dd = [for (var q = lo; q <= hi; q++) _dAt(q)]..sort(); + final rs = [for (var q = lo; q <= hi; q++) _rrAt(q)]..sort(); + tTh1.add(_thOf(dd)); + final m = _medExcl(rs, _rrAt(k)) ?? _rrAt(k); + final d = _rrAt(k) - m; + tMed.add(m); + tMrr.add(d < 0 ? d * 2 : d); + } + // ---- provisional classes for [c2, n) + final tc = []; + var prev = _lastFinal; + var normalTent = 0; + for (var g = _c2; g < n; g++) { + final lo = math.max(0, g - h), hi = math.min(end, g + h); + final ms = [for (var q = lo; q <= hi; q++) mrrAt(q)]..sort(); + final th2 = _thOf(ms); + final th1 = g < _c1 ? _th1[g - _so] : tTh1[g - _c1]; + final med = g < _c1 ? _med[g - _so] : tMed[g - _c1]; + final mrr = mrrAt(g); + var cls = _class0(_rrAt(g), _dAt(g), th1, med, mrr, th2); + cls = _reconcile( + g, cls, prev, _rrAt(g), _dAt(g), g > 0 ? _dAt(g - 1) : 0, med); + tc.add(cls.index); + prev = cls.index; + if (cls == BeatClass.normal) normalTent++; + } + // ---- provisional emission over [ce, n) + final fo = _ce; + final ln = List.of(_lastNormals); + final nn = [], times = []; + var dropped = _dropped, corrected = _corrected; + _emit( + ce: _ce, + limit: n, + isEnd: true, + clsAt: (g) => g < _c2 ? _fcls[g - fo] : tc[g - _c2], + lastNormals: ln, + put: (v, t) { + nn.add(v); + times.add(t); + }, + drop: (r) => dropped += r, + corrected: () => corrected++, + ); + return RrSnapshot( + n: n, + tailNn: nn, + tailNnTimes: times, + normalCount: _normalFinal + normalTent, + droppedCount: dropped, + correctedCount: corrected, + settledBeats: _ce, + classifiedBeats: _c2, + tailClasses: withClasses + ? [for (final c in tc) BeatClass.values[c]] + : null, + ); + } + + // --------------------------------------------------------------- compact + void _compact() { + final h = _h; + final newOff = math.max(0, math.min(_ce, _c2 - 2 * h - 1)); + if (newOff > _off) { + final drop = newOff - _off; + _rr.removeRange(0, drop); + _t.removeRange(0, drop); + _d.removeRange(0, drop); + _off = newOff; + } + final newSo = math.max(0, _c2 - h - 1); + if (newSo > _so && _med.isNotEmpty) { + final drop = math.min(newSo - _so, _med.length); + _med.removeRange(0, drop); + _mrr.removeRange(0, drop); + _th1.removeRange(0, drop); + _so += drop; + } + } + + // ------------------------------------------------------------------ json + Map toJson() => { + 'v': 1, + 'alpha': alpha, + 'win': windowBeats, + 'floor': minThresholdMs, + 'reanchor': reanchorGapMs, + 'n': n, + 'wall': _wall, + 'origin': _origin, + 'cur': _cur, + 'prevTs': _prevTs, + 'prevRr': _prevRr, + 'off': _off, + 'rr': _rr, + 't': _t, + 'd': _d, + 'c1': _c1, + 'c2': _c2, + 'ce': _ce, + 'fcls': _fcls, + 'lastFinal': _lastFinal, + 'normalFinal': _normalFinal, + 'dropped': _dropped, + 'corrected': _corrected, + 'lastNormals': _lastNormals, + }; + + factory RrCorrector.fromJson(Map j) { + List dl(Object? o) => [for (final x in o as List) (x as num).toDouble()]; + final c = RrCorrector( + alpha: (j['alpha'] as num).toDouble(), + windowBeats: j['win'] as int, + minThresholdMs: (j['floor'] as num).toDouble(), + reanchorGapMs: (j['reanchor'] as num).toDouble(), + ); + c.n = j['n'] as int; + c._wall = j['wall'] as bool?; + c._origin = (j['origin'] as num).toDouble(); + c._cur = (j['cur'] as num).toDouble(); + c._prevTs = (j['prevTs'] as num).toDouble(); + c._prevRr = (j['prevRr'] as num).toDouble(); + c._off = j['off'] as int; + c._rr.addAll(dl(j['rr'])); + c._t.addAll(dl(j['t'])); + c._d.addAll(dl(j['d'])); + c._c1 = j['c1'] as int; + c._c2 = j['c2'] as int; + c._ce = j['ce'] as int; + c._fcls.addAll((j['fcls'] as List).cast()); + c._lastFinal = j['lastFinal'] as int; + c._normalFinal = j['normalFinal'] as int; + c._dropped = j['dropped'] as int; + c._corrected = j['corrected'] as int; + c._lastNormals.addAll(dl(j['lastNormals'])); + c._rebuildStageArrays(); + return c; + } + + /// Stage arrays are pure functions of the buffered raw beats; recompute. + void _rebuildStageArrays() { + final h = _h; + _so = math.max(0, _c2 - h - 1); + _med.clear(); + _mrr.clear(); + _th1.clear(); + for (var g = _so; g < _c1; g++) { + final lo = math.max(0, g - h), hi = g + h; + final dd = [for (var q = lo; q <= hi; q++) _dAt(q)]..sort(); + final rs = [for (var q = lo; q <= hi; q++) _rrAt(q)]..sort(); + _th1.add(_thOf(dd)); + final m = _medExcl(rs, _rrAt(g)) ?? _rrAt(g); + final d = _rrAt(g) - m; + _med.add(m); + _mrr.add(d < 0 ? d * 2 : d); + } + _wPos = -1; + _w2Pos = -1; + } +} diff --git a/tool/incremental/synth.dart b/tool/incremental/synth.dart new file mode 100644 index 00000000..1c7bd90d --- /dev/null +++ b/tool/incremental/synth.dart @@ -0,0 +1,197 @@ +// Research scaffolding: real-shaped RR / accelerometer generators and the real +// night fixture loader. Deterministic (seeded), no wall clock. +// +// "Real-shaped" means what the app really feeds correctRr: +// * integer-millisecond RR, end-of-beat timestamps quantised to WHOLE seconds +// (rr_ts_ms = rec_ts * 1000, several beats can share one stamp), +// * ectopic pairs, missed beats (2x interval), extra beats (half interval), +// multi-beat noise runs, sensor dropouts (no beats for seconds..minutes), +// * optional counter-reset style timestamp steps backwards. +import 'dart:io'; +import 'dart:math' as math; + +class RrData { + final List rr; + final List ts; // epoch-ish ms, whole seconds + RrData(this.rr, this.ts); + int get length => rr.length; +} + +class SynthConfig { + final int seed; + final double hours; + final double startSec; + final double ectopicPerMin; // premature beat + compensatory pause + final double missedPerMin; + final double extraPerMin; + final double noiseRunPerMin; // run of 3..12 garbage beats + final double gapPerHour; // dropouts of 5..900 s + final double backwardsPerHour; // ts steps back 1..3 s (counter reset-ish) + final double nightFraction; // fraction of the span that is "asleep" + const SynthConfig({ + this.seed = 1, + this.hours = 24, + this.startSec = 1.7e9, + this.ectopicPerMin = 0.3, + this.missedPerMin = 0.1, + this.extraPerMin = 0.1, + this.noiseRunPerMin = 0.05, + this.gapPerHour = 1.5, + this.backwardsPerHour = 0.0, + this.nightFraction = 0.33, + }); +} + +/// Physiologically-flavoured RR: slow HR drift + RSA (0.25 Hz) + LF (0.1 Hz) +/// + white jitter, integer ms, with injected artefacts. +RrData synthRr(SynthConfig c) { + final rnd = math.Random(c.seed); + final rr = [], ts = []; + final endT = c.hours * 3600.0; + var t = 0.0; // seconds since start (beat clock) + var phase = rnd.nextDouble() * 6.28; + final nightStart = endT * 0.05; + final nightEnd = nightStart + endT * c.nightFraction; + void emit(double interval) { + final v = interval.roundToDouble(); + t += v / 1000.0; + var stamp = (c.startSec + t).floorToDouble() * 1000.0; + rr.add(v); + ts.add(stamp); + } + + while (t < endT) { + final asleep = t >= nightStart && t < nightEnd; + // Base HR: sleep ~52 bpm, day 65..120 with activity bouts. + final bout = (math.sin(t / 1900.0 + c.seed) + 1) / 2; // 0..1 + final bpm = asleep ? 52 + 3 * math.sin(t / 5400) : 66 + 55 * bout * bout; + var base = 60000.0 / bpm; + final rsaAmp = asleep ? 45.0 : 22.0 * (1 - 0.7 * bout); + final rsaHz = asleep ? 0.24 : 0.27; + final v = base + + rsaAmp * math.sin(2 * math.pi * rsaHz * t + phase) + + 28 * math.sin(2 * math.pi * 0.1 * t) + + 8 * (rnd.nextDouble() - .5) * 2; + final beatMin = v / 60000.0; // minutes per beat → per-beat probabilities + final u = rnd.nextDouble(); + var p = c.noiseRunPerMin * beatMin; + if (u < p) { + final len = 3 + rnd.nextInt(10); + for (var k = 0; k < len; k++) { + emit(250 + rnd.nextInt(2100).toDouble()); + } + continue; + } + p += c.ectopicPerMin * beatMin; + if (u < p) { + emit(v * (0.55 + 0.15 * rnd.nextDouble())); + emit(v * (1.3 + 0.2 * rnd.nextDouble())); + continue; + } + p += c.missedPerMin * beatMin; + if (u < p) { + emit(v * (1.9 + 0.2 * rnd.nextDouble())); + continue; + } + p += c.extraPerMin * beatMin; + if (u < p) { + final f = 0.35 + 0.3 * rnd.nextDouble(); + emit(v * f); + emit(v * (1 - f)); + continue; + } + p += c.gapPerHour * beatMin / 60.0; + if (u < p) { + final gap = 5 + rnd.nextInt(896); + t += gap.toDouble(); + continue; + } + p += c.backwardsPerHour * beatMin / 60.0; + if (u < p && ts.isNotEmpty) { + emit(v); + ts[ts.length - 1] = ts.last - 1000.0 * (1 + rnd.nextInt(3)); + continue; + } + emit(v); + } + return RrData(rr, ts); +} + +/// The checked-in anonymised 8.9 h WHOOP-4 night (rebased to 0 s). Returns +/// null if the fixture is missing (callers decide whether to fail). +RrData? realNightRr({double startSec = 1.7e9}) { + final f = File('test/onehz/fixtures/real_night_2026_07_rr.csv'); + if (!f.existsSync()) return null; + final rr = [], ts = []; + for (final line in f.readAsLinesSync().skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(','); + ts.add(startSec * 1000 + double.parse(p[0])); + rr.add(double.parse(p[1])); + } + return RrData(rr, ts); +} + +/// A whole 24 h-ish day: synthetic morning + REAL night + synthetic evening. +/// The synthetic parts carry the artefact load; the night is as recorded. +RrData realShapedDay({int seed = 7, double startSec = 1.7e9}) { + final real = realNightRr(startSec: startSec)!; + final nightSpanMs = real.ts.last - real.ts.first; + final morning = synthRr(SynthConfig( + seed: seed, hours: 5, startSec: startSec - 5 * 3600, nightFraction: 0)); + final evening = synthRr(SynthConfig( + seed: seed + 1, + hours: 9, + startSec: startSec + nightSpanMs / 1000 + 120, + nightFraction: 0)); + return RrData([...morning.rr, ...real.rr, ...evening.rr], + [...morning.ts, ...real.ts, ...evening.ts]); +} + +/// Per-second 1 Hz accel for [fromSec, toSec): stillness / motion bouts, +/// absent (all-zero) seconds and gaps. Rows carry epoch seconds. +class AccelSeries { + final List tsSec; + final List ax, ay, az; + AccelSeries(this.tsSec, this.ax, this.ay, this.az); + int get length => tsSec.length; +} + +AccelSeries synthAccel(int seed, int fromSec, int toSec, + {double stillFraction = 0.55, double gapPerHour = 2.0}) { + final rnd = math.Random(seed); + final ts = [], ax = [], ay = [], az = []; + var still = true; + var left = 60 + rnd.nextInt(1200); + var s = fromSec; + while (s < toSec) { + if (rnd.nextDouble() < gapPerHour / 3600.0) { + s += 5 + rnd.nextInt(300); + continue; + } + if (--left <= 0) { + still = rnd.nextDouble() < stillFraction; + left = 30 + rnd.nextInt(1500); + } + double x, y, z; + if (rnd.nextDouble() < 0.0008) { + x = y = z = 0; // fill row: accel absent + } else if (still) { + // gravity only: |g| within 0.02 g of 1 (some seconds slightly off) + final jitter = (rnd.nextDouble() - .5) * (rnd.nextDouble() < 0.1 ? .08 : .01); + x = 0.3 + (rnd.nextDouble() - .5) * .002; + y = 0.8 + (rnd.nextDouble() - .5) * .002; + z = math.sqrt(math.max(0.0, 1 - x * x - y * y)) + jitter; + } else { + x = (rnd.nextDouble() - .5) * 2.4; + y = (rnd.nextDouble() - .5) * 2.4; + z = 1 + (rnd.nextDouble() - .5) * 2.0; + } + ts.add(s); + ax.add(x); + ay.add(y); + az.add(z); + s++; + } + return AccelSeries(ts, ax, ay, az); +} From 51d33e99b658d498d3898f7390ca99347958904e Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 3 Nov 2026 16:11:01 -0700 Subject: [PATCH 2/4] Validate cap-only gap labels across streaming and checkpoint paths --- .../foundations/rr_correction_stream.dart | 5 ++- test/onehz/rr_artifact_cap_test.dart | 2 +- test/onehz/rr_correction_oracle_test.dart | 2 + test/onehz/support/rr_stream_driver.dart | 6 ++- .../review_stream_gap_test.dart | 38 +++++++++++++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 test/output_changes/review_stream_gap_test.dart diff --git a/lib/src/onehz/foundations/rr_correction_stream.dart b/lib/src/onehz/foundations/rr_correction_stream.dart index 1e034eb5..1f8b89d0 100644 --- a/lib/src/onehz/foundations/rr_correction_stream.dart +++ b/lib/src/onehz/foundations/rr_correction_stream.dart @@ -241,9 +241,10 @@ class RrCorrector { c._c2 <= c._c1 && c._c1 <= c._n && c._fcls.length == c._c2 - c._ce && - c._fcls.every((k) => k >= 0 && k < BeatClass.values.length) && + // Uncapped state never emits the cap-only gap label. + c._fcls.every((k) => k >= 0 && k <= BeatClass.extra.index) && c._lastFinal >= 0 && - c._lastFinal < BeatClass.values.length && + c._lastFinal <= BeatClass.extra.index && c._lastNormals.length <= 2 && c._off <= math.max(0, c._c2 - 2 * c._h - 1); if (!ok) throw const FormatException('inconsistent RrCorrector checkpoint'); diff --git a/test/onehz/rr_artifact_cap_test.dart b/test/onehz/rr_artifact_cap_test.dart index 773473c5..f2e70ca1 100644 --- a/test/onehz/rr_artifact_cap_test.dart +++ b/test/onehz/rr_artifact_cap_test.dart @@ -66,7 +66,7 @@ void _expectSplit(RrCorrectionResult got, RunSeries s, List<(int, int)> runs, St } for (final (a, b) in runs) { // the run's own beats sit between the pieces: artifacts, absent from nn - expect(got.classes.sublist(a, b).every((c) => c != BeatClass.normal), isTrue, + expect(got.classes.sublist(a, b), everyElement(BeatClass.gap), reason: '$why: run [$a, $b) is all artifact'); } expect(got.classes.length, n, reason: '$why: one class per input beat'); diff --git a/test/onehz/rr_correction_oracle_test.dart b/test/onehz/rr_correction_oracle_test.dart index 6fd13601..b999d692 100644 --- a/test/onehz/rr_correction_oracle_test.dart +++ b/test/onehz/rr_correction_oracle_test.dart @@ -25,7 +25,9 @@ void _check(List rr, List? ts, String why, windowBeats: win, minThresholdMs: floor, reanchorGapMs: reanchor); + // The frozen reference predates the cap; compare its explicit uncapped API. final got = correctRr(rr, + maxArtifactRun: null, rrTsMs: ts, alpha: alpha, windowBeats: win, diff --git a/test/onehz/support/rr_stream_driver.dart b/test/onehz/support/rr_stream_driver.dart index 6c8d07b9..43573acd 100644 --- a/test/onehz/support/rr_stream_driver.dart +++ b/test/onehz/support/rr_stream_driver.dart @@ -1,4 +1,5 @@ -// Driver + oracle comparison for the streaming RrCorrector tests. +// Driver for the uncapped RrCorrector against the frozen pre-cap oracle. +// rr_artifact_cap_test.dart and review_stream_gap_test.dart cover the default cap. import 'package:openstrap_analytics/onehz.dart'; import 'package:test/test.dart'; @@ -18,7 +19,7 @@ class RrStreamRun { int folded = 0; int restarts = 0; - RrStreamRun([RrCorrector? corrector]) : c = corrector ?? RrCorrector(); + RrStreamRun([RrCorrector? corrector]) : c = corrector ?? RrCorrector(maxArtifactRun: null); RrStreamRun.params( {double alpha = 5.2, @@ -26,6 +27,7 @@ class RrStreamRun { double floor = 100, double reanchor = 1000}) : c = RrCorrector( + maxArtifactRun: null, alpha: alpha, windowBeats: win, minThresholdMs: floor, diff --git a/test/output_changes/review_stream_gap_test.dart b/test/output_changes/review_stream_gap_test.dart new file mode 100644 index 00000000..d4c1a9d8 --- /dev/null +++ b/test/output_changes/review_stream_gap_test.dart @@ -0,0 +1,38 @@ +import 'dart:convert'; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + test('default capped stream retains explicit gaps across chunk and restart', () { + final rr = [...List.filled(100, 900), + ...List.filled(33, 100), ...List.filled(100, 900)]; + var stream = RrCorrector(); + final classes = []; + final nn = []; + for (var i = 0; i < rr.length; i++) { + final settled = stream.fold([rr[i]]); + classes.addAll(settled.classes); + nn.addAll(settled.nn); + if ([99, 119, 132].contains(i)) { + stream = RrCorrector.fromJson( + jsonDecode(jsonEncode(stream.toJson())) as Map); + } + final snap = stream.snapshot(); + final batch = correctRr(rr.sublist(0, i + 1)); + expect([...classes, ...snap.tailClasses], batch.classes, + reason: 'prefix ${i + 1}'); + expect([...nn, ...snap.tailNn], batch.nn, reason: 'prefix ${i + 1}'); + } + expect([...classes, ...stream.snapshot().tailClasses].sublist(100, 133), + everyElement(BeatClass.gap)); + expect(stream.snapshot().droppedCount, 33); + }); + + test('uncapped checkpoint rejects cap-only gap class', () { + final stream = RrCorrector(maxArtifactRun: null)..fold( + List.filled(300, 900)); + final state = jsonDecode(jsonEncode(stream.toJson())) as Map; + state['lastFinal'] = BeatClass.gap.index; + expect(() => RrCorrector.fromJson(state), throwsFormatException); + }); +} From 9ccf5b33cf02c4ee10250ca65d9250cb71c52dee Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 13 Nov 2026 17:42:42 +0530 Subject: [PATCH 3/4] calculation cache snapshots sets and keeps result types, property harness shrink and replay fixes, drop stale docs --- CONTRIBUTING.md | 7 +- lib/src/onehz/incremental_core.dart | 51 +++++++++- lib/src/onehz/sleep/sleep.dart | 3 - .../incremental/calculation_cache_test.dart | 94 +++++++++++++++++++ test/support/fold_law_support.dart | 2 +- test/support/law_registry.dart | 9 +- test/support/property.dart | 22 ++++- test/support/property_test.dart | 41 +++++++- 8 files changed, 209 insertions(+), 20 deletions(-) create mode 100644 test/onehz/incremental/calculation_cache_test.dart diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 447291f4..9f4cef0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,14 +68,9 @@ There is **one** sleep source (`segmentSleep`) and **one** headline readiness ## Tests -On Linux, `nix develop` supplies the pinned Flutter 3.41.6 / Dart 3.11.4 -toolchain used by Edge, plus Git, Clang, CMake, Ninja, and pkg-config. -The interactive shell enters an FHS environment for native build tools. -For a command without an interactive shell, use `nix develop -c dart test`; -`nix develop -c analytics-fhs ` also supplies the FHS environment. +Use a Dart SDK matching `pubspec.yaml`'s `environment` constraint. ```bash -nix develop dart pub get dart analyze dart test diff --git a/lib/src/onehz/incremental_core.dart b/lib/src/onehz/incremental_core.dart index 08e78d25..da25b60f 100644 --- a/lib/src/onehz/incremental_core.dart +++ b/lib/src/onehz/incremental_core.dart @@ -523,11 +523,19 @@ class CalculationCache { int get computations => _computations; int get hits => _hits; + /// Returns the cached result while [dependencies] are unchanged, otherwise + /// runs [calculate]. Results are copied in and out so callers cannot mutate + /// the cache. + /// + /// The default copier keeps the usual JSON-shaped lists and maps and their + /// element types. A result type it cannot rebuild exactly (for example a map + /// of sets) must pass [copy]; without it the first evaluation throws an + /// [ArgumentError] rather than handing back a differently typed value. T evaluate(String key, Object? dependencies, T Function() calculate, - {bool full = false}) { + {bool full = false, T Function(T value)? copy}) { final old = _entries[key]; if (!full && old != null && _deepEqual(old.dependencies, dependencies)) { - final result = _copyValue(old.value) as T; + final result = _copyResult(old.value, copy, key); _entries.remove(key); _entries[key] = old; _hits++; @@ -536,8 +544,8 @@ class CalculationCache { // Snapshot before calling user code, which may itself mutate dependencies. // Prepare both copies before committing, so exceptions preserve the old entry. final snapshot = _snapshotDependency(dependencies); - final value = _copyValue(calculate()); - final result = _copyValue(value) as T; + final value = _copyResult(calculate(), copy, key); + final result = _copyResult(value, copy, key); _entries.remove(key); _entries[key] = _CalculationEntry(snapshot, value); while (_entries.length > maxEntries) { @@ -550,15 +558,39 @@ class CalculationCache { void clear() => _entries.clear(); } +T _copyResult(Object? value, T Function(T value)? copy, String key) { + if (copy != null) return copy(value as T); + final copied = _copyValue(value); + if (copied is! T) { + throw ArgumentError( + 'CalculationCache "$key": cannot copy a ${value.runtimeType} as $T; ' + 'pass copy: to evaluate for this result type'); + } + return copied; +} + +/// Owned copy of a dependency. Lists, sets and maps are copied so an in-place +/// change by the caller is seen as a change. Any other collection is refused: +/// it would be stored by reference and a later mutation would compare equal to +/// itself. Object? _snapshotDependency(Object? value) { if (value is List) return [for (final item in value) _snapshotDependency(item)]; + if (value is Set) + return {for (final item in value) _snapshotDependency(item)}; if (value is Map) { return { for (final entry in value.entries) entry.key: _snapshotDependency(entry.value) }; } + if (value is Iterable) { + throw ArgumentError.value( + value, + 'dependencies', + 'use a List, Set or Map; a ${value.runtimeType} cannot be ' + 'snapshotted'); + } return value; } @@ -572,6 +604,13 @@ bool _deepEqual(Object? a, Object? b) { } return true; } + if (a is Set && b is Set) { + if (a.length != b.length) return false; + for (final item in a) { + if (!b.contains(item)) return false; + } + return true; + } if (a is Map && b is Map) { if (a.length != b.length) return false; for (final key in a.keys) { @@ -604,6 +643,10 @@ Object? _copyValue(Object? value) { if (value is Map) return _copyMap(value); if (value is Map) return _copyMap(value); if (value is Map) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); diff --git a/lib/src/onehz/sleep/sleep.dart b/lib/src/onehz/sleep/sleep.dart index e37b330c..7ceb3766 100644 --- a/lib/src/onehz/sleep/sleep.dart +++ b/lib/src/onehz/sleep/sleep.dart @@ -15,9 +15,6 @@ // honesty-bounded; stager.dart holds only the shared post-processing // (StagerResult, Webster rescore, consolidation) it reuses. The // autonomic-HR-only stager that used to live in stager.dart is DELETED. -// - CAUSAL (online) wake/NREM/REM stager (causal_stager.dart) — -// the same features/weights/cutoffs on past-only windows, abstains with a -// reason; for live decisions, NOT a replacement for cardioStager. // - Nonparametric circadian IS/IV/RA/L5/M10 (circadian_np.dart) // // Pure Dart, built on the package foundations (RR correction, Lomb-Scargle, diff --git a/test/onehz/incremental/calculation_cache_test.dart b/test/onehz/incremental/calculation_cache_test.dart new file mode 100644 index 00000000..72a13bc4 --- /dev/null +++ b/test/onehz/incremental/calculation_cache_test.dart @@ -0,0 +1,94 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + group('CalculationCache dependencies', () { + test('an unchanged list is a hit, a changed one recalculates', () { + final cache = CalculationCache(); + final deps = [1, 2, 3]; + expect(cache.evaluate('k', deps, () => 6), 6); + expect(cache.evaluate('k', deps, () => 99), 6); + expect(cache.hits, 1); + deps.add(4); + expect(cache.evaluate('k', deps, () => 10), 10); + expect(cache.computations, 2); + }); + + test('changing a set dependency in place invalidates the entry', () { + final cache = CalculationCache(); + final deps = {1, 2}; + expect(cache.evaluate('k', deps, () => 1), 1); + deps.add(3); + expect(cache.evaluate('k', deps, () => 2), 2); + expect(cache.computations, 2); + // The same contents in another set instance is still a hit. + expect(cache.evaluate('k', {3, 2, 1}, () => 3), 2); + expect(cache.hits, 1); + }); + + test('a set nested in a map is snapshotted too', () { + final cache = CalculationCache(); + final inner = {'a'}; + final deps = {'tags': inner}; + expect(cache.evaluate('k', deps, () => 1), 1); + inner.add('b'); + expect(cache.evaluate('k', deps, () => 2), 2); + }); + + test('a lazy iterable is refused instead of being stored by reference', () { + final cache = CalculationCache(); + expect( + () => cache.evaluate('k', [1, 2].map((v) => v), () => 1), + throwsArgumentError, + ); + }); + }); + + group('CalculationCache results', () { + test('a map of boolean lists keeps its declared type', () { + final cache = CalculationCache(); + final first = cache.evaluate>>( + 'flags', null, () => {'enabled': [true]}); + expect(first, {'enabled': [true]}); + first['enabled']!.add(false); + final second = cache.evaluate>>( + 'flags', null, () => {'enabled': [false]}); + // A hit, and the caller's edit did not reach the cache. + expect(second, {'enabled': [true]}); + }); + + test('a numeric list map keeps its declared type', () { + final cache = CalculationCache(); + final value = cache.evaluate>>( + 'n', null, () => {'x': [1, 2.5]}); + expect(value['x'], [1, 2.5]); + }); + + test('a type the default copier cannot rebuild says so', () { + final cache = CalculationCache(); + expect( + () => cache.evaluate>>( + 's', null, () => {'a': {1}}), + throwsA(isA() + .having((e) => e.message, 'message', contains('copy:'))), + ); + // A failed evaluation leaves nothing cached. + expect(cache.computations, 0); + }); + + test('a caller supplied copier handles any result type', () { + final cache = CalculationCache(); + Map> copy(Map> v) => + {for (final e in v.entries) e.key: {...e.value}}; + final first = cache.evaluate>>( + 's', null, () => {'a': {1}}, + copy: copy); + first['a']!.add(2); + final second = cache.evaluate>>( + 's', null, () => {'a': {9}}, + copy: copy); + expect(second, {'a': {1}}); + expect(cache.hits, 1); + }); + }); +} diff --git a/test/support/fold_law_support.dart b/test/support/fold_law_support.dart index 0443fde9..ea7b632c 100644 --- a/test/support/fold_law_support.dart +++ b/test/support/fold_law_support.dart @@ -104,7 +104,7 @@ class SizeGen extends Gen { } if (p < .40) return r.intIn(math.min(500, maxN), maxN); final cap = math.max(120, maxN * size ~/ 100); - if (p < .62) return r.intIn(0, math.min(cap, 400)); + if (p < .62) return r.intIn(0, math.min(math.min(cap, 400), maxN)); return r.intIn(0, math.min(cap, maxN)); } diff --git a/test/support/law_registry.dart b/test/support/law_registry.dart index 7f78f3a1..fe104039 100644 --- a/test/support/law_registry.dart +++ b/test/support/law_registry.dart @@ -75,7 +75,14 @@ class LawSet { milliseconds: int.tryParse( Platform.environment['PROPERTY_BUDGET_MS'] ?? '') ?? 60000)), - body: (v) => reach.observe(v, (k) => got[k] = (got[k] ?? 0) + 1), + // One count per situation per generated case: a body that reports the + // same situation many times for one input still reached it once. + body: (v) { + final seen = {}; + reach.observe(v, (k) { + if (seen.add(k)) got[k] = (got[k] ?? 0) + 1; + }); + }, genVersion: genVersion, ); if (!r.passed) fail(r.failure!.report); diff --git a/test/support/property.dart b/test/support/property.dart index 89456a3c..9d95bc0c 100644 --- a/test/support/property.dart +++ b/test/support/property.dart @@ -226,8 +226,16 @@ class _DoubleGen extends Gen { yield v - d / 8; } + // Only candidates strictly nearer the origin: truncating or halving a + // value below a nonzero origin can otherwise step away from it and the + // next round steps back, cycling until the shrink limit. for (final c in cand()) { - if (c >= min && c <= max && seen.add(c)) yield c; + if (c >= min && + c <= max && + (c - origin).abs() < d.abs() && + seen.add(c)) { + yield c; + } } } } @@ -305,7 +313,8 @@ class _NullableGen extends Gen { @override Iterable shrink(T? v) sync* { if (v == null) return; - yield null; + // Null is outside the domain when it can never be generated. + if (nullProbability > 0) yield null; yield* inner.shrink(v); } @@ -555,8 +564,12 @@ PropertyResult runProperty({ } } - String replayFor(String? caseId) => + // A budget failure depends on how many cases ran and how long they were + // given, so its replay pins both; the defaults would not reproduce an + // extended or tightened run. + String replayFor(String? caseId, {bool pinLimits = false}) => 'PROPERTY_SEED=$seed ${caseId == null ? '' : 'PROPERTY_CASE=$caseId '}' + "${pinLimits ? 'PROPERTY_ITERATIONS=${config.cases} PROPERTY_BUDGET_MS=${config.budget.inMilliseconds} ' : ''}" 'dart test ' "${config.testFile ?? ''} " "--plain-name '${name.replaceAll("'", r"'\''")}'"; @@ -567,7 +580,8 @@ PropertyResult runProperty({ int accepted = 0, String? original, bool overBudget = false}) { - final replay = replayFor(kind == FailureKind.budget ? null : caseId); + final replay = replayFor(kind == FailureKind.budget ? null : caseId, + pinLimits: kind == FailureKind.budget); final b = StringBuffer() ..writeln('Property "$name" FAILED (${kind.name})') ..writeln(' seed: $seed case: $caseId generator: $genVersion' diff --git a/test/support/property_test.dart b/test/support/property_test.dart index 69938a31..e2b829b1 100644 --- a/test/support/property_test.dart +++ b/test/support/property_test.dart @@ -6,6 +6,7 @@ // failure report (seed, case, input, generator version, replay command) and // the per-property wall budget. +import 'dart:io' show Platform; import 'dart:math' as math; import 'package:test/test.dart'; @@ -506,6 +507,25 @@ void main() { }); }); + group('shrinking domains', () { + test('a double never shrinks away from a nonzero origin', () { + final g = G.doubleIn(0, 100, origin: 60); + for (final v in [59.5, 59.25, 61.5, 10.0, 99.0]) { + for (final c in g.shrink(v)) { + expect((c - 60).abs() < (v - 60).abs(), isTrue, + reason: '$v shrank to $c, not nearer 60'); + } + } + }); + + test('a nullable that cannot generate null does not shrink to null', () { + final g = G.nullable(G.intIn(0, 9), nullProbability: 0); + expect(g.shrink(5), isNot(contains(null))); + final h = G.nullable(G.intIn(0, 9), nullProbability: .25); + expect(h.shrink(5).first, isNull); + }); + }); + group('shrinking lists, tuples, nullables', () { test('a list shrinks to the shortest failing length of zeros', () { final f = _fails(G.listOf(G.intIn(0, 9), maxLen: 20), (l) => l.length < 3); @@ -740,6 +760,20 @@ void main() { expect(r.failure?.kind, FailureKind.budget); }); + test('a budget failure replays with the limits it ran under', () { + final r = runProperty( + name: 'slow', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig( + seed: 1, cases: 7, budget: Duration(milliseconds: 123)), + elapsed: () => const Duration(hours: 1), + ); + expect(r.failure!.kind, FailureKind.budget); + expect(r.failure!.report, contains('PROPERTY_ITERATIONS=7')); + expect(r.failure!.report, contains('PROPERTY_BUDGET_MS=123')); + }); + test('a single-case replay is not held to the budget', () { final r = runProperty( name: 'replay', @@ -761,7 +795,12 @@ void main() { }, examples: const [0, 100], cases: 25); test('forAll ran its forced examples plus its cases', () { - expect(ran, anyOf(0, 27), + // The case count is whatever the environment makes it (PROPERTY_ITERATIONS + // replaces 25; PROPERTY_CASE replays one), so ask the same config. + final cfg = PropertyConfig.fromEnvironment(Platform.environment, + defaultCases: 25); + final expected = cfg.caseOnly != null ? null : 2 + cfg.cases; + expect(ran, expected == null ? lessThanOrEqualTo(1) : anyOf(0, expected), reason: '0 when this test alone is selected by name'); }); From cbc38fe95067482552cfa7af30badddf8689b0f6 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Fri, 13 Nov 2026 06:46:38 -0700 Subject: [PATCH 4/4] Fix cache copies and irregular screen checkpoints --- .../onehz/clinical/irregular_diagnostics.dart | 22 ++-- lib/src/onehz/clinical/irregular_rhythm.dart | 13 +- .../clinical/irregular_rhythm_state.dart | 27 ++-- lib/src/onehz/incremental_core.dart | 13 +- .../incremental/calculation_cache_test.dart | 73 +++++++++++ .../irregular_checkpoint_test.dart | 121 ++++++++++++++++++ 6 files changed, 245 insertions(+), 24 deletions(-) create mode 100644 test/onehz/incremental/irregular_checkpoint_test.dart diff --git a/lib/src/onehz/clinical/irregular_diagnostics.dart b/lib/src/onehz/clinical/irregular_diagnostics.dart index c2b91d2d..411505da 100644 --- a/lib/src/onehz/clinical/irregular_diagnostics.dart +++ b/lib/src/onehz/clinical/irregular_diagnostics.dart @@ -18,6 +18,7 @@ // batch call and the streaming `IrregularScreenState` must produce the same // diagnostics for the same beats, however chunked or restored. +import '../../aggregate/json_support.dart' show decodeDouble, encodeDouble; import '../types.dart'; import 'irregular_rhythm.dart'; @@ -152,7 +153,9 @@ class IrregularDiagnostics { }; /// Wire shape (version 1); see test/onehz/irregular_diagnostics_test.dart. - /// Doubles are written as they are held, so a JSON text round trip is exact. + /// Finite doubles stay numbers; non-finite ones use the shared tagged strings. + /// CLAIM(screen-json-nonfinite): fail-closed configs remain JSON-encodable + /// and restore without changing their gates [irregular_checkpoint_test.dart]. Map toJson() => { 'version': 1, 'abstain': abstain == null ? null : _abstainWire[abstain], @@ -162,7 +165,8 @@ class IrregularDiagnostics { 'nn_kept': nnKept, 'corrected': corrected, 'dropped': dropped, - 'artifact_fraction': artifactFraction, + 'artifact_fraction': + artifactFraction == null ? null : encodeDouble(artifactFraction!), }, 'windows': windows == null ? null @@ -176,13 +180,13 @@ class IrregularDiagnostics { }, 'thresholds': { 'min_beats': thresholds.minBeats, - 'max_artifact': thresholds.maxArtifact, - 'sd1sd2_flag': thresholds.sd1sd2Flag, - 'pnn_threshold_ms': thresholds.pnnThresholdMs, - 'pnn_flag_pct': thresholds.pnnFlagPct, - 'window_minutes': thresholds.windowMinutes, + 'max_artifact': encodeDouble(thresholds.maxArtifact), + 'sd1sd2_flag': encodeDouble(thresholds.sd1sd2Flag), + 'pnn_threshold_ms': encodeDouble(thresholds.pnnThresholdMs), + 'pnn_flag_pct': encodeDouble(thresholds.pnnFlagPct), + 'window_minutes': encodeDouble(thresholds.windowMinutes), 'min_window_beats': thresholds.minWindowBeats, - 'sustained_fraction': thresholds.sustainedFraction, + 'sustained_fraction': encodeDouble(thresholds.sustainedFraction), }, }; @@ -210,7 +214,7 @@ class IrregularDiagnostics { if (hit.isEmpty) throw FormatException('unknown abstain reason $a'); abstain = hit.first.key; } - double d(Map m, String k) => (m[k] as num).toDouble(); + double d(Map m, String k) => decodeDouble(m[k]); return IrregularDiagnostics( abstain: abstain, rrRaw: b['rr_raw'] as int?, diff --git a/lib/src/onehz/clinical/irregular_rhythm.dart b/lib/src/onehz/clinical/irregular_rhythm.dart index b4c9762e..9ab07191 100644 --- a/lib/src/onehz/clinical/irregular_rhythm.dart +++ b/lib/src/onehz/clinical/irregular_rhythm.dart @@ -161,7 +161,10 @@ IrregularScreenResult irregularBeatScreenDetailed( i > 0 && keep[i] && keep[i - 1] && - (!hasTimes || nnTimesMs[i] - nnTimesMs[i - 1] <= rrMs[i] + 0.5) + (!hasTimes || + (nnTimesMs[i].isFinite && + nnTimesMs[i - 1].isFinite && + nnTimesMs[i] - nnTimesMs[i - 1] <= rrMs[i] + 0.5)) ]; final nnAdjacent = [ for (var i = 0; i < rrMs.length; i++) @@ -336,7 +339,7 @@ IrregularWindowCounts _countWindows( var open = IrregularOpenWindow.none; var bucket = []; var bucketAdjacent = []; - var windowStart = timesMs.isEmpty ? 0.0 : timesMs.first; + double? windowStart; void flush() { if (bucket.isEmpty) return; final verdict = irregularWindowVerdict(bucket, bucketAdjacent, @@ -358,7 +361,11 @@ IrregularWindowCounts _countWindows( } for (var i = 0; i < rrMs.length; i++) { - if (timesMs[i] - windowStart >= windowMs) { + // Untimed beats cannot be placed in a window; the aggregate still counts + // them. See CLAIM(screen-finite-window-time) and its streaming counterpart. + if (!timesMs[i].isFinite) continue; + final start = windowStart ??= timesMs[i]; + if (timesMs[i] - start >= windowMs) { flush(); windowStart = timesMs[i]; } diff --git a/lib/src/onehz/clinical/irregular_rhythm_state.dart b/lib/src/onehz/clinical/irregular_rhythm_state.dart index 27b3c765..5719b57d 100644 --- a/lib/src/onehz/clinical/irregular_rhythm_state.dart +++ b/lib/src/onehz/clinical/irregular_rhythm_state.dart @@ -34,6 +34,7 @@ import 'dart:math' as math; +import '../../aggregate/json_support.dart' show decodeDouble, encodeDouble; import '../types.dart'; import 'irregular_diagnostics.dart'; import 'irregular_rhythm.dart'; @@ -103,7 +104,7 @@ class IrregularScreenState { throw FormatException( 'unsupported IrregularScreenState version ${j['version']}'); } - double d(String k) => (j[k] as num).toDouble(); + double d(String k) => decodeDouble(j[k]); final s = IrregularScreenState( sd1sd2Flag: d('sd1sd2Flag'), pnnThresholdMs: d('pnnThresholdMs'), @@ -127,6 +128,9 @@ class IrregularScreenState { s._lMean = d('lMean'); s._lM2 = d('lM2'); s._winStart = (j['winStart'] as num?)?.toDouble(); + if (s._winStart != null && !s._winStart!.isFinite) { + throw const FormatException('invalid winStart'); + } s._bk = [for (final x in j['bk'] as List) (x as num).toDouble()]; final bkAdj = j['bkAdj'] as List; if (bkAdj.any((x) => x is! int || (x != 0 && x != 1))) { @@ -167,12 +171,12 @@ class IrregularScreenState { Map toJson() => { 'version': 3, 'type': 'IrregularScreenState', - 'sd1sd2Flag': sd1sd2Flag, - 'pnnThresholdMs': pnnThresholdMs, - 'pnnFlagPct': pnnFlagPct, - 'windowMinutes': windowMinutes, + 'sd1sd2Flag': encodeDouble(sd1sd2Flag), + 'pnnThresholdMs': encodeDouble(pnnThresholdMs), + 'pnnFlagPct': encodeDouble(pnnFlagPct), + 'windowMinutes': encodeDouble(windowMinutes), 'minWindowBeats': minWindowBeats, - 'sustainedFraction': sustainedFraction, + 'sustainedFraction': encodeDouble(sustainedFraction), 'prevKept': _prevKept, 'prevV': _prevV, 'prevT': _prevT.isFinite ? _prevT : null, @@ -186,7 +190,9 @@ class IrregularScreenState { 'lMean': _lMean, 'lM2': _lM2, 'winStart': _winStart, - 'bk': _bk, + // CLAIM(screen-checkpoint): a saved map cannot change with a later + // fold or mutate the state [irregular_checkpoint_test.dart]. + 'bk': List.of(_bk), 'bkAdj': [for (final a in _bkAdj) a ? 1 : 0], 'total': _total, 'valid': _valid, @@ -243,7 +249,8 @@ class IrregularScreenState { // Adjacent = the previous INPUT beat was kept too and no time hole lies // between them: no difference is ever taken across a skipped beat or a // dropout (the batch's `successive`). - final adjacent = _prevKept && tm - _prevT <= v + 0.5; + final adjacent = + _prevKept && tm.isFinite && _prevT.isFinite && tm - _prevT <= v + 0.5; if (adjacent) { final d = v - _prevV; _dN++; @@ -257,7 +264,9 @@ class IrregularScreenState { _lMean += dl / _lN; _lM2 += dl * (v - _lMean); _nKept++; - if (_windowsOk) { + // CLAIM(screen-finite-window-time): untimed beats remain in aggregate + // counts but cannot anchor or join windows [irregular_checkpoint_test.dart]. + if (_windowsOk && tm.isFinite) { final start = _winStart ??= tm; if (tm - start >= windowMinutes * 60000) { _closeWindow(); diff --git a/lib/src/onehz/incremental_core.dart b/lib/src/onehz/incremental_core.dart index da25b60f..153207e6 100644 --- a/lib/src/onehz/incremental_core.dart +++ b/lib/src/onehz/incremental_core.dart @@ -606,8 +606,13 @@ bool _deepEqual(Object? a, Object? b) { } if (a is Set && b is Set) { if (a.length != b.length) return false; + // CLAIM(cache-set-equality): snapshots own nested collections; compare + // contents without reusing a match [calculation_cache_test.dart]. + final unmatched = b.toList(); for (final item in a) { - if (!b.contains(item)) return false; + final index = unmatched.indexWhere((other) => _deepEqual(item, other)); + if (index < 0) return false; + unmatched.removeAt(index); } return true; } @@ -644,14 +649,16 @@ Object? _copyValue(Object? value) { if (value is Map) return _copyMap(value); if (value is Map) return _copyMap(value); if (value is Map>) return _copyMap(value); - if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); - if (value is Map>) return _copyMap(value); + // CLAIM(cache-numeric-map-types): narrower lists must precede num lists + // under covariance [calculation_cache_test.dart]. if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); if (value is Map>) return _copyMap(value); diff --git a/test/onehz/incremental/calculation_cache_test.dart b/test/onehz/incremental/calculation_cache_test.dart index 72a13bc4..e5d0f1ef 100644 --- a/test/onehz/incremental/calculation_cache_test.dart +++ b/test/onehz/incremental/calculation_cache_test.dart @@ -35,6 +35,53 @@ void main() { expect(cache.evaluate('k', deps, () => 2), 2); }); + // why: snapshots copy nested collections, so set membership must compare + // their contents and consume each match once (CLAIM(cache-set-equality)). + test('nested set dependencies match by contents without reusing a match', + () { + final cache = CalculationCache(); + final list = [1, 2]; + final deps = { + list, + { + 'tags': {3, 4} + } + }; + expect(cache.evaluate('nested', deps, () => 10), 10); + expect( + cache.evaluate( + 'nested', + { + { + 'tags': {4, 3} + }, + [1, 2] + }, + () => 99), + 10); + expect(cache.computations, 1); + expect(cache.hits, 1); + list.add(5); + expect(cache.evaluate('nested', deps, () => 20), 20); + expect(cache.computations, 2); + + final duplicates = >{ + [1], + [1] + }; + expect(cache.evaluate('duplicates', duplicates, () => 30), 30); + expect( + cache.evaluate( + 'duplicates', + >{ + [1], + [2] + }, + () => 40), + 40); + expect(cache.computations, 4); + }); + test('a lazy iterable is refused instead of being stored by reference', () { final cache = CalculationCache(); expect( @@ -64,6 +111,32 @@ void main() { expect(value['x'], [1, 2.5]); }); + // why: Dart covariance lets a wider num-list branch swallow int/double + // maps and throw on copy (CLAIM(cache-numeric-map-types)). + void numericMap(String name, List items) { + test('a $name list map survives misses and hits without aliasing', () { + final cache = CalculationCache(); + final source = {'x': items.toList()}; + final result = + cache.evaluate>>('k', null, () => source); + expect(result, {'x': items}); + result['x']!.clear(); + expect(source, {'x': items}); + source['x']!.clear(); + final hit = cache.evaluate>>( + 'k', null, () => throw StateError('must hit')); + expect(hit, {'x': items}); + expect(cache.computations, 1); + expect(cache.hits, 1); + }); + } + + numericMap('int', [1, 2]); + numericMap('double', [1.5, 2.5]); + numericMap('nullable int', [1, null]); + numericMap('nullable double', [1.5, null]); + numericMap('nullable num', [1, 2.5, null]); + test('a type the default copier cannot rebuild says so', () { final cache = CalculationCache(); expect( diff --git a/test/onehz/incremental/irregular_checkpoint_test.dart b/test/onehz/incremental/irregular_checkpoint_test.dart new file mode 100644 index 00000000..5e30c18c --- /dev/null +++ b/test/onehz/incremental/irregular_checkpoint_test.dart @@ -0,0 +1,121 @@ +import 'dart:convert'; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +Map _wire(Map json) => + jsonDecode(jsonEncode(json)) as Map; + +void main() { + // why: a saved map must remain restorable while its source keeps folding, + // and edits to that map must not change the source (CLAIM(screen-checkpoint)). + test('open-window checkpoint owns its beat list in both directions', () { + final state = IrregularScreenState()..fold([800, 900], [800, 1700]); + final saved = state.toJson(); + state.fold([850], [2550]); + final restored = IrregularScreenState.fromJson(_wire(saved)); + expect(restored.toJson()['bk'], [800.0, 900.0]); + expect(restored.evaluateDetailed([], []).diagnostics.nnIn, 2); + (saved['bk'] as List).clear(); + expect(state.toJson()['bk'], [800.0, 900.0, 850.0]); + expect(state.evaluateDetailed([], []).diagnostics.windows!.openBeats, 3); + }); + + final nonFinite = { + 'NaN': double.nan, + 'Infinity': double.infinity, + '-Infinity': double.negativeInfinity, + }; + for (final entry in nonFinite.entries) { + for (final window in [true, false]) { + // why: failing closed is a supported config path; its evidence and + // checkpoint must persist (CLAIM(screen-json-nonfinite)). + test( + '${entry.key} ${window ? 'window' : 'fraction'} persists fail-closed', + () { + final rr = [for (var i = 0; i < 600; i++) 800.0 + i % 20]; + var clock = 0.0; + final times = [for (final v in rr) clock += v]; + final state = IrregularScreenState( + windowMinutes: window ? entry.value : 5, + sustainedFraction: window ? 0.5 : entry.value, + )..fold(rr, times); + final batch = irregularBeatScreenDetailed( + rr, + nnTimesMs: times, + windowMinutes: window ? entry.value : 5, + sustainedFraction: window ? 0.5 : entry.value, + ); + final stateJson = _wire(state.toJson()); + expect(stateJson[window ? 'windowMinutes' : 'sustainedFraction'], + entry.key); + final restored = IrregularScreenState.fromJson(stateJson); + expect(_wire(restored.toJson()), stateJson); + for (final result in [batch, restored.evaluateDetailed([], [])]) { + expect(result.metric.present, isTrue); + expect(result.metric.value!.flag, isFalse); + expect(result.diagnostics.windows, isNull); + final envelope = _wire(result.toJson()); + final diag = envelope['diagnostics'] as Map; + expect( + (diag['thresholds'] + as Map)[window ? 'window_minutes' : 'sustained_fraction'], + entry.key); + expect(_wire(IrregularDiagnostics.fromJson(diag).toJson()), diag); + } + }); + } + + // why: an unknown timestamp must not poison every later window or make + // checkpoint JSON fail (CLAIM(screen-finite-window-time)). + test( + '${entry.key} beat times are excluded from windows and break adjacency', + () { + final rr = List.filled(8, 800); + final times = [ + entry.value, + 0.0, + 800.0, + 300000.0, + entry.value, + 300800.0, + 600000.0, + entry.value + ]; + var state = IrregularScreenState(minWindowBeats: 2); + for (var i = 0; i < rr.length; i++) { + state.fold([rr[i]], [times[i]]); + state = IrregularScreenState.fromJson(_wire(state.toJson())); + } + final batch = irregularBeatScreenDetailed(rr, + nnTimesMs: times, minWindowBeats: 2, minBeats: 3); + final stream = state.evaluateDetailed([], [], minBeats: 3); + for (final result in [batch, stream]) { + final d = result.diagnostics; + // Five finite times make three buckets: [0,800], [300000,300800], + // [600000]. The unknown time splits adjacency in the middle bucket. + expect(d.nnKept, 8); + expect(d.abstain, IrregularAbstain.tooFewBeats); + expect(d.windows!.total, 3); + expect(d.windows!.valid, 1); + expect(d.windows!.flagged, 0); + expect(d.windows!.openBeats, 1); + expect(d.windows!.open, IrregularOpenWindow.thin); + } + expect(_wire(stream.toJson()), _wire(batch.toJson())); + expect((state.toJson()['bk'] as List).length, 1); + }); + } + + // why: a malformed in-memory checkpoint must not reintroduce an immortal + // window even when JSON text would reject it (CLAIM(screen-finite-window-time)). + test('restoring a non-finite window start is refused', () { + final state = IrregularScreenState()..fold([800], [800]); + for (final bad in nonFinite.values) { + expect( + () => IrregularScreenState.fromJson( + {...state.toJson(), 'winStart': bad}), + throwsFormatException); + } + }); +}