diff --git a/lib/app.dart b/lib/app.dart index c4bfc9b4e..d2c32afbd 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -76,10 +76,16 @@ class _OpenStrapAppState extends State with WidgetsBindingObserver WidgetsBinding.instance.platformDispatcher.platformBrightness); } + @override + void didHaveMemoryPressure() { + context.read().handleMemoryPressure(); + } + @override void didChangeAppLifecycleState(AppLifecycleState state) { final app = context.read(); if (state == AppLifecycleState.resumed) { + app.resumeDerivationRetention(); // The user may have flipped our notification switch either way in OS // Settings while we were backgrounded. Drop the cached authorization // decision so the next present/schedule re-reads reality — a denial used diff --git a/lib/compute/cardio_rows_cache.dart b/lib/compute/cardio_rows_cache.dart new file mode 100644 index 000000000..db39b7844 --- /dev/null +++ b/lib/compute/cardio_rows_cache.dart @@ -0,0 +1,46 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +/// Caller-owned bytes and provenance. Row objects exist only in the worker. +class CardioRowsChunk { + CardioRowsChunk(this.bytes, this.loSec, this.hiSec, this.revisions, + this.generation, this.algorithm); + final Uint8List bytes; + final int loSec, hiSec; + final Map revisions; + int generation, algorithm; + + bool overlaps(int lo, int hi) => hiSec >= lo && loSec <= hi; + + bool matches(Map current, int gen, int algo) => + generation == gen && algorithm == algo && + current.entries.every((e) => + !revisions.containsKey(e.key) || revisions[e.key] == e.value); +} + +/// Worker only. Damage or an unknown codec version costs a cold staging. +ana.CardioRowCache decodeCardioRows(List chunks) { + final rows = ana.CardioRowCache(); + for (final bytes in chunks) { + try { + rows.merge(ana.CardioRowCache.fromJson( + (jsonDecode(utf8.decode(bytes)) as Map).cast())); + } on FormatException { + // Refuse the whole chunk. + } on TypeError { + // A structurally invalid JSON value is also a miss. + } + } + return rows; +} + +/// Keep at most one search window plus slack and 8,000 row variants. A limit +/// only evicts reusable work; analytics always recomputes a missing row. +Uint8List encodeCardioRows(ana.CardioRowCache rows, int lo, int hi) { + var kept = 0; + rows.retainWhere((k) => + k.startSec >= lo && k.startSec <= hi && kept++ < 8000); + return Uint8List.fromList(utf8.encode(jsonEncode(rows.toJson()))); +} diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 2610e7726..a19711ab7 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -34,6 +34,7 @@ import 'readiness_calibration_backfill.dart' show backfillReadinessCalibration, seedRecentReadinessZ; import 'package:crypto/crypto.dart' show sha1; +import 'package:sqflite/sqflite.dart' show ConflictAlgorithm; import 'package:flutter/foundation.dart'; import 'findings.dart'; import 'nap_overlap_cleanup.dart'; @@ -70,6 +71,14 @@ import 'hr_max.dart' show estimatedMaxHr, kHrFloorBpm, smoothedMaxHr, smoothedMinHr; import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; +import 'ed2b_checkpoint_diagnostics.dart'; +import 'staged_sleep_cache.dart'; +import 'ed2b_stream_diagnostics.dart'; +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; +import 'night_checkpoint.dart'; +import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -1848,7 +1857,18 @@ import 'vendor_sleep.dart'; // temperature rhythm and alertness forecast use local time. Steps: the live // pedometer counts from stride periodicity. Workouts: a 5-15 min rest with HR // still raised stays inside one suggested workout. -const int kAlgoVersion = 119; +// 119 → 120 (analytics #99 @ f8e7d3c): RR artifact-run cap 32; +// toleranced batch aggregators hrvTime, banisterTrimp, ctlAtlTsb, cosinor, +// timeInZone and circadianNonparametric move last digits; Lomb–Scargle origin, +// singular-cosinor abstention, non-finite guards and IrregularScreenState's +// irregular-screen gaps change outputs. Retains the cosinor significance gate, +// nullable phase/amplitudeP, RSA floor guard and checkpoint v3, and SRI rules. +// Pooled-run batch staging labels unchanged; cardio caching respects run bounds. +// D35 queries coverage and resolves half-open ownership through inclusive to + 1, +// excluding non-owner rows at the last second. D36 folds only the target night's +// own observation, none when absent; forward-only, preserving existing profiles +// and folded_days. The restricted E15 walk now also applies to folding passes. +const int kAlgoVersion = 120; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2035,7 +2055,8 @@ const int kAlgoVersion = 119; // merge + spread calibration + capped re-centring + robust lnRMSSD // centre), for v112. // REPIN @ 3a7ca34: analytics main, #101 + #100 + #97 + #98, for v119. -const String kAnalyticsPin = '3a7ca34d30233ff11e6c2877083f319a907ccb31'; +// REPIN @ f8e7d3c: rebased analytics #99, for v120 above. +const String kAnalyticsPin = 'f8e7d3c478cc59707fde41f3edff7736026573fe'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both @@ -2958,6 +2979,13 @@ class _PrepareStats { int rows = 0; } +class _PrepareUnion { + RetainedSubstrate? load; + Substrate? search; + Map> ownership = const {}; + Map> priority = const {}; +} + /// Run [worker] over [items] with at most [concurrency] running at once. Each /// of up to [concurrency] "lanes" pulls the next unclaimed item as soon as /// it's free — a mix of fast (empty/mostly-empty day) and slow (heavy @@ -3092,6 +3120,259 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; + // ED-2b observations of the worker-owned caches and checkpoint path. + final debugCheckpoints = Ed2bCheckpointDiagnostics(); + @visibleForTesting + void Function(String day)? debugOnProfileFold; + @visibleForTesting + Future debugFoldObservation(String day, String observation) => + _foldObservationIntoProfile(day, observation); + @visibleForTesting + // Optional observations follow the real day-bundle worker and its RAM owner. + Ed2bStreamDiagnostics? debugNightStreams; + @visibleForTesting + void Function(Map input, Map output)? + debugOnNightBundle; + @visibleForTesting + Uint8List? get debugCardioRowBytes => + _cardioRowChunks.isEmpty ? null : _cardioRowChunks.last.bytes; + @visibleForTesting + bool debugDisableCardioRows = false; + @visibleForTesting + int debugCardioRowHits = 0; + @visibleForTesting + int debugCardioRowMisses = 0; + @visibleForTesting + List get debugCardioRowWorkerOperations => _cardioRowOperations.toList(); + @visibleForTesting + void debugRekeyCardioRows({required int inputRevision, + required int generation, required int algorithm}) { + for (final c in _cardioRowChunks) { + c.generation = generation; + c.algorithm = algorithm; + c.revisions.updateAll((_, fp) => (inputRevision, fp.$2, generation)); + } + } + @visibleForTesting + bool debugDisableUnionLoads = false; + @visibleForTesting + int get debugUnionWorkerCuts => _unionWorkerCuts; + @visibleForTesting + void debugRekeyUnionAlgorithm(int algorithm) { + for (final union in _retained.unions) { union.version = algorithm; } + } + @visibleForTesting + void Function(int from, int to, String load)? debugOnSubstrateLoad; + @visibleForTesting + Future<(Map>, Map>)> + debugResolveOwnership(int from, int to) => _resolveOwnership(from, to); + @visibleForTesting + Future debugLoadOwnedSubstrate(int from, int to, + {required String day}) async { + final (ownership, _) = await _resolveOwnership(from, to); + return _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + } + + + // ED-2a RED seams. Empty cache views and no-op controls describe the current + // uncached engine. Counters below observe real worker work, never results. + // E12/E13 back staging and retention; E18 backs finished-session reuse. + final _stagedSleepCache = StagedSleepCache(); + @visibleForTesting + bool debugDisableStagedSleepCache = false; + @visibleForTesting + int get debugStagedSleepCacheMaxDays => _stagedSleepCache.maxDays; + @visibleForTesting + set debugStagedSleepCacheMaxDays(int value) => _stagedSleepCache.maxDays = value; + @visibleForTesting + int get debugStagedSleepCacheSize => _stagedSleepCache.size; + @visibleForTesting + void debugStagedSleepCacheRekey(int version) => _stagedSleepCache.rekey(version); + @visibleForTesting + bool get debugDisableSubstrateRetention => _disableSubstrateRetention; + @visibleForTesting + set debugDisableSubstrateRetention(bool value) { + _disableSubstrateRetention = value; + if (value) _clearRetainedSubstrate(); + } + bool _disableSubstrateRetention = false; + @visibleForTesting + int get debugRetainedSubstrateMaxDays => _retained.maxDays; + @visibleForTesting + set debugRetainedSubstrateMaxDays(int value) => _retained.maxDays = value; + @visibleForTesting + Set get debugRetainedSubstrateDays => _retained.days; + @visibleForTesting + void debugRetainedSubstrateRekey(int version) => _retained.rekey(version); + void trimForBackground() { + _retentionInBackground = true; + _clearRetainedSubstrate(); + } + void resumeFromBackground() => _retentionInBackground = false; + void handleMemoryPressure() => _clearRetainedSubstrate(); + void dispose() { + _retentionDisposed = true; + _clearRetainedSubstrate(); + } + + final _retained = SubstrateRetention(); + final _unionLock = _AsyncLock(); + int _unionWorkerCuts = 0; + final _cardioRowChunks = []; + final _cardioRowOperations = {}; + final _nightStates = {}; + final _checkpointPlans = {}; + int? _checkpointGeneration; + static const _cardioRowsSlackSec = 3600; + static const _cardioRowsMaxBytes = 8 * 1024 * 1024; + int _retentionEpoch = 0; + int? _retentionPassEpoch; + bool _retentionInBackground = false; + bool _retentionDisposed = false; + final _retentionPreparedDays = {}; + bool get _canRetainSubstrate => !debugDisableSubstrateRetention && + !background && !_retentionInBackground && !_retentionDisposed && + (_retentionPassEpoch == null || _retentionPassEpoch == _retentionEpoch); + + void _clearRetainedSubstrate() { + _retentionEpoch++; + _retained.clear(); + _cardioRowChunks.clear(); + _nightStates.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; + debugNightStreams?.nightState.clear(); + } + + void _beginRetentionPass() { + _retentionPassEpoch = _retentionEpoch; + _retentionPreparedDays.clear(); + if (!_canRetainSubstrate) _retained.clear(); + } + + void _endRetentionPass() { + _retained.retainDays(_retentionPreparedDays); + _nightStates.removeWhere((day, state) { + if (_retentionPreparedDays.contains(day)) return false; + state.clear(); + return true; + }); + _retentionPreparedDays.clear(); + _retentionPassEpoch = null; + _checkpointPlans.clear(); + } + + void _dropRetainedDay(String day) { + _retained.remove(day); + _nightStates.remove(day)?.clear(); + _checkpointPlans.remove(day); + _retentionPreparedDays.remove(day); + } + @visibleForTesting + bool debugDisableSessionReuse = false; + @visibleForTesting + int debugCeilingComputed = 0; + @visibleForTesting + void debugSessionCacheRekey(int version) { + for (final e in _ceilingCache.entries.toList()) { + _ceilingCache[e.key] = (version: version, sig: e.value.sig, + result: e.value.result); + } + } + + // One complete envelope and the unrounded bpm used to choose the winner. + // Entries survive passes on this engine; oldest entries are evicted first. + final _ceilingCache = <(String, String), + ({int version, String sig, _SessionCeiling result})>{}; + static const _ceilingCacheMax = 128; + final _ceilingInput = Expando<_CeilingInput>(); + + Future<_CeilingInput?> _ceilingInputRevision(String day) async { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final lo = _localDayLabelToSec(day); + final hi = _localNextDayLabelToSec(day) - 1; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [lo ~/ 900, hi ~/ 900]); + return (generation: generation, revisions: {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}); + } catch (_) { + return null; // Unavailable provenance always recomputes. + } + } + + Future _sessionCeilingRevision() async { + try { + return await LocalDb.sessionCeilingRevision(); + } catch (_) { + return null; + } + } + + Future> _ceilingSignatures( + PreparedDerivationDay day, List> sessions, + int? sessionRevision) async { + final input = _ceilingInput[day]; + final sub = day.daySub; + if (input == null || sub.isEmpty || sessionRevision == null) return const {}; + final current = await _ceilingInputRevision(day.date); + if (current == null || input.generation != current.generation || + sessionRevision != await _sessionCeilingRevision()) { + return const {}; + } + final out = {}; + for (final row in sessions) { + final id = row['id']?.toString(); + final start = (row['start_ts'] as num?)?.toInt(); + final end = (row['end_ts'] as num?)?.toInt(); + if (id == null || row['status'] != 'done' || start == null || + end == null || end <= start) { + continue; + } + final lo = math.max(start, sub.tsSec.first); + final hi = math.min(end, sub.tsSec.last); + if (hi < lo) continue; + // E4 buckets are conservative at the clipped window's edges. Compare + // before/after the load so a racing decoded write cannot certify a hit. + String windowKey(_CeilingInput fp) => jsonEncode([ + fp.generation, for (final e in fp.revisions.entries) + if (e.key >= lo ~/ 900 && e.key <= hi ~/ 900) [e.key, e.value], + ]); + final revision = windowKey(input); + if (revision != windowKey(current)) continue; + out[id] = jsonEncode([ + revision, sessionRevision, id, row['type'], row['status'], start, end, + lo, hi, sub.deviceFamily, priorityKey(day.priority), + for (final signal in const [InputSignal.hr1Hz, InputSignal.accel1Hz]) + [signal.name, for (final span in day.ownership[signal] ?? const []) + if (span.end > lo && span.start <= hi) + [span.start, span.end, span.deviceId]], + ]); + } + return out; + } + @visibleForTesting + int debugStagingComputations = 0; + @visibleForTesting + void Function(bool mayFold, int segmentedDays, String? observation)? debugOnStagingWalk; + @visibleForTesting + bool debugDisableTargetDaySkip = false; + @visibleForTesting + Map debugRetainedSubstrate(String day) => _retained.extras(day); + @visibleForTesting + Future debugSleepCandidate(String day) => + _sleepCandidateForDay(day); + @visibleForTesting + Future debugPrepareDay(String day) => + _prepareTargetDay(day); + @visibleForTesting + Future debugLoadSubstrate(int from, int to, {required String day, + Map> ownership = const {}}) => + _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + /// True while a band offload is draining. The scheduler never STARTS a /// derive during one, but one can start mid-pass; the raw prune checks this /// before archiving each substrate bucket and stops for the pass, so a drain @@ -3151,9 +3432,11 @@ class DerivationEngine { static Future _withRunLock(T busy, Future Function() body) async { if (_running) return busy; _running = true; + beginWearableDayIndexPass(); try { return await body(); } finally { + endWearableDayIndexPass(); _running = false; } } @@ -3389,6 +3672,8 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3651,7 +3936,8 @@ class DerivationEngine { } else if (prepared != null) { _diag['prepared_days'] = (_diag['prepared_days'] as int) + 1; final complete = await _derivePreparedDay( - prepared, profile, dataNowSec, history); + prepared, profile, dataNowSec, history, + allowSessionReuse: !heavy && !force); done++; _diag['done_days'] = done; // Stored only when the derive was a fixed point; see @@ -3751,6 +4037,7 @@ class DerivationEngine { } catch (e, st) { _diag['last_error'] = '$e'; lastRunError = '$e'; + _clearRetainedSubstrate(); _log('derive ERROR: $e\n$st'); return 0; } finally { @@ -3763,90 +4050,95 @@ class DerivationEngine { } catch (e) { _log('[derive] outcome callback threw: $e'); } - // Storage housekeeping runs here, after everything, still holding - // `_running`. See _runStorageHousekeeping — this is the only place every - // entry path and every early return actually reaches. - _diag['stage'] = 'housekeeping'; - await _runStorageHousekeeping(long: heavy || force); - // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days - // inside the raw window re-derive from substrate above; everything older - // has none, so its headline is rebuilt from the stored TRIMP + wake - // window instead. No-ops after the first successful pass - // (`compute_freshness`), and never fatal — a failed rescale must not take - // the derive cycle down with it. - // - // AFTER the sweep, not before it. It used to be the first statement of - // run(), so the one launch where it actually fires — the first after the - // version bump, the launch the user is watching — spent a DB round trip - // and a bundle decode+encode PER DAY OF HISTORY before today's data was - // touched at all. Nothing in the sweep reads what it rewrites (it only - // ever touches days older than `rawRetentionDays`, which no pass - // re-derives), so the only thing the old ordering bought was that the - // FIRST cross-day rollup after the bump saw the rescaled history rather - // than the next one. Still holds `_running`, so nothing else can be - // reading day_result while it rewrites. - _diag['stage'] = 'strain_rescale'; try { - final rescaled = await backfillStrainScale( - female: workoutSex(profile.sex) == 'female', - ); - if (rescaled.didWork) { - _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' - 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' - 'window)'); + // Storage housekeeping runs here, after everything, still holding + // `_running`. See _runStorageHousekeeping — this is the only place every + // entry path and every early return actually reaches. + _diag['stage'] = 'housekeeping'; + _endRetentionPass(); + await _runStorageHousekeeping(long: heavy || force); + // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days + // inside the raw window re-derive from substrate above; everything older + // has none, so its headline is rebuilt from the stored TRIMP + wake + // window instead. No-ops after the first successful pass + // (`compute_freshness`), and never fatal — a failed rescale must not take + // the derive cycle down with it. + // + // AFTER the sweep, not before it. It used to be the first statement of + // run(), so the one launch where it actually fires — the first after the + // version bump, the launch the user is watching — spent a DB round trip + // and a bundle decode+encode PER DAY OF HISTORY before today's data was + // touched at all. Nothing in the sweep reads what it rewrites (it only + // ever touches days older than `rawRetentionDays`, which no pass + // re-derives), so the only thing the old ordering bought was that the + // FIRST cross-day rollup after the bump saw the rescaled history rather + // than the next one. Still holds `_running`, so nothing else can be + // reading day_result while it rewrites. + _diag['stage'] = 'strain_rescale'; + try { + final rescaled = await backfillStrainScale( + female: workoutSex(profile.sex) == 'female', + ); + if (rescaled.didWork) { + _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' + 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' + 'window)'); + } + } catch (e) { + _log('[derive] strain rescale failed (kept old values): $e'); } - } catch (e) { - _log('[derive] strain rescale failed (kept old values): $e'); - } - // ONE-SHOT, after the sweep for the same reason as the strain rescale: - // the rest of the readiness_z history, then a re-score of every day - // below v112 that has no raw substrate left (substrate days were just - // re-derived, or are retried next pass). A re-score rewrites stored - // days, so the baselines and the cross-day rollup are refreshed here — - // `finally` is reached by every early return, including "nothing to - // derive" and "no decoded data". - _diag['stage'] = 'readiness_calibration'; - try { - final r = await backfillReadinessCalibration( - // Both substrate sources, as `_deriveScope` reads them: a wearable - // day with rows is the derive's, not the re-score's. - rawDays: await substrateDays(), - zHistoryLoader: () async { - final h = await _BaselineHistoryCache.load(); - return (day, fam) => h.valuesBefore('readiness_z', day, - family: _baselineFamily('readiness_z', fam)); - }, - ); - if (r.seeded > 0 || r.rescored > 0) { - _log('[derive] readiness calibration: ${r.seeded} z seeded, ' - '${r.rescored} day(s) re-scored' - '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' - 'never landed, from stored inputs)' : ''}'); - await _refreshBaselines(); + // ONE-SHOT, after the sweep for the same reason as the strain rescale: + // the rest of the readiness_z history, then a re-score of every day + // below v112 that has no raw substrate left (substrate days were just + // re-derived, or are retried next pass). A re-score rewrites stored + // days, so the baselines and the cross-day rollup are refreshed here — + // `finally` is reached by every early return, including "nothing to + // derive" and "no decoded data". + _diag['stage'] = 'readiness_calibration'; + try { + final r = await backfillReadinessCalibration( + // Both substrate sources, as `_deriveScope` reads them: a wearable + // day with rows is the derive's, not the re-score's. + rawDays: await substrateDays(), + zHistoryLoader: () async { + final h = await _BaselineHistoryCache.load(); + return (day, fam) => h.valuesBefore('readiness_z', day, + family: _baselineFamily('readiness_z', fam)); + }, + ); + if (r.seeded > 0 || r.rescored > 0) { + _log('[derive] readiness calibration: ${r.seeded} z seeded, ' + '${r.rescored} day(s) re-scored' + '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' + 'never landed, from stored inputs)' : ''}'); + await _refreshBaselines(); + } + if (r.rescored > 0) await _runCrossDay(profile); + } catch (e) { + _log('[derive] readiness calibration failed (will retry): $e'); } - if (r.rescored > 0) await _runCrossDay(profile); - } catch (e) { - _log('[derive] readiness calibration failed (will retry): $e'); - } - // Restamp Today's freshness here, not in each caller: the iOS - // background pass has no caller to do it, and getToday reads the - // night from `overnight_day`, so a night settled in the background - // kept serving the previous one. - try { - await LocalDb.refreshComputeFreshness(); - } catch (e) { - _log('[derive] freshness refresh failed: $e'); - } - _running = false; - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['active_days'] = const [] - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; + // Restamp Today's freshness here, not in each caller: the iOS + // background pass has no caller to do it, and getToday reads the + // night from `overnight_day`, so a night settled in the background + // kept serving the previous one. + try { + await LocalDb.refreshComputeFreshness(); + } catch (e) { + _log('[derive] freshness refresh failed: $e'); + } + } finally { + endWearableDayIndexPass(); + _running = false; + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['active_days'] = const [] + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; - try { await runTrace?.stop(); } catch (_) {} + try { await runTrace?.stop(); } catch (_) {} + } } } @@ -3859,6 +4151,8 @@ class DerivationEngine { if (days.isEmpty) return 0; if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3969,25 +4263,47 @@ class DerivationEngine { } return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('derive selected ERROR: $e\n$st'); return 0; } finally { - await _runStorageHousekeeping(); - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; + } finally { + endWearableDayIndexPass(); + _running = false; + } } } static const int _rawDecodeBatchSize = 2000; static const int _maxDayRawRows = 500000; static const int _maxDayRawPages = 300; + static const int _maxUnionBeats = 4 * _maxDayRawRows; Future _prepareTargetDay(String dayId) async { + try { + final prepared = await _prepareTargetDayPayload(dayId); + if (prepared == null || prepared.daySub.isEmpty) { + _dropRetainedDay(dayId); + } else if (_retentionPassEpoch != null) { + _retentionPreparedDays.add(dayId); + } + return prepared; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future _prepareTargetDayPayload(String dayId) async { // Per-day page/row totals used to live in the shared `_diag` map (reset // then accumulated across this day's 2-3 substrate loads). Under // concurrent per-day processing (see `run()`), multiple days resetting/ @@ -3996,7 +4312,8 @@ class DerivationEngine { // is telemetry-only). Each day now gets its own local accumulator, // merged into the shared running max exactly once, below. final stats = _PrepareStats(); - final candidate = await _sleepCandidateForDay(dayId, stats: stats); + final union = debugDisableUnionLoads ? null : _PrepareUnion(); + final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union); final dayStart = _localDayLabelToSec(dayId); final dayEnd = _localNextDayLabelToSec(dayId); @@ -4020,7 +4337,29 @@ class DerivationEngine { if (candidate.present) candidate.sleepOffsetSec, ].reduce(math.max); - final (ownership, priority) = await _resolveOwnership(unionFrom, unionTo); + final (ownership, priority) = union?.load != null && + union!.load!.from == unionFrom && union.load!.to == unionTo + ? (Map>.of(union.ownership), + Map>.of(union.priority)) + : await _resolveOwnership(unionFrom, unionTo); + final ceilingInput = await _ceilingInputRevision(dayId); + if (union != null && (union.load == null || + unionFrom < union.load!.from! || unionTo > union.load!.to!)) { + // A banked candidate needs no search load. A live search already in + // the holder can extend to a wider selected night by borrowing itself. + final from = union.load == null + ? math.min(dayStart, candidate.present ? candidate.sleepOnsetSec : dayStart) + : unionFrom; + final to = union.load == null + ? math.max(dayEnd - 1 + napBoundaryBufferSec, + candidate.present ? candidate.sleepOffsetSec - 1 : 0) + : unionTo; + union.load = await _loadUnion(dayId, from, to, ownership, stats: stats); + } + final windows = await _cutUnionWindows(union?.load, [ + (dayStart, dayEnd - 1 + napBoundaryBufferSec), + (candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1), + ], ownership, dayStart: dayStart, dayEnd: dayEnd); // Load the day PLUS the nap boundary buffer in ONE pass (each // _loadSubstrateRange spawns its own isolate, so a second load would @@ -4028,12 +4367,13 @@ class DerivationEngine { // this, the live decoded path — run()/runDays()/rescanRecent(), i.e. every // non-import day — fell back to napSub == daySub and went on bisecting // naps at midnight. - var napSub = await _loadSubstrateRange( + var napSub = windows.$1[0] ?? await _loadSubstrateRange( dayStart, dayEnd - 1 + napBoundaryBufferSec, dayId: dayId, stats: stats, ownership: ownership, + load: 'day', ); // A day the primary band never saw is the ACTIVE WEARABLE's, when one is // chosen and flagged on (inputs/canonical.dart). Never mixed: a day with @@ -4041,27 +4381,29 @@ class DerivationEngine { // "Saw" is the calendar day, the same test the scope and the re-derive // use (`decodedRecTsMaxByDay`): band rows only in the next morning's nap // buffer do not keep the wearable off a day it alone covers. - final wearable = - napSub.slice(dayStart, dayEnd).isEmpty ? await activeWearable() : null; + final wearable = (windows.$2 ?? napSub.slice(dayStart, dayEnd)).isEmpty + ? await activeWearable() : null; if (wearable != null) { await _resolveSparseOwnership( unionFrom, unionTo, wearable.$1, ownership, priority); napSub = await wearableSubstrate( wearable, dayStart, dayEnd - 1 + napBoundaryBufferSec); } - final daySub = napSub.slice(dayStart, dayEnd); + final daySub = wearable == null && windows.$2 != null + ? windows.$2! : napSub.slice(dayStart, dayEnd); Substrate sleepSub = Substrate.empty; if (candidate.present && candidate.sleepOffsetSec > candidate.sleepOnsetSec) { sleepSub = wearable != null ? await wearableSubstrate(wearable, candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1) - : await _loadSubstrateRange( + : windows.$1[1] ?? await _loadSubstrateRange( candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1, dayId: dayId, stats: stats, ownership: ownership, + load: 'sleep', ); } // Single safe merge into the shared max-tracking diagnostics — one @@ -4088,7 +4430,7 @@ class DerivationEngine { dayEndSec: dayEnd, sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, ); - return candidate.toPreparedDay( + final prepared = candidate.toPreparedDay( daySub: daySub, napSub: napSub, sleepSub: sleepSub, @@ -4102,6 +4444,12 @@ class DerivationEngine { ownership: ownership, priority: priority, ); + // Wearable construction and supplied imports have other inputs; only the + // canonical decoded load above can carry this revision provenance. + if (wearable == null && ceilingInput != null) { + _ceilingInput[prepared] = ceilingInput; + } + return prepared; } /// [next] (the following day's main sleep) when it is TONIGHT's from this @@ -4129,6 +4477,8 @@ class DerivationEngine { /// candidate — so the spans differ, but the rule producing them does not. Future<(Map>, Map>)> _resolveOwnership(int from, int to) async { + // Loads include `to`; coverage queries and ownership spans are half-open. + final end = to + 1; final ownership = >{}; // The order each signal ACTUALLY resolved under, carried on the prepared // day so `priority_hash` names it by construction instead of re-reading @@ -4157,7 +4507,7 @@ class DerivationEngine { // gives it a rank — the conservative default, and the one that keeps // today's single-device installs byte-identical. final rawPriority = await LocalDb.signalPriority(sig); - final coverage = await LocalDb.coverageIntervals(sig, from, to); + final coverage = await LocalDb.coverageIntervals(sig, from, end); var resolved = rawPriority.isEmpty ? const [LocalDb.kPrimaryDeviceId] // A device with a coverage row here is definitionally declaring @@ -4188,7 +4538,7 @@ class DerivationEngine { coverage: coverage, priority: resolved, from: from, - to: to, + to: end, signal: sig, ); } @@ -4237,10 +4587,12 @@ class DerivationEngine { Future _sleepCandidateForDay( String dayId, { _PrepareStats? stats, + _PrepareUnion? union, }) async { // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is // not shadowed by a stale artifact). The auto path keeps its finalized cache. + // The RAM worker cache below keys the override itself, including edits. final overrideRow = await LocalDb.getSleepOverride(dayId); final override = overrideRow == null ? null @@ -4282,13 +4634,75 @@ class DerivationEngine { // exists — and it has to be: staging the night off both devices' rows // while `napSub`/`sleepSub` score it off one device's is a window found in // a night that was never scored. - final (searchOwnership, _) = await _resolveOwnership(range.$1, range.$2); - final searchSub = await _loadSubstrateRange( + final (searchOwnership, searchPriority) = + await _resolveOwnership(range.$1, range.$2); + final rowsEpoch = _retentionEpoch; + _checkpointPlans.remove(dayId); + NightCheckpointPlan? checkpointPlan; + if (!debugCheckpoints.disabled && _canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + if (_checkpointGeneration != null && _checkpointGeneration != generation) { + // Invalidate cached state while preserving the owners' work counters. + for (final state in _nightStates.values) { state.clear(); } + _cardioRowChunks.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; + } + _checkpointGeneration = generation; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [range.$1 ~/ 900, range.$2 ~/ 900]); + checkpointPlan = NightCheckpointPlan(range.$1, range.$2, + {for (final r in rows) r['bucket'] as int: r['rev'] as int}, + generation, rowsEpoch, await LocalDb.storeIdentity(executor: db)); + } catch (_) { + // Missing provenance costs a cold pass, never a partial adoption. + } + } + List>? revisions; + if (!debugDisableStagedSleepCache || + (!debugDisableCardioRows && _canRetainSubstrate)) { + try { + revisions = []; + final first = DateTime.fromMillisecondsSinceEpoch(range.$1 * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= range.$2; + d = DateTime(d.year, d.month, d.day + 1)) { + final label = dayLabelOf(d); + final fp = await LocalDb.inputDayFingerprint(label); + revisions.add([label, fp.revisionSum, fp.revisionMax, fp.generation]); + } + } catch (_) { + revisions = null; // Unavailable provenance costs a staging, never a hit. + } + } + Substrate? cutSearch; + if (union != null) { + final from = range.$1; + final to = _localNextDayLabelToSec(dayId) - 1 + napBoundaryBufferSec; + final (owned, ranked) = await _resolveOwnership(from, to); + union.ownership = owned; + union.priority = ranked; + union.load = await _loadUnion(dayId, from, to, owned, stats: stats, + search: union, window: (from: range.$1, to: range.$2, + ownership: searchOwnership)); + cutSearch = union.search; + union.search = null; + if (cutSearch == null) { + final cuts = await _cutUnionWindows(union.load, [range], searchOwnership); + cutSearch = cuts.$1.single; + } + } + final searchSub = cutSearch ?? await _loadSubstrateRange( range.$1, range.$2, dayId: dayId, stats: stats, ownership: searchOwnership, + load: 'search', ); // PERSONALIZED STAGER (v42): stage on a WORKER isolate, NOT the main/UI // thread. cardioStager reads analytics "ambient" globals — the rolling sleep @@ -4350,16 +4764,109 @@ class DerivationEngine { // ([_prepareTargetDay]): band rows from the previous afternoon, before // the ring went on, must not leave the ring's night unstaged. final ownNight = override == null && - searchSub.slice(_localDayLabelToSec(dayId), range.$2 + 1).isEmpty + (searchSub.lastTs ?? -1) < _localDayLabelToSec(dayId) ? await wearableNight(range.$1, range.$2) : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; + String? stagingKey; + if ((revisions != null && !debugDisableStagedSleepCache) || checkpointPlan != null) { + try { + final stagingWearable = override == null ? await activeWearable() : null; + String inputKey(bool prefixContext) => stagingInputKey( + fromSec: range.$1, + toSec: range.$2, + revisions: revisions ?? const [], + sub: searchSub, + ownership: searchOwnership, + priority: searchPriority, + override: override, + profileJson: profileJson, + priorSleep: priorSleep, + vendorNights: vendorNights, + activeWearable: stagingWearable, + prefixContext: prefixContext, + ); + if (revisions != null && !debugDisableStagedSleepCache) stagingKey = inputKey(false); + if (checkpointPlan != null) { + // Prefix reuse omits revision aggregates. The durable store id + // survives restart and changes when wipe/restore replaces inputs. + checkpointPlan.context = sha1.convert(utf8.encode(jsonEncode([ + checkpointPlan.storeId, inputKey(true), + ]))).toString(); + checkpointPlan.boundary = ((searchSub.lastTs ?? 0) + 1) ~/ 900 * 900; + } + } catch (_) { + // A context that cannot be keyed still takes the ordinary worker path. + checkpointPlan = null; + } + } // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run` // with no timeout at all, so a hung staging pass never completed its future // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e. // all derivation was dead until app restart. - final (candidateJson, observationJson) = - await _runIsolateCancellable(() { + final skipUnusedDays = !debugDisableTargetDaySkip; + // Read here: a `_perDayTimeout` inside `stage` would capture `this` into + // the context the worker closure ships, and the engine's callbacks (the + // app's own fields) are not sendable. + final stageTimeout = _perDayTimeout; + final cacheRows = !debugDisableCardioRows && _canRetainSubstrate && + rowsEpoch == _retentionEpoch && revisions != null; + final rowRevisions = { + for (final r in revisions ?? const >[]) + r[0] as String: (r[1] as int, r[2] as int, r[3] as int), + }; + final rowGeneration = LocalDb.storeGeneration; + final rowsLo = range.$1 - _cardioRowsSlackSec; + final rowsHi = range.$2 + _cardioRowsSlackSec; + _cardioRowChunks.removeWhere((c) => + c.generation != rowGeneration || c.algorithm != kAlgoVersion || + (c.overlaps(rowsLo, rowsHi) && + !c.matches(rowRevisions, rowGeneration, kAlgoVersion))); + final sentChunks = cacheRows ? [for (final c in _cardioRowChunks) + if (c.overlaps(rowsLo, rowsHi)) c] : []; + final rowBytes = [for (final c in sentChunks) c.bytes]; + if (checkpointPlan != null && checkpointPlan.boundary > range.$1 && + checkpointPlan.context != null) { + if (_nightStates[dayId]?.canStream != true && + _nightStates[dayId]?.acceptsCheckpointAlgorithm(kAlgoVersion) != false) { + try { + debugCheckpoints.reads++; + final row = await LocalDb.dayCheckpoint(dayId, kAlgoVersion, + maxBytes: maxNightCheckpointBytes); + if (row == null) { + debugCheckpoints.misses++; + } else { + final context = checkpointPlan.context!; + final from = checkpointPlan.from, through = checkpointPlan.through; + final inputRevisions = checkpointPlan.revisions; + final searchRr = searchSub.rrMs, searchTimes = searchSub.rrTsMs; + final restored = await _runIsolateCancellable( + () => decodeNightCheckpoint(row, dayId, kAlgoVersion, context, + from, through, inputRevisions, + searchRr: searchRr, searchTimes: searchTimes), _perDayTimeout, + label: 'checkpoint-read $dayId'); + if (restored.rejection != null) { + debugCheckpoints.rejections.add(restored.rejection!); + debugCheckpoints.misses++; + } else if (LocalDb.storeGeneration == checkpointPlan.generation && + rowsEpoch == _retentionEpoch) { + checkpointPlan.restored = restored.night; + debugCheckpoints.lastDecodedPayload = restored.payload; + if (cacheRows && restored.cardio != null) rowBytes.add(restored.cardio!); + } + } + } catch (_) { + debugCheckpoints.misses++; + } + } + checkpointPlan.cardio = rowBytes; + _checkpointPlans[dayId] = checkpointPlan; + while (_checkpointPlans.length > 3) { + _checkpointPlans.remove(_checkpointPlans.keys.first); + } + } + Future<(String, String?, int, Uint8List?, int, int, List)> stage() => + _runIsolateCancellable(() { try { final p = profileJson == null ? null @@ -4380,14 +4887,26 @@ class DerivationEngine { } ana.cardioRecordObservations = true; ana.resetCardioObservations(); + debugSegmentedDayCount = 0; + final rowOperations = []; + final ana.CardioRowCache? rowCache; + if (cacheRows) { + rowCache = decodeCardioRows(rowBytes); + rowOperations.addAll(['decode:worker', 'merge:worker']); + } else { + rowCache = null; + } final candidate = prepareSleepSessionCandidate( searchSub, targetDay: dayId, + skipUnusedDays: skipUnusedDays, + recordTargetOnly: true, override: override, priorSleep: priorSleep, vendorNights: vendorNights, + rowCache: rowCache, ); - // Fold the MAIN sleep (most epochs) of a freshly-staged night into the + // Fold the TARGET sleep (most epochs of its own observations) into the // rolling profile — done here in the worker because the observations live // in THIS isolate's globals. Skipped for overrides. EWMA self-seeds. String? observationJson; @@ -4400,7 +4919,7 @@ class DerivationEngine { // onto whichever day was re-derived last. Measured effect of that // corrupt profile on the same nights: wake 4.3% -> 36.4%, deep 1.9% -> // 0.0%. One fold per day_id, ever. - if (mayFold) { + if (mayFold && candidate.present) { final obs = ana.takeCardioObservations(); if (obs.isNotEmpty) { obs.sort((a, b) => b.epochs.compareTo(a.epochs)); @@ -4427,8 +4946,60 @@ class DerivationEngine { } } } - return (jsonEncode(candidate.toJson()), observationJson); - }, _perDayTimeout, label: 'sleep-staging $dayId'); + final rowsOut = rowCache == null ? null : + encodeCardioRows(rowCache, rowsLo, rowsHi); + if (rowCache != null) rowOperations.add('prune:worker'); + return (jsonEncode(candidate.toJson()), observationJson, + debugSegmentedDayCount, rowsOut, rowCache?.hits ?? 0, + rowCache?.misses ?? 0, rowOperations); + }, stageTimeout, label: 'sleep-staging $dayId'); + final hit = stagingKey == null ? null : _stagedSleepCache.lookup( + dayId, kAlgoVersion, stagingKey, mayFold: mayFold); + final String candidateJson; + final String? observationJson; + // Reuse worker output only; banked-night and fold checks below still run. + if (hit != null) { + candidateJson = hit.candidateJson; + observationJson = mayFold ? hit.observationJson : null; + } else { + debugStagingComputations++; + final result = await stage(); + if (checkpointPlan != null && result.$4 != null) { + checkpointPlan.cardio = [result.$4!]; + } + candidateJson = result.$1; + observationJson = result.$2; + debugCardioRowHits += result.$5; + debugCardioRowMisses += result.$6; + _cardioRowOperations.addAll(result.$7); + if (_canRetainSubstrate && rowsEpoch == _retentionEpoch && + LocalDb.storeGeneration == rowGeneration && !debugDisableCardioRows) { + _cardioRowChunks.removeWhere(sentChunks.contains); + if (result.$4 != null && result.$4!.length <= _cardioRowsMaxBytes) { + _cardioRowChunks.add(CardioRowsChunk(result.$4!, rowsLo, rowsHi, + rowRevisions, rowGeneration, kAlgoVersion)); + } + // Bound both disjoint windows and repeated fingerprint variants. + final newest = _cardioRowChunks.fold(rowsHi, + (hi, c) => math.max(hi, c.hiSec)); + _cardioRowChunks.removeWhere((c) => c.hiSec < newest - 4 * 24 * 3600); + while (_cardioRowChunks.length > 4 || + _cardioRowChunks.fold(0, (n, c) => n + c.bytes.length) > + _cardioRowsMaxBytes) { + _cardioRowChunks.removeAt(0); + } + } + debugOnStagingWalk?.call(mayFold, result.$3, observationJson); + if (stagingKey != null) { + _stagedSleepCache.store(dayId, StagedSleepEntry( + algoVersion: kAlgoVersion, + key: stagingKey, + mayFold: mayFold, + candidateJson: candidateJson, + observationJson: observationJson, + )); + } + } final candidate = SleepSessionCandidate.fromJson( (jsonDecode(candidateJson) as Map).cast()); for (final f in candidate.flags) { @@ -4557,6 +5128,7 @@ class DerivationEngine { // profile, fold, and clobber the other, losing both the fold and its // day_id from folded_days. SQLite's write lock is cross-connection and // therefore cross-isolate. + debugOnProfileFold?.call(dayId); await LocalDb.updateBaseline('sleep_user_profile', (current) { // Re-derive freshness INSIDE the transaction: the value this day read // before staging is stale by definition, another lane may have folded @@ -4656,7 +5228,174 @@ class DerivationEngine { return (startSec: startSec, endSec: endSec); } + /// The E13 owner holds one union per day. Only the loading turn is serial: + /// a neighbouring day can borrow its completed overlap while staging runs. + Future _loadUnion(String dayId, int from, int to, + Map> ownership, {_PrepareStats? stats, + _PrepareUnion? search, SubstrateUnionWindow? window}) { + final epoch = _retentionEpoch; + return _unionLock.run(() async { + int generation; + Map revisions; + try { + final db = await LocalDb.instance; + generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [from ~/ 900, to ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + } catch (_) { + _dropRetainedDay(dayId); + return null; + } + final context = substrateLoadContext(from, to, generation, ownership); + final bases = _canRetainSubstrate ? [for (final base in _retained.unions) + if (base.version == kAlgoVersion && base.generation == generation && + base.context == substrateLoadContext(base.from!, base.to!, + generation, base.ownership)) base] : []; + final stretches = reusableUnionStretches(bases, from, to, revisions, ownership); + final timeout = _perDayTimeout; + final whole = stretches.length == 1 && + stretches.single.$1 == stretches.single.$3.from && + stretches.single.$2 == stretches.single.$3.to; + final borrowed = whole ? [stretches.single.$3] : + stretches.isEmpty ? [] : + await _runIsolateCancellable(borrowUnionJob(stretches, ownership), + timeout, label: 'union-borrow $dayId'); + if (!whole) _unionWorkerCuts += borrowed.whereType().length; + final parts = []; + Future read(int a, int b) async { + debugOnSubstrateLoad?.call(a, b, 'union'); + final (sub, extras) = await _readSubstrateRange(a, b, dayId: dayId, + stats: stats, ownership: ownership, retain: true, + unionWindow: a == from && b == to ? window : null); + if (extras.isEmpty) return null; + final cut = extras.remove('union_window') as Substrate?; + if (cut != null) { + search?.search = cut; + _unionWorkerCuts++; + } + return RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: sub, extras: extras, cut: a, + from: a, to: b, generation: generation, ownership: ownership); + } + var pos = from; + for (var i = 0; i < stretches.length; i++) { + final part = borrowed[i]; + if (part == null) continue; + if (part.from! > pos) { + final gap = await read(pos, part.from! - 1); + if (gap == null) return null; + parts.add(gap); + } + parts.add(part); + pos = part.to! + 1; + } + if (pos <= to) { + final gap = await read(pos, to); + if (gap == null) return null; + parts.add(gap); + } + var loaded = parts.length == 1 ? parts.single : + await _runIsolateCancellable( + joinUnionJob(parts, context, revisions, ownership), timeout, + label: 'union-join $dayId'); + loaded ??= await read(from, to); + if (loaded == null) return null; + final rows = (loaded.extras['raw_rows_by_bucket'] as Map) + .values.fold(0, (int n, v) => n + v); + _enforcePrepareBudget(dayId: dayId, fromRecTs: from, toRecTs: to, + rangePages: (rows / _rawDecodeBatchSize).ceil(), rangeRows: rows); + if (_canRetainSubstrate && epoch == _retentionEpoch && + LocalDb.storeGeneration == generation && + loaded.sub.length <= _maxDayRawRows && + loaded.sub.rrMs.length <= _maxUnionBeats) { + // A sole borrowed part still carries its original, wider context. + loaded = RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: loaded.sub, extras: loaded.extras, + cut: from, from: from, to: to, generation: generation, + ownership: ownership); + _retained.store(dayId, 'union', loaded); + } + return loaded; + }); + } + + /// The calendar day deliberately keeps the buffered load's provenance and + /// beat-time slice semantics, just as the separate-window path does. + Future<(List, Substrate?)> _cutUnionWindows( + RetainedSubstrate? load, List<(int, int)> ranges, + Map> ownership, + {int? dayStart, int? dayEnd}) async { + if (load == null) return ([for (final _ in ranges) null], null); + final timeout = _perDayTimeout; + final cuts = await _runIsolateCancellable( + cutUnionJob(load, ranges, ownership, dayStart, dayEnd), + timeout, label: 'union-windows'); + _unionWorkerCuts += cuts.$1.whereType().length; + if (cuts.$2 != null) _unionWorkerCuts++; + return cuts; + } + Future _loadSubstrateRange( + int fromRecTs, + int toRecTs, { + required String dayId, + _PrepareStats? stats, + Map> ownership = const {}, + String load = 'direct', + }) async { + debugOnSubstrateLoad?.call(fromRecTs, toRecTs, load); + if (toRecTs < fromRecTs) { + _dropRetainedDay(dayId); + return Substrate.empty; + } + final epoch = _retentionEpoch; + String? context; + Map? revisions; + if (_canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [fromRecTs ~/ 900, toRecTs ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + context = substrateLoadContext(fromRecTs, toRecTs, generation, ownership); + } catch (_) { + // Missing revision provenance always takes the ordinary cold path. + _dropRetainedDay(dayId); + } + } else { + _dropRetainedDay(dayId); + } + final old = context == null ? null : _retained.lookup(dayId, load); + final readFrom = old?.readFrom(kAlgoVersion, context!, revisions!, toRecTs); + if (readFrom != null && readFrom > toRecTs) return old!.sub; + try { + final (sub, extras) = await _readSubstrateRange( + readFrom ?? fromRecTs, toRecTs, dayId: dayId, stats: stats, + ownership: ownership, seed: readFrom == null ? null : old, + retain: context != null); + if (_canRetainSubstrate && epoch == _retentionEpoch && + context != null && extras.isNotEmpty) { + _retained.store(dayId, load, RetainedSubstrate( + version: kAlgoVersion, context: context, revisions: revisions!, + sub: sub, extras: extras, + cut: math.max(fromRecTs, ((sub.lastTs ?? fromRecTs) ~/ 900) * 900))); + } else { + _dropRetainedDay(dayId); + } + return sub; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future<(Substrate, Map)> _readSubstrateRange( int fromRecTs, int toRecTs, { required String dayId, @@ -4667,8 +5406,11 @@ class DerivationEngine { // single-device install resolves to anyway and what the direct-load tests // pass. Map> ownership = const {}, + RetainedSubstrate? seed, + bool retain = false, + SubstrateUnionWindow? unionWindow, }) async { - if (toRecTs < fromRecTs) return Substrate.empty; + if (toRecTs < fromRecTs) return (Substrate.empty, const {}); final port = ReceivePort(); // onError/onExit are LOAD-BEARING. Without them, an uncaught throw inside // the worker (a malformed SQLite row reaching one of the numeric reads in @@ -4686,6 +5428,7 @@ class DerivationEngine { ); final ready = Completer(); final result = Completer(); + Map extras = const {}; // A failure completes BOTH completers, but we may bail out via `ready` and // never await `result` — register a listener so that error is never an // unobserved async error. (The real error still propagates via `ready`.) @@ -4706,7 +5449,13 @@ class DerivationEngine { if (kind == 'substrate') { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); - if (!result.isCompleted) result.complete(Substrate.fromJson(payload)); + extras = ((message['retain'] as Map?) ?? const {}).cast(); + final window = message['union_window']; + if (window is Map) { + extras = {...extras, + 'union_window': Substrate.fromTransfer(window.cast())}; + } + if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; } @@ -4737,11 +5486,20 @@ class DerivationEngine { } try { final worker = await ready.future; - worker.send(const {'type': 'config', 'mode': 'substrate'}); + worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain, + if (unionWindow != null) ...{'union_window': unionWindow, + 'load_from': fromRecTs, 'load_to': toRecTs, 'ownership': ownership}, + }); + if (seed != null) worker.send(seed.seed(fromRecTs)); int? afterRecTs; int? afterCursor; - var rangePages = 0; - var rangeRows = 0; + final rawRowsByBucket = { + if (seed != null) + for (final e in (seed.extras['raw_rows_by_bucket'] as Map).entries) + if (e.key < fromRecTs ~/ 900) e.key: e.value, + }; + var rangeRows = rawRowsByBucket.values.fold(0, (int n, v) => n + v); + var rangePages = rangeRows ~/ _rawDecodeBatchSize; // Low water mark for the BEAT read, which is NOT the same window as the // frame read. Paging is driven by `decoded_onehz`, but a beat can exist // for a second with no 1 Hz row at all (gen4 R10-lite is hr-only and @@ -4773,6 +5531,25 @@ class DerivationEngine { ); if (decodedRows.isNotEmpty) { _trackPrepareBatch(decodedRows.length); + if (retain) { + // Count sorted bucket runs, including rows ownership will drop. + // This keeps the existing full-load row budget exact on a tail read. + var i = 0; + while (i < decodedRows.length) { + final bucket = (decodedRows[i]['rec_ts'] as num).toInt() ~/ 900; + var lo = i + 1, hi = decodedRows.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if ((decodedRows[mid]['rec_ts'] as num).toInt() < (bucket + 1) * 900) { + lo = mid + 1; + } else { + hi = mid; + } + } + rawRowsByBucket[bucket] = (rawRowsByBucket[bucket] ?? 0) + lo - i; + i = lo; + } + } rangePages += 1; rangeRows += decodedRows.length; if (stats != null) { @@ -4871,12 +5648,15 @@ class DerivationEngine { // BOUNDED. `result.future` had no timeout at all, so any path that left // the worker unable to answer hung this call — and with it the whole // engine — permanently. - return await result.future.timeout( + final loaded = await result.future.timeout( _perDayTimeout, onTimeout: () => throw TimeoutException( 'substrate prepare for $dayId timed out after $_perDayTimeout', ), ); + return (loaded, { + ...extras, if (extras.isNotEmpty) 'raw_rows_by_bucket': rawRowsByBucket, + }); } finally { // ALWAYS tear down: on success, on error, and on timeout. The isolate is // killed rather than abandoned so a wedged worker can never outlive the @@ -5311,6 +6091,8 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); try { await refreshSessionSensorSources(); // as [run] does // Baseline gate: compute the CURRENT signature and compare to the stored @@ -5435,11 +6217,17 @@ class DerivationEngine { await LocalDb.refreshComputeFreshness(); return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('rescan ERROR: $e\n$st'); return 0; } finally { - await _runStorageHousekeeping(); - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + } finally { + endWearableDayIndexPass(); + _running = false; + } } } @@ -5753,6 +6541,7 @@ class DerivationEngine { Future refreshActivityReviews(Profile profile) async { if (_running) return false; _running = true; + beginWearableDayIndexPass(); try { final revisions = await LocalDb.applyPendingActivityReviews(); if (revisions.isEmpty) return true; @@ -5763,6 +6552,7 @@ class DerivationEngine { await LocalDb.finishActivityReviews(revisions); return (await (await LocalDb.instance).query('activity_review_days', limit: 1)).isEmpty; } finally { + endWearableDayIndexPass(); _running = false; } } @@ -5851,6 +6641,26 @@ class DerivationEngine { _BaselineHistoryCache history, { bool forceFinalize = false, bool suppliedSubstrate = false, + bool allowSessionReuse = false, + }) async { + try { + return await _derivePreparedDayPayload(day, profile, dataNowSec, history, + forceFinalize: forceFinalize, suppliedSubstrate: suppliedSubstrate, + allowSessionReuse: allowSessionReuse); + } catch (_) { + _dropRetainedDay(day.date); + rethrow; + } + } + + Future _derivePreparedDayPayload( + PreparedDerivationDay day, + Profile profile, + int dataNowSec, + _BaselineHistoryCache history, { + bool forceFinalize = false, + bool suppliedSubstrate = false, + bool allowSessionReuse = false, }) async { final daySub = day.daySub; final sleepSub = day.sleepSub; @@ -5931,11 +6741,106 @@ class DerivationEngine { // Cancellable: on timeout the isolate is KILLED, not merely abandoned to // keep burning a core behind the worker pool's back. - final bundle = await _runIsolateCancellable( - () => deriveDayBundle(withHistory), + final nightEpoch = _retentionEpoch; + final retainNight = _canRetainSubstrate && !suppliedSubstrate; + final diagnostics = debugNightStreams; + final checkpointPlan = _checkpointPlans.remove(day.date); + final checkpointEnabled = !debugCheckpoints.disabled && retainNight && + checkpointPlan?.context != null && checkpointPlan!.epoch == nightEpoch && + checkpointPlan.generation == LocalDb.storeGeneration && + day.sleepOnsetSec >= checkpointPlan.from && + day.sleepOffsetSec <= checkpointPlan.through + 1; + var nightRevision = 0, nightGeneration = LocalDb.storeGeneration; + var nightProvenance = false; + // Keep DB reads and engine fields outside the shipped closure. A changed + // revision rejects even a byte-identical replacement; raw prefix proofs in + // the worker additionally detect moved windows and in-memory mutations. + NightRrState? nightState; + if ((retainNight && sleepSub.rrMs.isNotEmpty) || diagnostics != null) { + nightState = _nightStates.remove(day.date) ?? NightRrState(); + try { + var revision = 0; + var generation = LocalDb.storeGeneration; + final first = DateTime.fromMillisecondsSinceEpoch( + day.sleepOnsetSec * 1000); + final last = day.sleepOffsetSec > day.sleepOnsetSec + ? day.sleepOffsetSec - 1 : day.sleepOnsetSec; + for (var date = DateTime(first.year, first.month, first.day); + date.millisecondsSinceEpoch ~/ 1000 <= last; + date = DateTime(date.year, date.month, date.day + 1)) { + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + revision += fp.revisionSum; + generation = fp.generation; + } + nightState.context(inputRevision: revision, generation: generation, + algorithm: kAlgoVersion); + nightRevision = revision; + nightGeneration = generation; + nightProvenance = true; + } catch (_) { + nightState.clear(); // Unavailable provenance never certifies reuse. + } + diagnostics?.nightState = nightState; + } + final canCheckpoint = checkpointEnabled && nightProvenance && + nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation; + final restoredNight = canCheckpoint ? checkpointPlan.restored : null; + final checkpointContext = checkpointEnabled ? checkpointPlan.context! : ''; + final checkpointBoundary = checkpointEnabled ? checkpointPlan.boundary : 0; + final checkpointFrom = checkpointEnabled ? checkpointPlan.from : 0; + final checkpointVector = checkpointEnabled ? checkpointRevisions( + checkpointPlan.revisions, checkpointFrom, checkpointBoundary) : Uint8List(0); + final checkpointRows = checkpointEnabled ? checkpointPlan.cardio : []; + final checkpointDay = day.date; + final workerResult = await _runIsolateCancellable(() { + var state = nightState; + var resumed = false; + if (restoredNight != null && restoredNight.acceptsPrefix( + input.sleepRrMs, input.sleepRrTsMs, input.date, + input.sleepOnsetSec)) { + restoredNight.bindRestoredContext(nightRevision, nightGeneration, kAlgoVersion, + previous: state); + state = restoredNight; + resumed = true; + if (diagnostics != null) diagnostics.nightState = state; + } + final result = deriveDayBundle(withHistory, streamDiagnostics: diagnostics, + nightState: state); + NightCheckpointWrite? checkpoint; + if (canCheckpoint) { + try { + checkpoint = encodeNightCheckpoint(state, checkpointRows, checkpointDay, + kAlgoVersion, checkpointContext, checkpointFrom, checkpointBoundary, + checkpointVector); + } catch (_) { + // Scratch serialization cannot turn a successful derive into an error. + } + } + return (result, state, diagnostics, resumed, + restoredNight != null && !resumed, checkpoint); + }, _perDayTimeout, label: 'day-bundle ${day.date}', ); + final bundle = workerResult.$1; + if (workerResult.$4) debugCheckpoints.hits++; + if (workerResult.$5) { + debugCheckpoints.rejections.add('checkpoint night prefix/window'); + debugCheckpoints.misses++; + } + if (diagnostics != null) debugNightStreams = workerResult.$3; + final returnedNight = workerResult.$2; + if (returnedNight != null && retainNight && _canRetainSubstrate && + nightEpoch == _retentionEpoch) { + _nightStates[day.date] = returnedNight; + while (_nightStates.length > math.min(2, _retained.maxDays)) { + _nightStates.remove(_nightStates.keys.first)?.clear(); + } + } else { + returnedNight?.clear(); + } + debugOnNightBundle?.call(withHistory, bundle); // Readiness came back absent for TODAY specifically (not a historical // backfill day, which would just be noise) — log why. This ran inside // Isolate.run so it couldn't call Firebase itself; it just returned the @@ -6043,6 +6948,7 @@ class DerivationEngine { if (_isRealDayResult(existing)) { _log('derive ${day.date}: no substrate (raw pruned) — kept the ' 'existing result rather than blanking it'); + _dropRetainedDay(day.date); return false; } } @@ -6118,6 +7024,7 @@ class DerivationEngine { "under a day that already had real night scalars — kept the " 'existing result rather than overwriting the readiness baseline ' '(edge#305)'); + _dropRetainedDay(day.date); return false; } } @@ -6159,10 +7066,24 @@ class DerivationEngine { // sample, so a completed session the band's PPG lost contact for // entirely (zero HR samples at all, e.g. a wrist-gripping lift) can // fall outside them and never reach the zero-coverage credit below. + final sessionRevision = await _sessionCeilingRevision(); final savedSessions = await LocalDb.sessionsInRange( _localDayLabelToSec(day.date), localNextMidnightSecForDayLabel(day.date), ); + final ceilingSigs = debugDisableSessionReuse ? const {} + : await _ceilingSignatures(day, savedSessions, sessionRevision); + final ceilingReuse = {}; + if (allowSessionReuse && !debugDisableSessionReuse) { + for (final e in ceilingSigs.entries) { + final hit = _ceilingCache[(day.date, e.key)]; + if (hit != null && hit.version == kAlgoVersion && hit.sig == e.value) { + ceilingReuse[e.key] = hit.result; + } + } + } + // Removed, live or changed sessions cannot resurrect a former result. + _ceilingCache.removeWhere((key, _) => key.$1 == day.date); // Off-wrist / charging spans over the NAP window (which runs past this // day's end), read here because the isolate has no DB handle. These are @@ -6261,6 +7182,7 @@ class DerivationEngine { } final napEdits = await ActivityStore.napEdits(reviewDb, day.date); final blocksInput = _DayBlocksInput( + ceilingReuse: ceilingReuse, daySub: daySub, napSub: day.napSub, napEdits: napEdits, @@ -6324,6 +7246,19 @@ class DerivationEngine { final blocks = await _runDayBlocksCancellable(blocksInput, _perDayTimeout); + debugCeilingComputed += blocks.ceilingComputations; + if (!debugDisableSessionReuse) { + for (final e in {...ceilingReuse, ...blocks.ceilingComputed}.entries) { + final sig = ceilingSigs[e.key]; + if (sig == null) continue; + _ceilingCache[(day.date, e.key)] = + (version: kAlgoVersion, sig: sig, result: e.value); + while (_ceilingCache.length > _ceilingCacheMax) { + _ceilingCache.remove(_ceilingCache.keys.first); + } + } + } + // Merge the computed blocks back into the isolate-1 bundle. scMap is the // CastMap view over bundle['scalars'], so addAll writes through — nap_min / // hrr_bpm reach the persisted series map below. @@ -6458,6 +7393,7 @@ class DerivationEngine { await _persistWakeDayFeatures(dayId: day.date, wake: blocks.wake); } catch (e, st) { secondHalfOk = false; + _dropRetainedDay(day.date); _log('day-blocks (offloaded second half) failed for ${day.date} — ' 'persisting headline day (partial): $e'); TelemetryService.instance.recordNonFatal(e, st, reason: 'day_blocks_failed'); @@ -6724,6 +7660,40 @@ class DerivationEngine { 'hr_ceiling_bpm': sc('hr_ceiling_bpm'), }, ); + if (effectiveFinalized || effectivePartial) _dropRetainedDay(day.date); + if (!debugCheckpoints.disabled && !suppliedSubstrate) { + try { + final db = await LocalDb.instance; + if (effectiveFinalized || effectivePartial) { + await db.delete('day_checkpoint', where: 'day_id = ?', whereArgs: [day.date]); + } else if (checkpointEnabled && workerResult.$6 != null && + _canRetainSubstrate && nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation) { + final checkpoint = workerResult.$6!; + await db.transaction((txn) async { + if (LocalDb.storeGeneration != checkpointPlan.generation || + await LocalDb.storeIdentity(executor: txn) != checkpointPlan.storeId) { + return; + } + await txn.insert('day_checkpoint', { + 'day_id': day.date, 'algo_version': kAlgoVersion, + 'fmt': nightCheckpointFormat, 'ctx_sig': checkpointContext, + 'cp_rec_ts': checkpointBoundary, 'rev_vec': checkpointVector, + 'state': checkpoint.bytes, 'night_ref': null, + 'computed_at': DateTime.now().millisecondsSinceEpoch, + }, conflictAlgorithm: ConflictAlgorithm.replace); + await txn.rawDelete('DELETE FROM day_checkpoint WHERE day_id NOT IN ' + '(SELECT day_id FROM day_checkpoint GROUP BY day_id ' + 'ORDER BY day_id DESC LIMIT 3) OR algo_version != ?', [kAlgoVersion]); + }); + debugCheckpoints.writes++; + debugCheckpoints.lastEncodedPayload = checkpoint.bytes; + debugCheckpoints.lastDecodedPayload = checkpoint.payload; + } + } catch (_) { + // Disposable scratch is never required to commit a day result. + } + } // AFTER the row: retired only once the row that proves it is durable. if (quietSettled) await LocalDb.deleteComputeFreshness(quietKey); // NOTE: the sweep's `history` snapshot is deliberately NOT updated here. @@ -10572,7 +11542,11 @@ class DerivationEngine { bundlePatch['workout_suggestions'] = wc.boutJson; if (wc.hrrBpm != null) scMap['hrr_bpm'] = wc.hrrBpm; if (wc.hrrTauS != null) scMap['hrr_tau_s'] = wc.hrrTauS; - final ceiling = _dayHrCeiling(daySub, inp.savedSessions); + var ceilingComputations = 0; + final ceilingComputed = {}; + final ceiling = _dayHrCeiling(daySub, inp.savedSessions, + reuse: inp.ceilingReuse, computed: ceilingComputed, + onCompute: () => ceilingComputations++); bundlePatch['hr_ceiling'] = ceiling; // TS-03 — the day's observed ceiling as a SCALAR too, so the app-wide // "highest we've seen" is a max over `metric_series` (one small query the @@ -10629,6 +11603,8 @@ class DerivationEngine { // than no diagnostic. return _DayBlocksOutput( + ceilingComputations: ceilingComputations, + ceilingComputed: ceilingComputed, bundlePatch: bundlePatch, seriesPatch: seriesPatch, scalarPatch: scMap, @@ -10819,50 +11795,66 @@ class DerivationEngine { ) => _dayHrCeiling(s, saved); + // Reuse carries the full Metric JSON, including absence, and its unrounded + // bpm. Comparing serialized/rounded values could choose a different session. static Map _dayHrCeiling( Substrate s, - List> saved, - ) { - ana.Metric? best; + List> saved, { + Map reuse = const {}, + Map? computed, + void Function()? onCompute, + }) { + Map? bestJson; + double? bestBpm; String? bestId; String? bestType; - ana.Metric? lastAbsent; + Map? lastAbsentJson; for (final row in saved) { final start = (row['start_ts'] as num?)?.toInt(); final end = (row['end_ts'] as num?)?.toInt(); if (start == null || end == null || end <= start) continue; - final hr = []; - final accel = []; - for (var i = 0; i < s.length; i++) { - final t = s.tsSec[i]; - if (t < start) continue; - if (t > end) break; - final tsMs = t * 1000.0; - hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); - accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], - valid: s.accelPresentAt(i))); - } - if (hr.isEmpty) continue; - final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); - if (!m.present) { - lastAbsent = m; + final id = row['id']?.toString(); + var result = row['status'] == 'done' ? reuse[id] : null; + if (result == null) { + final hr = []; + final accel = []; + for (var i = 0; i < s.length; i++) { + final t = s.tsSec[i]; + if (t < start) continue; + if (t > end) break; + final tsMs = t * 1000.0; + hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); + accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], + valid: s.accelPresentAt(i))); + } + if (hr.isEmpty) continue; + onCompute?.call(); + final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); + result = (json: m.toJson((v) => v.toJson()), + bpm: m.present ? m.value!.bpm : null); + if (id != null && row['status'] == 'done') computed?[id] = result; + } + final bpm = result.bpm; + if (bpm == null) { + lastAbsentJson = result.json; continue; } - if (best == null || m.value!.bpm > best.value!.bpm) { - best = m; - bestId = row['id']?.toString(); + if (bestBpm == null || bpm > bestBpm) { + bestJson = result.json; + bestBpm = bpm; + bestId = id; bestType = row['type']?.toString(); } } - final m = best ?? - lastAbsent ?? + final json = bestJson ?? + lastAbsentJson ?? const ana.Metric.absent( tier: ana.Tier.high, inputs_used: ['hr_1hz', 'accel_1hz', 'device_family'], note: 'no saved session on this day to observe a ceiling in', - ); + ).toJson((v) => v.toJson()); return { - ...m.toJson((v) => v.toJson()), + ...json, 'session_id': ?bestId, 'session_type': ?bestType, }; @@ -11214,7 +12206,7 @@ class _IsolateValue { /// Test seam for [DerivationEngine._runIsolateCancellable] — the isolate /// lifecycle guarantees (value / error / killed-on-timeout) are what the engine /// depends on to never hang, so they're pinned directly. -@visibleForTesting +/// Also used by wearable and session-window preparation in production. Future runCancellableIsolate( FutureOr Function() compute, Duration timeout, { @@ -11222,11 +12214,15 @@ Future runCancellableIsolate( }) => DerivationEngine._runIsolateCancellable(compute, timeout, label: label); +typedef _SessionCeiling = ({Map json, double? bpm}); +typedef _CeilingInput = ({int generation, Map revisions}); + /// Sendable input for [DerivationEngine._computeDayBlocks] — crosses the /// `Isolate.run` boundary, so every field is plain data (Substrate is int/double /// lists; Profile is a primitive data class). DB reads that the /// pure compute needs are performed by the caller and passed in here. class _DayBlocksInput { + final Map ceilingReuse; final Substrate daySub; final Substrate napSub; final Substrate sleepSub; @@ -11321,6 +12317,7 @@ class _DayBlocksInput { /// `[startSec, endSec]`. Nap suggestions overlapping one are dropped. final List> rejectedSleepSpans; const _DayBlocksInput({ + this.ceilingReuse = const {}, required this.daySub, required this.napSub, required this.sleepSub, @@ -11360,6 +12357,8 @@ class _DayBlocksInput { /// isolate; [wake] is persisted; candidates are reconciled and session HRR /// writes are applied by the database-owning caller. class _DayBlocksOutput { + final Map ceilingComputed; + final int ceilingComputations; final List> napSuggestions; final Map bundlePatch; final Map seriesPatch; @@ -11368,6 +12367,8 @@ class _DayBlocksOutput { final List> suggestionsToPersist; final List<(String, double)> sessionHrrWrites; const _DayBlocksOutput({ + this.ceilingComputed = const {}, + this.ceilingComputations = 0, required this.bundlePatch, this.napSuggestions = const [], required this.seriesPatch, diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 316077e2b..12845ca04 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -6,6 +6,8 @@ import 'package:openstrap_analytics/onehz.dart' as ana; import '../ble/adapters/signals.dart'; import '../data/coverage_resolver.dart'; import 'substrate.dart'; +import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'vendor_sleep.dart'; class PreparedDerivationDay { @@ -317,6 +319,9 @@ void derivationPrepareWorker(SendPort mainSendPort) { final state = _PrepareAccumulator(); String? targetDay; var mode = 'prepared_day'; + SubstrateUnionWindow? unionWindow; + var loadFrom = 0, loadTo = -1; + Map> loadOwnership = const {}; mainSendPort.send(port.sendPort); void handle(Object? message) { @@ -347,16 +352,35 @@ void derivationPrepareWorker(SendPort mainSendPort) { targetDay = message['target_day']?.toString(); final cfgMode = message['mode']?.toString(); if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode; + state.retain = message['retain'] == true; + unionWindow = message['union_window'] as SubstrateUnionWindow?; + loadFrom = message['load_from'] as int? ?? 0; + loadTo = message['load_to'] as int? ?? -1; + loadOwnership = message['ownership'] as Map>? ?? const {}; + return; + } + if (type == 'seed') { + state.seed(message); return; } if (type == 'finish') { try { final substrate = state.buildSubstrate(); if (mode == 'substrate') { + final extras = state.retainable ? state.retainExtras() : null; + final request = unionWindow; + final window = request == null || extras == null ? null : + cutSubstrateUnion(RetainedSubstrate(version: 0, context: '', + revisions: const {}, sub: substrate, extras: extras, cut: loadFrom, + from: loadFrom, to: loadTo, ownership: loadOwnership), + request.from, request.to, request.ownership, + retainMetadata: false)?.sub; mainSendPort.send({ 'type': 'result', 'kind': 'substrate', 'payload': substrate.toJson(), + 'retain': ?extras, + if (window != null) 'union_window': window.toJson(), }); } else { final payload = prepareDerivationPayload( @@ -408,6 +432,9 @@ PreparedDerivationPayload prepareDerivationPayload( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + bool recordTargetOnly = false, }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -415,7 +442,11 @@ PreparedDerivationPayload prepareDerivationPayload( final days = []; for (final day in calendarDays( sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, override: override, + targetDay: targetDay, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: vendorNights, )) { @@ -470,6 +501,9 @@ SleepSessionCandidate prepareSleepSessionCandidate( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + bool recordTargetOnly = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches @@ -513,8 +547,11 @@ SleepSessionCandidate prepareSleepSessionCandidate( claimed.add(n); } final payload = prepareDerivationPayload(sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, targetDay: targetDay, override: override, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: claimed); if (payload.days.isEmpty) return SleepSessionCandidate.absent(targetDay); @@ -635,10 +672,75 @@ class _PrepareAccumulator { /// worker used to hang the whole engine (see [derivationPrepareWorker]). static num? _num(Object? v) => v is num ? v : null; + // Retention is recorded only for decoded loads requested by the engine. + bool retain = false; + bool _retainBroken = false; + bool get retainable => retain && !_retainBroken; + final List _rrStart = []; + final Map _buckets = {}; + Float64List? _rrTsRaw; + final List> _provenance = []; + + void seed(Map message) { + List ints(String k) => message[k] as List; + List dbls(String k) => message[k] as List; + final ts = ints('ts_sec'); + final cut = message['cut'] as int; + var n = 0; + while (n < ts.length && ts[n] < cut) { n++; } + final starts = ints('rr_start'); + final beats = n < ts.length ? starts[n] : dbls('rr_ms').length; + void take(List into, String k) => into.addAll(ints(k).getRange(0, n)); + take(tsSec, 'ts_sec'); + take(hr, 'hr'); + take(spo2Red, 'spo2_red'); + take(spo2Ir, 'spo2_ir'); + take(skinTemp, 'skin_temp'); + take(skinContact, 'skin_contact'); + take(stepCount, 'step_count'); + take(hrValid, 'hr_valid'); + take(bandSleepState, 'band_sleep_state'); + ax.addAll(dbls('ax').getRange(0, n)); + ay.addAll(dbls('ay').getRange(0, n)); + az.addAll(dbls('az').getRange(0, n)); + // Start from the raw axis: buildSubstrate applies the running max once + // across the entire rebuilt load, including the prefix/tail seam. + rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats)); + rrMs.addAll(dbls('rr_ms').getRange(0, beats)); + _rrStart.addAll(starts.getRange(0, n)); + for (final run in (message['provenance'] as List?) ?? const []) { + final r = (run as List).cast(); + if ((r[0] as int) >= n) break; + _provenance.add(List.from(r)); + } + for (final wire in message['meta'] as List) { + final m = wire as List; + final bucket = m[0] as int; + if (bucket >= cut ~/ 900) continue; + final acc = _buckets[bucket] = _BucketAcc() + ..families.addAll((m[1] as List).cast()) + ..deviceIds.addAll((m[2] as List).cast()) + ..skinUnit = m[3] as String?; + _families.addAll(acc.families); + _deviceIds.addAll(acc.deviceIds); + _skinTempUnit ??= acc.skinUnit; + } + } + + Map retainExtras() => { + 'rr_start': Int32List.fromList(_rrStart), + 'rr_ts_raw': _rrTsRaw!, + 'provenance': _provenance, + 'meta': [for (final e in _buckets.entries) + [e.key, e.value.families.toList(), e.value.deviceIds.toList(), + e.value.skinUnit]], + }; + void addRawPage(List hexes) { if (hexes.isEmpty) return; final sub = decodeSubstrate(hexes); if (sub.isEmpty) return; + _retainBroken = true; tsSec.addAll(sub.tsSec); hr.addAll(sub.hr); rrTsMs.addAll(sub.rrTsMs); @@ -707,6 +809,20 @@ class _PrepareAccumulator { final row = frameByRecTs[recTs]; _noteFamily(row?['device_family']); _noteDeviceId(row?['device_id']); + if (retain) { + _rrStart.add(rrMs.length); + _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row); + final family = row?['device_family']; + final id = row?['device_id']; + final unit = row?['skin_temp_raw'] is num ? 'raw' : + (row?['skin_temp_c'] is num ? 'c' : null); + final f = family is String && family.isNotEmpty ? family : null; + final d = id is String ? id : null; + final last = _provenance.isEmpty ? null : _provenance.last; + if (last == null || last[1] != f || last[2] != d || last[3] != unit) { + _provenance.add([tsSec.length, f, d, unit]); + } + } tsSec.add(recTs); // NULL (absent), a beat-only second (no row at all) and an impossible // byte all land on the same 0 — the array's one "no usable HR this @@ -794,6 +910,7 @@ class _PrepareAccumulator { // `addRawPage` each place their own beats correctly and then APPEND, so an // overlap between the last record of one page and the first of the next is // only visible once every page is in. See [monotonizeBeatAxis]. + if (retainable) _rrTsRaw = Float64List.fromList(rrTsMs); monotonizeBeatAxis(rrTsMs); // `_skinTempUnitDrops` (see [_skinTempFor]) is structurally 0 today — // `derivableSourceSql()` renders `source IS NULL` (db.dart:116), so a @@ -807,23 +924,41 @@ class _PrepareAccumulator { // which M0 must not take; M5 — which is what makes two-device admission // reachable for real — turns this into a refusal note on // `wellness.skin_temp` instead. + List packed(List values) => + retainable ? Int64List.fromList(values) : values; return Substrate( - tsSec: tsSec, - hr: hr, + tsSec: packed(tsSec), + hr: packed(hr), rrTsMs: rrTsMs, rrMs: rrMs, ax: ax, ay: ay, az: az, - spo2Red: spo2Red, - spo2Ir: spo2Ir, - skinTemp: skinTemp, - skinContact: skinContact, - stepCount: stepCount, - hrValid: hrValid, - bandSleepState: bandSleepState, + spo2Red: packed(spo2Red), + spo2Ir: packed(spo2Ir), + skinTemp: packed(skinTemp), + skinContact: packed(skinContact), + stepCount: packed(stepCount), + hrValid: packed(hrValid), + bandSleepState: packed(bandSleepState), deviceFamily: deviceFamily, deviceIds: _deviceIds, ); } } + +class _BucketAcc { + final Set families = {}; + final Set deviceIds = {}; + String? skinUnit; + + void note(Map? row) { + if (row == null) return; + final family = row['device_family']; + if (family is String && family.isNotEmpty) families.add(family); + final id = row['device_id']; + if (id is String) deviceIds.add(id); + skinUnit ??= row['skin_temp_raw'] is num + ? 'raw' : (row['skin_temp_c'] is num ? 'c' : null); + } +} diff --git a/lib/compute/ed2b_checkpoint_diagnostics.dart b/lib/compute/ed2b_checkpoint_diagnostics.dart new file mode 100644 index 000000000..a39c0ce58 --- /dev/null +++ b/lib/compute/ed2b_checkpoint_diagnostics.dart @@ -0,0 +1,24 @@ +import 'dart:typed_data'; + +/// Observations of the real checkpoint path. No storage, codec or resume policy +/// lives here. Counters remain zero until the GREEN implementation reports work. +class Ed2bCheckpointDiagnostics { + bool disabled = false; + int reads = 0; + int writes = 0; + int hits = 0; + int misses = 0; + final List rejections = []; + Uint8List? lastEncodedPayload; + Map? lastDecodedPayload; + + void reset() { + reads = 0; + writes = 0; + hits = 0; + misses = 0; + rejections.clear(); + lastEncodedPayload = null; + lastDecodedPayload = null; + } +} diff --git a/lib/compute/ed2b_stream_diagnostics.dart b/lib/compute/ed2b_stream_diagnostics.dart new file mode 100644 index 000000000..76ff21597 --- /dev/null +++ b/lib/compute/ed2b_stream_diagnostics.dart @@ -0,0 +1,45 @@ +import 'night_rr_state.dart'; + +// Work is counted beside the real owner and the remaining batch metric calls. +class Ed2bStreamDiagnostics { + int passes = 0; + NightRrState nightState = NightRrState(); + int get processedNightBeats => nightState.processedNightBeats; + int get nightRebuilds => nightState.nightRebuilds; + final Map _batchBeats = {}; + final Map _batchCalls = {}; + bool disabled = false; + void Function(List nn, List times, double cleanFraction)? + onCorrected; + void Function()? onBeforePassReturn; + void Function(List)? onRespWindows; + + void beginPass() { passes++; } + void endPass() { onBeforePassReturn?.call(); } + void context({required int inputRevision, required int generation, + required int algorithm}) => nightState.context(inputRevision: inputRevision, + generation: generation, algorithm: algorithm); + // Total input work: GREEN counts only newly appended/settled beats here, + // regardless of whether the operation uses a batch call or an addSlice. + int get retainedNightBeats => nightState.retainedNightBeats; + int get settledNightBeats => nightState.settledNightBeats; + int get identityMemoEntries => nightState.identityMemoEntries; + int get fullListDigests => nightState.fullListDigests; + int get maximumListDigestsPerIdentity => nightState.maximumListDigestsPerIdentity; + int retainedStreamBeats(String stream) => nightState.retainedStreamBeats(stream); + int processedBeats(String stream) => nightState.processedBeats(stream) + + (_batchBeats[stream] ?? 0); + int slicesProcessed(String stream) => nightState.slicesProcessed(stream) + + (_batchCalls[stream] ?? 0); + int streamRestarts(String stream) => nightState.streamRestarts(stream); + + void observeNightCorrection(int rawBeats, List nn, + List times, double cleanFraction) { + onCorrected?.call(nn, times, cleanFraction); + } + + void observeBatch(String stream, int beats) { + _batchCalls[stream] = (_batchCalls[stream] ?? 0) + 1; + _batchBeats[stream] = (_batchBeats[stream] ?? 0) + beats; + } +} diff --git a/lib/compute/inputs/canonical.dart b/lib/compute/inputs/canonical.dart index ff4b2ca8b..17950e9fc 100644 --- a/lib/compute/inputs/canonical.dart +++ b/lib/compute/inputs/canonical.dart @@ -18,7 +18,7 @@ import '../../ble/adapters/_registry.dart' import '../../data/day_label.dart'; import '../../data/db.dart'; import '../crossday_pipeline.dart' show buildCrossDayBundle; -import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion; +import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion, runCancellableIsolate; import '../substrate.dart'; import '../vendor_sleep.dart'; import 'colmi_inputs.dart'; @@ -390,6 +390,36 @@ Future refreshSessionSensorSources() async { LocalDb.sessionWearableSource = (await activeWearable())?.$2; } +bool debugDisableWearableWorker = false; +void Function(String)? debugOnWearableConstruction; +// Called beside the actual row loops. A worker extraction must return its +// execution marker to this observer, not label a dispatch on the caller. +void debugRecordWearableConstruction([String? isolateName]) => + debugOnWearableConstruction?.call( + isolateName ?? Isolate.current.debugName ?? 'unnamed'); + +// Only plain rows, columns and bounds are captured by these builders. Return +// the execution marker with the value; observers stay on the calling isolate. +Future prepareWearable(R Function() build) async { + (R, String) construct() => (build(), Isolate.current.debugName ?? 'unnamed'); + final result = debugDisableWearableWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'wearable construction'); + debugRecordWearableConstruction(result.$2); + return result.$1; +} + +Future constructWearable(Substrate Function() build, + {Substrate? unchanged}) async { + final columns = await prepareWearable(() { + final value = build(); + return identical(value, unchanged) ? null : value.toJson(); + }); + // A no-op overlay preserves even a direct caller's empty optional columns. + return columns == null ? unchanged! : Substrate.fromTransfer(columns); +} + /// The active wearable's rows over `[from, to]` as a substrate, for a day the /// primary band left empty, with a strap's session windows laid over them /// ([withStrapSessions]). Empty for an adapter with no inputs mapping. A day @@ -460,6 +490,12 @@ Future withStrapSessions(Substrate sub, int from, int to) async { 'AND d.rec_ts >= ? AND d.rec_ts <= ? ORDER BY d.rec_ts', [from, to], ); + return constructWearable(() => _overlayStrapSessions(sub, beats, all), + unchanged: sub); +} + +Substrate _overlayStrapSessions(Substrate sub, + List> beats, List> all) { final owner = ownerBy([for (final r in all) (r['sid'], r['src'])]); final rows = [ for (final r in all) @@ -616,10 +652,32 @@ Future> flagOnOrPrimary(List ids, return out; } +// Pass-lifetime index. The hook runs after an actual scan, so a test can +// count scans and inject a post-scan failure. +Map<(String, String), Future>>? _passDayIndex; +bool _disableWearableDayIndex = false; +bool get debugDisableWearableDayIndex => _disableWearableDayIndex; +set debugDisableWearableDayIndex(bool value) { + _disableWearableDayIndex = value; + if (value) _passDayIndex?.clear(); +} +void beginWearableDayIndexPass() => _passDayIndex = {}; +void endWearableDayIndexPass() => _passDayIndex = null; +int get debugWearableDayIndexSize => _passDayIndex?.length ?? 0; +void debugWearableDayIndexRekey(int version) => _passDayIndex?.clear(); +void Function((String, String), Map)? debugOnWearableDayIndexScan; + /// `{local day -> last second}` the active wearable has rows for, and the /// days a flagged-on strap recorded a session in while the wearable was off /// or charging ([wearableSubstrate] lays the session over the empty day). Future> wearableRecTsMaxByDay( + (String, String) wearable) { + final cache = debugDisableWearableDayIndex ? null : _passDayIndex; + if (cache == null) return _scanWearableRecTsMaxByDay(wearable); + return cache.putIfAbsent(wearable, () => _scanWearableRecTsMaxByDay(wearable)); +} + +Future> _scanWearableRecTsMaxByDay( (String, String) wearable) async { final out = { ...await switch (wearable.$2) { @@ -637,6 +695,7 @@ Future> wearableRecTsMaxByDay( in (await _strapSessionRecTsMaxByDay()).entries) { out[day] = math.max(out[day] ?? t, t); } + debugOnWearableDayIndexScan?.call(wearable, out); return out; } diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index b9823df78..cc7ebb2a7 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -51,10 +51,9 @@ Future> colmiSkinTemps( int to, ) async { final db = await LocalDb.instance; - final out = {}; // A reply is banked on or after the day it describes and at most the // ring's history (kColmiHistoryDays) later: scan only those captures. - for (final r in await db.rawQuery( + final rows = await db.rawQuery( 'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms ' 'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? ' "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' " @@ -64,7 +63,14 @@ Future> colmiSkinTemps( (from - 86400) * 1000, (to + (kColmiHistoryDays + 1) * 86400) * 1000, ], - )) { + ); + return prepareWearable(() => _colmiSkinTemps(rows, from, to)); +} + +List<(int, int, int)> _colmiSkinTemps( + List> rows, int from, int to) { + final out = {}; + for (final r in rows) { final hex = r['hex'] as String; final bytes = [ for (var i = 0; i + 1 < hex.length; i += 2) @@ -99,6 +105,10 @@ Future> colmiSkinTemps( Future colmiHr(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kColmiFamily, deviceId, from, to); if (sub.length < 2) return sub; + return constructWearable(() => _colmiHr(sub)); +} + +Substrate _colmiHr(Substrate sub) { final gaps = [ for (var i = 1; i < sub.length; i++) sub.tsSec[i] - sub.tsSec[i - 1], ]..sort(); @@ -162,6 +172,10 @@ Future colmiSubstrate(String deviceId, int from, int to) async { final sub = await colmiHr(deviceId, from, to); if (sub.isEmpty) return sub; final temps = await colmiSkinTemps(deviceId, from, to); + return constructWearable(() => _colmiSubstrate(sub, temps)); +} + +Substrate _colmiSubstrate(Substrate sub, List<(int, int, int)> temps) { final skin = []; var j = 0; for (final t in sub.tsSec) { diff --git a/lib/compute/inputs/garmin_inputs.dart b/lib/compute/inputs/garmin_inputs.dart index 47c771e2b..ebe695dce 100644 --- a/lib/compute/inputs/garmin_inputs.dart +++ b/lib/compute/inputs/garmin_inputs.dart @@ -107,6 +107,11 @@ Future sparseHrSubstrate( [deviceId, family, from, to], ); if (rows.isEmpty) return Substrate.empty; + return constructWearable(() => _sparseHrSubstrate(family, deviceId, rows)); +} + +Substrate _sparseHrSubstrate(String family, String deviceId, + List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/oura_inputs.dart b/lib/compute/inputs/oura_inputs.dart index 2eed4d12f..aaa578aaf 100644 --- a/lib/compute/inputs/oura_inputs.dart +++ b/lib/compute/inputs/oura_inputs.dart @@ -68,6 +68,18 @@ Future ouraSubstrate(String deviceId, int from, int to) async { if (sub.isEmpty) return _ouraTempSubstrate(deviceId, from, to); final c = kHrCadenceSec[kOuraFamily]!; final temps = await _ouraTempRows(deviceId, from - c, to + c); + final db = await LocalDb.instance; + final beats = await db.rawQuery( + 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' + 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' + 'ORDER BY t, beat_index', + [deviceId, kOuraFamily, from, to], + ); + return constructWearable(() => _ouraSubstrate(sub, temps, beats, c)); +} + +Substrate _ouraSubstrate(Substrate sub, List> temps, + List> beats, int c) { final skin = []; var j = 0; for (final t in sub.tsSec) { @@ -80,13 +92,6 @@ Future ouraSubstrate(String deviceId, int from, int to) async { ((temps[j]['rec_ts'] as num) - t).abs() <= c; skin.add(near ? ((temps[j]['skin_temp_c'] as num) * 100).round() : 0); } - final db = await LocalDb.instance; - final beats = await db.rawQuery( - 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' - 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' - 'ORDER BY t, beat_index', - [deviceId, kOuraFamily, from, to], - ); final rr = [ for (final b in beats) if (plausibleRrOrNull(b['rr'] as num) case final ms?) @@ -104,6 +109,10 @@ Future ouraSubstrate(String deviceId, int from, int to) async { Future _ouraTempSubstrate(String deviceId, int from, int to) async { final rows = await _ouraTempRows(deviceId, from, to); if (rows.isEmpty) return Substrate.empty; + return constructWearable(() => _ouraTempColumns(deviceId, rows)); +} + +Substrate _ouraTempColumns(String deviceId, List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index dd467784d..300c022e8 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -47,13 +47,18 @@ Future> ultrahumanRecords( int to, ) async { final db = await LocalDb.instance; - final out = []; - for (final r in await db.rawQuery( + final rows = await db.rawQuery( "SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? " "AND reason = 'ultrahuman_record' AND rec_ts >= ? AND rec_ts <= ? " 'ORDER BY rec_ts', [deviceId, from, to], - )) { + ); + return prepareWearable(() => _ultrahumanRecords(rows)); +} + +List _ultrahumanRecords(List> rows) { + final out = []; + for (final r in rows) { final hex = r['hex'] as String; final rec = parseUltrahumanRecord([ for (var i = 0; i + 1 < hex.length; i += 2) @@ -77,8 +82,13 @@ Future ultrahumanSubstrate(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kUltrahumanFamily, deviceId, from, to); if (sub.isEmpty) return sub; + final records = await ultrahumanRecords(deviceId, from, to); + return constructWearable(() => _ultrahumanSubstrate(sub, records)); +} + +Substrate _ultrahumanSubstrate(Substrate sub, List records) { final temp = { - for (final r in await ultrahumanRecords(deviceId, from, to)) + for (final r in records) if (_skinC(r) case final c?) r.tsA: (c * 100).round(), }; return Substrate( diff --git a/lib/compute/night_checkpoint.dart b/lib/compute/night_checkpoint.dart new file mode 100644 index 000000000..d30e9491a --- /dev/null +++ b/lib/compute/night_checkpoint.dart @@ -0,0 +1,115 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; + +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; + +// Format 2 carries RSA v3 and closed-bin spectral states, not saved rates. +// Older closed windows cannot be certified under the current RSA floor gate. +const nightCheckpointFormat = 2; +const maxNightCheckpointBytes = 8 * 1024 * 1024; + +/// One request's provenance, read before loading substrate. No decoded rows. +class NightCheckpointPlan { + NightCheckpointPlan(this.from, this.through, this.revisions, this.generation, + this.epoch, this.storeId); + final int from, through, generation, epoch; + final String storeId; + final Map revisions; + String? context; + int boundary = 0; + List cardio = []; + NightRrState? restored; +} + +Uint8List checkpointRevisions(Map revisions, int from, int boundary) => + Uint8List.fromList(utf8.encode(jsonEncode([ + for (var b = from ~/ 900; b < boundary ~/ 900; b++) + [b, revisions[b] ?? 0], + ]))); + +typedef NightCheckpointRead = ({NightRrState? night, Uint8List? cardio, + Map? payload, String? rejection}); +typedef NightCheckpointWrite = ({Uint8List bytes, Map payload}); + +/// Worker only. SHA-256 covers both sections and all SQL provenance. No section +/// is returned unless every section and the complete revision vector validate. +NightCheckpointRead decodeNightCheckpoint(Map row, + String day, int algorithm, String context, int from, int through, + Map revisions, {List? searchRr, List? searchTimes}) { + try { + final boundary = row['cp_rec_ts']; + if (row['day_id'] != day || row['algo_version'] != algorithm || + row['fmt'] != nightCheckpointFormat || row['ctx_sig'] != context || + row['night_ref'] != null || boundary is! int || boundary % 900 != 0 || + boundary <= from || boundary > through + 1) { + throw const FormatException('checkpoint provenance'); + } + final expectedRevisions = checkpointRevisions(revisions, from, boundary); + if (base64Encode(row['rev_vec'] as List) != base64Encode(expectedRevisions)) { + throw const FormatException('checkpoint revisions'); + } + final bytes = row['state'] as List; + if (bytes.length <= 32 || bytes.length > maxNightCheckpointBytes) { + throw const FormatException('checkpoint length'); + } + final body = bytes.sublist(32); + if (base64Encode(sha256.convert(body).bytes) != base64Encode(bytes.sublist(0, 32))) { + throw const FormatException('checkpoint checksum'); + } + final payload = (jsonDecode(utf8.decode(body)) as Map).cast(); + if (payload.length != 4 || payload['v'] != nightCheckpointFormat || + jsonEncode(payload['provenance']) != jsonEncode([ + day, algorithm, context, boundary, base64Encode(expectedRevisions), + ])) { + throw const FormatException('checkpoint envelope'); + } + final night = NightRrState.fromCheckpoint( + (payload['night'] as Map).cast(), boundary); + if (((payload['night'] as Map)['tag'] as Map)['day'] != day || + (searchRr != null && (searchTimes == null || + !night.acceptsSearchPrefix(searchRr, searchTimes, day)))) { + throw const FormatException('checkpoint raw prefix'); + } + final cardioJson = (payload['cardio'] as Map).cast(); + final cardio = ana.CardioRowCache.fromJson(cardioJson); + if (cardio.length > 8000) throw const FormatException('checkpoint cardio size'); + var outside = false; + cardio.retainWhere((k) { + if (k.startSec < from - 3600 || k.startSec + k.epochSec > boundary) outside = true; + return true; + }); + if (outside) throw const FormatException('checkpoint cardio window'); + return (night: night, + cardio: Uint8List.fromList(utf8.encode(jsonEncode(cardioJson))), + payload: payload, rejection: null); + } catch (e) { + return (night: null, cardio: null, payload: null, + rejection: 'checkpoint refused: ${e.runtimeType}'); + } +} + +/// Worker only. Oversized or absent nights stay on the ordinary cold path. +NightCheckpointWrite? encodeNightCheckpoint(NightRrState? night, + List chunks, String day, int algorithm, String context, + int from, int boundary, Uint8List revisions) { + final nightJson = night?.checkpoint(boundary); + if (nightJson == null) return null; + final cardio = decodeCardioRows(chunks); + var kept = 0; + cardio.retainWhere((k) => k.startSec >= from - 3600 && + k.startSec + k.epochSec <= boundary && kept++ < 8000); + final payload = { + 'v': nightCheckpointFormat, + 'provenance': [day, algorithm, context, boundary, base64Encode(revisions)], + 'night': nightJson, 'cardio': cardio.toJson(), + }; + final body = utf8.encode(jsonEncode(payload)); + if (body.length + 32 > maxNightCheckpointBytes) return null; + final bytes = Uint8List.fromList([...sha256.convert(body).bytes, ...body]); + return (bytes: bytes, + payload: (jsonDecode(utf8.decode(body)) as Map).cast()); +} diff --git a/lib/compute/night_prsa.dart b/lib/compute/night_prsa.dart new file mode 100644 index 000000000..14f9e2502 --- /dev/null +++ b/lib/compute/night_prsa.dart @@ -0,0 +1,48 @@ +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; + +final _dc = prsaAggregator(deceleration: true); +final _ac = prsaAggregator(deceleration: false); + +/// Settled night NN in reference order. Provisional tails only touch copies. +class NightPrsa { + NightPrsa(); + Object? _dcState = _dc.empty(), _acState = _ac.empty(); + + Map toJson() => { + 'v': 1, 'dc': _dc.encodeState(_dcState), 'ac': _ac.encodeState(_acState), + }; + + factory NightPrsa.fromJson(Map json, int fed) { + if (json['v'] != 1) throw const FormatException('PRSA version'); + final dc = json['dc'], ac = json['ac']; + if (dc is! List || ac is! List || dc.first != fed || ac.first != fed) { + throw const FormatException('PRSA feed count'); + } + return NightPrsa() + .._dcState = _dc.decodeState(dc) + .._acState = _ac.decodeState(ac); + } + + void fold(List nn) { + final column = DoubleColumn(Float64List.fromList(nn)); + _dcState = _dc.addSlice(_dcState, column, 0, column.length); + _acState = _ac.addSlice(_acState, column, 0, column.length); + } + + (Metric, Metric) snapshot(List tailNn) { + final tail = DoubleColumn(Float64List.fromList(tailNn)); + Metric finish( + Aggregator> agg, + Object? state) { + final copy = agg.decodeState(agg.encodeState(state)); + return agg.finish(agg.addSlice(copy, tail, 0, tail.length)); + } + // Sequential addSlice preserves anchor order; never combine partial sums. + // #99 prsa_aggregator.dart declares profile/capacity abs+rel 1e-9, + // confidence abs 1e-12 parity with the batch functions. + return (finish(_dc, _dcState), finish(_ac, _acState)); + } +} diff --git a/lib/compute/night_resp_bins.dart b/lib/compute/night_resp_bins.dart new file mode 100644 index 000000000..728769538 --- /dev/null +++ b/lib/compute/night_resp_bins.dart @@ -0,0 +1,109 @@ +import 'package:openstrap_analytics/onehz.dart'; + +/// Thirty-minute bins anchored at the first corrected beat, like the batch. +/// Keep closed spectral states so every snapshot applies the current night's +/// artifact gate through analytics, including to bins closed on a prior pass. +class NightRespBins { + NightRespBins(); + double? _origin; + int _openIndex = 0, _openCount = 0; + RsaRespStream _open = RsaRespStream(); + final List<({int count, RsaRespStream stream})> _closed = []; + + Map toJson() => { + 'v': 2, 'origin': _origin, 'index': _openIndex, 'count': _openCount, + 'open': _open.toJson(), + 'closed': [for (final bin in _closed) + {'count': bin.count, 'state': bin.stream.toJson()}], + }; + + factory NightRespBins.fromJson(Map json, int fed) { + final origin = json['origin'], index = json['index'], count = json['count']; + final closed = json['closed'] as List; + if (json['v'] != 2 || index is! int || index < 0 || count is! int || + count < 0 || count > fed || closed.length > fed ~/ 60 || + (fed == 0 ? origin != null : origin is! num || !origin.isFinite)) { + throw const FormatException('RSA bins'); + } + final open = (json['open'] as Map).cast(); + if (open['n'] != count) throw const FormatException('RSA bin count'); + final result = NightRespBins() + .._origin = (origin as num?)?.toDouble() + .._openIndex = index + .._openCount = count + .._open = RsaRespStream.fromJson(open); + var total = count; + for (final bin in closed) { + final n = (bin as Map)['count']; + final state = (bin['state'] as Map).cast(); + if (n is! int || n < 60 || n > fed || state['n'] != n) { + throw const FormatException('closed RSA bin count'); + } + total += n; + if (total > fed) throw const FormatException('RSA bins feed count'); + result._closed.add((count: n, stream: RsaRespStream.fromJson(state))); + } + return result; + } + + int _bin(double time, double origin) => ((time - origin) / 1800000.0).floor(); + + void _appendRate(List out, Metric metric) { + final rate = metric.present ? metric.value!.brpm : null; + if (rate != null) out.add(rate); + } + + /// Returns the number of nonempty bin slices actually folded. + int fold(List nn, List times) { + var from = 0, slices = 0; + void feed(int end) { + if (end == from) return; + _open.fold(nn.sublist(from, end), times.sublist(from, end)); + _openCount += end - from; + slices++; + } + for (var i = 0; i < times.length; i++) { + final index = _bin(times[i], _origin ??= times[i]); + if (index == _openIndex) continue; + feed(i); + if (_openCount >= 60) _closed.add((count: _openCount, stream: _open)); + _open = RsaRespStream(); + _openCount = 0; + _openIndex = index; + from = i; + } + feed(nn.length); + return slices; + } + + List snapshot(List tailNn, List tailTimes, + {required double artifactFraction}) { + if (_origin == null && tailNn.isEmpty) return []; + final origin = _origin ?? tailTimes.first; + final out = []; + for (final bin in _closed) { + _appendRate(out, bin.stream.snapshot(artifactFraction: artifactFraction)); + } + var i = 0; + while (i < tailNn.length && _bin(tailTimes[i], origin) == _openIndex) { + i++; + } + if (_openCount + i >= 60) { + _appendRate(out, _open.snapshot(tailNnMs: tailNn.sublist(0, i), + tailNnTimesMs: tailTimes.sublist(0, i), artifactFraction: artifactFraction)); + } + while (i < tailNn.length) { + final index = _bin(tailTimes[i], origin); + var end = i + 1; + while (end < tailNn.length && _bin(tailTimes[end], origin) == index) { + end++; + } + if (end - i >= 60) { + _appendRate(out, rsaRespRate(tailNn.sublist(i, end), + tailTimes.sublist(i, end), artifactFraction: artifactFraction)); + } + i = end; + } + return out; + } +} diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart new file mode 100644 index 000000000..10e4d682c --- /dev/null +++ b/lib/compute/night_rr_state.dart @@ -0,0 +1,389 @@ +import 'dart:collection'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +import 'night_prsa.dart'; +import 'night_resp_bins.dart'; + +typedef NightRr = ({List nn, List nnTimesMs, + double cleanFraction, int droppedCount, int correctedCount}); +typedef NightListProof = ({int beats, int times, (int, int) nn, (int, int) ts}); + +NightRr nightRrBatch(List rr, List times) { + final c = ana.correctRr(rr, rrTsMs: times); + return (nn: c.nn, nnTimesMs: c.nnTimesMs, cleanFraction: c.cleanFraction, + droppedCount: c.droppedCount, correctedCount: c.correctedCount); +} + +/// One RAM night, copied into a worker and returned with its result. Raw +/// prefix copies prove append-only input; settled output is never re-corrected. +class NightRrState { + NightRrState(); + static const maxRetainedBeats = 200000; + ana.RrCorrector? _corrector; + final List _rr = [], _times = [], _nn = [], _nnTimes = []; + (int, int, int)? _context; + (String, int)? _night; + final _memos = {}; + final _digests = _PassListDigests(); + List _tailNn = const [], _tailTimes = const []; + ana.HrvFreqStream? _frequency; + ana.RsaRespStream? _rsa; + ana.CvhrStream? _cvhr; + NightPrsa? _prsa; + NightRespBins? _rsaBins; + final Map _fed = {}, _processed = {}, _slices = {}, _restarts = {}; + int processedNightBeats = 0, nightRebuilds = 0; + + bool get canStream => _corrector != null; + // Settled beats represented by each stream, including compacted summaries. + int retainedStreamBeats(String key) => _fed[key] ?? 0; + int processedBeats(String key) => _processed[key] ?? 0; + int slicesProcessed(String key) => _slices[key] ?? 0; + int streamRestarts(String key) => _restarts[key] ?? 0; + int get retainedNightBeats => _rr.length; + int get settledNightBeats => _nn.length; + int get identityMemoEntries => _digests.entries; + int get fullListDigests => _digests.computed; + int get maximumListDigestsPerIdentity => _digests.maximumPerIdentity; + + // A cleared owner still records its algorithm. A transition must rebuild; + // only a fresh owner or a same-algorithm restart may adopt disk state. + bool acceptsCheckpointAlgorithm(int algorithm) => + _context == null || _context!.$3 == algorithm; + + bool acceptsPrefix(List rr, List times, String day, int onset) { + if (_night != (day, onset) || rr.length < _rr.length || + rr.length != times.length) { + return false; + } + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[i], _rr[i]) || !identical(times[i], _times[i])) { + return false; + } + } + return true; + } + + bool acceptsSearchPrefix(List rr, List times, String day) { + if (_night?.$1 != day || rr.length != times.length || _times.isEmpty) { + return false; + } + var start = 0; + while (start < times.length && times[start] < _times.first) { start++; } + if (start + _rr.length > rr.length) return false; + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[start + i], _rr[i]) || !identical(times[start + i], _times[i])) { + return false; + } + } + return true; + } + + void bindRestoredContext(int revision, int generation, int algorithm, + {NightRrState? previous}) { + _context = (revision, generation, algorithm); + // Disk state carries no work counters. Adopting it after clearing a RAM + // owner must preserve that owner's cumulative observations. + if (previous == null) return; + processedNightBeats = previous.processedNightBeats; + nightRebuilds = previous.nightRebuilds; + _processed.addAll(previous._processed); + _slices.addAll(previous._slices); + _restarts.addAll(previous._restarts); + _digests.computed = previous._digests.computed; + _digests.maximumPerIdentity = previous._digests.maximumPerIdentity; + } + + /// Worker only. A checkpoint stops before the closed input boundary. Cutting + /// a live tail requires a separate fold; it must never rewind the live owner. + Map? checkpoint(int boundary) { + if (_corrector == null || _night == null) return null; + var n = 0; + while (n < _times.length && _times[n] < boundary * 1000) { n++; } + if (n == 0) return null; + final NightRrState state; + if (n == _rr.length) { + state = this; + } else { + state = NightRrState(); + state.correct(_rr.sublist(0, n), _times.sublist(0, n), + day: _night!.$1, onsetSec: _night!.$2); + if (_frequency != null) { + final stream = state._frequency = ana.HrvFreqStream(); + state._feed('frequency', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_rsa != null) { + final stream = state._rsa = ana.RsaRespStream(); + state._feed('rsa', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_cvhr != null) { + final stream = state._cvhr = ana.CvhrStream(); + state._feed('cvhr', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_prsa != null) { + final stream = state._prsa = NightPrsa(); + state._feed('prsa', (nn, ts) { stream.fold(nn); return 1; }); + } + if (_rsaBins != null) { + final stream = state._rsaBins = NightRespBins(); + state._feed('rsa_bins', stream.fold); + } + } + return { + 'v': 1, 'tag': {'day': state._night!.$1, 'onset': state._night!.$2, + 'count': state._rr.length}, + 'rr': state._rr, 'times': state._times, + 'corrector': state._corrector!.toJson(), + 'nn': state._nn, 'nnTimes': state._nnTimes, 'fed': state._fed, + 'frequency': state._frequency?.toJson(), 'rsa': state._rsa?.toJson(), + 'cvhr': state._cvhr?.toJson(), 'prsa': state._prsa?.toJson(), + 'rsa_bins': state._rsaBins?.toJson(), + }; + } + + factory NightRrState.fromCheckpoint(Map json, int boundary) { + List doubles(Object? v) { + final list = (v as List).map((x) => (x as num).toDouble()).toList(); + if (list.length > maxRetainedBeats || list.any((x) => !x.isFinite)) { + throw const FormatException('night column'); + } + return list; + } + final tag = json['tag'] as Map; + final rr = doubles(json['rr']), times = doubles(json['times']); + final nn = doubles(json['nn']), nnTimes = doubles(json['nnTimes']); + final correctorJson = (json['corrector'] as Map).cast(); + final count = correctorJson['maxArtifactRun'] == null + ? correctorJson['n'] : (correctorJson['capped'] as Map)['n']; + if (json['v'] != 1 || tag['day'] is! String || tag['onset'] is! int || + rr.isEmpty || rr.length != times.length || rr.length != tag['count'] || + count != rr.length || nn.length != nnTimes.length || nn.length > rr.length || + times.any((t) => t >= boundary * 1000) || + correctorJson['maxArtifactRun'] != 32 || correctorJson['alpha'] != 5.2 || + correctorJson['windowBeats'] != 91 || correctorJson['minThresholdMs'] != 100 || + correctorJson['reanchorGapMs'] != 1000) { + throw const FormatException('night shape/count'); + } + for (var i = 1; i < times.length; i++) { + if (times[i] < times[i - 1]) throw const FormatException('night times'); + } + for (var i = 1; i < nnTimes.length; i++) { + if (nnTimes[i] < nnTimes[i - 1]) throw const FormatException('NN times'); + } + final state = NightRrState() + .._night = (tag['day'] as String, tag['onset'] as int) + .._corrector = ana.RrCorrector.fromJson(correctorJson) + .._rr.addAll(rr).._times.addAll(times).._nn.addAll(nn).._nnTimes.addAll(nnTimes); + final fed = (json['fed'] as Map).cast(); + const keys = ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']; + if (fed.keys.any((k) => !keys.contains(k))) { + throw const FormatException('night stream key'); + } + for (final key in keys) { + final data = json[key]; + if (data == null) { + if (fed.containsKey(key)) throw const FormatException('missing stream'); + continue; + } + if (fed[key] != nn.length) throw const FormatException('stream feed count'); + final map = (data as Map).cast(); + if (key == 'prsa') { + state._prsa = NightPrsa.fromJson(map, fed[key]!); + } else if (key == 'rsa_bins') { + state._rsaBins = NightRespBins.fromJson(map, fed[key]!); + } else { + if (map['n'] != fed[key]) throw const FormatException('stream count'); + if (key == 'frequency' && map['oversample'] != 4.0) { + throw const FormatException('frequency config'); + } + if (key == 'cvhr') { + final defaults = ana.CvhrStream().toJson(); + for (final name in ['minWidthSec', 'maxWidthSec', 'depthWidthRatioMin', + 'envWindowSec', 'maxGapSec']) { + if (map[name] != defaults[name]) throw const FormatException('CVHR config'); + } + } + if (key == 'frequency') state._frequency = ana.HrvFreqStream.fromJson(map); + if (key == 'rsa') state._rsa = ana.RsaRespStream.fromJson(map); + if (key == 'cvhr') state._cvhr = ana.CvhrStream.fromJson(map); + } + } + state._fed.addAll(fed); + return state; + } + + void beginPass() => _digests.clear(); + void endPass() => _digests.clear(); + + void context({required int inputRevision, required int generation, + required int algorithm}) { + final next = (inputRevision, generation, algorithm); + if (_context != next) clear(); + _context = next; + } + + void clear() { + _resetNight(); + _digests.clear(); + } + + void _resetNight() { + _corrector = null; + _rr.clear(); _times.clear(); _nn.clear(); _nnTimes.clear(); + _night = null; + _memos.clear(); + _tailNn = const []; _tailTimes = const []; + _frequency = null; _rsa = null; _cvhr = null; + _prsa = null; _rsaBins = null; + _fed.clear(); + } + + /// Shared by the night metric memos. No digest outlives this pass's + /// identities; only its integer words may be kept as a dependency proof. + NightListProof dependencyProof(List nn, List times) => + (beats: nn.length, times: times.length, + nn: _digests.digest(nn), ts: _digests.digest(times)); + + NightRr correct(List rr, List times, + {required String day, required int onsetSec}) { + final night = (day, onsetSec); + var held = _rr.length; + var extendsPrefix = _night == night && rr.length >= held && + times.length == rr.length; + for (var i = 0; extendsPrefix && i < held; i++) { + extendsPrefix = identical(rr[i], _rr[i]) && identical(times[i], _times[i]); + } + if (!extendsPrefix) { + _resetNight(); + held = 0; + } + // Keep malformed/oversized input on the unchanged batch path. Eviction + // affects work only, never absence rules or numeric output. + if (rr.isEmpty || times.length != rr.length || + rr.length > maxRetainedBeats) { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + _night = night; + if (_corrector == null) nightRebuilds++; + final corrector = _corrector ??= ana.RrCorrector(); + if (rr.length > held) { + final addRr = held == 0 ? rr : rr.sublist(held); + final addTimes = held == 0 ? times : times.sublist(held); + final ana.RrSettled settled; + try { + settled = corrector.fold(addRr, tsMs: addTimes); + } on ArgumentError { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + processedNightBeats += addRr.length; + _rr.addAll(addRr); _times.addAll(addTimes); + _nn.addAll(settled.nn); _nnTimes.addAll(settled.nnTimes); + } + final tail = corrector.snapshot(); + _tailNn = tail.tailNn; _tailTimes = tail.tailNnTimes; + return (nn: [..._nn, ...tail.tailNn], + nnTimesMs: [..._nnTimes, ...tail.tailNnTimes], + cleanFraction: tail.cleanFraction, droppedCount: tail.droppedCount, + correctedCount: tail.correctedCount); + } + + /// Exact result reuse. Growing lists snapshot incremental metric streams; + /// unchanged lists also avoid re-evaluating their provisional tails. + T memo(String key, Object dependencies, T Function() compute) { + if (_corrector == null) return compute(); + final old = _memos[key]; + if (old != null && old.dependencies == dependencies) return old.value as T; + final result = compute(); + if (!_memos.containsKey(key) && _memos.length >= 8) { + _memos.remove(_memos.keys.first); + } + _memos[key] = (dependencies: dependencies, value: result); + return result; + } + + void _feed(String key, int Function(List, List) fold) { + final held = _fed[key] ?? 0; + if (!_fed.containsKey(key)) { + _restarts[key] = streamRestarts(key) + 1; + _fed[key] = 0; + } + if (held == _nn.length) return; + final slices = fold(_nn.sublist(held), _nnTimes.sublist(held)); + _processed[key] = processedBeats(key) + _nn.length - held; + _slices[key] = slicesProcessed(key) + slices; + _fed[key] = _nn.length; + } + + ana.Metric nightHrvFreq(double artifactFraction) { + final stream = _frequency ??= ana.HrvFreqStream(); + _feed('frequency', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightResp(double artifactFraction) { + final stream = _rsa ??= ana.RsaRespStream(); + _feed('rsa', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightCvhr(double artifactFraction) { + final stream = _cvhr ??= ana.CvhrStream(); + _feed('cvhr', (nn, times) { stream.fold(nn, times); return 1; }); + // #99 handles a fresh stream's entirely provisional night too. + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + (ana.Metric, ana.Metric) nightPrsa() { + final stream = _prsa ??= NightPrsa(); + _feed('prsa', (nn, times) { stream.fold(nn); return 1; }); + return stream.snapshot(_tailNn); + } + + List nightRespWindows(double artifactFraction) { + final stream = _rsaBins ??= NightRespBins(); + _feed('rsa_bins', stream.fold); + return stream.snapshot(_tailNn, _tailTimes, artifactFraction: artifactFraction); + } +} + +/// Short window lists are walked directly. Only long shared lists occupy +/// identity keys, and the pipeline releases all keys in finally. +class _PassListDigests { + final HashMap, (int, int, int, int)> _byList = HashMap.identity(); + final _bits = ByteData(8); + int computed = 0, maximumPerIdentity = 0; + int get entries => _byList.length; + void clear() => _byList.clear(); + + (int, int) digest(List list) { + final hit = _byList[list]; + if (hit != null && hit.$1 == list.length) return (hit.$2, hit.$3); + var a = 0x243f6a8885a308d3, b = 0x13198a2e03707344; + for (final value in list) { + _bits.setFloat64(0, value); + final word = _bits.getInt64(0); + a = _mix(a + word + 0x9e3779b97f4a7c15); + b = _mix((b ^ word) * 0xd6e8feb86659fd93 + 0x632be59bd9b4e019); + } + computed++; + final count = (hit?.$4 ?? 0) + 1; + if (count > maximumPerIdentity) maximumPerIdentity = count; + if (list.length >= 2048) _byList[list] = (list.length, a, b, count); + return (a, b); + } + + static int _mix(int z) { + z = (z ^ (z >>> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >>> 27)) * 0x94d049bb133111eb; + return z ^ (z >>> 31); + } +} diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 9f188d117..54a946d4f 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -32,6 +32,8 @@ import 'package:openstrap_analytics/onehz.dart'; // instead of two that can drift. import 'inputs/garmin_inputs.dart' show isWearableFamily, kHrCadenceSec, kOneHzRunGapSec; +import 'ed2b_stream_diagnostics.dart'; +import 'night_rr_state.dart'; import 'hr_max.dart' show estimatedMaxHr, @@ -545,7 +547,25 @@ class DayBundleInput { /// Pure: takes the serialized [DayBundleInput] map, returns a plain JSON map (the /// full derived bundle). Call directly + synchronously in tests, or via /// `Isolate.run(() => deriveDayBundle(input))` in production. -Map deriveDayBundle(Map inputJson) { +Map deriveDayBundle(Map inputJson, { + Ed2bStreamDiagnostics? streamDiagnostics, + NightRrState? nightState, +}) { + final state = streamDiagnostics?.disabled == true + ? null : nightState ?? streamDiagnostics?.nightState; + streamDiagnostics?.beginPass(); + state?.beginPass(); + try { + final bundle = _deriveDayBundle(inputJson, state, streamDiagnostics); + streamDiagnostics?.endPass(); + return bundle; + } finally { + state?.endPass(); + } +} + +Map _deriveDayBundle(Map inputJson, + NightRrState? nightState, Ed2bStreamDiagnostics? streamDiagnostics) { final d = DayBundleInput.fromJson(inputJson); // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── @@ -591,10 +611,22 @@ Map deriveDayBundle(Map inputJson) { // HRV/RHR are rest/sleep-only per the catalog. Running correctRr+hrvTime over // the SLEEP RR (not the whole day) is what brings RMSSD back to physiological // tens-of-ms instead of the whole-day ~166 ms inflated value. - final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); + final corrected = nightState == null + ? nightRrBatch(d.sleepRrMs, d.sleepRrTsMs) + : nightState.correct(d.sleepRrMs, d.sleepRrTsMs, + day: d.date, onsetSec: d.sleepOnsetSec); + streamDiagnostics?.observeNightCorrection(d.sleepRrMs.length, corrected.nn, + corrected.nnTimesMs, corrected.cleanFraction); final nn = corrected.nn; final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); + final streams = nightState?.canStream == true ? nightState : null; + final nnProof = nightState?.dependencyProof(nn, nnTimes); + T nightMemo(String key, Object? extra, T Function() compute) => + nnProof == null ? compute() : + nightState!.memo(key, (nnProof, extra), compute); + Map nightJson(Map json) => + nightState == null ? json : _copyNightJson(json); // Σ RR ÷ wall span of the raw sleep RR: above 1.10 the stream holds more // beat-time than elapsed (duplicated or interleaved beats), and every RMSSD // below refuses it. Null under 10 min of span. @@ -653,7 +685,11 @@ Map deriveDayBundle(Map inputJson) { final sleepSessionRmssd = sleepSessionRmssdMetric.present ? sessionDetail!.rmssd : null; final hrvF = nn.length >= 20 - ? hrvFreq(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('frequency', artifactFraction, () { + if (streams != null) return streams.nightHrvFreq(artifactFraction); + streamDiagnostics?.observeBatch('frequency', nn.length); + return hrvFreq(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.high, inputs_used: const ['rr_cleaned'], @@ -706,8 +742,13 @@ Map deriveDayBundle(Map inputJson) { // HR dip: day-side = waking HR outside the sleep window; night-side = sleep HR. final dayOnly = _dayHrOutsideSleep(d); final dip = hrDip(dayOnly, sleepHr); - final dc = decelerationCapacity(nn); - final ac = accelerationCapacity(nn); + final prsa = nightMemo('prsa', null, () { + if (streams != null) return streams.nightPrsa(); + streamDiagnostics?.observeBatch('prsa', nn.length); + return (decelerationCapacity(nn), accelerationCapacity(nn)); + }); + final dc = prsa.$1; + final ac = prsa.$2; // Baevsky Stress Index over the sleep NN — resting autonomic tension (a // transparent RR-histogram metric; no ML). Daily resting-stress indicator. // nnTimesMs lets it segment at a charging/off-wrist gap instead of letting @@ -717,14 +758,22 @@ Map deriveDayBundle(Map inputJson) { // ── RESPIRATION (sleep-windowed) ─────────────────────────────────────────── final resp = nn.length >= 30 - ? rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('rsa', artifactFraction, () { + if (streams != null) return streams.nightResp(artifactFraction); + streamDiagnostics?.observeBatch('rsa', nn.length); + return rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 30), ); final cvhr = nn.length >= 60 - ? cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('cvhr', artifactFraction, () { + if (streams != null) return streams.nightCvhr(artifactFraction); + streamDiagnostics?.observeBatch('cvhr', nn.length); + return cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], @@ -749,8 +798,13 @@ Map deriveDayBundle(Map inputJson) { // ── BREATHING-RATE VARIABILITY (per-window RSA over the sleep NN) ────────── // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. - final respWindows = - respPerWindow(nn, nnTimes, artifactFraction: artifactFraction); + final cachedRespWindows = nightMemo('rsa_bins', artifactFraction, () => + streams != null ? streams.nightRespWindows(artifactFraction) : + respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, + diagnostics: streamDiagnostics)); + final respWindows = nightState == null + ? cachedRespWindows : cachedRespWindows.toList(); + streamDiagnostics?.onRespWindows?.call(respWindows); final brv = respWindows.length >= 3 ? breathingRateVariability(respWindows) : Metric.absent( @@ -1318,11 +1372,11 @@ Map deriveDayBundle(Map inputJson) { 'min_diffs_per_window': sessionDetail?.minDiffsPerWindow, }, 'rmssd_nocturnal': robustRmssd.toJson(), - 'hrv_freq': hrvF.toJson((v) => v.toJson()), + 'hrv_freq': nightJson(hrvF.toJson((v) => v.toJson())), 'resting_hr': rhr.toJson((v) => v.toJson()), 'hr_dip': dip.toJson((v) => v.toJson()), - 'prsa_dc': dc.toJson((v) => v.toJson()), - 'prsa_ac': ac.toJson((v) => v.toJson()), + 'prsa_dc': nightJson(dc.toJson((v) => v.toJson())), + 'prsa_ac': nightJson(ac.toJson((v) => v.toJson())), 'readiness_lnrmssd': lnReadiness.toJson((v) => v.toJson()), 'readiness_composite': { ...composite.toJson((v) => v.toJson()), @@ -1427,8 +1481,8 @@ Map deriveDayBundle(Map inputJson) { }; final respiration = { - 'rsa': resp.toJson((v) => v.toJson()), - 'cvhr_apnea': cvhr.toJson((v) => v.toJson()), + 'rsa': nightJson(resp.toJson((v) => v.toJson())), + 'cvhr_apnea': nightJson(cvhr.toJson((v) => v.toJson())), 'odi': odi.toJson((v) => v.toJson()), }; @@ -1958,6 +2012,7 @@ List respPerWindow( List nn, List nnTimes, { required double artifactFraction, + Ed2bStreamDiagnostics? diagnostics, double windowMs = 1800000.0, int minBeats = 60, }) { @@ -1977,12 +2032,24 @@ List respPerWindow( if (segNn.length < minBeats) continue; final r = rsaRespRate(segNn, binsTs[idx]!, artifactFraction: artifactFraction); + diagnostics?.observeBatch('rsa_bins', segNn.length); final b = r.present ? r.value!.brpm : null; if (b != null) out.add(b); } return out; } +// Metric serializers can expose lists owned by the cached value or envelope. +// Detach these small JSON blocks before returning them to a caller. +Map _copyNightJson(Map json) { + Object? copy(Object? value) { + if (value is Map) return value.map((k, v) => MapEntry(k, copy(v))); + if (value is List) return [for (final v in value) copy(v)]; + return value; + } + return json.map((k, v) => MapEntry(k, copy(v))); +} + /// Wrap a value sub-map in the {value,confidence,tier,inputs_used} envelope the /// serve seam reads via `.value`. Null inner → honest "—". Map _envelope( diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart new file mode 100644 index 000000000..67174684f --- /dev/null +++ b/lib/compute/staged_sleep_cache.dart @@ -0,0 +1,134 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'sleep_profile_policy.dart'; +import 'substrate.dart'; +import 'vendor_sleep.dart'; + +class StagedSleepEntry { + StagedSleepEntry({ + required this.algoVersion, + required this.key, + required this.mayFold, + required this.candidateJson, + required this.observationJson, + }); + + int algoVersion; + final String key; + final bool mayFold; + final String candidateJson; + final String? observationJson; +} + +/// Per-engine RAM only. One worker result per day, least recently used first. +class StagedSleepCache { + final _byDay = {}; + int _maxDays = 14; + + int get size => _byDay.length; + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_byDay.length > _maxDays) { + _byDay.remove(_byDay.keys.first); + } + } + + StagedSleepEntry? lookup(String day, int algoVersion, String key, + {required bool mayFold}) { + final entry = _byDay[day]; + if (entry == null || entry.algoVersion != algoVersion || entry.key != key || + (mayFold && !entry.mayFold)) { + return null; + } + _byDay.remove(day); + _byDay[day] = entry; + return entry; + } + + void store(String day, StagedSleepEntry entry) { + _byDay.remove(day); + if (_maxDays == 0) return; + _byDay[day] = entry; + maxDays = _maxDays; + } + + void rekey(int version) { + for (final entry in _byDay.values) { + entry.algoVersion = version; + } + } +} + +/// Revisions are captured before loading rows. A concurrent write then makes +/// the next request miss. No sample columns are walked on the UI isolate. +String stagingInputKey({ + required int fromSec, + required int toSec, + required List> revisions, + required Substrate sub, + required Map> ownership, + required Map> priority, + required SleepWindowOverride? override, + required String? profileJson, + required List<({int startSec, int endSec, String dayKey})> priorSleep, + required List vendorNights, + required (String, String)? activeWearable, + bool prefixContext = false, +}) { + // Count local midnights, including DST days, rather than dividing by 86400. + var walked = 0; + if (sub.tsSec.isNotEmpty) { + final first = DateTime.fromMillisecondsSinceEpoch(sub.tsSec.first * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= sub.tsSec.last; + d = DateTime(d.year, d.month, d.day + 1)) { + walked++; + } + } + final effectiveHistory = + {for (final h in priorSleep) h.dayKey}.length + walked < 14 + ? const <({int startSec, int endSec, String dayKey})>[] + : priorSleep; + + // Fold bookkeeping is re-read by the caller. Only a warmed-up profile can + // reach staging, but all of its fields (including updated_at_ms) are keyed. + Map? blendedProfile; + if (profileJson != null) { + final p = (jsonDecode(profileJson) as Map).cast(); + if (SleepProfilePolicy.shouldBlend((p['nights'] as num?)?.toInt())) { + blendedProfile = {...p}..remove(SleepProfilePolicy.foldedDaysKey); + } + } + + // Local day boundaries and historical midsleeps read offsets at their own + // instants. Key those too: equal current zone names/offsets are insufficient. + final zoneOffsets = >[]; + for (var t = fromSec - 86400; t <= toSec + 86400; t += 900) { + zoneOffsets.add([t, tzOffsetSecondsAt(t)]); + } + for (final h in effectiveHistory) { + final mid = (h.startSec + h.endSec) ~/ 2; + zoneOffsets.add([mid, tzOffsetSecondsAt(mid)]); + } + final now = DateTime.now(); + return jsonEncode([ + fromSec, toSec, prefixContext ? const [] : revisions, + [for (final sig in InputSignal.values) [sig.name, + [for (final o in ownership[sig] ?? const []) + [o.start, o.end, o.deviceId]], priority[sig]]], + sub.deviceFamily, (sub.deviceIds.toList()..sort()), + if (prefixContext) ...[null, null, null, null] + else ...[sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length], + now.timeZoneName, now.timeZoneOffset.inSeconds, zoneOffsets, + override == null ? null : + [override.dayId, override.onsetSec, override.offsetSec, override.source], + blendedProfile, + [for (final h in effectiveHistory) [h.startSec, h.endSec, h.dayKey]], + activeWearable == null ? null : [activeWearable.$1, activeWearable.$2], + [for (final n in vendorNights) [n.deviceId, n.source, n.decodedAtSec, + [for (final e in n.epochs) [e.startSec, e.endSec, e.stage]]]], + ]); +} diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 08d8664f3..8e1403f06 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -621,15 +621,27 @@ class Substrate { 'device_ids': deviceIds.toList(), }; - static Substrate fromJson(Map m) { - List ints(Map m, String k) => - ((m[k] as List?) ?? const []).map((e) => (e as num).toInt()).toList(); - // Straight into a Float64List — see [_packed]. This runs on the RECEIVING - // (main) isolate for every substrate handed back from a worker, so the - // growable intermediate was ~400 000 boxes allocated and immediately - // thrown away, on the UI thread. + /// Adopt the columns received from a worker. Packing/copies happen in the + /// worker or isolate transport; correctly typed arrays need no element walk + /// here. Loose numeric lists retain the same normalization as [fromJson]. + static Substrate fromTransfer(Map m) => + _fromColumns(m, adopt: true); + + static Substrate fromJson(Map m) => + _fromColumns(m, adopt: false); + + static Substrate _fromColumns(Map m, {required bool adopt}) { + List ints(Map m, String k) { + final src = m[k]; + if (adopt && src is List) return src; + return ((src as List?) ?? const []) + .map((e) => (e as num).toInt()).toList(); + } + // Convert loose lists straight into packed doubles, without a boxed + // intermediate. The worker already packs these through the constructor. Float64List dbls(String k) { final src = (m[k] as List?) ?? const []; + if (adopt && src is Float64List) return src; final out = Float64List(src.length); for (var i = 0; i < src.length; i++) { out[i] = (src[i] as num).toDouble(); @@ -1208,6 +1220,11 @@ class SleepWindowOverride { String localDateLabel(int epochSec) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); +// Walk controls: the counter observes the actual segmentation body; the full +// walk remains available to check the restricted walk's target output. +int debugSegmentedDayCount = 0; +bool debugStageEveryWalkedDay = false; + /// Split the substrate into CALENDAR days (local midnight → next local midnight). /// /// Each day owns its 24 h of data. A day's SLEEP is the main sleep that ENDED @@ -1237,12 +1254,23 @@ String localDateLabel(int epochSec) => /// always fell back to the fixed 03:30 cold-start anchor, and a night-shift /// sleeper's 4 h main block lost the alignment bonus to a shorter block nearer /// 03:30. Days found in THIS call still win (a full restage sees them all). +/// +/// With [skipUnusedDays], only [targetDay] is needed. Earlier days can be +/// skipped when even one new history key per day cannot reach the prior's +/// 14-night threshold. Callers needing profile observations can use +/// [recordTargetOnly] to record the target night even during a full walk. List calendarDays( Substrate sub, { SleepWindowOverride? override, int Function(int epochSec)? tzOffsetAt, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + String? targetDay, + bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + // D36: other walked nights must not train under the target's ID. + // Restore the caller's recording flag on every exit. + bool recordTargetOnly = false, }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; @@ -1256,6 +1284,23 @@ List calendarDays( ...priorSleep, ]; var dayStart = _localMidnight(dataStart); + final prunable = targetDay != null && + skipUnusedDays && + !debugStageEveryWalkedDay && + (override == null || override.dayId == targetDay); + var skipBefore = false; + if (prunable) { + var earlier = 0; + for (var d = dayStart, g = 0; + d < dataEnd && g++ < 400 && localDateLabel(d).compareTo(targetDay) < 0; + d = _nextLocalMidnight(d)) { + earlier++; + } + skipBefore = {for (final h in sleepHistory) h.dayKey}.length + earlier < 14; + } + final recording = ana.cardioRecordObservations; + final onlyTarget = recordTargetOnly && targetDay != null; + try { var guard = 0; while (dayStart < dataEnd && guard++ < 400) { final dayEnd = _nextLocalMidnight(dayStart); @@ -1266,6 +1311,17 @@ List calendarDays( continue; } + final dayLabel = localDateLabel(dayStart); + if (prunable) { + final order = dayLabel.compareTo(targetDay); + if (order > 0) break; + if (order < 0 && skipBefore) { + dayStart = dayEnd; + continue; + } + } + debugSegmentedDayCount++; + if (onlyTarget) ana.cardioRecordObservations = recording && dayLabel == targetDay; // The main sleep that ENDS in this calendar day: search from the previous // local noon through this local midnight, then let the sleep selector pick // the overnight main block from any naps / split fragments it sees. The old @@ -1281,7 +1337,6 @@ List calendarDays( var sleepLo = 0, sleepHi = 0; var sleepSource = 'none'; final vendorIgnored = []; - final dayLabel = localDateLabel(dayStart); // Does the user have an override (manual / confirmed) for THIS day? final ov = (override != null && override.dayId == dayLabel) ? override : null; @@ -1339,6 +1394,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: ov.onsetSec, offsetSec: ov.offsetSec), @@ -1362,6 +1418,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, habitualMidsleepSec: habitualMidsleepSec, @@ -1395,6 +1452,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: @@ -1429,6 +1487,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: vn.onsetSec, offsetSec: vn.offsetSec), @@ -1534,6 +1593,9 @@ List calendarDays( )); dayStart = dayEnd; } + } finally { + ana.cardioRecordObservations = recording; + } return days; } diff --git a/lib/compute/substrate_retention.dart b/lib/compute/substrate_retention.dart new file mode 100644 index 000000000..2cfdd50fe --- /dev/null +++ b/lib/compute/substrate_retention.dart @@ -0,0 +1,111 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; + +/// One exact load, plus the worker metadata needed to rebuild its open bucket. +class RetainedSubstrate { + RetainedSubstrate({required this.version, required this.context, + required this.revisions, required this.sub, required this.extras, + required this.cut, this.from, this.to, this.generation, + this.ownership = const {}}); + + int version; + final String context; + final Map revisions; + final Substrate sub; + final Map extras; + final int cut; + final int? from, to, generation; + final Map> ownership; + + /// A changed closed bucket requires a cold load. Otherwise rebuild the tail + /// from [cut], or return the whole load when every bucket is unchanged. + int? readFrom(int version, String context, Map revisions, int to) { + if (version != this.version || context != this.context) return null; + var changed = false; + for (final b in {...this.revisions.keys, ...revisions.keys}) { + if ((revisions[b] ?? 0) == (this.revisions[b] ?? 0)) continue; + if (b < cut ~/ 900) return null; + changed = true; + } + return changed ? cut : to + 1; + } + + Map seed(int cut) => { + ...sub.toJson(), ...extras, 'type': 'seed', 'cut': cut, + }; +} + +/// At most three loads per day: separate windows, or their shared union. +/// Both days and loads are ordered least recently used first. +class SubstrateRetention { + final _days = >{}; + int _maxDays = 2; + + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_days.length > _maxDays) { + _days.remove(_days.keys.first); + } + } + Set get days => _days.keys.toSet(); + Iterable get unions sync* { + for (final loads in _days.values) { + final union = loads['union']; + if (union != null) yield union; + } + } + + RetainedSubstrate? lookup(String day, String load) { + final loads = _days.remove(day); + if (loads == null) return null; + _days[day] = loads; + final entry = loads.remove(load); + if (entry != null) loads[load] = entry; + return entry; + } + + void store(String day, String load, RetainedSubstrate entry) { + if (_maxDays == 0) return; + final loads = _days.remove(day) ?? {}; + loads.remove(load); + loads[load] = entry; + while (loads.length > 3) { + loads.remove(loads.keys.first); + } + _days[day] = loads; + maxDays = _maxDays; + } + + Map extras(String day) => + _days[day]?.values.last.extras ?? const {}; + void remove(String day) => _days.remove(day); + void clear() => _days.clear(); + void retainDays(Set days) => + _days.removeWhere((day, _) => !days.contains(day)); + void rekey(int version) { + for (final loads in _days.values) { + for (final entry in loads.values) { + entry.version = version; + } + } + } +} + +/// Only inputs consumed by the decoded loader belong here. Resolved spans +/// already carry priority, coverage and device eligibility decisions. +String substrateLoadContext(int from, int to, int generation, + Map> ownership) { + final now = DateTime.now(); + return jsonEncode([ + from, to, generation, now.timeZoneName, now.timeZoneOffset.inSeconds, + [for (var t = from - 86400; t <= to + 86400; t += 900) + [t, tzOffsetSecondsAt(t)]], + [for (final sig in InputSignal.values) [sig.name, + [for (final s in ownership[sig] ?? const []) + [s.start, s.end, s.deviceId]]]], + ]); +} diff --git a/lib/compute/substrate_union.dart b/lib/compute/substrate_union.dart new file mode 100644 index 000000000..c52be1148 --- /dev/null +++ b/lib/compute/substrate_union.dart @@ -0,0 +1,271 @@ +import 'dart:typed_data'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; +import 'substrate_retention.dart'; + +typedef SubstrateUnionWindow = ({int from, int to, + Map> ownership}); + +/// Compare the loader's decisions, including null-owner (unfiltered) spans. +/// Wider resolver windows can seed hysteresis differently. +bool sameSubstrateOwnership(Map> a, + Map> b, int from, int to) { + List clip(List spans) { + final out = []; + for (final s in spans) { + if (s.deviceId == null || s.end <= from || s.start > to) continue; + final lo = s.start < from ? from : s.start; + final hi = s.end > to + 1 ? to + 1 : s.end; + if (out.isNotEmpty && out.last.end == lo && + out.last.deviceId == s.deviceId) { + out[out.length - 1] = + (start: out.last.start, end: hi, deviceId: s.deviceId); + } else { + out.add((start: lo, end: hi, deviceId: s.deviceId)); + } + } + return out; + } + for (final signal in InputSignal.values) { + final x = clip(a[signal] ?? const []), y = clip(b[signal] ?? const []); + if (x.length != y.length) return false; + for (var i = 0; i < x.length; i++) { + if (x[i] != y[i]) return false; + } + } + return true; +} + +int _lowerBound(List ts, int value) { + var lo = 0, hi = ts.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if (ts[mid] < value) { lo = mid + 1; } else { hi = mid; } + } + return lo; +} + +List> _runs(RetainedSubstrate load) => + (load.extras['provenance'] as List).cast>(); +String? _unit(List> runs) { + for (final r in runs) { + if (r[3] != null) return r[3] as String; + } + return null; +} +(Set, Set) _sources(List> runs) { + final families = {}, ids = {}; + for (final r in runs) { + if (r[1] != null) families.add(r[1] as String); + if (r[2] != null) ids.add(r[2] as String); + } + return (families, ids); +} + +/// Worker-only: reproduce a decoded load by record second, restarting the +/// beat running maximum from the original placements. Substrate.slice instead +/// selects beats by their repaired time and keeps the parent's provenance. +RetainedSubstrate? cutSubstrateUnion(RetainedSubstrate load, int from, int to, + Map> ownership, {bool retainMetadata = true}) { + if (load.from == null || from < load.from! || to > load.to! || to < from || + load.extras['provenance'] == null || + !sameSubstrateOwnership(load.ownership, ownership, from, to)) { + return null; + } + final s = load.sub, n = s.length; + final starts = load.extras['rr_start'] as List; + final raw = load.extras['rr_ts_raw'] as List; + if (starts.length != n || raw.length != s.rrMs.length) return null; + final lo = _lowerBound(s.tsSec, from), hi = _lowerBound(s.tsSec, to + 1); + final r0 = lo < n ? starts[lo] : s.rrMs.length; + final r1 = hi < n ? starts[hi] : s.rrMs.length; + final runs = _runs(load), selected = >[]; + for (var i = 0; i < runs.length; i++) { + final start = runs[i][0] as int; + final end = i + 1 < runs.length ? runs[i + 1][0] as int : n; + if (end <= lo || start >= hi) continue; + selected.add([start < lo ? 0 : start - lo, ...runs[i].skip(1)]); + } + // The union has already suppressed other temperature units. A later + // window that would choose another unit must take the ordinary loader. + final unit = _unit(selected); + if (unit != null && unit != _unit(runs)) return null; + final (families, ids) = _sources(selected); + final axis = Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0); + monotonizeBeatAxis(axis); + final columns = s.toJson(); + final out = { + for (final key in _intColumns) + key: (columns[key] as List).sublist(lo, hi), + for (final key in _motionColumns) + key: (columns[key] as List).sublist(lo, hi), + 'rr_ts_ms': axis, 'rr_ms': s.rrMs.sublist(r0, r1), + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList(), + }; + return RetainedSubstrate(version: load.version, context: load.context, + revisions: load.revisions, from: from, to: to, + generation: load.generation, ownership: ownership, + sub: Substrate.fromTransfer(out), cut: from, + extras: !retainMetadata ? const {} : { + 'rr_start': Int32List.fromList([ + for (var i = lo; i < hi; i++) starts[i] - r0]), + 'rr_ts_raw': Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0), + 'provenance': selected, + 'meta': [for (final m in load.extras['meta'] as List) + if ((m as List)[0] as int >= from ~/ 900 && + m[0] as int <= to ~/ 900) m], + 'raw_rows_by_bucket': { + for (final e in ((load.extras['raw_rows_by_bucket'] as Map?) ?? + const {}).entries) + if (e.key >= from ~/ 900 && e.key <= to ~/ 900) e.key: e.value}, + }); +} + +const _intColumns = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _motionColumns = ['ax', 'ay', 'az']; + +/// Worker-only: join disjoint loads and repair their raw beat axis once across +/// the seams. Refuse independently selected temperature units that cannot be +/// reconstructed from already suppressed columns. +RetainedSubstrate? joinSubstrateUnions(List parts, + String context, Map revisions, + Map> ownership) { + if (parts.isEmpty) return null; + final units = {for (final p in parts) _unit(_runs(p))}..remove(null); + if (units.length > 1) return null; + var n = 0, beats = 0; + for (var i = 0; i < parts.length; i++) { + if (i > 0 && parts[i].from != parts[i - 1].to! + 1) return null; + n += parts[i].sub.length; + beats += parts[i].sub.rrMs.length; + } + final columns = { + for (final key in _intColumns) key: Int64List(n), + for (final key in _motionColumns) key: Float64List(n), + 'rr_ms': Float64List(beats), + }; + final raw = Float64List(beats), starts = Int32List(n); + final runs = >[], meta = >{}; + final rawRows = {}; + var at = 0, rat = 0; + for (final p in parts) { + final source = p.sub.toJson(), pn = p.sub.length, rn = p.sub.rrMs.length; + for (final key in [..._intColumns, ..._motionColumns]) { + (columns[key] as List).setRange(at, at + pn, source[key] as List); + } + (columns['rr_ms'] as List).setRange(rat, rat + rn, p.sub.rrMs); + raw.setRange(rat, rat + rn, p.extras['rr_ts_raw'] as List); + final rs = p.extras['rr_start'] as List; + for (var i = 0; i < pn; i++) { starts[at + i] = rs[i] + rat; } + for (final r in _runs(p)) { + final last = runs.isEmpty ? null : runs.last; + if (last == null || last[1] != r[1] || last[2] != r[2] || last[3] != r[3]) { + runs.add([(r[0] as int) + at, ...r.skip(1)]); + } + } + for (final m in p.extras['meta'] as List) { + final wire = m as List; + final target = meta.putIfAbsent(wire[0] as int, + () => [wire[0], {}, {}, null]); + (target[1] as Set).addAll((wire[1] as List).cast()); + (target[2] as Set).addAll((wire[2] as List).cast()); + target[3] ??= wire[3]; + } + for (final e in (p.extras['raw_rows_by_bucket'] as Map).entries) { + rawRows[e.key] = (rawRows[e.key] ?? 0) + e.value; + } + at += pn; rat += rn; + } + final axis = Float64List.fromList(raw); + monotonizeBeatAxis(axis); + final (families, ids) = _sources(runs); + columns.addAll({'rr_ts_ms': axis, + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList()}); + return RetainedSubstrate(version: parts.first.version, context: context, + revisions: revisions, from: parts.first.from, to: parts.last.to, + generation: parts.first.generation, ownership: ownership, + sub: Substrate.fromTransfer(columns), cut: parts.first.from!, + extras: {'rr_start': starts, 'rr_ts_raw': raw, 'provenance': runs, + 'raw_rows_by_bucket': rawRows, + 'meta': [for (final m in meta.values) + [m[0], (m[1] as Set).toList(), (m[2] as Set).toList(), m[3]]], + }); +} + +/// Revision/ownership proofs are small; column cuts and joins run in workers. +List<(int, int, RetainedSubstrate)> reusableUnionStretches( + Iterable bases, int from, int to, + Map revisions, Map> ownership) { + final candidates = <(int, int, RetainedSubstrate)>[]; + for (final base in bases) { + final lo = from > base.from! ? from : base.from!; + final hi = to < base.to! ? to : base.to!; + int? start; + void close(int end) { + if (start == null) return; + final a = start! < lo ? lo : start!; + final b = end > hi ? hi : end; + start = null; + if (b < a || (a != base.from && a % 900 != 0) || + (b != base.to && (b + 1) % 900 != 0)) { return; } + if (sameSubstrateOwnership(base.ownership, ownership, a, b)) { + candidates.add((a, b, base)); + } + } + for (var b = lo ~/ 900; b <= hi ~/ 900; b++) { + if ((revisions[b] ?? 0) == (base.revisions[b] ?? 0)) { + start ??= b * 900; + } else { close(b * 900 - 1); } + } + close(hi); + } + final out = <(int, int, RetainedSubstrate)>[]; + var pos = from; + while (pos <= to) { + (int, int, RetainedSubstrate)? best; + int? next; + for (final c in candidates) { + if (c.$1 <= pos && c.$2 >= pos && + (best == null || c.$2 > best.$2)) { best = c; } + if (c.$1 > pos && (next == null || c.$1 < next)) next = c.$1; + } + if (best == null) { + if (next == null) break; + pos = next; + } else { + if (pos != best.$3.from && pos % 900 != 0) { + pos = (pos ~/ 900 + 1) * 900; + continue; + } + out.add((pos, best.$2, best.$3)); + pos = best.$2 + 1; + } + } + return out; +} + +// Factories keep dispatch closures in a scope containing only plain inputs. +List Function() borrowUnionJob( + List<(int, int, RetainedSubstrate)> stretches, + Map> ownership) => () => [ + for (final s in stretches) cutSubstrateUnion(s.$3, s.$1, s.$2, ownership)]; + +RetainedSubstrate? Function() joinUnionJob(List parts, + String context, Map revisions, + Map> ownership) => () => + joinSubstrateUnions(parts, context, revisions, ownership); + +(List, Substrate?) Function() cutUnionJob(RetainedSubstrate load, + List<(int, int)> ranges, Map> ownership, + int? dayStart, int? dayEnd) => () { + final windows = [for (final r in ranges) + cutSubstrateUnion(load, r.$1, r.$2, ownership, retainMetadata: false)?.sub]; + final day = dayStart == null || dayEnd == null ? null : + windows.first?.slice(dayStart, dayEnd); + return (windows, day); +}; diff --git a/lib/data/db.dart b/lib/data/db.dart index 578df4b9c..8d7d3cb61 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -33,6 +33,7 @@ import '../coach/coach_db.dart' show CoachDb; import '../compute/derivation_engine.dart' show kAlgoVersion, + runCancellableIsolate, kOvernightGiveUpSec, kPinWakeToleranceSec, pinnableNight, @@ -64,6 +65,9 @@ import 'observation.dart'; import 'series_codec.dart'; import 'substrate_archive.dart'; +part 'db_input_revisions.dart'; +part 'db_session_ceiling_revisions.dart'; + /// The outcome of a database rebuild: why the old file would not open, where it /// was parked, and how many rows came back per table. typedef DbRebuild = ({ @@ -72,6 +76,11 @@ typedef DbRebuild = ({ Map salvaged, }); +typedef SessionHrReads = ({ + Map> stats, + Map> samples, +}); + // ── SUBSTRATE ADMISSION — the one predicate that decides what may become a // number ─────────────────────────────────────────────────────────────────── // @@ -166,7 +175,13 @@ String derivableSourceSql([String col = 'source']) => kDerivableSources.isEmpty /// ponytail: on that gap the workout screen and the day disagree; give the /// band day the strap's window too (or drop the override there) once the /// owner decides whether a strap may fill a band gap. -List sessionWindowRows>(List rows) { +List sessionWindowRows>(List rows) => + _sessionWindowRowsFor(rows, sensors: LocalDb.sessionSensorSources, + wearable: LocalDb.sessionWearableSource); + +// Workers receive the flags explicitly; isolate-local statics start empty. +List _sessionWindowRowsFor>( + List rows, {required Set sensors, required String? wearable}) { bool derivable(Object? s) => s == null || kDerivableSources.contains(s); List plain() => [ for (final r in rows) @@ -177,7 +192,7 @@ List sessionWindowRows>(List rows) { for (final r in rows) { final s = r['source']; if (s == null) return plain(); - if (LocalDb.sessionSensorSources.contains(s) && worn(r)) { + if (sensors.contains(s) && worn(r)) { n[s] = (n[s] ?? 0) + 1; } } @@ -195,7 +210,6 @@ List sessionWindowRows>(List rows) { if (lo == null || t < lo) lo = t; if (hi == null || t > hi) hi = t; } - final wearable = LocalDb.sessionWearableSource; return [ for (final r in rows) if (r['source'] == owner || @@ -479,7 +493,46 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 58; + static const int schemaVersion = 60; + + /// In-memory cache identity, advanced after open, wipe or committed import. + /// It is local to this isolate; no cache survives a process restart. + static int _storeGeneration = 0; + static int get storeGeneration => _storeGeneration; + + static void _markStoreReplaced() => _storeGeneration++; + + /// Revisions over every absolute 15-minute bucket intersecting the local + /// calendar day. Legacy/missing buckets read zero without a raw-row backfill. + /// RR-only days participate, and all devices share these conservative buckets. + static Future<({int revisionSum, int revisionMax, int generation})> + inputDayFingerprint(String dayId) async { + final (lo, hi) = _localDayWindow(dayId); + final db = await instance; + final generation = storeGeneration; + final rows = await db.rawQuery( + 'SELECT COALESCE(SUM(rev), 0) AS revision_sum, ' + 'COALESCE(MAX(rev), 0) AS revision_max FROM input_rev ' + 'WHERE bucket >= ? AND bucket <= ?', + [lo ~/ 900, (hi - 1) ~/ 900], + ); + return ( + revisionSum: (rows.single['revision_sum'] as num).toInt(), + revisionMax: (rows.single['revision_max'] as num).toInt(), + generation: generation, + ); + } + + /// Session identity/window writes invalidate ceiling reuse, including an + /// identical REPLACE or UPDATE. Derived score writes do not affect it. + static Future sessionCeilingRevision() async { + final rows = await (await instance).query('session_ceiling_rev'); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); + } + + @visibleForTesting + static Future repairOpenSchemaForTest(Database db) => + _repairOpenSchema(db); /// SQLite caps host parameters per statement (`SQLITE_MAX_VARIABLE_NUMBER` — /// only 999 on the builds shipped with older Android/iOS). Any `IN (?, ?, …)` @@ -540,7 +593,7 @@ class LocalDb { static Future _open() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); - return openDatabase( + final opened = await openDatabase( path, onConfigure: (db) async { // WRITE PERFORMANCE. The default rollback journal (journal_mode=delete) @@ -1249,6 +1302,16 @@ class LocalDb { // A band's own hypnogram (`vendor_staged`). New table only. await _createVendorSleepEpoch(db); } + if (oldV < 59) { + // Additive revision counters only: no decoded scan or backfill. + // Attach after any earlier rung that rebuilds decoded tables. + await _createInputRev(db); + await _createSessionCeilingRev(db); + } + if (oldV < 60) { + await _createStoreIdentity(db); + await _createDayCheckpoint(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -1264,6 +1327,8 @@ class LocalDb { }, version: schemaVersion, ); + _markStoreReplaced(); + return opened; } static Future _repairOpenSchema(Database db) async { @@ -1366,12 +1431,76 @@ class LocalDb { await _addColumnIfMissing(db, 'device', 'battery_ts', 'INTEGER'); await _createStepCalibration(db); await _ensureLiveStepCalibrationCounterVersion(db); + // A decoded-table rebuild above drops its triggers; attach them last. + await _createInputRev(db); + await _createSessionCeilingRev(db); + await _createStoreIdentity(db); + await _createDayCheckpoint(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); } + // Stable across opens, unique to this file's current input history. Neither + // restore nor salvage imports it: input_rev is local and can repeat on reset. + static Future _createStoreIdentity(DatabaseExecutor db) async { + await db.execute('CREATE TABLE IF NOT EXISTS store_identity (' + 'singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL)'); + await db.execute('INSERT OR IGNORE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + } + + static Future storeIdentity({DatabaseExecutor? executor}) async => + (await (executor ?? await instance).rawQuery( + 'SELECT id FROM store_identity WHERE singleton = 1')).single['id'] as String; + + static Future _rotateStoreIdentity(DatabaseExecutor db) async { + await db.execute('INSERT OR REPLACE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + await db.delete('day_checkpoint'); + } + + static Future _createDayCheckpoint(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS day_checkpoint (' + 'day_id TEXT NOT NULL, algo_version INTEGER NOT NULL, ' + 'fmt INTEGER NOT NULL, ctx_sig TEXT NOT NULL, cp_rec_ts INTEGER NOT NULL, ' + 'rev_vec BLOB NOT NULL, state BLOB NOT NULL, night_ref TEXT, ' + 'computed_at INTEGER NOT NULL, PRIMARY KEY(day_id, algo_version))'); + } + + /// Chunk the disposable blob like substrate_archive: a whole night can + /// exceed Android's cursor window. A racing replacement fails codec checks. + static Future?> dayCheckpoint( + String day, int algorithm, {required int maxBytes}) async { + final db = await instance; + const where = 'day_id = ? AND algo_version = ?'; + final rows = await db.rawQuery('SELECT day_id, algo_version, fmt, ctx_sig, ' + 'cp_rec_ts, rev_vec, night_ref, computed_at, length(state) AS state_length ' + 'FROM day_checkpoint WHERE $where', [day, algorithm]); + if (rows.isEmpty) return null; + final row = Map.of(rows.single); + final length = row.remove('state_length') as int; + final bytes = Uint8List(length <= maxBytes ? length : 0); + row['state'] = bytes; + if (length > maxBytes) return row; + for (var off = 0; off < length; off += 128 * 1024) { + final parts = await db.rawQuery( + 'SELECT substr(state, ?, ?) AS part FROM day_checkpoint WHERE $where', + [off + 1, 128 * 1024, day, algorithm]); + if (parts.isEmpty) return null; + final part = parts.single['part'] as Uint8List; + if (part.length > length - off) return null; + bytes.setRange(off, off + part.length, part); + } + return row; + } + + /// One revision counter per absolute 15-minute input bucket, shared by the + /// canonical 1 Hz rows and sparse beats. Triggers cover every write path and + /// roll back with the input transaction. No raw rows are read at creation. + static Future _createInputRev(Database db) => _ldbCreateInputRev(db); + /// Older WHOOP CSV imports filed the export's absolute skin temp (°C) under /// `scalars.skin_temp_z`, which every reader treats as SDs off baseline, so /// 33 °C read as +33 SD in the cross-day temp inputs and the skin temp card. @@ -5567,6 +5696,10 @@ class LocalDb { '''); } + // Schema 59 and every-open repair; no session scan or backfill. + static Future _createSessionCeilingRev(Database db) => + _ldbCreateSessionCeilingRev(db); + static Future _createSyncState(DatabaseExecutor db) async { await db.execute(''' CREATE TABLE IF NOT EXISTS sync_ledger ( @@ -11042,6 +11175,9 @@ class LocalDb { } await db.transaction((txn) async { + // Scratch prefixes can include an evening owned by another day label. + // Clear every copy atomically with deletion so deleted raw stays deleted. + await _rotateStoreIdentity(txn); // `raw_archive` frames are kept, but a later archive re-drive must not // decode them back into a day the user deleted: record the windows. final prevRow = await txn.query('compute_freshness', @@ -11199,7 +11335,9 @@ class LocalDb { final db = await instance; final rows = await db.rawQuery( "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'", + "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata' " + // Clear revisions after the decoded/session DELETE triggers have finished. + "ORDER BY name IN ('input_rev', 'session_ceiling_rev')", ); var deleted = 0; await db.transaction((txn) async { @@ -11207,7 +11345,9 @@ class LocalDb { final t = r['name']; if (t is String) deleted += await txn.delete(t); } + await _rotateStoreIdentity(txn); }); + _markStoreReplaced(); return deleted; } @@ -11588,7 +11728,10 @@ class LocalDb { /// file saved mid-move). Every row is rebuilt through [rawArchiveRow], so a /// frame already here, or in the backup twice, lands once. Paged on rowid, /// like every other table the merge reads. Returns rows offered. - static Future _mergeRawArchive(Database src, Database db) async { + static Future _mergeRawArchive(Database src, Database db, { + required Future Function(Transaction) beforePageCommit, + required void Function() onPageCommitted, + }) async { var copied = 0; Future pageFrom(String table, String hexSql) async { if (await _objectType(src, table) != 'table') return; @@ -11607,26 +11750,30 @@ class LocalDb { ); if (page.isEmpty) return; after = (page.last['_r'] as num).toInt(); - final batch = db.batch(); - for (final r in page) { - final hex = r['hex']; - if (hex is! String) continue; - batch.insert( - rawArchiveTable, - rawArchiveRow( - deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, - hex: hex, - counter: (r['counter'] as num?)?.toInt(), - packetType: (r['packet_type'] as num?)?.toInt() ?? 0, - recTs: (r['rec_ts'] as num?)?.toInt(), - capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, - reason: r['reason'] as String? ?? 'unknown', - ), - conflictAlgorithm: ConflictAlgorithm.ignore, - ); - copied++; - } - await batch.commit(noResult: true); + await db.transaction((txn) async { + await beforePageCommit(txn); + final batch = txn.batch(); + for (final r in page) { + final hex = r['hex']; + if (hex is! String) continue; + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: hex, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + copied++; + } + await batch.commit(noResult: true); + }); + onPageCommitted(); } } @@ -11661,6 +11808,34 @@ class LocalDb { String path, { List? only, bool tolerant = false, + }) async { + var pageCommitted = false; + try { + return await _mergeFromDbFileBody(path, + only: only, + tolerant: tolerant, + beforePageCommit: (txn) async { + if (!pageCommitted) await _rotateStoreIdentity(txn); + }, + onPageCommitted: () => pageCommitted = true, + ); + } finally { + // A later page, source close or follow-up write can fail after a durable + // replacement. Invalidate on every such exit, but never on a rolled-back + // first page. Keep the flag local so concurrent merges cannot share it. + if (pageCommitted) { + _markStoreReplaced(); + await (await instance).delete('day_checkpoint'); + } + } + } + + static Future> _mergeFromDbFileBody( + String path, { + List? only, + bool tolerant = false, + required Future Function(Transaction) beforePageCommit, + required void Function() onPageCommitted, }) async { final db = await instance; final src = await openDatabase(path, readOnly: true); @@ -11934,6 +12109,7 @@ class LocalDb { for (final upper in ends) { final first = lower == 0; await db.transaction((txn) async { + await beforePageCommit(txn); if (t == 'day_result' && first) { // Inside the first chunk's transaction, so a day the engine // finalizes while the restore runs is still protected. @@ -11979,6 +12155,7 @@ class LocalDb { await txn.execute('DROP TABLE temp._restore_rr_carried'); } }); + onPageCommitted(); lower = upper; } failed = false; @@ -12051,7 +12228,9 @@ class LocalDb { for (final t in (only ?? tables)) { if (t == 'raw_archive') { try { - counts[t] = await _mergeRawArchive(src, db); + counts[t] = await _mergeRawArchive(src, db, + beforePageCommit: beforePageCommit, + onPageCommitted: onPageCommitted); } catch (_) { if (!tolerant) rethrow; counts[t] = 0; @@ -12255,6 +12434,9 @@ class LocalDb { } } await db.transaction((txn) async { + // Rotate atomically with the first committed page. A killed + // process must not leave old scratch valid after partial restore. + await beforePageCommit(txn); if (t == SubstrateArchive.table) { final r = await SubstrateArchive.restoreRows( txn, @@ -12562,6 +12744,7 @@ class LocalDb { } await flush(); }); + onPageCommitted(); // Advance past the last row this page actually delivered. Read the // cursor BEFORE dropping the page, and stop on a short page rather // than issuing one more query to discover the end. @@ -14901,13 +15084,38 @@ class LocalDb { ]; } + @visibleForTesting + static bool debugDisableSessionWindowsWorker = false; + @visibleForTesting + static void Function(String)? debugOnSessionWindowsConstruction; + + // Test forwarding seam for the production combined read. + @visibleForTesting + static Future sessionHrReadsForTest(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) => + sessionHrReads(from, to, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor); + + /// Both HR reads for the workout list. With a sensor enabled, fetch each + /// window once and construct its stats and samples together in a worker. + static Future sessionHrReads(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) async { + if (sessionSensorSources.isEmpty) { + return (stats: await _sessionHrStatsSql(from, to, maxHrCeiling, minHrFloor), + samples: await _sessionHrSamplesSql(from, to)); + } + return _sessionWindows(from, to, maxHrCeiling, minHrFloor); + } + /// Every session in [fromTs, toTs] (epoch SECONDS) with the worn rows of /// its window a session read keeps ([sessionWindowRows]), ascending. Only /// with a workout sensor flagged on; a session with none is absent, as from /// the band-only joins. - static Future>>> _sessionWindows( + static Future _sessionWindows( int fromTs, int toTs, + int maxHrCeiling, + int minHrFloor, ) async { final db = await instance; final rows = await db.rawQuery( @@ -14919,20 +15127,53 @@ class LocalDb { 'ORDER BY s.id, d.rec_ts ASC', [fromTs, toTs], ); + final sensors = {...sessionSensorSources}; + final wearable = sessionWearableSource; + (SessionHrReads, String) construct() => _prepareSessionWindows( + rows, sensors, wearable, maxHrCeiling, minHrFloor); + final result = debugDisableSessionWindowsWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'session windows'); + debugOnSessionWindowsConstruction?.call(result.$2); + return result.$1; + } + + static (SessionHrReads, String) _prepareSessionWindows( + List> rows, Set sensors, String? wearable, + int maxHrCeiling, int minHrFloor) { final by = >>{}; for (final r in rows) { final id = r['id'] as String?; if (id != null) (by[id] ??= []).add(r); } - return { - for (final MapEntry(key: id, value: rs) in by.entries) - if ([ - for (final r in sessionWindowRows(rs)) - if (((r['hr'] as num?) ?? 0) > 0) r, - ] - case final kept when kept.isNotEmpty) - id: kept, - }; + final stats = >{}; + final samples = >{}; + for (final MapEntry(key: id, value: rs) in by.entries) { + final hr = [ + for (final r in _sessionWindowRowsFor(rs, + sensors: sensors, wearable: wearable)) + if (((r['hr'] as num?) ?? 0) > 0) (r['hr'] as num).toInt(), + ]; + if (hr.isEmpty) continue; + samples[id] = hr; + final low = [ + for (final h in hr) + if (minHrFloor <= 0 || h >= minHrFloor) h, + ]; + final high = [ + for (final h in hr) + if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, + ]; + stats[id] = { + 'n': hr.length, + 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, + 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), + 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), + }; + } + return ((stats: stats, samples: samples), + Isolate.current.debugName ?? 'unnamed'); } /// Per-session HR aggregates over the 1 Hz substrate for every session in @@ -14952,28 +15193,14 @@ class LocalDb { int minHrFloor = 0, }) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: () { - final hr = [for (final r in rs) (r['hr'] as num).toInt()]; - final low = [ - for (final h in hr) - if (minHrFloor <= 0 || h >= minHrFloor) h, - ]; - final high = [ - for (final h in hr) - if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, - ]; - return { - 'n': hr.length, - 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, - 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), - 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), - }; - }(), - }; + return (await sessionHrReads(fromTs, toTs, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor)).stats; } + return _sessionHrStatsSql(fromTs, toTs, maxHrCeiling, minHrFloor); + } + + static Future>> _sessionHrStatsSql( + int fromTs, int toTs, int maxHrCeiling, int minHrFloor) async { final db = await instance; final ceilClause = maxHrCeiling > 0 ? 'AND d.hr <= $maxHrCeiling ' : ''; final floorClause = minHrFloor > 0 ? 'AND d.hr >= $minHrFloor ' : ''; @@ -15015,12 +15242,13 @@ class LocalDb { int toTs, ) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: [for (final r in rs) (r['hr'] as num).toInt()], - }; + return (await sessionHrReads(fromTs, toTs)).samples; } + return _sessionHrSamplesSql(fromTs, toTs); + } + + static Future>> _sessionHrSamplesSql( + int fromTs, int toTs) async { final db = await instance; final rows = await db.rawQuery( 'SELECT s.id AS id, d.hr AS hr ' diff --git a/lib/data/db_input_revisions.dart b/lib/data/db_input_revisions.dart new file mode 100644 index 000000000..251cebcb3 --- /dev/null +++ b/lib/data/db_input_revisions.dart @@ -0,0 +1,73 @@ +// Revision DDL executes here so schema creation stays additive and cheap. +part of 'db.dart'; + +Future _ldbCreateInputRev(Database db) async { + await db.execute( + 'CREATE TABLE IF NOT EXISTS input_rev (' + 'bucket INTEGER PRIMARY KEY, rev INTEGER NOT NULL)', + ); + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_onehz'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ai + AFTER INSERT ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au + AFTER UPDATE ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au_old + AFTER UPDATE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ad + AFTER DELETE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_rr'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ai + AFTER INSERT ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au + AFTER UPDATE ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au_old + AFTER UPDATE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ad + AFTER DELETE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } +} diff --git a/lib/data/db_session_ceiling_revisions.dart b/lib/data/db_session_ceiling_revisions.dart new file mode 100644 index 000000000..cb3d07d8c --- /dev/null +++ b/lib/data/db_session_ceiling_revisions.dart @@ -0,0 +1,29 @@ +part of 'db.dart'; + +Future _ldbCreateSessionCeilingRev(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS session_ceiling_rev (' + 'id INTEGER PRIMARY KEY CHECK (id = 1), rev INTEGER NOT NULL)'); + // Sparse legacy schemas get their sessions table during onOpen repair. + final present = await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sessions'", + ); + if (present.isEmpty) return; + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ai + AFTER INSERT ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_au + AFTER UPDATE OF id, type, status, start_ts, end_ts ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ad + AFTER DELETE ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); +} diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index ceb77017d..caeb96ee1 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2405,7 +2405,7 @@ class LocalRepositoryImpl extends LocalRepository { // "no data" (avg_hr == 0) even when the window is full of worn HR. try { final age = _profileAge(); - final stats = await LocalDb.sessionHrStats( + final (:stats, samples: rawBySession) = await LocalDb.sessionHrReads( fromTs, nowSec, maxHrCeiling: hrCeilingForAge(age), @@ -2414,10 +2414,6 @@ class LocalRepositoryImpl extends LocalRepository { // Spike-suppressed max/min per session (issue #127): smooth the raw 1 Hz // over one batched join so the list agrees with getWorkout's on-read // recompute. - final rawBySession = await LocalDb.sessionHrSamplesBySession( - fromTs, - nowSec, - ); for (final w in workouts) { final s = stats[w['id']]; final raw = rawBySession[w['id']]; diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index a97a59523..9355d2165 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -82,6 +82,10 @@ const Map kSliceTables = { 'baselines': (SliceKeep.all, null, 'personal baselines'), 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), + 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), + 'session_ceiling_rev': (SliceKeep.all, null, 'session ceiling revisions'), + 'store_identity': (SliceKeep.none, null, 'local checkpoint store identity'), + 'day_checkpoint': (SliceKeep.none, null, 'disposable night/cardio checkpoints'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart new file mode 100644 index 000000000..7b3d10443 --- /dev/null +++ b/lib/data/restore_classification.dart @@ -0,0 +1,81 @@ +// Restore policy for each schema table. Revision counters are local, +// rebuildable bookkeeping: imported inputs fire the destination triggers. +enum RestoreClass { merge, localOnly, rebuildable } + +const kRestoreClassification = { + 'decoded_onehz': RestoreClass.merge, + 'decoded_rr': RestoreClass.merge, + 'samples': RestoreClass.merge, + 'raw_archive': RestoreClass.merge, + 'raw_archive_frames': RestoreClass.localOnly, + 'raw_archive_legacy': RestoreClass.localOnly, + 'events': RestoreClass.merge, + 'band_events': RestoreClass.merge, + 'band_battery': RestoreClass.merge, + 'band_backlog': RestoreClass.localOnly, + 'sync_ledger': RestoreClass.localOnly, + 'sync_quarantine': RestoreClass.localOnly, + 'external_hr': RestoreClass.merge, + 'imported_measurement': RestoreClass.merge, + 'observation': RestoreClass.merge, + 'vendor_sleep_epoch': RestoreClass.merge, + 'live_coverage': RestoreClass.merge, + 'device_coverage': RestoreClass.merge, + 'workout_route': RestoreClass.merge, + 'live_workout_tally': RestoreClass.localOnly, + 'compute_jobs': RestoreClass.localOnly, + 'ecg_reading': RestoreClass.merge, + 'ecg_reading_packet': RestoreClass.merge, + 'ecg_raw_packet': RestoreClass.merge, + 'sleep_session_candidates': RestoreClass.localOnly, + 'wake_day_features': RestoreClass.localOnly, + 'substrate_archive': RestoreClass.merge, + 'day_result': RestoreClass.merge, + 'derived_day': RestoreClass.localOnly, + 'metric_series': RestoreClass.merge, + 'metric_series_version': RestoreClass.merge, + 'metric_method': RestoreClass.merge, + 'baselines': RestoreClass.merge, + 'compute_freshness': RestoreClass.localOnly, + 'derive_fingerprint': RestoreClass.rebuildable, + 'input_rev': RestoreClass.rebuildable, + 'day_checkpoint': RestoreClass.rebuildable, + 'store_identity': RestoreClass.localOnly, + 'session_ceiling_rev': RestoreClass.rebuildable, + 'sleep_nap': RestoreClass.merge, + 'sleep_override': RestoreClass.merge, + 'sessions': RestoreClass.merge, + 'session_sensor': RestoreClass.merge, + 'session_score_mask': RestoreClass.localOnly, + 'workout_split': RestoreClass.merge, + 'strength_set': RestoreClass.merge, + 'workout_suggestions': RestoreClass.merge, + 'imported_workout': RestoreClass.merge, + 'activity_suggestions': RestoreClass.merge, + 'activity_review_days': RestoreClass.merge, + 'activity_review_meta': RestoreClass.merge, + 'breathing_session': RestoreClass.merge, + 'exercise_def': RestoreClass.merge, + 'journal': RestoreClass.merge, + 'journal_metric': RestoreClass.merge, + 'journal_field_def': RestoreClass.merge, + 'cycle_log': RestoreClass.merge, + 'cycle_symptom': RestoreClass.merge, + 'lab_result': RestoreClass.merge, + 'lab_marker_def': RestoreClass.merge, + 'med_def': RestoreClass.merge, + 'med_dose': RestoreClass.merge, + 'food_def': RestoreClass.merge, + 'food_entry': RestoreClass.merge, + 'device': RestoreClass.merge, + 'signal_priority': RestoreClass.merge, + 'step_calibration': RestoreClass.merge, + 'step_calibration_day': RestoreClass.merge, + 'live_step_calibration': RestoreClass.merge, + 'live_step_calibration_run': RestoreClass.merge, + 'sync_cursor': RestoreClass.merge, + 'alarm_schedule': RestoreClass.localOnly, + 'notifications': RestoreClass.merge, + 'notif_fired': RestoreClass.localOnly, + 'notif_slots': RestoreClass.localOnly, +}; diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index b581d5318..83c947fef 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2030,6 +2030,7 @@ class AppState extends ChangeNotifier { } _sync.cancelQuietTimer(); _disposed = true; + _derive.dispose(); _ecg?.dispose(); _ecgTransport?.dispose(); // EVERY timer this object owns, not just three of them. @@ -3486,7 +3487,13 @@ class AppState extends ChangeNotifier { /// drops (band out of range / app jettisoned) — see [_onEngineState] / [_armRecovery]. /// /// On Android the Edge Tracking foreground service keeps the process + connection alive. - Future pauseForBackground() => _sync.pauseForBackground(); + Future pauseForBackground() { + _derive.trimForBackground(); + return _sync.pauseForBackground(); + } + + void resumeDerivationRetention() => _derive.resumeFromBackground(); + void handleMemoryPressure() => _derive.handleMemoryPressure(); /// A feature session (workout, breathing, ECG capture) is running — the /// "nothing else in flight" bar the one-off VACUUM waits for. ECG matters diff --git a/pubspec.lock b/pubspec.lock index eee7705f5..3376bc858 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -18,7 +18,7 @@ packages: source: hosted version: "1.3.77" analyzer: - dependency: transitive + dependency: "direct dev" description: name: analyzer sha256: de7148ed2fcec579b19f122c1800933dfa028f6d9fd38a152b04b1516cec120b @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" - resolved-ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" + ref: f8e7d3c478cc59707fde41f3edff7736026573fe + resolved-ref: f8e7d3c478cc59707fde41f3edff7736026573fe url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" @@ -946,8 +946,8 @@ packages: dependency: "direct main" description: path: "." - ref: "89080bcfac7767b3a871fd3a7d9307f5f30dad26" - resolved-ref: "89080bcfac7767b3a871fd3a7d9307f5f30dad26" + ref: 89080bcfac7767b3a871fd3a7d9307f5f30dad26 + resolved-ref: 89080bcfac7767b3a871fd3a7d9307f5f30dad26 url: "https://github.com/OpenStrap/protocol.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index c20440bfd..ff205c53c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -515,7 +515,8 @@ dependencies: # analytics main @ 3a7ca34: adds #101 (HRV/readiness/wellness fixes), #100 # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). - ref: 3a7ca34d30233ff11e6c2877083f319a907ccb31 + # Rebased analytics #99: incremental kernels, RSA v3 and cosinor gate (v120). + ref: f8e7d3c478cc59707fde41f3edff7736026573fe # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 @@ -670,6 +671,8 @@ dev_dependencies: flutter_test: sdk: flutter flutter_lints: ^6.0.0 + # DB part guards inspect declarations and SQL arguments through the Dart AST. + analyzer: ^10.0.1 test: ^1.25.8 # no_runtime_font_fetch_test parses pubspec.yaml to assert every family the # type scale uses is bundled rather than fetched at runtime. diff --git a/test/data_ownership_test.dart b/test/data_ownership_test.dart index 576523a81..aca6e952c 100644 --- a/test/data_ownership_test.dart +++ b/test/data_ownership_test.dart @@ -4,7 +4,8 @@ // The reset test is the load-bearing one. The old reset walked the derived // days and deleted those, which left about twenty tables standing behind a // dialog that said otherwise — so the assertion here is deliberately -// "EVERY table in sqlite_master is empty", not a list of table names. A list +// "EVERY user-data table in sqlite_master is empty", not a list of table names. +// The store_identity row is rotated on wipe to invalidate old checkpoints. A list // would have to be updated when a table is added, which is exactly the failure // mode being tested for. @@ -60,7 +61,7 @@ void main() { }); group('wipeAll', () { - test('leaves EVERY table empty, including the ones a list would forget', + test('leaves EVERY user-data table empty and rotates local store identity', () async { await _seedEverything(); final db = await LocalDb.instance; @@ -83,11 +84,14 @@ void main() { expect(n, greaterThan(0), reason: '$t was never seeded'); } + final storeId = await LocalDb.storeIdentity(); await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); for (final t in tables) { final n = _count(await db.rawQuery('SELECT COUNT(*) AS n FROM $t')); - expect(n, 0, reason: '$t survived "Delete everything"'); + expect(n, t == 'store_identity' ? 1 : 0, + reason: '$t survived "Delete everything"'); } }); diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 89194293e..9e49ccf66 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -49,6 +49,15 @@ Future _freshDb(String name) async { } } +// The fixture starts at 18:00 on the previous date. That short first day +// finalizes on its first derive, while its daytime summaries read the NEXT +// day's sleep window. Bank that real window before either parity run so the +// foreground worker order cannot decide whether the first day's evening is +// excluded as tonight's sleep. Every stored output is still compared below. +Future _primeFirstNight() async { + expect(await DerivationEngine().runDays(_profile, {'2026-07-13'}), 1); +} + Future<({int derived, int unchanged})> _pass({Profile profile = _profile}) async { final e = DerivationEngine(); final n = await e.run(profile, heavy: true); @@ -160,6 +169,7 @@ void main() { () async { const passes = 5; await _freshDb('unchanged_skip_on.db'); + await _primeFirstNight(); final log = <({int derived, int unchanged})>[]; for (var i = 0; i < passes; i++) { log.add(await _pass()); @@ -167,6 +177,7 @@ void main() { final withSkip = await _snapshot(); await _freshDb('unchanged_skip_off.db'); + await _primeFirstNight(); DerivationEngine.debugDeriveEveryDay = true; for (var i = 0; i < passes; i++) { final r = await _pass(); diff --git a/test/ed2a_e12_staging_cache_test.dart b/test/ed2a_e12_staging_cache_test.dart new file mode 100644 index 000000000..692f34b8c --- /dev/null +++ b/test/ed2a_e12_staging_cache_test.dart @@ -0,0 +1,275 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _profile({String field = 'hr_floor_p5', int delta = 1}) async { + final p = {...const ana.SleepUserProfile(nights: 14, hrFloorP5: 48, + hrFloorP25: 51, hrSleepMedian: 55, hrArousal: 70, rmssdMed: 40, + rmssdMad: 5, enmoStillCut: 0.02, enmoMoveCut: 0.1, + lfhfMed: 1, rkMed: 2).toJson(), 'folded_days': [edDay]}; + p[field] = (p[field] as num) + delta; + await LocalDb.putBaseline('sleep_user_profile', jsonEncode(p)); +} + +Future _history({int shift = 0}) async { + final db = await LocalDb.instance; + for (var i = 1; i <= 14; i++) { + final d = DateTime(2020, 1, 2 - i); + final start = DateTime(d.year, d.month, d.day, 1).millisecondsSinceEpoch; + await db.insert('day_result', {'day_id': dayLabelOf(d), + 'algo_version': kAlgoVersion, 'payload_json': '{}', 'computed_at': 1, + 'window_json': jsonEncode({'onset_ms': start + shift * 1000, + 'offset_ms': start + (6 * 3600 + shift) * 1000})}, + conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +Future _vendor({String stage = 'light', String id = ''}) async { + final db = await LocalDb.instance; + for (final (offset, duration, label) in [(0, 7200, stage), + (7200, 3600, 'deep'), (10800, 3600, 'rem')]) { + await db.insert('vendor_sleep_epoch', { + 'device_id': id, 'night_onset_ts': edStart, 'start_ts': edStart + offset, + 'end_ts': edStart + offset + duration, 'stage': label, + 'source': 'gen4', 'decoded_at': 1, + }, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +void main() { + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e12.db'); + await edRows(edStart, edStart + 1200, hr: 60, az: 1); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged staging input is computed once, including absent sleep', () async { + final e = DerivationEngine(); + final first = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, 1); + final again = await e.debugSleepCandidate(edDay); + expect(jsonEncode(again.toJson()), jsonEncode(first.toJson())); + expect(e.debugStagingComputations, 1, + reason: 'the second call must reuse the worker result, even absence'); + }); + + test('the staging worker closure does not drag the engine into the isolate', () async { + // AppState hands its engine a callback over its own fields (BLE phase + // futures among them). If the worker closure captured the engine, that + // callback would ride into Isolate.run and the send would be refused. + final unsendable = Completer(); + final e = DerivationEngine(offloadActive: () => unsendable.isCompleted); + final c = await e.debugSleepCandidate(edDay); + expect(c.toJson(), isNotNull); + expect(e.debugStagingComputations, 1); + }); + + final changes = Function(DerivationEngine)>{ + 'identical input replacement': (_) => edRows(edStart, edStart + 1, hr: 60, az: 1), + 'late RR revision': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'search lookback revision on previous local day': (_) => + edRows(DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000 + 1), + 'store generation with unchanged revisions': (_) async { + final before = await LocalDb.inputDayFingerprint(edDay); + await LocalDb.close(); await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.generation, greaterThan(before.generation)); + }, + 'algorithm': (e) async { e.debugStagedSleepCacheRekey(kAlgoVersion - 1); }, + 'HR ownership': (_) => edOwnership(signals: ['hr1Hz']), + 'accel ownership': (_) => edOwnership(signals: ['accel1Hz']), + 'PPG ownership': (_) => edOwnership(signals: ['ppgRedIr']), + 'skin-temperature ownership': (_) => edOwnership(signals: ['skinTempRaw']), + 'RR ownership': (_) async { + await (await LocalDb.instance).insert('device_coverage', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'start_ts': edStart, + 'end_ts': edStart + 600}); + await (await LocalDb.instance).insert('signal_priority', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'rank': 0, 'user_set': 1}); + }, + 'priority': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 4, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'history at 14 distinct nights': (_) => _history(shift: 60), + 'manual override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'manual'), + 'confirmed wake override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'confirmed'), + 'override onset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart + 30, offsetTs: edStart + 900, source: 'manual'), + 'override offset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 960, source: 'manual'), + 'override source edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'rejected'), + 'override removal': (_) => LocalDb.deleteSleepOverride(edDay), + 'vendor source': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'source': 'miband234'}); }, + 'vendor decode revision': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'decoded_at': 2}); }, + 'vendor window': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'end_ts': edStart + 14460}, + where: 'start_ts = ?', whereArgs: [edStart + 10800]); }, + 'vendor stages': (_) => _vendor(stage: 'deep'), + 'vendor device': (_) => _vendor(id: 'vendor-other'), + 'vendor flag': (_) => LocalDb.setCursor('wearable_enabled:garmin', '1'), + 'active wearable': (_) => LocalDb.setCursor('active_wearable', 'garmin-y'), + 'zone': (_) async { edSetZone('Pacific/Auckland'); }, + // Denver and Phoenix agree in winter, but disagree in the summer search + // window. The separate historical-zone case below pins that distinction. + for (final field in ['nights', 'hr_floor_p5', 'hr_floor_p25', + 'hr_sleep_median', 'hr_arousal', 'rmssd_med', 'rmssd_mad', + 'enmo_still_cut', 'enmo_move_cut', 'lfhf_med', 'rk_med', 'updated_at_ms']) + 'profile $field': (_) => _profile(field: field), + }; + for (final entry in changes.entries) { + test('${entry.key}: miss, exact output, then hit on the new key', () async { + if (entry.key == 'override removal' || entry.key.startsWith('override ') && entry.key.endsWith(' edit')) { + await LocalDb.putSleepOverride(dayId: edDay, onsetTs: edStart, + offsetTs: edStart + 900, source: 'manual'); + } + if (entry.key == 'priority') await edPriorityOrder(['', 'other']); + if (entry.key.startsWith('profile ')) await _profile(delta: 0); + if (entry.key.startsWith('history ')) await _history(); + if (entry.key.startsWith('vendor ') || entry.key == 'active wearable') { + await _vendor(); + await LocalDb.upsertDevice(id: 'garmin-x', adapterId: 'garmin', label: 'watch'); + await LocalDb.setCursor('active_wearable', 'garmin-x'); + await _vendor(id: 'garmin-x'); + } + if (entry.key == 'vendor flag' || entry.key == 'active wearable') { + await (await LocalDb.instance).delete('decoded_onehz'); + await (await LocalDb.instance).delete('vendor_sleep_epoch', + where: 'device_id = ?', whereArgs: ['']); + } + if (entry.key == 'vendor device') { + await LocalDb.upsertDevice(id: 'vendor-other', adapterId: 'garmin', label: 'new vendor'); + await LocalDb.setCursor('active_wearable', 'vendor-other'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + await (await LocalDb.instance).delete('decoded_onehz'); + } + if (entry.key == 'active wearable') { + await LocalDb.upsertDevice(id: 'garmin-y', adapterId: 'garmin', label: 'watch2'); + await _vendor(id: 'garmin-y'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + } + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final count = e.debugStagingComputations; + await entry.value(e); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'changed ${entry.key} must invalidate the staged result'); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true; + expect(jsonEncode(changed.toJson()), + jsonEncode((await cold.debugSleepCandidate(edDay)).toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'the replacement entry must now hit'); + }); + } + + test('same present-day zone name/offset cannot hide historical night offsets', () async { + edSetZone('America/Chihuahua'); + final nowOffset = DateTime.now().timeZoneOffset; + final nowName = DateTime.now().timeZoneName; + final historical = DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset; + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final before = e.debugStagingComputations; + edSetZone('America/Guatemala'); + expect(DateTime.now().timeZoneOffset, nowOffset); + expect(DateTime.now().timeZoneName, nowName); + expect(DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset, + isNot(historical), reason: 'fixture zones agree now but differed in January 2020'); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1, reason: 'new historical-offset key must hit'); + }); + + test('selected derive followed by baseline rescan reuses actual staging work', () async { + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final before = e.debugStagingComputations; + expect(before, greaterThan(0)); + await LocalDb.setCursor('baseline_sig', 'force-ed2a-rescan'); + expect(await e.rescanRecent(edProfile), greaterThan(0)); + expect(e.debugStagingComputations, before, + reason: 'baseline recompute must not rerun the unchanged staging worker'); + }); + + test('a nonfolding entry cannot satisfy a later mayFold request', () async { + final e = DerivationEngine(); + final modes = []; + e.debugOnStagingWalk = (fold, _, _) => modes.add(fold); + await e.debugSleepCandidate(edDay); + expect(modes, [false]); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final changed = await e.debugSleepCandidate(edDay); + expect(modes, [false, true], reason: 'a hit without an observation cannot suppress a fold'); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + final count = e.debugStagingComputations; + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count, reason: 'fold-capable result is reusable even when observation is absent'); + }); + + test('bounded staging cache retains only the most recent two days', () async { + final e = DerivationEngine()..debugStagedSleepCacheMaxDays = 2; + for (final day in ['2020-01-01', edDay, '2020-01-03']) { + await e.debugSleepCandidate(day); + expect(e.debugStagedSleepCacheSize, lessThanOrEqualTo(2)); + } + expect(e.debugStagedSleepCacheSize, 2, + reason: 'three distinct staging keys must populate the bounded cache'); + final count = e.debugStagingComputations; + await e.debugSleepCandidate('2020-01-03'); + expect(e.debugStagingComputations, count); + await e.debugSleepCandidate('2020-01-01'); + expect(e.debugStagingComputations, count + 1, reason: 'oldest day was evicted'); + }); + + test('existing fixture: cached and uncached derive JSON stays exact after invalidation', () async { + await edExistingFixture(); + final cached = DerivationEngine(); + final uncached = DerivationEngine()..debugDisableStagedSleepCache = true; + await edPreparedOutput(cached); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + await edRows(edStart, edStart + 1, hr: 87); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + }); + + test('profile fold bookkeeping remains idempotent (passing guard)', () { + final p = SleepProfilePolicy.withFoldedDays( + const ana.SleepUserProfile(nights: 3).toJson(), {}, edDay); + expect(SleepProfilePolicy.shouldFold(alreadyFolded: {edDay}, + dayId: edDay, hasOverride: false), isFalse); + expect(SleepProfilePolicy.foldedDays(jsonEncode(p)), {edDay}); + }); +} diff --git a/test/ed2a_e13_substrate_retention_test.dart b/test/ed2a_e13_substrate_retention_test.dart new file mode 100644 index 000000000..8a72ee80a --- /dev/null +++ b/test/ed2a_e13_substrate_retention_test.dart @@ -0,0 +1,247 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +int _reads(DerivationEngine e) => e.snapshot()['raw_rows'] as int; +final _lo = (edStart ~/ 900) * 900; +final _hi = _lo + 5 * 900 - 1; +Future _load(DerivationEngine e, {int? lo, int? hi, + Map> ownership = const {}}) => + e.debugLoadSubstrate(lo ?? _lo, hi ?? _hi, day: edDay, ownership: ownership); + +void main() { + final originalZone = Platform.environment["TZ"] ?? "UTC"; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e13.db'); + await edRows(_lo, _lo + 4 * 900 + 200); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged load reuses retained arrays without rereading rows', () async { + final e = DerivationEngine(); + final first = await _load(e); + final reads = _reads(e); + expect(reads, greaterThan(0)); + final next = await _load(e); + expect(jsonEncode(next.toJson()), jsonEncode(first.toJson())); + expect(_reads(e), reads, reason: 'unchanged load must not decode rows twice'); + }); + + test('append rereads at most the open revision bucket and new tail', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e) - reads, lessThanOrEqualTo(900), + reason: 'four closed buckets must survive the tail append'); + }); + + final changes = Function()>{ + 'identical old-bucket revision': () => edRows(_lo + 30, _lo + 31), + 'old-bucket replacement': () => edRows(_lo + 30, _lo + 31, hr: 83), + 'delete seconds and beats': () async { + final db = await LocalDb.instance; + for (final table in ['decoded_onehz', 'decoded_rr']) { + await db.delete(table, where: 'rec_ts >= ? AND rec_ts < ?', + whereArgs: [_lo + 10, _lo + 20]); + } + }, + 'late beat in retained bucket': () async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': (_lo + 30) * 1000, 'rec_ts': _lo + 30, + 'rr_ts_ms': (_lo + 30) * 1000, 'beat_index': 0, 'rr_ms': 850}); + }, + 'zone': () async { edSetZone('Pacific/Auckland'); }, + 'generation': () async { await LocalDb.close(); await LocalDb.instance; }, + 'family': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + 'device provenance': () => edRows(_lo + 30, _lo + 31, id: 'second'), + 'skin unit bucket': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'skin_temp_c': 33.1}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + }; + for (final change in changes.entries) { + test('${change.key}: invalidate, preserve every byte, then retain again', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await change.value(); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads), reason: 'closed prefix must be reloaded'); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + final afterReload = _reads(e); + await _load(e); + expect(_reads(e), afterReload, reason: 'newly retained key must hit'); + }); + } + + test('algorithm rekey rejects retained work and next load hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + e.debugRetainedSubstrateRekey(kAlgoVersion - 1); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads)); + expect(jsonEncode(warm.toJson()), jsonEncode((await _load(DerivationEngine())).toJson())); + final after = _reads(e); + await _load(e); + expect(_reads(e), after, reason: 'current-version retention should now hit'); + }); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal ownership miss even if revisions are unchanged', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final ownership = >{signal: [ + (start: _lo, end: _hi, deviceId: 'other')]}; + final warm = await _load(e, ownership: ownership); + final cold = await _load(DerivationEngine(), ownership: ownership); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final again = _reads(e); + await _load(e, ownership: ownership); + expect(_reads(e), again, reason: 'ownership belongs to the retained key'); + }); + } + + for (final window in ['from', 'to']) { + test('$window window change reloads then hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final lo = window == 'from' ? _lo + 1 : _lo; + final hi = window == 'to' ? _hi - 1 : _hi; + final warm = await _load(e, lo: lo, hi: hi); + final cold = await _load(DerivationEngine(), lo: lo, hi: hi); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final after = _reads(e); + await _load(e, lo: lo, hi: hi); + expect(_reads(e), after); + }); + } + + test('raw beat times, per-second beat offsets and bucket metadata survive retention', () async { + final db = await LocalDb.instance; + final rawTimes = []; + final intervals = []; + final starts = []; + final b = db.batch(); + b.delete('decoded_onehz', where: 'rec_ts = ?', whereArgs: [_lo + 1]); + // Include beat-only seconds, missing beats, multiple beats and a backward + // raw timestamp at the cut. Monotonizing a retained prefix twice loses it. + for (var t = _lo; t < _lo + 4 * 900 + 200; t++) { + starts.add(rawTimes.length); + for (var i = 0; i < (t % 3 == 0 ? 0 : 1 + t % 2); i++) { + final raw = t * 1000 - (i == 0 && t % 11 == 0 ? 1500 : i * 400); + rawTimes.add(raw.toDouble()); + intervals.add((700 + t % 90).toDouble()); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'beat_index': i, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': raw, 'rr_ms': 700 + t % 90}); + } + } + await b.commit(noResult: true); + final e = DerivationEngine(); + final before = await _load(e); + var runningMax = double.negativeInfinity; + final axis = []; + for (final raw in rawTimes) { + if (raw > runningMax) runningMax = raw; + axis.add(runningMax); + } + expect(before.rrTsMs, axis, reason: 'hold backward placements; never sort or shift beats'); + expect(before.rrMs, intervals, reason: 'interval order is the recorded rhythm'); + final extras = e.debugRetainedSubstrate(edDay); + expect(extras['rr_ts_raw'], rawTimes, + reason: 'the raw axis must be retained separately from its running max'); + expect(extras['rr_start'], starts); + expect((extras['meta'] as List).map((m) => (m as List).first), + [for (var i = 0; i < 5; i++) _lo ~/ 900 + i]); + expect(before.rrTsMs, isNot(rawTimes)); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + }); + + test('retention respects its day bound and evicts the oldest day', () async { + final e = DerivationEngine()..debugRetainedSubstrateMaxDays = 2; + for (var i = 0; i < 3; i++) { + final lo = DateTime(2020, 1, 2 + i, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(lo, lo + 1200); + await e.debugLoadSubstrate(lo, lo + 1200, day: edLabel(lo)); + expect(e.debugRetainedSubstrateDays.length, lessThanOrEqualTo(2)); + } + expect(e.debugRetainedSubstrateDays, {'2020-01-03', '2020-01-04'}, + reason: 'a zero-sized stub must not satisfy the bound'); + }); + + for (final boundary in ['background', 'memory pressure', 'dispose']) { + test('$boundary clears populated retention and next load stays exact', () async { + final e = DerivationEngine(); + final before = await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay), reason: 'populated before boundary'); + switch (boundary) { + case 'background': e.trimForBackground(); + case 'memory pressure': e.handleMemoryPressure(); + case 'dispose': e.dispose(); + } + expect(e.debugRetainedSubstrateDays, isEmpty); + final receiver = boundary == 'dispose' ? DerivationEngine() : e; + expect(jsonEncode((await _load(receiver)).toJson()), jsonEncode(before.toJson())); + }); + } + + test('real day finalization drops retained day and beat data', () async { + final e = DerivationEngine(); + await e.runDays(edProfile, {edDay}); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final later = DateTime(2020, 1, 6, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1200); + await e.runDays(edProfile, {edDay}); + expect((await LocalDb.dayResult(edDay))!['finalized'], 1); + expect(e.debugRetainedSubstrateDays, isNot(contains(edDay))); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('a pass with no preparable day releases previously retained rows', () async { + final e = DerivationEngine(); + await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); + expect(await e.runDays(edProfile, {edDay}), 0); + expect(e.debugRetainedSubstrateDays, isEmpty); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('existing fixture: retained and cold derive output exact before and after revision', () async { + await edExistingFixture(); + final warm = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSubstrateRetention = true; + await edPreparedOutput(warm); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + await edRows(edStart, edStart + 1, hr: 89); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + }); +} diff --git a/test/ed2a_e15_restricted_target_walk_test.dart b/test/ed2a_e15_restricted_target_walk_test.dart new file mode 100644 index 000000000..61135f276 --- /dev/null +++ b/test/ed2a_e15_restricted_target_walk_test.dart @@ -0,0 +1,166 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +List<({int startSec, int endSec, String dayKey})> _history(int distinct, + {bool duplicates = false}) => [ + for (var i = 0; i < (duplicates ? 30 : distinct); i++) + (startSec: DateTime(2019, 12, 1 + i % distinct, 1).millisecondsSinceEpoch ~/ 1000, + endSec: DateTime(2019, 12, 1 + i % distinct, 7).millisecondsSinceEpoch ~/ 1000, + dayKey: dayLabelOf(DateTime(2019, 12, 1 + i % distinct))), +]; + +PreparedDerivationPayload _walk(Substrate sub, {required bool skip, + List<({int startSec, int endSec, String dayKey})> history = const [], + SleepWindowOverride? override}) { + debugSegmentedDayCount = 0; + return prepareDerivationPayload(sub, targetDay: edDay, + priorSleep: history, override: override, skipUnusedDays: skip); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh("ed2a_e15.db")); + tearDown(() async { + debugStageEveryWalkedDay = false; + debugSegmentedDayCount = 0; + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + // Two local days, with enough seconds for the segmentation body on each. + final sub = edSmallSub(start: DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + count: 10 * 3600); + for (final h in [0, 1, 12]) { + test('$h stored nights cannot reach 14: skip earlier day, equal target JSON', () { + final history = _history(h); + final full = _walk(sub, skip: false, history: history); + expect(debugSegmentedDayCount, 2); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), + jsonEncode(edBundle(full.days.single))); + expect(debugSegmentedDayCount, 1, + reason: '$h distinct stored nights + one earlier day is below 14'); + }); + } + + test('30 rows with only 2 distinct night keys still permit the skip', () { + final history = _history(2, duplicates: true); + final full = _walk(sub, skip: false, history: history); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(debugSegmentedDayCount, 1, reason: 'count distinct nights, not rows'); + }); + + for (final h in [13, 14, 15]) { + test('$h stored nights may reach 14: preserve full earlier walk (guard)', () { + final full = _walk(sub, skip: false, history: _history(h)); + final fast = _walk(sub, skip: true, history: _history(h)); + expect(debugSegmentedDayCount, 2, + reason: '13 prior nights plus one walked night can reach the prior threshold'); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); + } + + test('override on an earlier day preserves its history contribution (guard)', () { + final override = SleepWindowOverride(dayId: '2020-01-01', + onsetSec: sub.tsSec.first, offsetSec: sub.tsSec.first + 3600, source: 'manual'); + final full = _walk(sub, skip: false, override: override); + final fast = _walk(sub, skip: true, override: override); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + }); + + test('full-walk control preserves upstream observations (guard)', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = _walk(sub, skip: false); + final fullEpochs = [for (final o in ana.takeCardioObservations()) o.epochs]; + ana.resetCardioObservations(); + debugStageEveryWalkedDay = true; + final forced = _walk(sub, skip: true); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(forced.toJson()), jsonEncode(full.toJson())); + expect([for (final o in ana.takeCardioObservations()) o.epochs], fullEpochs); + }); + + test('production nonfolding worker actually requests the restricted skip', () async { + await edFresh('ed2a_e15_nonfold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final walked = <(bool, int)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, _) => walked.add((fold, count)); + final got = await e.debugSleepCandidate(edDay); + final full = DerivationEngine()..debugDisableTargetDaySkip = true; + expect(jsonEncode(got.toJson()), + jsonEncode((await full.debugSleepCandidate(edDay)).toJson())); + expect(walked, [(false, 1)], + reason: 'nonfolding production staging must skip the irrelevant earlier day'); + }); + + test('mayFold skips unused earlier days and folds the target once (D36)', () async { + await edFresh('ed2a_e15_fold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final search = await DerivationEngine().debugLoadSubstrate( + sub.tsSec.first, sub.tsSec.last, day: edDay); + ana.cardioUserProfile = null; + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + final observations = ana.takeCardioObservations(); + expect(observations, hasLength(1)); + final o = observations.single; + final targetObservation = jsonEncode({'epochs': o.epochs, + 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + final walked = <(bool, int, String?)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, obs) => walked.add((fold, count, obs)); + final candidate = await e.debugSleepCandidate(edDay); + expect(jsonEncode(candidate.toJson()), jsonEncode(full.toJson())); + expect(walked.single.$3, targetObservation, reason: "D36 folds the target's own observation"); + expect(walked.single.$1, isTrue, reason: 'fixture reaches a folding pass'); + expect(walked.single.$2, 1, + reason: 'D36 permits the output-neutral E15 skip on folding passes'); + expect(walked.single.$3, isNotNull, reason: 'fixture must produce a real observation'); + final profile = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + final p = jsonDecode(profile as String) as Map; + expect(p['nights'], 1); + expect(p['folded_days'], contains(edDay)); + await e.debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], profile, + reason: 'repeated staging must preserve every profile byte'); + }); + + test('existing fixture has exact target output with skip on and off (guard)', () async { + await edFresh('ed2a_e15_fixture.db'); + await edExistingFixture(); + final e = DerivationEngine(); + final sub = await e.debugLoadSubstrate(1577880000, 1578038400, day: edDay); + final full = _walk(sub, skip: false); + final fast = _walk(sub, skip: true); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); +} diff --git a/test/ed2a_e18_session_reuse_test.dart b/test/ed2a_e18_session_reuse_test.dart new file mode 100644 index 000000000..b6e75cb44 --- /dev/null +++ b/test/ed2a_e18_session_reuse_test.dart @@ -0,0 +1,271 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _light(DerivationEngine e) async { + expect(await e.run(edProfile), greaterThan(0), reason: 'light pass must reach the real derive'); + expect(e.snapshot()['last_error'], isNull); +} + +Future _sessionEdit(Map patch) async { + await (await LocalDb.instance).update('sessions', patch, + where: 'id = ?', whereArgs: ['s1']); +} + +void main() { + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e18.db'); + await edRows(edStart, edStart + 1200); + await edSession(); + // These worker-level tests must reach the worker even after main's + // day fingerprint settles. The integration test below keeps both enabled. + DerivationEngine.debugDeriveEveryDay = true; + }); + tearDown(() async { + DerivationEngine.debugDeriveEveryDay = false; + await edDrop(); + }); + + test('session revisions upgrade from 58 and repair without backfilling sessions', () async { + var db = await LocalDb.instance; + final sessions = await db.query('sessions'); + for (final event in ['ai', 'au', 'ad']) { + await db.execute('DROP TRIGGER sessions_ceiling_$event'); + } + await db.execute('DROP TABLE session_ceiling_rev'); + await db.setVersion(58); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await db.getVersion(), LocalDb.schemaVersion); + expect(await db.query('sessions'), sessions); + expect(await db.query('session_ceiling_rev'), isEmpty, + reason: 'upgrade never scans or backfills existing sessions'); + await _sessionEdit({'type': 'run'}); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.setSessionHrr('s1', 20); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'derived-only writes do not invalidate input identity'); + await expectLater(db.transaction((txn) async { + await txn.update('sessions', {'type': 'bike'}); + throw StateError('rollback'); + }), throwsStateError); + expect(await LocalDb.sessionCeilingRevision(), 1); + expect((await db.query('sessions')).single['type'], 'run'); + }); + + test('session revisions travel in slices, rebuild on restore, and empty on wipe', () async { + final db = await LocalDb.instance; + expect(kRestoreClassification['session_ceiling_rev'], RestoreClass.rebuildable); + expect(kSliceTables['session_ceiling_rev']?.$1, SliceKeep.all); + expect(LocalDb.restoreTablesForTest, isNot(contains('session_ceiling_rev'))); + expect(await db.query('session_ceiling_rev'), hasLength(1)); + expect(await db.query('input_rev'), isNotEmpty); + final generation = LocalDb.storeGeneration; + final identity = await LocalDb.storeIdentity(); + await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(identity)); + for (final table in await LocalDb.tableNames()) { + if (table == 'store_identity') { + final rows = await db.query(table); + expect(rows, hasLength(1)); + expect(rows.single['singleton'], 1); + expect(rows.single['id'], matches(RegExp(r'^[0-9a-f]{32}$'))); + } else { + expect(await db.query(table, limit: 1), isEmpty, reason: table); + } + } + expect(LocalDb.storeGeneration, greaterThan(generation)); + await edSession(); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'wipe preserves working triggers'); + }); + + test('main skips settled days and changed samples still replace reused ceilings', () async { + DerivationEngine.debugDeriveEveryDay = false; + final e = DerivationEngine(); + Future settle() async { + for (var i = 0; i < 6; i++) { + final derived = await e.run(edProfile); + expect(e.snapshot()['last_error'], isNull); + if (derived == 0 && e.lastRunUnchangedDays == 1) return; + } + fail('main fingerprint never settled'); + } + await settle(); + final count = e.debugCeilingComputed; + expect(count, 1); + final first = await edStored(edDay); + expect(await e.run(edProfile), 0); + expect(e.lastRunUnchangedDays, 1); + expect(e.debugCeilingComputed, count); + expect(await edStored(edDay), first); + await edRows(edStart, edStart + 1200, hr: 160); + expect(await e.run(edProfile), 1, + reason: 'changed input must defeat main skip and the old ceiling key'); + expect(e.debugCeilingComputed, count + 1); + final changed = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 160); + await settle(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); + + test('finished session ceiling is scanned once across unchanged light passes', () async { + final e = DerivationEngine(); + await _light(e); + expect(e.debugCeilingComputed, 1); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 170, + reason: 'motion fixture must contain a real held ceiling'); + final first = await edStored(edDay); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'a hit must bypass sessionHrCeiling'); + }); + + final edits = Function(DerivationEngine)>{ + 'decoded sample revision (identical REPLACE)': (_) => edRows(edStart + 30, edStart + 31), + 'decoded sample revision (changed HR)': (_) => edRows(edStart + 30, edStart + 300, hr: 160), + 'late RR input revision': (_) async { + final t = edStart + 30; + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'session revision (identical REPLACE)': (_) => edSession(), + 'session revision (identical UPDATE)': (_) => _sessionEdit({'type': 'run'}), + 'session identity': (_) => _sessionEdit({'id': 's2'}), + 'session type': (_) => _sessionEdit({'type': 'bike'}), + 'session start': (_) => _sessionEdit({'start_ts': edStart + 1}), + 'session end': (_) => _sessionEdit({'end_ts': edStart + 1000}), + 'clipped sample window': (_) async { + await (await LocalDb.instance).delete('decoded_onehz', + where: 'rec_ts < ?', whereArgs: [edStart + 10]); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'HR ownership': (_) async { + // The primary needs coverage too: as the only covering candidate, + // `other` would own the whole window and leave the day without HR. + await (await LocalDb.instance).insert('device_coverage', {'device_id': '', + 'signal': 'hr1Hz', 'start_ts': edStart, 'end_ts': edStart + 1200}); + await edOwnership(signals: ['hr1Hz']); + }, + 'accel ownership': (_) async { + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': 'other', 'signal': 'accel1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + await db.insert('signal_priority', {'device_id': 'other', 'signal': 'accel1Hz', + 'rank': 0, 'user_set': 1}); + }, + 'priority without changed input revisions': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 2, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'store generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugSessionCacheRekey(kAlgoVersion - 1); }, + }; + for (final edit in edits.entries) { + test('${edit.key}: miss, exact result, then reuse new key', () async { + if (edit.key.startsWith('priority ')) await edPriorityOrder(['', 'other']); + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + expect(count, 1); + await edit.value(e); + await _light(e); + expect(e.debugCeilingComputed, count + 1, reason: 'changed ${edit.key} must rescan'); + final warm = await edStored(edDay); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), warm, + reason: 'includes the full metric envelope and unrounded winning session'); + final after = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, after, reason: 'new key should now hit'); + }); + } + + test('an absent ceiling is reusable without inventing a number', () async { + await edRows(edStart, edStart + 1200, az: 1); + final e = DerivationEngine(); + await _light(e); + final first = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['hr_ceiling']['value'], '—'); + expect(payload['scalars']['hr_ceiling_bpm'], isNull); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'absence is a cached result too'); + }); + + test('live sessions always recompute, including done to live transition (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await _sessionEdit({'status': 'live'}); + final before = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, before + 1); + await _light(e); + expect(e.debugCeilingComputed, before + 2); + await _sessionEdit({'status': 'done'}); + await _light(e); + expect(e.debugCeilingComputed, before + 3, reason: 'return to done must not resurrect stale entry'); + }); + + for (final mode in ['heavy', 'force', 'selected']) { + test('$mode pass always recomputes (guard)', () async { + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + final done = switch (mode) { + 'heavy' => await e.run(edProfile, heavy: true), + 'force' => await e.run(edProfile, force: true), + _ => await e.runDays(edProfile, {edDay}), + }; + expect(done, greaterThan(0)); + expect(e.debugCeilingComputed, greaterThan(count)); + }); + } + + test('session removal does not keep its former ceiling (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await (await LocalDb.instance).delete('sessions', where: 'id = ?', whereArgs: ['s1']); + await _light(e); + final got = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(got['hr_ceiling']['value'], '—'); + expect(got['scalars']['hr_ceiling_bpm'], isNull); + }); + + test('existing fixture: reuse and forced output byte equality after invalidation (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + await _light(e); + await _light(e); + final before = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), before); + await edRows(edStart + 30, edStart + 300, hr: 160); + await _light(e); + final changed = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), changed); + }); +} diff --git a/test/ed2a_e19_wearable_day_index_test.dart b/test/ed2a_e19_wearable_day_index_test.dart new file mode 100644 index 000000000..e96b0dff7 --- /dev/null +++ b/test/ed2a_e19_wearable_day_index_test.dart @@ -0,0 +1,201 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/compute/readiness_calibration_backfill.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; +import 'readiness_calibration_backfill_test.dart' show bundle; + +const _watch = ('garmin-a', 'garmin'); +Future _wearable() async { + await LocalDb.upsertDevice(id: _watch.$1, adapterId: _watch.$2, label: 'Watch A'); + await LocalDb.setCursor(kActiveWearableCursor, _watch.$1); + await LocalDb.setCursor(wearableEnabledCursor(_watch.$2), '1'); + await edRows(edStart, edStart + 1200, id: _watch.$1, family: _watch.$2, + source: _watch.$2, hr: 70, az: 0); +} + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2a_e19.db'); await _wearable(); }); + tearDown(() async { + endWearableDayIndexPass(); + debugOnWearableDayIndexScan = null; + debugDisableWearableDayIndex = false; + await edDrop(); + }); + + test('one real wearable scan per pass; new pass sees newly appended days', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { + final first = await wearableRecTsMaxByDay(_watch); + expect(first, {edDay: edStart + 1199}); + expect(await wearableRecTsMaxByDay(_watch), first); + expect(scans, 1, reason: 'second request in the same pass must hit'); + } finally { endWearableDayIndexPass(); } + expect(debugWearableDayIndexSize, 0); + final later = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + beginWearableDayIndexPass(); + try { + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2, reason: 'an index belongs to one pass, not to the engine lifetime'); + } finally { endWearableDayIndexPass(); } + }); + + for (final change in ['input revision', 'generation', 'algorithm']) { + test('$change cannot lend a stale index to the next request', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + switch (change) { + case 'input revision': await edRows(edStart, edStart + 1, + id: _watch.$1, family: _watch.$2, source: _watch.$2, hr: 80); + case 'generation': await LocalDb.close(); await LocalDb.instance; + case 'algorithm': debugWearableDayIndexRekey(kAlgoVersion - 1); + } + beginWearableDayIndexPass(); + try { + final fresh = await wearableRecTsMaxByDay(_watch); + expect(fresh, {edDay: edStart + 1199}); + expect(scans, 2, reason: 'the changed context belongs to a fresh pass'); + expect(await wearableRecTsMaxByDay(_watch), fresh); + expect(scans, 2, reason: 'unchanged second request in that pass must hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + for (final key in ['device', 'family']) { + test('different wearable $key gets its own pass key', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final other = key == 'device' ? ('garmin-b', 'garmin') : ('garmin-a', 'oura'); + beginWearableDayIndexPass(); + try { + final original = await wearableRecTsMaxByDay(_watch); + final changed = await wearableRecTsMaxByDay(other); + expect(changed, isEmpty); + expect(original, isNotEmpty); + expect(scans, 2, reason: 'a different $key must miss'); + expect(await wearableRecTsMaxByDay(_watch), original); + expect(await wearableRecTsMaxByDay(other), changed); + expect(scans, 2, reason: 'both keys, including empty index, should now hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + test('index is discarded when body throws after a populated lookup', () async { + var scans = 0; + var populated = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + Future failedPass() async { + beginWearableDayIndexPass(); + try { + expect(await wearableRecTsMaxByDay(_watch), isNotEmpty); + populated = debugWearableDayIndexSize; + throw StateError('after index fill'); + } finally { endWearableDayIndexPass(); } + } + await expectLater(failedPass(), throwsStateError); + expect(debugWearableDayIndexSize, 0); + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + expect(scans, 2, reason: 'the failed pass cannot lend its index to a retry'); + expect(populated, 1, reason: 'exception cleanup must be tested with an occupied index'); + }); + + test('real run clears the index and running latch on an injected scan failure', () async { + var scans = 0; + var occupiedDuringFailure = 0; + debugOnWearableDayIndexScan = (_, _) { + scans++; + if (scans == 1) { + occupiedDuringFailure = debugWearableDayIndexSize; + throw StateError('injected after wearable scan'); + } + }; + final e = DerivationEngine(); + await e.run(edProfile); + expect(e.running, isFalse, reason: 'failure must release the process-wide latch'); + expect(e.snapshot()['last_error'], contains('injected after wearable scan')); + expect(debugWearableDayIndexSize, 0, reason: 'outer finally must release the pass index'); + final beforeRetry = scans; + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, greaterThan(beforeRetry), reason: 'retry must build a fresh index'); + expect(occupiedDuringFailure, 1, reason: 'the fault must occur after population'); + }); + + test('actual light run scans the wearable index once through readiness backfill', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final e = DerivationEngine(); + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, 1, reason: 'scope, edge and readiness backfill share the pass index'); + expect(debugWearableDayIndexSize, 0); + }); + + test('lookups outside a pass stay fresh (passing guard)', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + await wearableRecTsMaxByDay(_watch); + final t = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(t, t + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2); + expect(debugWearableDayIndexSize, 0); + }); + + test('readiness backfill defers wearable substrate and still rescores pruned days (guard)', () async { + for (final day in [edDay, '2020-01-01']) { + final payload = bundle(0.5, 0.5, 0); + await LocalDb.putDayResult(dayId: day, algoVersion: 111, + payloadJson: jsonEncode(payload), windowJson: '{}', finalized: true, + readiness: (payload['scalars'] as Map)['readiness'] as double); + } + final before = await edStored(edDay); + beginWearableDayIndexPass(); + late final ({int seeded, int rescored, int forced}) result; + try { + final raw = await substrateDays(); + expect(raw, contains(edDay)); + result = await backfillReadinessCalibration(rawDays: raw, + zHistoryLoader: () async => (day, family) => []); + } finally { endWearableDayIndexPass(); } + expect(result.rescored, 1, reason: 'pruned history still needs calibration'); + expect(result.forced, 0); + expect((await LocalDb.dayResult('2020-01-01'))!['algo_version'], kAlgoVersion); + expect((await LocalDb.dayResult(edDay))!['algo_version'], 111, + reason: 'wearable substrate must wait for its own derive'); + // readiness_z seeding is intentional; the day bundle must remain untouched. + final row = await LocalDb.dayResult(edDay); + expect(jsonDecode(before)['day_result'][0]['payload_json'], row!['payload_json']); + }); + + test('existing fixture: pass index enabled/disabled derive JSON remains exact (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final indexed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), indexed); + debugDisableWearableDayIndex = false; + await edRows(edStart, edStart + 1, hr: 89); + expect(await e.runDays(edProfile, {edDay}), 1); + final changed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); +} diff --git a/test/ed2a_e20_worker_preparation_test.dart b/test/ed2a_e20_worker_preparation_test.dart new file mode 100644 index 000000000..76b8a2641 --- /dev/null +++ b/test/ed2a_e20_worker_preparation_test.dart @@ -0,0 +1,204 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String get _mainIsolate => Isolate.current.debugName ?? 'unnamed'; +Future> _sessionReads() async { + final r = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200); + return {'stats': r.stats, 'samples': r.samples}; +} + +void main() { + final sql = []; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e20.db'); + // Log the real join, rather than counting calls to a mock query function. + await LocalDb.close(); + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions(type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? v) { + if (v is Map) { + if (v['sql'] case final String s) sql.add(s); + for (final x in v.values) { collect(x); } + } else if (v is List) { + for (final x in v) { collect(x); } + } + } + collect(event.arguments); + })); + await LocalDb.instance; + sql.clear(); + }); + tearDown(() async { + debugDisableWearableWorker = false; + debugOnWearableConstruction = null; + LocalDb.debugDisableSessionWindowsWorker = false; + LocalDb.debugOnSessionWindowsConstruction = null; + await edDrop(); + databaseFactory = databaseFactoryFfi; + }); + + for (final family in ['garmin', 'colmi', 'oura', 'ultrahuman', 'pebble', 'miband234']) { + test('$family construction runs off the caller isolate with exact output', () async { + final id = '$family-a'; + await edRows(edStart, edStart + 1200, id: id, family: family, source: family, hr: 71, az: 0); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(inline.isEmpty, isFalse, reason: 'fixture must enter a real builder'); + final locations = []; + debugOnWearableConstruction = locations.add; + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + final a = prepareDerivationPayload(inline, targetDay: edDay).days.single; + final b = prepareDerivationPayload(worker, targetDay: edDay).days.single; + expect(jsonEncode(edBundle(b)), jsonEncode(edBundle(a))); + expect(locations, isNotEmpty, reason: 'actual construction must report execution location'); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'row construction still executes on the UI/caller isolate'); + }); + } + + test('two wearable devices at one second remain separate (passing guard)', () async { + await edRows(edStart, edStart + 1200, id: 'a', family: 'garmin', source: 'garmin', hr: 65, az: 0); + await edRows(edStart, edStart + 1200, id: 'b', family: 'garmin', source: 'garmin', hr: 95, az: 0); + for (final id in ['a', 'b']) { + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr.toSet(), {id == 'a' ? 65 : 95}); + expect(worker.deviceIds, {id}); + } + }); + + test('two straps keep owner, beat timing and session overlay exact (guard)', () async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70, az: 0); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150, az: 0); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180, az: 0); + await edSession(start: edStart, end: edStart + 800); + final db = await LocalDb.instance; + for (final (id, source, rr) in [('strap-a', 'ble_hrs', 850), ('strap-b', 'polar', 900)]) { + for (var i = 0; i < (id == 'strap-a' ? 3 : 1); i++) { + final t = edStart + 150 + i; + await db.insert('decoded_rr', {'device_id': id, 'source': source, + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': t * 1000 - 123, 'rr_ms': rr, 'beat_index': 0}); + } + } + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr, isNot(contains(180)), reason: 'minority source must not mix into a session'); + expect(worker.rrMs, [850, 850, 850]); + expect(worker.rrTsMs.first, (edStart + 150) * 1000 - 123); + }); + + test('existing two-device fixture: on/off worker derive JSON is byte-identical (guard)', () async { + await edExistingFixture(); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + expect(inline.isEmpty, isFalse); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.skinTemp.where((v) => v > 0), isNotEmpty); + expect(jsonEncode(edBundle(prepareDerivationPayload(worker, targetDay: edLabel(worker.tsSec.last)).days.single)), + jsonEncode(edBundle(prepareDerivationPayload(inline, targetDay: edLabel(inline.tsSec.last)).days.single))); + }); + + Future sessionFixture() async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edSession(); + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180); + } + + test('combined session stats and samples use one SELECT', () async { + await sessionFixture(); + sql.clear(); + final r = await _sessionReads(); + expect((r['samples'] as Map)['s1'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'stats and samples currently reread the same window twice'); + }); + + test('session grouping executes off the caller isolate with exact JSON', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await _sessionReads(); + final locations = []; + LocalDb.debugOnSessionWindowsConstruction = locations.add; + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + expect(locations, isNotEmpty); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'grouping and owner selection still loop over rows on the caller'); + }); + + test('workout repository uses the combined read in its actual list flow', () async { + await sessionFixture(); + final repo = LocalRepositoryImpl(getProfileMap: () => edProfile.toMap()); + sql.clear(); + final got = await repo.getWorkouts(range: 'all'); + expect(got['workouts'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'GREEN must wire the real getWorkouts consumer too'); + }); + + test('bounded session aggregates and empty windows stay exact (guard)', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + expect(jsonEncode({'stats': worker.stats, 'samples': worker.samples}), + jsonEncode({'stats': inline.stats, 'samples': inline.samples})); + expect(worker.stats['s1']!['min_hr'], 150); + expect(worker.stats['s1']!['max_hr'], 70); + expect(worker.samples['s1'], containsAll([70, 150])); + final empty = await LocalDb.sessionHrReadsForTest(edStart - 100, edStart - 1); + expect(empty.stats, isEmpty); + expect(empty.samples, isEmpty); + }); + + test('session windows keep two-device ownership and last-second samples (guard)', () async { + await sessionFixture(); + final inline = await _sessionReads(); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + final samples = (worker['samples'] as Map)['s1'] as List; + expect(samples, isNot(contains(180))); + expect(samples.first, 70); + expect(samples.last, 70, reason: 'inclusive session end must survive preparation'); + final stats = (worker['stats'] as Map)['s1'] as Map; + expect(stats['n'], 1101); + }); +} diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart new file mode 100644 index 000000000..b21e0092e --- /dev/null +++ b/test/ed2a_input_revision_test.dart @@ -0,0 +1,556 @@ +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +// ED-2a E4 RED contract. Run with flutter test --concurrency=1. +// Only the input portion of fork 561f2838 and generation portions of +// 74c0b37b/30cd944e apply. No store_rev, row_rev, scheduler or checkpoints. +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/models.dart'; + +const _inputs = ['decoded_onehz', 'decoded_rr']; +final _midnight = DateTime(2026, 10, 20); +final _lo = _midnight.millisecondsSinceEpoch ~/ 1000; +final _day = dayLabelOf(_midnight); + +Map _row(String table, int ts, {String device = ''}) => { + 'device_id': device, 'ts_ms': ts * 1000, 'rec_ts': ts, + if (table == 'decoded_onehz') ...{'counter': 1, 'hr': 61}, + if (table == 'decoded_rr') ...{ + 'beat_index': 0, 'rr_ts_ms': ts * 1000, 'rr_ms': 850, + }, +}; + +Future _hasRevisions(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='input_rev'", +)).isNotEmpty; + +// Missing infrastructure is revision zero, so mutation tests fail an assertion +// about an unchanged revision, never a spurious 'no such table' exception. +Future _rev(Database db, int ts) async { + if (!await _hasRevisions(db)) return 0; + final rows = await db.query('input_rev', columns: ['rev'], + where: 'bucket = ?', whereArgs: [ts ~/ 900]); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); +} + +Future>> _revisionRows(Database db) async => + await _hasRevisions(db) ? db.query('input_rev', orderBy: 'bucket') : []; + +Future>> _legacyFingerprint(Database db) => + db.rawQuery('SELECT MAX(rec_ts) AS mx, COUNT(*) AS n FROM decoded_onehz ' + 'WHERE rec_ts >= ? AND rec_ts < ?', [_lo, _lo + 7200]); + +Future> _schema(Database db) async { + final out = {}; + final entries = await db.rawQuery( + "SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' " + "AND name != 'android_metadata' ORDER BY type, name", + ); + for (final entry in entries) { + final name = entry['name'] as String; + // Compare column definitions rather than CREATE text: ALTER and CREATE + // paths have different whitespace and IF NOT EXISTS spelling. + out[name] = entry['type'] == 'table' + ? await db.rawQuery('PRAGMA table_info("$name")') + : (entry['sql'] as String?)?.replaceAll(RegExp(r'\s+'), ' ').trim(); + } + return out; +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory tmp; + late String livePath; + late String sourcePath; + late DatabaseFactory? previousFactory; + late String previousName; + final sql = []; + final legacyRows = {}; + final legacyColumns = {}; + + setUp(() async { + sqfliteFfiInit(); + previousFactory = databaseFactoryOrNull; + previousName = LocalDb.dbName; + await LocalDb.close(); + SharedPreferences.setMockInitialValues({}); + tmp = await Directory.systemTemp.createTemp('ed2a_rev_'); + livePath = p.join(tmp.path, 'live.db'); + sourcePath = p.join(tmp.path, 'source.db'); + // Public sqflite logger observes actual SQL, including nested batches. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + LocalDb.dbName = livePath; + LocalDb.lastRebuild = null; + sql.clear(); + }); + + tearDown(() async { + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await databaseFactory.deleteDatabase(sourcePath); + databaseFactoryOrNull = previousFactory; + LocalDb.dbName = previousName; + await tmp.delete(recursive: true); + }); + + Future open() async { + final db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a broken rung must not pass by quarantining the original DB'); + return db; + } + + Future seed58() async { + final db = await databaseFactory.openDatabase(livePath, + options: OpenDatabaseOptions(version: 58)); + // Frozen v58 decoded DDL is below. Other tables use the existing repair + // seam; remove E4 infrastructure before setting the old version and seeding. + for (final ddl in _v58DecodedDdl) { await db.execute(ddl); } + await LocalDb.repairOpenSchemaForTest(db); + final triggers = await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type='trigger' " + "AND tbl_name IN ('decoded_onehz','decoded_rr')", + ); + for (final t in triggers) { await db.execute('DROP TRIGGER "${t['name']}"'); } + await db.execute('DROP TABLE IF EXISTS input_rev'); + await db.setVersion(58); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 3)); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 3)); + for (final table in _inputs) { + legacyRows[table] = await db.query(table); + legacyColumns[table] = await db.rawQuery('PRAGMA table_info($table)'); + } + await db.close(); + } + + test('every table is classified for restore and diagnostic slices', () async { + await open(); + final tables = await LocalDb.tableNames(); + expect(tables.toSet().difference(kRestoreClassification.keys.toSet()), isEmpty); + expect(tables.toSet().difference(kSliceTables.keys.toSet()), isEmpty); + expect(kRestoreClassification['input_rev'], RestoreClass.rebuildable); + expect(LocalDb.restoreTablesForTest, isNot(contains('input_rev'))); + }); + + test('rung 59 is reserved for additive input revisions', () async { + expect(LocalDb.schemaVersion, 60, reason: 'rung 60 follows the additive revisions at 59'); + final db = await open(); + expect(await db.getVersion(), LocalDb.schemaVersion); + expect(await _hasRevisions(db), isTrue); + final tables = await LocalDb.tableNames(); + expect(tables, isNot(contains('store_rev'))); + expect(tables, isNot(contains('row_rev'))); + }); + + test('58 to 59 preserves raw rows and creates all mutation triggers', () async { + await seed58(); + final db = await open(); + expect((await db.query('decoded_onehz')).single['hr'], 61); + expect((await db.query('decoded_rr')).single['rr_ms'], 850); + for (final table in _inputs) { + expect(await db.query(table), legacyRows[table], reason: '$table data unchanged'); + expect(await db.rawQuery('PRAGMA table_info($table)'), legacyColumns[table], + reason: 'rung 59 must not change raw columns/defaults'); + } + expect(await db.getVersion(), LocalDb.schemaVersion); + expect(await _hasRevisions(db), isTrue); + final triggers = await db.rawQuery( + "SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger'", + ); + for (final table in _inputs) { + final body = triggers.where((t) => t['tbl_name'] == table) + .map((t) => (t['sql'] as String).toUpperCase()).join('\n'); + for (final op in ['INSERT', 'UPDATE', 'DELETE']) { + expect(body, contains('AFTER $op'), reason: '$table $op revision trigger'); + } + } + expect(await _revisionRows(db), isEmpty, + reason: 'legacy rows must not be backfilled into input_rev'); + expect(await LocalDb.inputDayFingerprint(_day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + }); + + test('upgrade and every-open repair do not scan or rewrite raw input', () async { + await seed58(); + sql.clear(); + final db = await open(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + await open(); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final dataAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false, + ); + expect(openingSql.where((s) { + final ddl = s.trimLeft().toUpperCase(); + // Declaring a trigger/view does not execute its SELECT/UPDATE body. + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + // Main probes whether legacy decoded input exists before metadata repair. + // This bounded probe is unrelated to revision creation or backfill. + if (s == 'SELECT 1 FROM decoded_onehz LIMIT 1') return false; + return dataAccess.hasMatch(s); + }), + isEmpty, reason: 'opening may inspect schema, never scan/backfill raw rows'); + expect(await _revisionRows(await LocalDb.instance), isEmpty); + }); + + test('fresh and upgraded schema agree; repair and reopen are idempotent', () async { + await seed58(); + var db = await open(); + expect(await _hasRevisions(db), isTrue, reason: 'input_rev must exist first'); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 100)); + final schema = await _schema(db); + final revs = await _revisionRows(db); + final rows = await db.query('decoded_onehz', orderBy: 'rec_ts'); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + db = await open(); + expect(await _schema(db), schema); + expect(await _revisionRows(db), revs); + expect(await db.query('decoded_onehz', orderBy: 'rec_ts'), rows); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + db = await open(); + expect(await _schema(db), schema, reason: 'onCreate and onUpgrade must agree'); + }); + + test('absent day and legacy bucket revisions read zero', () async { + await seed58(); + await open(); + for (final day in [_day, '1999-01-01']) { + expect(await LocalDb.inputDayFingerprint(day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + } + }); + + for (final table in _inputs) { + for (final op in ['INSERT', 'REPLACE', 'UPDATE', 'DELETE']) { + test('$table $op advances only its 15-minute bucket', () async { + final db = await open(); + final ts = _lo + 930; + if (op != 'INSERT') { await db.insert(table, _row(table, ts)); } + final before = await _rev(db, ts); + final adjacent = await _rev(db, ts - 900); + switch (op) { + case 'INSERT': await db.insert(table, _row(table, ts)); + case 'REPLACE': + // Identical values still invalidate; repeat to catch revision reset + // under the outer statement's OR REPLACE conflict policy. + for (var i = 0; i < 3; i++) { + final last = await _rev(db, ts); + await db.insert(table, _row(table, ts), + conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _rev(db, ts), greaterThan(last), + reason: '$table identical REPLACE must never reset the bucket'); + } + case 'UPDATE': + await db.update(table, table == 'decoded_onehz' ? {'hr': 62} : {'rr_ms': 900}); + case 'DELETE': await db.delete(table); + } + expect(await _rev(db, ts), greaterThan(before), reason: '$table $op was invisible'); + expect(await _rev(db, ts - 900), adjacent, reason: 'unrelated bucket changed'); + }); + } + + test('$table timestamp UPDATE invalidates both old and new buckets', () async { + final db = await open(); + final ts = _lo + 899; + await db.insert(table, _row(table, ts)); + final old = await _rev(db, ts), next = await _rev(db, ts + 1); + await db.update(table, {'rec_ts': ts + 1, 'ts_ms': (ts + 1) * 1000}); + expect(await _rev(db, ts), greaterThan(old), reason: 'old window lost a row'); + expect(await _rev(db, ts + 1), greaterThan(next), reason: 'new window gained a row'); + }); + + test('$table invalid timestamps do not create revision buckets', () async { + final db = await open(); + for (final ts in [0, -1]) { await db.insert(table, _row(table, ts)); } + expect(await _revisionRows(db), isEmpty); + }); + } + + test('identical newest-wins replacement defeats MAX/count but moves revision', () async { + final db = await open(); + final row = _row('decoded_onehz', _lo + 10); + await db.insert('decoded_onehz', row); + final old = await _legacyFingerprint(db); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', row, conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _legacyFingerprint(db), old, reason: 'MAX/count blind spot reproduced'); + expect((await db.query('decoded_onehz')).length, 1); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'identical REPLACE must invalidate cached input'); + }); + + test('late RR without any 1 Hz row changes the day fingerprint', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 20)); + expect(await db.query('decoded_onehz'), isEmpty); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'RR-only inputs must not require MAX(rec_ts)'); + }); + + for (final date in [DateTime(2026, 10, 20), DateTime(2026, 3, 8), DateTime(2026, 11, 1)]) { + test('fingerprint sums/maxes local day buckets for ${dayLabelOf(date)}', () async { + final db = await open(); + final lo = date.millisecondsSinceEpoch ~/ 1000; + final hi = DateTime(date.year, date.month, date.day + 1).millisecondsSinceEpoch ~/ 1000; + // Unequal bucket revisions pin SUM and MAX independently. Outside rows + // must not contribute, including precisely the following midnight. + for (final ts in [lo - 1, lo, lo + 900, hi - 1, hi]) { + await db.insert('decoded_onehz', _row('decoded_onehz', ts)); + } + await db.update('decoded_onehz', {'hr': 63}, where: 'rec_ts = ?', whereArgs: [lo]); + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + expect(fp.revisionSum, 4, reason: 'two writes in first bucket, one in each of two others'); + expect(fp.revisionMax, 2); + expect(fp.generation, LocalDb.storeGeneration); + expect(await LocalDb.inputDayFingerprint(dayLabelOf(date)), fp, + reason: 'reading the fingerprint must be idempotent'); + }); + } + + test('rolled-back raw writes roll back revisions as well', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await expectLater(db.transaction((txn) async { + await txn.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await txn.insert('decoded_rr', _row('decoded_rr', _lo + 10)); + throw StateError('abort before commit'); + }), throwsStateError); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await _revisionRows(db), isEmpty); + expect(await LocalDb.inputDayFingerprint(_day), before); + }); + + test('reopen advances generation without changing input revisions', () async { + var db = await open(); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + final before = await LocalDb.inputDayFingerprint(_day); + await LocalDb.close(); + db = await open(); + final after = await LocalDb.inputDayFingerprint(_day); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await db.query('decoded_onehz'), hasLength(1)); + }); + + test('wipe increments generation even when the revision key returns to zero', () async { + final db = await open(); + final before = LocalDb.storeGeneration; + final emptyKey = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await LocalDb.wipeAll(); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await db.query('input_rev'), isEmpty); + expect(LocalDb.storeGeneration, greaterThan(before)); + expect(await LocalDb.inputDayFingerprint(_day), isNot(emptyKey), + reason: '(day, revision, generation) must reject a pre-wipe cache entry'); + }); + + Future source({bool failAfterCommit = false}) async { + final src = await databaseFactory.openDatabase(sourcePath); + // A valid replacement in an EARLY table, followed by a destination failure + // in a later table. This drives the public importer, not a test-only merge. + await src.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await src.insert('journal', {'date': _day, 'note': 'replacement', 'updated_at': 2}); + if (failAfterCommit) { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + } + + for (final partial in [false, true]) { + test('${partial ? 'partial failing' : 'successful replacement'} import advances generation', () async { + final db = await open(); + await db.insert('journal', {'date': _day, 'note': 'old', 'updated_at': 1}); + await source(failAfterCommit: partial); + final before = LocalDb.storeGeneration; + final key = await LocalDb.inputDayFingerprint(_day); + if (partial) { + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(sourcePath); + } + expect((await db.query('journal')).single['note'], 'replacement', + reason: 'prove a page committed before checking invalidation'); + expect(LocalDb.storeGeneration, greaterThan(before), + reason: 'every exit after durable replacement must invalidate caches'); + expect(await LocalDb.inputDayFingerprint(_day), isNot(key)); + }); + } + + for (final mode in ['success', 'later failure', 'first failure']) { + test('bulk restore $mode invalidates only after a durable page', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute(_v58DecodedDdl.first); + await src.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + if (mode == 'later failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure') { + await db.execute("CREATE TRIGGER refuse_decoded BEFORE INSERT ON decoded_onehz " + "BEGIN SELECT RAISE(ABORT, 'test bulk first-page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + expect(await db.query('decoded_onehz'), + hasLength(mode == 'first failure' ? 0 : 1)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + for (final mode in ['success', 'later table failure', + 'later archive-page failure', 'first failure']) { + test('archive-only restore $mode tracks durable pages', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, ' + 'captured_at INTEGER, reason TEXT)'); + final sourceRows = mode == 'later archive-page failure' ? 501 : 1; + final batch = src.batch(); + for (var i = 1; i <= sourceRows; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': _lo, + 'captured_at': _lo * 1000, 'reason': 'test_archive'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final rejected = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $rejected ' + "BEGIN SELECT RAISE(ABORT, 'test archive page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + test('failed first import page leaves generation unchanged', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + }); + + test('failed first import page can retry and invalidate the next generation', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + await db.execute('DROP TRIGGER refuse_journal'); + await LocalDb.importFromDbFile(sourcePath); + expect((await db.query('journal')).single['note'], 'replacement'); + expect(LocalDb.storeGeneration, greaterThan(before), reason: 'retry must not leave a sticky latch'); + }); + + test('GUARD sync REPLACE shrinks/clears beats only for its device and second', () async { + final db = await open(); + Future commit(String device, int counter, List rr, {int? ts}) { + final at = ts ?? _lo; + return LocalDb.commitSyncBatch( + [RawRecord(counter: counter, packetType: 47, hex: 'feed$counter', + capturedAt: at * 1000, recTs: at)], + [Sample(tsEpoch: at, counter: counter, hr: 60 + counter, + rrIntervalsMs: rr, ax: 0, ay: 0, az: 1, + spo2RedRaw: 0, spo2IrRaw: 0, skinTempRaw: 30000)], + deviceId: device, deviceFamily: 'gen4', + ); + } + await commit('', 10, [800, 810, 820]); + await commit('secondary', 11, [700, 710]); + await commit('', 12, [780], ts: _lo + 1); + await commit('', 1, [900]); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['counter'], 1); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['hr'], 61); + expect((await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo")).map((r) => r['rr_ms']), [900]); + await commit('', 2, []); + expect(await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo"), isEmpty); + expect((await db.query('decoded_rr', where: 'rec_ts = ?', whereArgs: [_lo + 1])) + .single['rr_ms'], 780, reason: 'the neighboring second must keep its beats'); + expect((await db.query('decoded_rr', where: "device_id = 'secondary'", orderBy: 'beat_index')) + .map((r) => r['rr_ms']), [700, 710]); + }); +} + +// Frozen c8e63556 v58 base table definitions; optional columns added by the +// existing repair path are captured before upgrade. Never seed input_rev here. +const _v58DecodedDdl = [ + '''CREATE TABLE decoded_onehz ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER, counter INTEGER NOT NULL, hr INTEGER, ax REAL, ay REAL, az REAL, + spo2_red_raw INTEGER, spo2_ir_raw INTEGER, skin_temp_raw INTEGER, + step_count INTEGER, step_cadence INTEGER, activity_class INTEGER, skin_temp_c REAL, + on_wrist INTEGER, hr_valid INTEGER, hr_alt INTEGER, ambient_raw INTEGER, + device_family TEXT, source TEXT, temp_ch2_c REAL, temp_ch3_c REAL, + signal_quality_logvar REAL, dyn_accel_g REAL, PRIMARY KEY(device_id, ts_ms))''', + '''CREATE TABLE decoded_rr ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER NOT NULL, beat_index INTEGER NOT NULL, rr_ts_ms INTEGER NOT NULL, + rr_ms INTEGER NOT NULL, device_family TEXT, source TEXT, + PRIMARY KEY(device_id, ts_ms, beat_index))''', +]; diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart new file mode 100644 index 000000000..8ce9485fe --- /dev/null +++ b/test/ed2a_substrate_transfer_test.dart @@ -0,0 +1,258 @@ +// ED-2a E5: upstream's existing worker, no E3 annotations/registry/AST guard. +// fromTransfer delegates to fromJson in RED: parity passes, adoption fails. +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart' show kAlgoVersion; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; + +import 'support/dart_source.dart'; + +const _intKeys = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _doubleKeys = ['rr_ts_ms', 'rr_ms', 'ax', 'ay', 'az']; +const _start = 1577923200; // existing two_device_day fixture, UTC + +Map _columns() => { + 'ts_sec': [_start, _start + 1, _start + 2], + 'hr': [60, 0, 61], + 'rr_ts_ms': Float64List.fromList([1577923199999.5, 1577923201000.25]), + 'rr_ms': Float64List.fromList([850.125, 900.5]), + 'ax': Float64List.fromList([-0.0, double.minPositive, double.nan]), + 'ay': Float64List.fromList([double.infinity, double.negativeInfinity, 0.1 + 0.2]), + 'az': Float64List.fromList([1, -1, 0.5]), + 'spo2_red': [0, 1, 65535], 'spo2_ir': [123, 0, 65534], + 'skin_temp': [30001, 0, 30002], 'skin_contact': [1, 0, -1], + 'step_count': [65535, 0, -1], 'hr_valid': [1, 0, -1], + 'band_sleep_state': [1, 0, -1], + 'device_family': 'gen5', 'device_ids': ['secondary', '', 'secondary'], +}; + +void _same(Substrate got, Substrate oracle) { + final a = got.toJson(), b = oracle.toJson(); + expect(a.keys, b.keys, reason: 'every serialized field must survive'); + for (final key in _intKeys) { expect(a[key], b[key], reason: key); } + for (final key in _doubleKeys) { + expect(a[key], isA(), reason: '$key must remain packed'); + final g = a[key] as Float64List, w = b[key] as Float64List; + expect(g.buffer.asUint64List(g.offsetInBytes, g.length), + w.buffer.asUint64List(w.offsetInBytes, w.length), reason: '$key IEEE-754 bits'); + } + expect(got.deviceFamily, oracle.deviceFamily); + expect(got.deviceIds, oracle.deviceIds); +} + +// Exercise the actual upstream prepare worker. Own every port/subscription and +// kill on every exit, including a failed handshake or malformed-page error. +Future> _worker(List> frames, + List> rr, {Map? page}) async { + final port = ReceivePort(); + Isolate? isolate; + StreamSubscription? subscription; + final ready = Completer(); + final done = Completer>(); + // Either future can fail before the caller reaches it. + unawaited(ready.future.then((_) {}, onError: (Object _, StackTrace _) {})); + unawaited(done.future.then((_) {}, onError: (Object _, StackTrace _) {})); + void fail(Object error) { + if (!ready.isCompleted) ready.completeError(error); + if (!done.isCompleted) done.completeError(error); + } + try { + subscription = port.listen((message) { + if (message is SendPort) { + if (!ready.isCompleted) ready.complete(message); + } else if (message is Map && message['type'] == 'result') { + if (!done.isCompleted) done.complete((message['payload'] as Map).cast()); + } else if (message is Map && message['type'] == 'error') { + fail(StateError('${message['error']}')); + } else if (message is List) { + fail(StateError('uncaught worker error: $message')); + } else if (message == null && !done.isCompleted) { + fail(StateError('worker exited before returning a payload')); + } + }); + isolate = await Isolate.spawn(derivationPrepareWorker, port.sendPort, + onError: port.sendPort, onExit: port.sendPort); + final worker = await ready.future.timeout(const Duration(seconds: 10)); + worker.send({'type': 'config', 'mode': 'substrate'}); + worker.send(page ?? {'type': 'page', 'frames': frames, 'rr': rr}); + worker.send({'type': 'finish'}); + return await done.future.timeout(const Duration(seconds: 10)); + } finally { + await subscription?.cancel(); + port.close(); + isolate?.kill(priority: Isolate.immediate); + } +} + +Map _bundle(Substrate sub) { + final day = prepareDerivationPayload(sub).days.first; + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput( + date: day.date, dayTsSec: d.tsSec, dayHr: d.hr, + dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, + sleepJson: day.sleepJson, hypnoStages: day.hypnoStages, + sleepOnsetSec: day.sleepOnsetSec, sleepOffsetSec: day.sleepOffsetSec, + profile: const Profile().toMap(), dayConfidence: day.confidence, + dayFlags: day.flags, deviceFamily: d.deviceFamily, + sleepSource: day.sleepSource, + ).toJson()); +} + +// Same generator as case 1 in two_device_fixture_test.dart, driven by the +// committed region spec. Only the documented gen4 regions feed that golden. +({List> frames, List> rr}) _fixture() { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + int offset(String hhmmss) { + final p = hhmmss.split(':').map(int.parse).toList(); + return p[0] * 3600 + p[1] * 60 + p[2]; + } + final frames = >[], rr = >[]; + for (final region in spec['regions'] as List) { + final r = region as Map; + if (!(r['devices'] as List).contains('gen4')) continue; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = _start + i; + frames.add({'rec_ts': ts, 'device_family': 'gen4', 'hr': 60 + ts % 7, + 'ax': 0.0, 'ay': 0.0, 'az': 1.0, 'skin_temp_raw': 30000 + ts % 11}); + rr.add({'rec_ts': ts, 'rr_ms': 1000 - ts % 5 * 10}); + } + } + return (frames: frames, rr: rr); +} + +void main() { + test('engine receives prepare-worker substrate through the transfer seam', () { + final code = stripCommentsAndStrings(File('lib/compute/derivation_engine.dart').readAsStringSync()); + expect(code, matches(RegExp(r'\bSubstrate\.fromTransfer\(\s*payload\s*\)')), + reason: 'the actual receiver still calls fromJson, which copies every column'); + }); + + for (final key in [..._intKeys, ..._doubleKeys]) { + test('transfer adopts correctly typed $key without an element copy', () { + final message = _columns(); + final got = Substrate.fromTransfer(message).toJson(); + expect(identical(got[key], message[key]), isTrue, + reason: '$key was copied on the receiving/UI isolate'); + }); + } + + test('transfer preserves every field and double bit, including band sleep state', () { + final message = _columns(); + final got = Substrate.fromTransfer(message); + _same(got, Substrate.fromJson(message)); + expect(got.bandSleepState, [1, 0, -1], reason: '0 is wake, -1 is absent'); + expect(got.deviceIds, {'secondary', ''}); + }); + + final variants = Function()>{ + 'empty': () => {}, + 'timestamps only': () => {'ts_sec': [_start, _start + 1]}, + 'missing timestamps': () => {..._columns()}..remove('ts_sec'), + 'empty optional columns': () => {..._columns(), 'hr': [], 'ax': Float64List(0), + 'step_count': [], 'hr_valid': [], 'band_sleep_state': []}, + 'short columns and sparse unpaired RR': () => {..._columns(), 'hr': [62], + 'az': Float64List.fromList([0.25]), 'step_count': [1], + 'hr_valid': [1], 'band_sleep_state': [0], + 'rr_ms': Float64List.fromList([850])}, + 'null provenance': () => {..._columns(), 'device_family': null, 'device_ids': null}, + 'loose numeric columns': () => { + for (final e in _columns().entries) + e.key: e.value is List ? List.from(e.value as List) : e.value, + }, + 'mixed integer and double numbers': () => {'ts_sec': [_start.toDouble(), _start + 1], + 'hr': [60.0, 61], 'rr_ms': [850, 900.5], + 'rr_ts_ms': [1577923200123, 1577923201000.5], + 'band_sleep_state': [0.0, 1]}, + 'wide integers': () => {..._columns(), 'ts_sec': [1 << 40, (1 << 40) + 1, (1 << 40) + 2], + 'step_count': [1 << 40, -(1 << 40), 0]}, + }; + for (final variant in variants.entries) { + test('transfer defaults equal fromJson: ${variant.key}', () { + final m = variant.value(); + _same(Substrate.fromTransfer(m), Substrate.fromJson(m)); + }); + } + + test('GUARD constructor optional arrays remain empty; decoder defaults remain absent', () { + final direct = Substrate(tsSec: [_start], hr: [60], rrTsMs: [], rrMs: [], + ax: [0], ay: [0], az: [1], spo2Red: [0], spo2Ir: [0], skinTemp: [0], skinContact: [0]); + expect(direct.stepCount, isEmpty); + expect(direct.hrValid, isEmpty); + expect(direct.bandSleepState, isEmpty); + final decoded = Substrate.fromTransfer({'ts_sec': [_start]}); + expect(decoded.stepCount, [-1]); + expect(decoded.hrValid, [-1]); + expect(decoded.bandSleepState, [-1]); + expect(decoded.hr, [0]); + expect(decoded.ax, [0.0]); + expect(decoded.rrMs, isEmpty); + }); + + test('actual prepare worker emits columnar typed doubles and preserves band sleep', () async { + final frames = [for (var i = 0; i < 3; i++) { + 'rec_ts': _start + i, 'hr': 60 + i, 'device_family': 'gen5', + 'ax': 0.25, 'ay': -0.0, 'az': 1.0, 'step_count': 20 + i, + 'hr_valid': i % 2, 'band_sleep_state': [1, 0, -1][i], + 'device_id': '', 'skin_temp_raw': 30000 + i, + }]; + final rr = [{'rec_ts': _start, 'rr_ms': 850, + 'rr_ts_ms': _start * 1000, 'beat_ts_ms': _start * 1000 + 123}]; + final message = await _worker(frames, rr); + for (final key in _doubleKeys) { expect(message[key], isA(), reason: key); } + for (final key in _intKeys) { expect(message[key], isA>(), reason: key); } + expect(message.values.any((v) => v is List && v.any((e) => e is Map)), isFalse, + reason: 'worker substrate contains columns, never per-row maps'); + final got = Substrate.fromTransfer(message); + _same(got, substrateFromDecodedPage(frames, rr)); + expect(got.bandSleepState, [1, 0, -1]); + expect(got.rrTsMs, [_start * 1000.0 + 123]); + }); + + test('actual worker result adopts its received arrays', () async { + final message = await _worker([{'rec_ts': _start, 'hr': 60, 'az': 1.0}], []); + final adopted = Substrate.fromTransfer(message).toJson(); + for (final key in [..._intKeys, ..._doubleKeys]) { + expect(identical(adopted[key], message[key]), isTrue, reason: '$key copied after worker receive'); + } + }); + + test('a worker error does not prevent the next transfer', () async { + await expectLater(_worker([], [], page: {'type': 'page', 'frames': 'malformed transport'}), throwsStateError); + final good = await _worker([{'rec_ts': _start, 'hr': 60}], []); + expect(Substrate.fromTransfer(good).hr, [60]); + }); + + test('GUARD existing golden is byte-equal with transfer off, on, and across the worker', () async { + // Same timezone condition as the existing golden test; no regeneration. + if (DateTime.fromMillisecondsSinceEpoch(_start * 1000).timeZoneOffset != Duration.zero) { + markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); + return; + } + expect(kAlgoVersion, 120, reason: 'ED-2b repins analytics; transfer remains exact'); + final fixture = _fixture(); + final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); + final transferred = Substrate.fromTransfer(baseline.toJson()); + final workerMessage = await _worker(fixture.frames, fixture.rr); + final acrossWorker = Substrate.fromTransfer(workerMessage); + _same(transferred, baseline); + _same(acrossWorker, baseline); + final golden = jsonDecode(File('test/fixtures/two_device_day_expected.json').readAsStringSync()) as Map; + final expected = jsonEncode(golden['single_device']); + expect(jsonEncode(_bundle(baseline)), expected, reason: 'transfer off'); + expect(jsonEncode(_bundle(transferred)), expected, reason: 'transfer on'); + expect(jsonEncode(_bundle(acrossWorker)), expected, reason: 'actual worker round trip'); + }, timeout: const Timeout(Duration(minutes: 2))); +} diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart new file mode 100644 index 000000000..9031d5423 --- /dev/null +++ b/test/ed2b_algo_version_114_test.dart @@ -0,0 +1,21 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + test('ED-2b takes one version bump and names the output changes', () { + expect(kAlgoVersion, 120); + expect(kAnalyticsPin, 'f8e7d3c478cc59707fde41f3edff7736026573fe'); + final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); + final at = src.indexOf('// 119 → 120'); + expect(at, isNonNegative); + final entry = src.substring(at, src.indexOf('const int kAlgoVersion', at)); + for (final change in ['f8e7d3c', '32', 'hrvTime', 'banisterTrimp', + 'ctlAtlTsb', 'cosinor', 'timeInZone', 'circadianNonparametric', + 'Lomb–Scargle', 'non-finite', 'irregular-screen gaps', 'D35', 'D36', + 'forward-only', 'staging labels unchanged']) { + expect(entry, contains(change)); + } + }); +} diff --git a/test/ed2b_cardio_cache_test.dart b/test/ed2b_cardio_cache_test.dart new file mode 100644 index 000000000..2339b257a --- /dev/null +++ b/test/ed2b_cardio_cache_test.dart @@ -0,0 +1,280 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/compute/vendor_sleep.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +const _day = '2020-01-02'; +int _sec(int hour) => DateTime(2020, 1, 2, hour).millisecondsSinceEpoch ~/ 1000; + +// RR plus accelerometry forces cardio rows through the real stager; a sleepy +// HR-only fixture could pass output guards without ever exercising the cache. +Substrate _night({int endHour = 7, int fromHour = -2}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + final rr = [], rt = []; + var beat = _sec(fromHour) * 1000.0; + for (var t = _sec(fromHour); t < _sec(endHour); t++) { + final asleep = t >= _sec(-1) && t < _sec(6); + final bpm = asleep ? 52 + t % 7 : 85 + t % 13; + ts.add(t); hr.add(bpm); + ax.add(asleep ? 0 : .3 * math.sin(t * .21)); + ay.add(asleep ? 0 : .2 * math.cos(t * .13)); + az.add(asleep ? 1 : 1 + .05 * math.sin(t * .07)); + while (beat < (t + 1) * 1000) { + final ibi = 60000 / bpm + (asleep ? 40 : 12) * math.sin(t * .35); + rt.add(beat); rr.add(ibi); beat += ibi; + } + } + return Substrate(tsSec: ts, hr: hr, ax: ax, ay: ay, az: az, + rrMs: rr, rrTsMs: rt, skinTemp: List.filled(ts.length, 3000), + skinContact: List.filled(ts.length, 1), spo2Red: List.filled(ts.length, 1), + spo2Ir: List.filled(ts.length, 1), deviceFamily: 'gen4'); +} + +String _candidate(Substrate sub, ana.CardioRowCache? cache, + {SleepWindowOverride? override}) => jsonEncode(prepareSleepSessionCandidate( + sub, targetDay: _day, rowCache: cache, override: override).toJson()); + +Future _save(Substrate sub) async { + final db = await LocalDb.instance; final b = db.batch(); + for (var i = 0; i < sub.tsSec.length; i++) { + final t = sub.tsSec[i]; + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'counter': i, 'hr': sub.hr[i], 'ax': sub.ax[i], + 'ay': sub.ay[i], 'az': sub.az[i], 'device_family': 'gen4'}); + } + final indexes = {}; + for (var i = 0; i < sub.rrMs.length; i++) { + final t = sub.rrTsMs[i] ~/ 1000; + final index = indexes.update(t, (v) => v + 1, ifAbsent: () => 0); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': sub.rrTsMs[i], 'rr_ms': sub.rrMs[i], + 'beat_index': index}); + } + await b.commit(noResult: true); +} + +void _nearPrsa(Object? got, Object? want, [String path = r'$']) { + if (got is double && want is double) { + final tolerance = path.endsWith('.confidence') ? 1e-12 : 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tolerance), reason: path); + } else if (want is Map) { + final g = got as Map; expect(g.keys.toList(), want.keys.toList()); + for (final k in want.keys) { _nearPrsa(g[k], want[k], '$path.$k'); } + } else if (want is List) { + final g = got as List; expect(g.length, want.length); + for (var i = 0; i < want.length; i++) { _nearPrsa(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _checkNightOutput(Map input, Map output) { + final cold = deriveDayBundle(input); + final a = Map.from(output['clinical'] as Map); + final b = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _nearPrsa(a.remove(k), b.remove(k)); } + expect(jsonEncode({...output, 'clinical': a}), jsonEncode({...cold, 'clinical': b})); +} + +void main() { + for (final path in ['candidate auto', 'candidate manual', 'candidate confirmed', 'payload', 'calendar']) { + test('E6 $path shares the supplied CardioRowCache and stays byte-identical', () { + final sub = _night(); final cache = ana.CardioRowCache(); + String run(ana.CardioRowCache? rows) { + if (path == 'candidate auto') { return _candidate(sub, rows); } + if (path == 'candidate confirmed' || path == 'candidate manual') { return _candidate(sub, rows, + override: SleepWindowOverride(dayId: _day, onsetSec: _sec(0), + offsetSec: _sec(5), source: path == 'candidate manual' ? 'manual' : 'confirmed')); } + if (path == 'payload') { return jsonEncode(prepareDerivationPayload(sub, + targetDay: _day, rowCache: rows).days.map((d) => d.toJson()).toList()); } + return jsonEncode(calendarDays(sub, rowCache: rows) + .map((d) => d.sleep.toJson()).toList()); + } + final plain = run(null); expect(run(cache), plain); expect(run(cache), plain); + expect(cache.length, greaterThan(0), reason: 'every actual staging call must populate shared rows'); + expect(cache.hits, greaterThan(0)); + }); + } + + test('E6 HR-led fallback shares the cache through its forced stager', () { + // Include waking HR before the previous local noon so the fallback uses + // an actual daytime baseline rather than a flat night percentile. + final original = _night(fromHour: -13); + final sub = Substrate.fromJson({...original.toJson(), + // Keep valid moving vectors: all-zero missing accel would leave the + // forced stager with no feature rows to cache even after E6 lands. + 'ax': [for (final t in original.tsSec) .3 * math.sin(t * .21)], + 'ay': [for (final t in original.tsSec) .2 * math.cos(t * .13)], + 'az': [for (final t in original.tsSec) 1 + .05 * math.sin(t * .07)]}); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + expect(plain.sleepSource, 'auto_fallback', reason: 'fixture must reach the HR-led forced-window call'); + final cache = ana.CardioRowCache(); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 vendor-owned forced staging shares the same auto-stage row cache', () { + final sub = _night(); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + final start = plain.sleepOnsetSec, end = plain.sleepOffsetSec - 1; + final step = (end - start) ~/ 3; + final vendor = VendorNight(deviceId: '', source: 'gen4', decodedAtSec: _sec(7), + epochs: [VendorEpoch(start, start + step, 'light'), + VendorEpoch(start + step, start + step * 2, 'deep'), + VendorEpoch(start + step * 2, end, 'rem')]); + String run(ana.CardioRowCache? cache) { + final got = prepareSleepSessionCandidate(sub, targetDay: _day, + vendorNights: [vendor], rowCache: cache); + expect(got.sleepSource, 'vendor_staged', reason: 'fixture must reach the vendor forced-window call'); + return jsonEncode(got.toJson()); + } + final cold = run(null), cache = ana.CardioRowCache(); + expect(run(cache), cold); expect(run(cache), cold); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 an append reuses settled rows and preserves cold candidate bytes', () { + final cache = ana.CardioRowCache(); + _candidate(_night(endHour: 3), cache); + final full = _night(); final got = _candidate(full, cache); + expect(got, _candidate(full, null)); + expect(cache.hits, greaterThan(0), reason: 'earlier epochs survive a growing night'); + }); + + test('guard: #99 malformed cardio cache refuses whole payload', () { + for (final bad in [{'version': 99, 'rows': []}, + {'version': 2, 'rows': 'invalid'}, + {'version': 2, 'rows': [{'startSec': 1}]}]) { + expect(() => ana.CardioRowCache.fromJson(bad), throwsFormatException); + } + }); + + group('E6 engine owns encoded bytes and workers own row objects', () { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_cardio.db'); await _save(_night()); }); + tearDown(edDrop); + + test('real engine worker decodes, merges and prunes the shared row cache', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final second = await e.debugSleepCandidate(_day); + expect(jsonEncode(second.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowBytes, isA(), reason: 'UI isolate retains only the encoded payload'); + expect(e.debugCardioRowHits, greaterThan(0)); + expect(e.debugCardioRowWorkerOperations, containsAll(['decode:worker', 'merge:worker', 'prune:worker'])); + expect(e.debugCardioRowWorkerOperations.where((s) => s.endsWith(':caller')), isEmpty); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true + ..debugDisableCardioRows = true; + expect(jsonEncode((await cold.debugSleepCandidate(_day)).toJson()), + jsonEncode(first.toJson())); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('$key rejects bytes then replacement cache hits with cold-exact staging', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final firstMisses = e.debugCardioRowMisses; + if (key == 'input revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (key == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugRekeyCardioRows(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + final changed = await e.debugSleepCandidate(_day); + expect(jsonEncode(changed.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowMisses, greaterThan(firstMisses), + reason: 'new provenance must reject existing encoded rows'); + final misses = e.debugCardioRowMisses, hits = e.debugCardioRowHits; + await e.debugSleepCandidate(_day); + expect(e.debugCardioRowMisses, misses); + expect(e.debugCardioRowHits, greaterThan(hits), reason: 'new provenance must become reusable'); + }); + } + test('E7-E11 real runDays retains night streams across worker dispatches', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + var observed = 0; + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + e.debugOnNightBundle = (input, output) { + observed++; + _checkNightOutput(input, output); + }; + expect(await e.runDays(edProfile, {_day}), 1); + expect(observed, greaterThan(0)); + final before = e.debugNightStreams!; + expect(before.processedNightBeats, greaterThan(0)); + final work = {for (final name in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + name: before.processedBeats(name)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, before.processedNightBeats, + reason: 'the production dispatch must return and retain the RAM night owner'); + for (final entry in work.entries) { + expect(work[entry.key], greaterThan(0)); + expect(e.debugNightStreams!.processedBeats(entry.key), entry.value); + } + expect(e.debugNightStreams!.fullListDigests, greaterThan(0)); + expect(e.debugNightStreams!.maximumListDigestsPerIdentity, 1); + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + + for (final change in ['revision', 'generation', 'algorithm']) { + test('E7-E11 engine $change misses then retains the replacement worker state', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics() + ..debugOnNightBundle = _checkNightOutput; + // Isolate the RAM generation contract: disk checkpoints legitimately + // survive reopen and would otherwise hide the required RAM rebuild. + if (change == 'generation') e.debugCheckpoints.disabled = true; + expect(await e.runDays(edProfile, {_day}), 1); + final before = e.debugNightStreams!.processedNightBeats; + if (change == 'revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', where: 'rec_ts = ?', + whereArgs: [_sec(1) + 60], limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (change == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugNightStreams!.context(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + expect(await e.runDays(edProfile, {_day}), 1); + final miss = e.debugNightStreams!.processedNightBeats; + expect(miss, greaterThan(before)); + final metricWork = {for (final s in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + s: e.debugNightStreams!.processedBeats(s)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, miss); + for (final s in metricWork.keys) { + expect(e.debugNightStreams!.processedBeats(s), metricWork[s]); + } + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + } + }); +} diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart new file mode 100644 index 000000000..afd534a8e --- /dev/null +++ b/test/ed2b_checkpoints_test.dart @@ -0,0 +1,854 @@ +// E16 RED contracts. The standalone night/cardio payload uses format 2; +// fork day/minute/awake accumulator formats and schema numbers do not transfer. +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:openstrap_edge/compute/night_checkpoint.dart'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +class _CheckpointPaths extends PathProviderPlatform { + _CheckpointPaths(this.root); + final String root; + @override + Future getTemporaryPath() async => root; +} + +Map _legacyRsa(Map row, {required bool bins}) { + final bytes = row['state'] as List; + final payload = jsonDecode(utf8.decode(bytes.sublist(32))) as Map; + final night = payload['night'] as Map; + final state = bins ? (night['rsa_bins'] as Map)['open'] as Map : night['rsa'] as Map; + state['version'] = 2; + final body = utf8.encode(jsonEncode(payload)); + return {'state': Uint8List.fromList([...sha256.convert(body).bytes, ...body])}; +} + +Future _hasCheckpoint(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='day_checkpoint'")) + .isNotEmpty; + +Future _storage() async { + final db = await LocalDb.instance; + expect(await _hasCheckpoint(db), isTrue, + reason: 'E16 needs disposable day_checkpoint storage at rung 60'); + expect(await db.query('store_identity'), hasLength(1), + reason: 'rung 60 and repair must also create the durable identity row'); + return db; +} + +Map _sampleRow({String day = edDay, int? version}) => { + 'day_id': day, 'algo_version': version ?? kAlgoVersion, 'fmt': nightCheckpointFormat, + 'ctx_sig': 'night-cardio-context', 'cp_rec_ts': edStart ~/ 900 * 900, + 'rev_vec': Uint8List.fromList([0, 1, 2, 3, 4, 5, 6, 7]), + 'state': Uint8List.fromList([0, 255, 9, 10, 13, 128]), + 'night_ref': null, 'computed_at': 7, +}; + +Future>> _rows(Database db) => db.query( + 'day_checkpoint', where: 'day_id = ? AND algo_version = ?', + whereArgs: [edDay, kAlgoVersion]); + +// Supply a real sleep window and RR, not an absent-night fixture that could +// satisfy persistence by writing an empty payload. Two closed 15-minute buckets +// plus a tail exercise the resume boundary. +const _nightSeconds = 2100, _manualOffsetSeconds = 2000; +// One beat per second, including both manual-window endpoints. Clamp to the +// seeded RR extent; later 1 Hz-only appends add no beats. +int get _manualBeatCount => math.min(_nightSeconds, _manualOffsetSeconds + 1); + +Future _night() async { + await edRows(edStart, edStart + _nightSeconds, hr: 60, az: 1); + await LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + _manualOffsetSeconds, source: 'manual'); + final db = await LocalDb.instance; + final batch = db.batch(); + for (var i = 0; i < _nightSeconds; i++) { + final ts = edStart + i; + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'beat_index': 0, 'rr_ts_ms': ts * 1000, + 'rr_ms': 950.0 + 35 * math.sin(i * .13) + 13 * math.sin(i * .041)}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future _derive(DerivationEngine engine) async { + expect(await engine.runDays(edProfile, {edDay}), 1, + reason: 'the checkpoint assertions require a real persisted derivation'); +} + +void main() { + final originalName = LocalDb.dbName; + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + edSetZone('UTC'); + await edFresh('ed2b_checkpoints.db'); + LocalDb.lastRebuild = null; + }); + tearDown(() async { + DerivationEngine.debugHoldTonight = {}; + await edDrop(); + LocalDb.dbName = originalName; + edSetZone(originalZone); + }); + + test('E16 fresh storage uses rung 60 and a versioned composite key', () async { + expect(LocalDb.schemaVersion, 60, + reason: '59 belongs to ED-2a revisions; fork rungs do not transfer'); + final db = await _storage(); + expect(await db.getVersion(), 60); + expect(await db.query('store_identity'), hasLength(1)); + expect(await LocalDb.storeIdentity(), matches(RegExp(r'^[0-9a-f]{32}$'))); + final columns = {for (final row in await db.rawQuery( + 'PRAGMA table_info(day_checkpoint)')) row['name']: row}; + expect(columns.keys.toSet(), {'day_id', 'algo_version', 'fmt', 'ctx_sig', + 'cp_rec_ts', 'rev_vec', 'state', 'night_ref', 'computed_at'}); + expect(columns['day_id']!['pk'], 1); + expect(columns['algo_version']!['pk'], 2); + for (final name in ['rev_vec', 'state']) { + expect(columns[name]!['type'], 'BLOB'); + expect(columns[name]!['notnull'], 1); + } + expect(await db.query('day_checkpoint'), isEmpty); + }); + + test('E16 59 to 60 preserves decoded rows and revisions without backfill', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.insert('decoded_rr', {'device_id': '', 'ts_ms': edStart * 1000, + 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + final raw = await db.query('decoded_onehz'); + final rr = await db.query('decoded_rr'); + final revs = await db.query('input_rev', orderBy: 'bucket'); + final rawShape = await db.rawQuery('PRAGMA table_info(decoded_onehz)'); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(59); + await LocalDb.close(); + db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a migration must preserve the original DB, not quarantine it'); + expect(await db.query('decoded_onehz'), raw); + expect(await db.query('decoded_rr'), rr); + expect(await db.query('input_rev', orderBy: 'bucket'), revs); + expect(await db.rawQuery('PRAGMA table_info(decoded_onehz)'), rawShape); + await _storage(); + expect(await db.getVersion(), 60); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'migration creates empty scratch storage without deriving old days'); + }); + + test('E16 same-version repair creates storage and is idempotent', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await LocalDb.repairOpenSchemaForTest(db); + await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await db.query('day_checkpoint'); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint'), before); + expect(await LocalDb.storeIdentity(), storeId); + }); + + test('E16 upgrade and open repair create scratch without scanning raw input', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(59); + await LocalDb.close(); + final sql = []; + // Observe the actual sqflite calls, including batches, without changing + // migration code or providing a synthetic checkpoint implementation. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + try { + db = await LocalDb.instance; + await LocalDb.repairOpenSchemaForTest(db); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final rawAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false); + expect(openingSql.where((statement) { + final ddl = statement.trimLeft().toUpperCase(); + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + return rawAccess.hasMatch(statement); + }), isEmpty, reason: 'rung 60 and repair must perform no raw scan/backfill/rewrite'); + await _storage(); + expect(await db.query('day_checkpoint'), isEmpty); + } finally { + await LocalDb.close(); + databaseFactory = databaseFactoryFfi; + } + }); + + test('E16 actual same-version reopen repairs missing checkpoint storage', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + final version = await db.getVersion(); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.getVersion(), version); + await _storage(); + expect(await reopened.query('day_checkpoint'), isEmpty); + }); + + test('E16 rerunning rung 60 keeps checkpoint blobs byte for byte', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(version: kAlgoVersion - 1)); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final before = await db.query('day_checkpoint', orderBy: 'day_id,algo_version'); + final storeId = await LocalDb.storeIdentity(); + await db.setVersion(59); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint', orderBy: 'day_id,algo_version'), before); + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(reopened), hasLength(1)); + expect((await _rows(reopened)).single['state'], _sampleRow()['state']); + }); + + test('E16 standalone payload carries night and cardio only, format 2', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final diagnostics = engine.debugCheckpoints; + expect(diagnostics.writes, greaterThan(0), + reason: 'the worker must emit a checkpoint from real derived inputs'); + expect(diagnostics.lastEncodedPayload, isNotNull); + expect(diagnostics.lastEncodedPayload, isNotEmpty); + final payload = diagnostics.lastDecodedPayload; + expect(payload, isNotNull, + reason: 'observe the worker decode, without decoding on the UI isolate'); + expect(payload!['v'], nightCheckpointFormat, + reason: 'RSA v3 and closed-bin states require the new format'); + expect(payload['night'], isNotNull, reason: 'fixture supplies a settled RR night'); + expect(payload['cardio'], isNotNull); + // Inspect the actual worker-emitted inner checkpoint. In #99 capped + // correctors carry their consumed input count at capped.n, not top-level n. + final night = payload['night'] as Map; + final tag = night['tag'] as Map; + final corrector = (night['corrector'] as Map).cast(); + expect(corrector['version'], 2); + expect((night['rsa'] as Map)['version'], 3); + expect((night['rsa_bins'] as Map)['v'], 2); + expect(corrector['type'], 'RrCorrector'); + expect(corrector['maxArtifactRun'], 32); + expect((corrector['capped'] as Map)['n'], tag['count'], + reason: 'the serialized capped corrector must describe exactly the tagged night prefix'); + expect(tag['count'], greaterThan(0)); + expect(ana.RrCorrector.fromJson(corrector).toJson(), corrector, + reason: '#99 must accept the real inner checkpoint without changing its state'); + for (final key in ['day', 'minutes', 'bills', 'awake', 'activity', 'curves']) { + expect(payload, isNot(contains(key)), reason: '$key is outside ED-2b adoption'); + } + }); + + test('E16 process restart resumes and equals cold derive byte for byte', () async { + await _night(); + // Close the entire selected night before the checkpoint boundary, so a + // restarted worker has no tail beats left to fold. + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final first = DerivationEngine(); + await _derive(first); + final before = await edStored(edDay); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.close(); + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), storeId); + final resumed = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(resumed); + final resumedBytes = await edStored(edDay); + expect(resumedBytes, before); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), resumedBytes); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'a fresh engine after reopen must adopt the persisted night/cardio state'); + expect(resumed.debugNightStreams!.processedNightBeats, 0); + expect(cold.debugCheckpoints.hits, 0); + }); + + test('E16 resumed append equals cold derive on the identical extended input', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart + 2100, edStart + 2400, hr: 61, az: 1); + final resumed = DerivationEngine(); + await _derive(resumed); + final actual = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), actual); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'new open-bucket tail must not discard the valid closed prefix'); + }); + + final damaged = Function(Map)>{ + 'truncated blob': (_) => {'state': Uint8List.fromList([1, 0, 255])}, + 'checksum': (row) { + final bytes = Uint8List.fromList(row['state'] as List); + bytes[bytes.length ~/ 2] ^= 1; + return {'state': bytes}; + }, + 'old format': (_) => {'fmt': 1}, + 'future format': (_) => {'fmt': 999}, + 'legacy RSA v2': (row) => _legacyRsa(row, bins: false), + 'legacy RSA-bin v2': (row) => _legacyRsa(row, bins: true), + 'old algorithm': (_) => {'algo_version': kAlgoVersion - 1}, + 'foreign context': (_) => {'ctx_sig': 'foreign-owner-profile-and-window'}, + 'misaligned boundary': (row) => {'cp_rec_ts': (row['cp_rec_ts'] as int) + 1}, + 'corrupt revision vector': (_) => {'rev_vec': Uint8List.fromList([1])}, + }; + for (final entry in damaged.entries) { + test('E16 ${entry.key} is refused whole and recomputed exactly', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final rows = await _rows(db); + expect(rows, hasLength(1), reason: 'real derive must persist a resume point'); + await db.update('day_checkpoint', entry.value(rows.single), + where: 'day_id = ? AND algo_version = ?', whereArgs: [edDay, kAlgoVersion]); + final engine = DerivationEngine(); + await _derive(engine); + final rejectedOutput = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), rejectedOutput); + expect(engine.debugCheckpoints.hits, 0, + reason: 'never salvage a night or cardio subset of an invalid checkpoint'); + if (entry.key == 'old algorithm') { + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'version-keyed lookup must miss the old algorithm row'); + } else { + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'rejection must be observable after the real reader attempts it'); + } + expect(engine.debugCheckpoints.writes, greaterThan(0), reason: 'cold recompute replaces bad state'); + }); + } + + test('E16 changed closed input revision refuses an otherwise valid checkpoint', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart, edStart + 1, hr: 60, az: 1); + final engine = DerivationEngine(); + await _derive(engine); + final revised = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), revised); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'identical newest-wins replacement changes input_rev, so prefix must rebuild'); + }); + + for (final sameEngine in [true, false]) { + test('E16 reopen reuses checkpoints with unchanged revisions, ' + '${sameEngine ? 'retained' : 'fresh'} engine', () async { + await _night(); + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final storeId = await LocalDb.storeIdentity(); + final first = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(first); + final before = await LocalDb.inputDayFingerprint(edDay); + final output = await edStored(edDay); + final beats = first.debugNightStreams!.retainedNightBeats; + final work = first.debugNightStreams!.processedNightBeats; + final streamWork = {for (final key in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + key: first.debugNightStreams!.processedBeats(key)}; + expect(beats, _manualBeatCount, + reason: 'one seeded RR beat per second, including the manual-window end'); + final saved = await _rows(await _storage()); + expect(saved, hasLength(1)); + await LocalDb.close(); + final db = await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await _rows(db), saved, + reason: 'ordinary reopen preserves the durable checkpoint'); + expect(await LocalDb.storeIdentity(), storeId); + final engine = sameEngine ? first : + (DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics()); + engine.debugCheckpoints.reset(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 1, + reason: 'the same store id permits disk adoption after RAM generation changes'); + expect(engine.debugCheckpoints.rejections, isEmpty); + expect(engine.debugNightStreams!.processedNightBeats, + sameEngine ? work : 0, + reason: 'resume adds no night work and preserves cumulative RAM counters'); + for (final entry in streamWork.entries) { + expect(engine.debugNightStreams!.processedBeats(entry.key), sameEngine ? entry.value : 0, + reason: '${entry.key} counters survive same-engine disk adoption'); + } + expect(await edStored(edDay), output); + final replacement = await _rows(db); + expect(replacement, hasLength(1)); + expect(replacement.single['ctx_sig'], saved.single['ctx_sig']); + final rebuilt = engine.debugNightStreams!.processedNightBeats; + await _derive(engine); + expect(engine.debugNightStreams!.processedNightBeats, rebuilt, + reason: 'the replacement RAM owner must be retained'); + expect(await edStored(edDay), output); + final resumed = DerivationEngine(); + await _derive(resumed); + expect(resumed.debugCheckpoints.hits, 1, + reason: 'the checkpoint remains reusable in the same store'); + expect(await edStored(edDay), output); + }); + } + + test('E16 committed import rejects stale disk state with unchanged raw revisions', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final saved = (await _rows(await _storage())).single; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_generation_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'generation-only', 'updated_at': 1}); + await source.close(); + await LocalDb.importFromDbFile(path); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation), + reason: 'committed store replacement differs from a legitimate checkpoint restart'); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + final db = await _storage(); + expect(await _rows(db), isEmpty); + // Defeat scratch clearing deliberately: identity must reject stale bytes + // even when the revision vector and raw inputs are unchanged. + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + engine.debugCheckpoints.reset(); + await _derive(engine); + final replaced = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), replaced); + expect(engine.debugCheckpoints.hits, 0, + reason: 'the original engine must not revive its pre-restore resume state'); + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'the new store identity must force a cold night/cardio lookup'); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 wipe and re-import reject stale checkpoint when revisions repeat', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_wipe_'); + final path = p.join(dir.path, 'source.db'); + try { + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await File(db.path).copy(path); + await LocalDb.wipeAll(); + final wipedId = await LocalDb.storeIdentity(); + expect(wipedId, isNot(storeId)); + await LocalDb.importFromDbFile(path); + expect(await LocalDb.storeIdentity(), isNot(wipedId)); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum, + reason: 'input_rev is local, so reset and identical restore can repeat it'); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(db), isEmpty); + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + final actual = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), actual); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 replaced DB file rejects old checkpoint with identical inputs', () async { + await _night(); + await _derive(DerivationEngine()); + final original = await _storage(); + final saved = (await _rows(original)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final liveName = LocalDb.dbName; + final livePath = original.path; + final replacementPath = p.join(p.dirname(livePath), 'ed2b_cp_replacement.db'); + await LocalDb.close(); + try { + await edFresh('ed2b_cp_replacement.db'); + await _night(); + final replacement = await _storage(); + final replacementId = await LocalDb.storeIdentity(); + expect(replacementId, isNot(storeId)); + await replacement.insert('day_checkpoint', saved); + await replacement.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await File(replacementPath).copy(livePath); + LocalDb.dbName = liveName; + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), replacementId); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(await _storage()), [saved]); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'an intact old blob must not certify a different database file'); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + await LocalDb.close(); + LocalDb.dbName = liveName; + await databaseFactory.deleteDatabase(replacementPath); + } + }); + + test('E16 rebuild with tolerant salvage rotates identity and rejects old state', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final storeId = await LocalDb.storeIdentity(); + final before = await LocalDb.inputDayFingerprint(edDay); + // Force the real ladder-rebuild path. The rename collision rolls back the + // failed open; all input tables remain available to tolerant salvage. + await db.execute('CREATE TABLE _raw_old (hex TEXT)'); + await db.execute('DROP TABLE baselines'); + await db.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await db.insert('baselines', {'key': 'unreadable-salvage-shape'}); + await db.setVersion(2); + await LocalDb.close(); + try { + final rebuilt = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNotNull); + final rebuild = LocalDb.lastRebuild!; + expect(rebuild.salvaged['baselines'], 0, + reason: 'tolerant salvage must continue past a failed table'); + expect(rebuild.salvaged['decoded_onehz'], _nightSeconds); + expect(rebuild.salvaged['decoded_rr'], _nightSeconds); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await _rows(rebuilt), isEmpty); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + await rebuilt.insert('day_checkpoint', saved); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + final rebuild = LocalDb.lastRebuild; + if (rebuild != null) await databaseFactory.deleteDatabase(rebuild.quarantinePath); + } + }); + + test('E16 wipe clears checkpoints with every other table', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + expect(await _hasCheckpoint(db), isTrue, reason: 'wipe clears data and preserves schema'); + }); + + for (final fail in [false, true]) { + test('E16 ${fail ? 'partially failing' : 'successful'} restore clears local scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await db.execute("CREATE TRIGGER require_rotated_identity BEFORE INSERT ON journal " + "BEGIN SELECT CASE WHEN (SELECT id FROM store_identity WHERE singleton = 1) " + "= '$storeId' OR EXISTS(SELECT 1 FROM day_checkpoint) " + "THEN RAISE(ABORT, 'old checkpoint identity inside restore page') END; END"); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_restore_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'restored', 'updated_at': 1}); + // Even a source with matching schema/version must not import its + // checkpoint into a different store's revision history. + final ddl = (await db.rawQuery("SELECT sql FROM sqlite_master " + "WHERE type='table' AND name='day_checkpoint'")).single['sql'] as String; + await source.execute(ddl); + await source.insert('day_checkpoint', { + ..._sampleRow(), 'ctx_sig': 'foreign-store-checkpoint', + }); + if (fail) { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'invalid-import'}); + } + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect((await db.query('journal')).single['note'], 'restored', + reason: 'at least one replacement page committed'); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a committed restore invalidates local checkpoint inputs even if a later page fails'); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + for (final fail in [false, true]) { + test('E16 ${fail ? 'rolled-back first page' : 'empty import'} preserves identity and scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final saved = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + final generation = LocalDb.storeGeneration; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_no_commit_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + if (fail) await source.insert('baselines', {'key': 'invalid-first-page'}); + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), storeId); + expect(LocalDb.storeGeneration, generation); + expect(await _rows(db), saved); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 issue-report slices explicitly classify disposable checkpoints', () { + expect(kSliceTables, contains('day_checkpoint'), + reason: 'new storage must be classified, not silently use fallback'); + expect(kSliceTables['day_checkpoint']!.$1, SliceKeep.none); + expect(kSliceTables['store_identity']!.$1, SliceKeep.none); + }); + + test('GUARD restore allow-list excludes disposable checkpoints', () { + expect(LocalDb.restoreTablesForTest, isNot(contains('day_checkpoint'))); + expect(LocalDb.restoreTablesForTest, isNot(contains('store_identity'))); + expect(LocalDb.salvageTablesForTest, isNot(contains('store_identity'))); + }); + + test('GUARD disabled checkpoint observations are inert and output-neutral', () async { + await _night(); + final normal = DerivationEngine(); + await _derive(normal); + final expected = await edStored(edDay); + final disabled = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(disabled); + expect(await edStored(edDay), expected); + expect(disabled.debugCheckpoints.hits, 0); + expect(disabled.debugCheckpoints.writes, 0); + disabled.debugCheckpoints.reset(); + expect(disabled.debugCheckpoints.disabled, isTrue); + expect(disabled.debugCheckpoints.lastEncodedPayload, isNull); + expect(jsonEncode(disabled.debugCheckpoints.lastDecodedPayload), 'null'); + }); + for (final mode in ['success', 'later table failure', 'later archive-page failure', 'first failure']) { + test('E16 archive-only $mode rotates identity atomically with committed pages', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + await db.execute('CREATE TRIGGER require_archive_identity BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} BEGIN SELECT CASE WHEN ' + "(SELECT id FROM store_identity WHERE singleton = 1) = '$storeId' " + "OR EXISTS(SELECT 1 FROM day_checkpoint) THEN RAISE(ABORT, 'old archive identity') END; END"); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final refused = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive_page BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $refused ' + "BEGIN SELECT RAISE(ABORT, 'archive page failure'); END"); + } + final dir = await Directory.systemTemp.createTemp('ed2b_archive_identity_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, captured_at INTEGER, reason TEXT)'); + final batch = source.batch(); + final count = mode == 'later archive-page failure' ? 501 : 1; + for (var i = 1; i <= count; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': edStart, + 'captured_at': edStart * 1000, 'reason': 'identity_test'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'missing-payload'}); + } + await source.close(); + if (mode == 'success') { + await LocalDb.importFromDbFile(path); + } else { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + if (mode == 'first failure') { + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(db), before); + } else { + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + } + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 deleting a day removes every disposable raw prefix and keeps unrelated input', () async { + await _night(); + final db = await _storage(); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final identity = await LocalDb.storeIdentity(); + const learning = '{"nights":7,"folded_days":["2020-01-01"]}'; + await LocalDb.putBaseline('sleep_user_profile', learning); + await LocalDb.deleteDays({edDay}); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a neighbouring checkpoint can contain the deleted evening too'); + expect(await LocalDb.storeIdentity(), isNot(identity)); + expect(await db.query('decoded_rr'), isEmpty); + final remaining = await db.query('decoded_onehz'); + expect(remaining.map((row) => row['rec_ts']), [other, other + 1]); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], learning, + reason: 'learned state cannot be rebuilt once old input is pruned'); + }); + + test('E16 selected-day export omits scratch and uses a fresh identity', () async { + final db = await _storage(); + await edRows(edStart, edStart + 2); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + final identity = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_selected_export_'); + final provider = PathProviderPlatform.instance; + PathProviderPlatform.instance = _CheckpointPaths(dir.path); + try { + final path = await LocalDb.exportDaysDb({edDay}); + final out = await databaseFactory.openDatabase(path, options: OpenDatabaseOptions(readOnly: true)); + try { + expect(await _hasCheckpoint(out), isTrue); + expect(await out.query('day_checkpoint'), isEmpty); + expect((await out.query('store_identity')).single['id'], isNot(identity)); + expect((await out.query('decoded_onehz')).map((r) => r['rec_ts']), [edStart, edStart + 1]); + expect(await LocalDb.storeIdentity(), identity); + expect(await _rows(db), hasLength(1)); + } finally { + await out.close(); + } + } finally { + PathProviderPlatform.instance = provider; + await dir.delete(recursive: true); + } + }); + + test('E16 checkpoint and RAM reuse cannot finalize or fingerprint a held night', () async { + await _night(); + final later = DateTime(2020, 1, 5, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1); + DerivationEngine.debugHoldTonight = {edDay}; + await _derive(DerivationEngine()); + expect(await _rows(await _storage()), hasLength(1)); + final engine = DerivationEngine(); + for (var i = 0; i < 2; i++) { + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), isNot(contains(edDay))); + expect(await LocalDb.deriveFingerprint(edDay), isNull); + } + expect(engine.debugCheckpoints.hits, greaterThan(0), + reason: 'the hold must survive actual disk adoption'); + DerivationEngine.debugHoldTonight = {}; + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(edDay)); + }); + +} diff --git a/test/ed2b_d35_ownership_endpoint_test.dart b/test/ed2b_d35_ownership_endpoint_test.dart new file mode 100644 index 000000000..07eb87861 --- /dev/null +++ b/test/ed2b_d35_ownership_endpoint_test.dart @@ -0,0 +1,85 @@ +@Timeout(Duration(minutes: 5)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d35.db')); + tearDown(edDrop); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal inclusive endpoint is covered by the ownership read and spans', () async { + final end = edStart + 600; + final db = await LocalDb.instance; + // This interval starts exactly at the inclusive last sample. A query + // ending at `end` silently omits it, even if resolution is later fixed. + await db.insert('device_coverage', {'device_id': '', 'signal': signal.name, + 'start_ts': end, 'end_ts': end + 1}); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart, end); + expect(spanAt(ownership[signal]!, end)?.deviceId, '', + reason: 'both coverage query and resolution must extend through to+1'); + expect(ownership[signal]!.last.end, end + 1); + }); + } + + test('non-owner last-second HR does not leak into a single-second load', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final last = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(last.hr, [61], reason: 'a zero-width half-open resolver must not disable the filter'); + }); + + test('non-owner last-second RR beats are excluded by the same inclusive ownership', () async { + await edRows(edStart, edStart + 601, hr: 61); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'rrIntervals', + 'start_ts': edStart, 'end_ts': edStart + 601}); + for (final id in ['', 'unranked']) { + await db.insert('decoded_rr', {'device_id': id, + 'ts_ms': (edStart + 600) * 1000, 'rec_ts': edStart + 600, + 'rr_ts_ms': (edStart + 600) * 1000, 'beat_index': 0, + 'rr_ms': id.isEmpty ? 900 : 1500}); + } + final last = await DerivationEngine().debugLoadOwnedSubstrate( + edStart + 600, edStart + 600, day: edDay); + expect(last.rrMs, [900], reason: 'the other device must not add a beat at to'); + }); + + test('same endpoint loaded alone or inside a larger range has identical owner data', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final wide = await e.debugLoadOwnedSubstrate(edStart, edStart + 601, day: edDay); + final alone = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(jsonEncode(alone.toJson()), + jsonEncode(wide.slice(edStart + 600, edStart + 601).toJson())); + }); + + test('interior ownership stays primary and absent owner stays absent (guard)', () async { + await edRows(edStart, edStart + 600, hr: 61); + await edRows(edStart, edStart + 600, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + final sub = await DerivationEngine().debugLoadOwnedSubstrate(edStart, edStart + 600, day: edDay); + expect(sub.hr, everyElement(61)); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart - 60, edStart - 1); + expect(ownership[InputSignal.hr1Hz], everyElement(predicate((s) => s.deviceId == null))); + }); +} diff --git a/test/ed2b_d36_target_profile_test.dart b/test/ed2b_d36_target_profile_test.dart new file mode 100644 index 000000000..843f67f51 --- /dev/null +++ b/test/ed2b_d36_target_profile_test.dart @@ -0,0 +1,195 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +final _mid = DateTime(2020, 1, 2).millisecondsSinceEpoch ~/ 1000; +int _h(int hour) => _mid + hour * 3600; + +// Longer previous night, shorter target, real motion and HR contrast. No +// forced/manual window: those deliberately cannot train the profile. +Substrate _night({bool target = true, bool earlierEvening = false}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + for (var t = _h(-36); t < _h(30); t++) { + final still = (t >= _h(earlierEvening ? -12 : -26) && + t < _h(earlierEvening ? -2 : -17)) || + (target && t >= _h(-1) && t < _h(5)); + ts.add(t); + hr.add(still ? 52 + t % 7 : 80 + (t ~/ 60) % 20 + t % 3); + ax.add(still ? 0 : .3 * math.sin(t * .21)); + ay.add(still ? 0 : .2 * math.cos(t * .13)); + az.add(still ? 1 : 1 + .05 * math.sin(t * .07)); + } + return Substrate.fromJson({'ts_sec': ts, 'hr': hr, 'ax': ax, 'ay': ay, + 'az': az, 'device_family': 'gen4'}); +} + +String _observation(ana.SleepNightObservation o) => jsonEncode({ + 'epochs': o.epochs, 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + +Future _save(Substrate s) async { + final b = (await LocalDb.instance).batch(); + for (var i = 0; i < s.length; i++) { + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': s.tsSec[i] * 1000, + 'rec_ts': s.tsSec[i], 'counter': i, 'hr': s.hr[i], 'ax': s.ax[i], + 'ay': s.ay[i], 'az': s.az[i], 'device_family': 'gen4'}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await b.commit(noResult: true); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d36.db')); + tearDown(() async { + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + for (final target in [true, false]) { + test('recordTargetOnly: ${target ? "one target observation" : "no target means none"}', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final sub = _night(target: target); + // An existing exact restricted walk supplies an independent oracle; the + // full walk must retain history while excluding its training side effects. + calendarDays(sub, targetDay: edDay, skipUnusedDays: true); + final expected = ana.takeCardioObservations().map(_observation).toList(); + expect(expected.length, target ? 1 : 0, reason: 'fixture sanity'); + ana.resetCardioObservations(); + final days = calendarDays(sub, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + expect(days.length, greaterThan(1), reason: 'history walk remains real'); + expect(ana.takeCardioObservations().map(_observation).toList(), expected, + reason: 'earlier and later walked nights must never train under the target ID'); + expect(ana.cardioRecordObservations, isTrue); + }); + } + + test('production mayFold worker returns the target observation, never the longest walked night', () async { + // Production's folding pass must use the same exact target-only walk as + // the nonfolding path, and report that target's own training input. + final sub = _night(); + await _save(sub); + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final search = await e.debugLoadSubstrate(range.$1, range.$2, day: edDay); + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: true); + final target = ana.takeCardioObservations(); + expect(target.length, 1, reason: 'the target has a usable real night'); + final observed = []; + e.debugOnStagingWalk = (fold, count, observation) { + expect(fold, isTrue); + expect(count, 1, reason: 'target-only folding permits the E15 skip during training'); + observed.add(observation); + }; + await e.debugSleepCandidate(edDay); + expect(observed, [_observation(target.single)]); + }); + + test('production absent target never folds the earlier walked evening under its ID', () async { + await _save(_night(target: false, earlierEvening: true)); + final observations = []; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, _, observation) { + expect(fold, isTrue); + observations.add(observation); + }; + final candidate = await e.debugSleepCandidate(edDay); + expect(candidate.present, isFalse, reason: 'the only sleep ended before target midnight'); + expect(observations, [null], reason: 'no target observation means no training'); + expect(await LocalDb.baseline('sleep_user_profile'), isNull); + }); + + test('already folded profile and folded_days survive version adoption byte-for-byte (guard)', () async { + await _save(_night()); + final profile = jsonEncode({...const ana.SleepUserProfile().toJson(), + 'nights': 7, 'folded_days': ['2019-12-31', edDay]}); + await LocalDb.putBaseline('sleep_user_profile', profile); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + await DerivationEngine().debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'D36 is forward-only; existing accumulated evidence is not reset or rebuilt'); + }); + + test('fold flags restored when segmentation throws (guard)', () { + for (final initial in [true, false]) { + ana.cardioRecordObservations = initial; + expect(() => calendarDays(_night(), targetDay: edDay, recordTargetOnly: true, + priorSleep: [for (var i = 1; i <= 14; i++) + (startSec: _h(-24 * i - 1), endSec: _h(-24 * i + 5), + dayKey: '2019-12-${i.toString().padLeft(2, '0')}')], + tzOffsetAt: (_) => throw StateError('offset failure')), throwsStateError); + expect(ana.cardioRecordObservations, initial); + } + }); + + test('disabled recording remains disabled on successful target walk (guard)', () { + ana.cardioRecordObservations = false; + ana.resetCardioObservations(); + calendarDays(_night(), targetDay: edDay, recordTargetOnly: true); + expect(ana.cardioRecordObservations, isFalse); + expect(ana.takeCardioObservations(), isEmpty); + }); + + + test('actual transactional folds preserve explicit deterministic order and idempotent replay (guard)', () async { + final inputs = [ + ('2020-01-01', _observation(const ana.SleepNightObservation( + epochs: 480, hrFloorP5: 51, hrSleepMedian: 56))), + (edDay, _observation(const ana.SleepNightObservation( + epochs: 720, hrFloorP5: 58, hrSleepMedian: 63))), + ]; + final profiles = []; + for (var repeat = 0; repeat < 2; repeat++) { + final db = await LocalDb.instance; + await db.delete('baselines', where: 'key = ?', whereArgs: ['sleep_user_profile']); + final order = []; + final e = DerivationEngine()..debugOnProfileFold = order.add; + for (final input in inputs) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect(order, inputs.map((input) => input.$1).toList()); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + for (final input in inputs.reversed) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'already folded nights leave the whole profile byte-identical'); + final profile = jsonDecode(before as String) as Map; + profile.remove('updated_at_ms'); // operational fold timestamp only + profiles.add(jsonEncode(profile)); + } + expect(profiles.last, profiles.first); + }); + + test('folded day bookkeeping is deterministic regardless of insertion order (guard)', () { + final a = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2020-01-01', '2019-12-30'}, edDay); + final b = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2019-12-30', '2020-01-01'}, edDay); + expect(jsonEncode(a), jsonEncode(b)); + expect(a['folded_days'], ['2019-12-30', '2020-01-01', edDay]); + }); +} diff --git a/test/ed2b_e14_union_loading_test.dart b/test/ed2b_e14_union_loading_test.dart new file mode 100644 index 000000000..c62758a8c --- /dev/null +++ b/test/ed2b_e14_union_loading_test.dart @@ -0,0 +1,128 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String _payload(PreparedDerivationDay day) => jsonEncode({ + 'prepared': day.toJson(), 'bundle': edBundle(day)}); + +Future _night() async { + final start = DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000; + await edRows(start, start + 10 * 3600, hr: 55, az: 1); + await edRows(edStart, edStart + 1200, hr: 85); + final batch = (await LocalDb.instance).batch(); + // Multiple beats, beat-only seconds and backwards raw placements straddle + // every revision bucket; worker cuts must restart the running maximum. + for (var t = start; t < start + 10 * 3600; t += 17) { + for (var i = 0; i < 2; i++) { + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': i, + 'beat_ts_ms': t * 1000 - (t % 11 == 0 ? 9000 : i * 400), + 'rr_ms': 900 + t % 90}, conflictAlgorithm: ConflictAlgorithm.replace); + } + } + await batch.commit(noResult: true); +} + +DerivationEngine _cold() => DerivationEngine() + ..debugDisableUnionLoads = true + ..debugDisableSubstrateRetention = true + ..debugDisableStagedSleepCache = true; + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_e14.db'); await _night(); }); + tearDown(edDrop); + + test('E14 search, calendar, nap and night come from one union load with worker cuts', () async { + final loads = <(int, int, String)>[]; + final e = DerivationEngine()..debugDisableSubstrateRetention = true + ..debugOnSubstrateLoad = (a, b, label) => loads.add((a, b, label)); + final day = await e.debugPrepareDay(edDay); + expect(day, isNotNull); + expect(day!.sleepSub.isEmpty, isFalse, reason: 'fixture must reach the staged-night cut'); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(day), _payload(cold!)); + expect(loads.length, 1, + reason: 'overlapping search/day/night rows must be loaded once as a union'); + expect(e.debugUnionWorkerCuts, greaterThanOrEqualTo(3), + reason: 'UI transfers encoded substrate; workers cut all requested windows'); + }); + + test('E14 neighbouring days borrow overlap rather than loading it twice', () async { + final next = DateTime(2020, 1, 3, 5).millisecondsSinceEpoch ~/ 1000; + await edRows(next, next + 1200, hr: 57); + final e = DerivationEngine(); + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + final warm = await e.debugPrepareDay('2020-01-03'); + final cold = await _cold().debugPrepareDay('2020-01-03'); + expect(_payload(warm!), _payload(cold!)); + final reread = (e.snapshot()['raw_rows'] as int) - before; + final coldEngine = _cold(); + await coldEngine.debugPrepareDay('2020-01-03'); + expect(reread, lessThan(coldEngine.snapshot()['raw_rows'] as int), + reason: 'neighbour union must retain its earlier shared hours'); + }); + + final changes = Function(DerivationEngine)>{ + 'input revision': (_) => edRows(edStart, edStart + 1, hr: 91), + 'late RR': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, + 'beat_index': 0, 'rr_ms': 850}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugRekeyUnionAlgorithm(kAlgoVersion - 1); }, + }; + for (final change in changes.entries) { + test('E14 ${change.key}: refuse stale union, cold bytes exact, then reuse worker cuts', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + await change.value(e); + final warm = await e.debugPrepareDay(edDay); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(warm!), _payload(cold!)); + final refreshed = e.snapshot()['raw_rows'] as int; + expect(refreshed, greaterThan(before), + reason: 'changed provenance must reload rather than hit a stale union'); + final cuts = e.debugUnionWorkerCuts; + await e.debugPrepareDay(edDay); + expect(e.snapshot()['raw_rows'], refreshed, + reason: 'unchanged replacement provenance must reuse the refreshed union'); + expect(e.debugUnionWorkerCuts, greaterThan(cuts), + reason: 'valid refreshed union supplies worker cuts after invalidation'); + }); + } + + test('E14 D35 endpoint: union cuts agree with a separately owned single second', () async { + final db = await LocalDb.instance; + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final end = range.$2; + await edRows(end, end + 1, hr: 61); + await edRows(end, end + 1, hr: 199, id: 'non-owner'); + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': range.$1, 'end_ts': end + 1}); + final day = await e.debugPrepareDay(edDay); + expect(day!.daySub.hr.last, 61); + final last = await DerivationEngine().debugLoadOwnedSubstrate(end, end, day: edDay); + expect(last.hr, [61], reason: 'D35 is a prerequisite for exact union cuts'); + expect(last.hr, day.daySub.slice(end, end + 1).hr); + }); + + test('E14 disabled union diagnostic is behavior-neutral (guard)', () async { + final ordinary = await DerivationEngine().debugPrepareDay(edDay); + final disabled = await _cold().debugPrepareDay(edDay); + expect(_payload(ordinary!), _payload(disabled!)); + }); +} diff --git a/test/ed2b_streams_test.dart b/test/ed2b_streams_test.dart new file mode 100644 index 000000000..4de8c4931 --- /dev/null +++ b/test/ed2b_streams_test.dart @@ -0,0 +1,361 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/aggregate.dart' as agg; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/night_resp_bins.dart'; + +// Deliberately no fork DayCalculationState, mode or revision framework. The +// optional diagnostics observe the real upstream batch calls until GREEN. +Map _input({int beats = 950, bool gaps = false, + bool artifacts = false}) { + const start = 1577930400; + final rr = [], times = []; + var t = start * 1000.0; + for (var i = 0; i < beats; i++) { + final x = 970 + 38 * math.sin(i * .071) + 24 * math.sin(i * 1.31); + rr.add(artifacts && i >= 430 && i < 465 ? 2400.0 : x); + times.add(t); + t += x; + if (gaps && i == 510) t += 90000; + } + return DayBundleInput(date: '2020-01-02', dayTsSec: const [], + dayHr: const [], sleepTsSec: const [], sleepHr: const [], + sleepRrMs: rr, sleepRrTsMs: times, sleepSkinTemp: const [], + sleepJson: const {}, hypnoStages: const [], sleepOnsetSec: start, + sleepOffsetSec: t ~/ 1000 + 1, profile: const {}, + deviceFamily: 'gen4').toJson(); +} + +Map _prefix(Map input, int beats) => { + ...input, + 'sleep_rr_ms': (input['sleep_rr_ms'] as List).take(beats).toList(), + 'sleep_rr_ts_ms': (input['sleep_rr_ts_ms'] as List).take(beats).toList(), +}; + +String _wire(Object? value) => jsonEncode(value); +List _rr(Map x) => x['sleep_rr_ms'] as List; +List _times(Map x) => x['sleep_rr_ts_ms'] as List; + +void _near(Object? got, Object? want, [String path = r'$']) { + if (got is num && want is num) { + // #99 PRSA profile/capacity abs+relative 1e-9; confidence abs 1e-12. + final tol = path.endsWith('.confidence') ? 1e-12 : + 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tol), reason: path); + } else if (want is Map) { + expect(got, isA(), reason: path); + final g = got as Map; + expect(g.keys.toList(), want.keys.toList(), reason: path); + for (final k in want.keys) { _near(g[k], want[k], '$path.$k'); } + } else if (want is List) { + expect(got, isA(), reason: path); + final g = got as List; + expect(g.length, want.length, reason: path); + for (var i = 0; i < want.length; i++) { _near(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _oracles(Map input, Map bundle) { + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final nn = c.nn, ts = c.nnTimesMs; + final af = (1 - c.cleanFraction).clamp(0.0, 1.0); + final h = ana.HrvFreqStream(), r = ana.RsaRespStream(), v = ana.CvhrStream(); + for (var i = 0; i < nn.length; i += 73) { + final end = math.min(i + 73, nn.length); + final a = nn.sublist(i, end), b = ts.sublist(i, end); + h.fold(a, b); r.fold(a, b); v.fold(a, b); + } + final clinical = bundle['clinical'] as Map; + final respiration = bundle['respiration'] as Map; + if (nn.length >= 20) { + final batch = ana.hrvFreq(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(h.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(clinical['hrv_freq']), _wire(batch)); + } + if (nn.length >= 30) { + final batch = ana.rsaRespRate(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(r.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['rsa']), _wire(batch)); + } + if (nn.length >= 60) { + final batch = ana.cvhrApneaScreen(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(v.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['cvhr_apnea']), _wire(batch)); + // #99 contains 914bb6a: an entirely provisional tail on a FRESH stream. + expect(_wire(ana.CvhrStream().snapshot(tailNnMs: nn, + tailNnTimesMs: ts, artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + } + for (final deceleration in [true, false]) { + final a = agg.prsaAggregator(deceleration: deceleration); + var state = a.empty(); + final column = agg.DoubleColumn(Float64List.fromList(nn)); + for (var i = 0; i < nn.length; i += 37) { + state = a.addSlice(state, column, i, math.min(i + 37, nn.length)); + } + final batch = (deceleration ? ana.decelerationCapacity(nn) : + ana.accelerationCapacity(nn)).toJson((x) => x.toJson()); + _near(a.finish(state).toJson((x) => x.toJson()), batch); + _near(clinical[deceleration ? 'prsa_dc' : 'prsa_ac'], batch); + } +} + +void _bundleSame(Map got, Map cold) { + // #99 documents PRSA capacity/profile abs+relative 1e-9 and confidence + // abs 1e-12. Every other bundle byte, including notes and absence, is exact. + final gc = Map.from(got['clinical'] as Map); + final cc = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _near(gc.remove(k), cc.remove(k)); } + expect(_wire({...got, 'clinical': gc}), _wire({...cold, 'clinical': cc})); +} + +void _exact(Map input, Ed2bStreamDiagnostics diagnostics) { + final cold = deriveDayBundle(input); + final got = deriveDayBundle(input, streamDiagnostics: diagnostics); + _bundleSame(got, cold); + _oracles(input, got); +} + +void main() { + for (final style in ['smooth', 'gaps + capped artifact run']) { + final input = _input(gaps: style != 'smooth', artifacts: style != 'smooth'); + test('E7 $style chunked night NN is batch-exact and folds each beat once', () { + final d = Ed2bStreamDiagnostics(); + var observed = false; + d.onCorrected = (nn, times, fraction) { + observed = true; + final c = ana.correctRr(_rr(current), rrTsMs: _times(current)); + expect(_wire(nn), _wire(c.nn)); expect(_wire(times), _wire(c.nnTimesMs)); + expect(fraction, c.cleanFraction); + }; + // Cuts straddle the 248-beat capped corrector's provisional tail and + // the >32 artifact run. No cut is rounded to an epoch boundary. + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + current = _prefix(input, n); + _exact(current, d); + } + expect(observed, isTrue); + expect(d.processedNightBeats, _rr(input).length, + reason: 'RAM corrector must process only the appended raw beats'); + expect(d.retainedNightBeats, greaterThan(0)); + expect(d.nightRebuilds, 1); + }); + + for (final stream in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + final contract = {'frequency': 'E8', 'rsa': 'E8', 'cvhr': 'E9', + 'prsa': 'E10', 'rsa_bins': 'E10'}[stream]; + test('$contract $style $stream streams across growing and repeated passes', () { + final d = Ed2bStreamDiagnostics(); + for (final n in [300, 641, 950]) { _exact(_prefix(input, n), d); } + final work = d.processedBeats(stream); + expect(work, greaterThan(0), reason: 'real metric path must be exercised'); + _exact(input, d); + expect(d.processedBeats(stream), work, + reason: 'repeat must reuse settled stream state'); + expect(d.retainedStreamBeats(stream), greaterThan(0)); + expect(d.streamRestarts(stream), 1); + }); + } + } + + for (final feature in ['rr', 'frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + for (final key in ['input revision', 'store generation', 'algorithm']) { + test('E7-E10 $feature refuses changed $key, equals cold, then reuses replacement', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); + final before = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'store generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _exact(input, d); + final miss = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + expect(miss, greaterThan(before), reason: 'changed context must reject retained work'); + _exact(input, d); + expect(feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature), miss, + reason: 'the replacement context must subsequently hit'); + }); + } + } + + for (final mutation in ['old value', 'old timestamp', 'shorter prefix', 'shifted onset']) { + test('E7-E10 $mutation rebuilds the night and preserves every bundle byte', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); _exact(input, d); + final changed = _prefix(input, mutation == 'shorter prefix' ? 700 : 950); + if (mutation == 'old value') (_rr(changed))[100] += 7; + if (mutation == 'old timestamp') (_times(changed))[100] += .125; + if (mutation == 'shifted onset') { + changed['sleep_rr_ms'] = _rr(changed).sublist(10); + changed['sleep_rr_ts_ms'] = _times(changed).sublist(10); + changed['sleep_onset_sec'] = (changed['sleep_onset_sec'] as int) + 10; + } + _exact(changed, d); + final miss = d.processedNightBeats; + _exact(changed, d); + expect(d.processedNightBeats, miss, reason: 'changed prefix must be retained after rebuild'); + expect(d.nightRebuilds, 2); + }); + } + + test('E7 RAM night owner survives a real isolate hop without changed output', () async { + final d = Ed2bStreamDiagnostics(); final input = _input(); + deriveDayBundle(input, streamDiagnostics: d); + final before = d.processedNightBeats; + final hopped = await Isolate.run(() { + final bundle = deriveDayBundle(input, streamDiagnostics: d); + return (bundle, d); + }); + _bundleSame(hopped.$1, deriveDayBundle(input)); + expect(hopped.$2.processedNightBeats, before); + expect(hopped.$2.retainedNightBeats, greaterThan(0)); + }); + + test('guard: #99 capped RR settles and snapshots equal batch at every chunk', () { + final input = _input(gaps: true, artifacts: true); + final stream = ana.RrCorrector(); + final settled = [], times = []; + var previous = 0; + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + final next = stream.fold(_rr(input).sublist(previous, n), + tsMs: _times(input).sublist(previous, n)); + settled.addAll(next.nn); times.addAll(next.nnTimes); + final tail = stream.snapshot(); + final batch = ana.correctRr(_rr(input).sublist(0, n), + rrTsMs: _times(input).sublist(0, n)); + expect(_wire([...settled, ...tail.tailNn]), _wire(batch.nn)); + expect(_wire([...times, ...tail.tailNnTimes]), _wire(batch.nnTimesMs)); + previous = n; + } + }); + + test('E11 exceptional exit releases occupied identity memo and retry is fresh', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + var digestsAtExit = 0, memoAtExit = 0; + d.onBeforePassReturn = () { + digestsAtExit = d.fullListDigests; memoAtExit = d.identityMemoEntries; + throw StateError('injected after the actual analytics pass'); + }; + expect(() => deriveDayBundle(input, streamDiagnostics: d), throwsStateError); + expect(digestsAtExit, greaterThan(0), reason: 'must exercise actual hash work before failure'); + expect(memoAtExit, greaterThan(0), reason: 'must fail while the pass owns real identity keys'); + expect(d.identityMemoEntries, 0, reason: 'finally must release lists after exceptions'); + d.onBeforePassReturn = null; + _rr(input)[90] += 13; _exact(input, d); + expect(d.identityMemoEntries, 0); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('E11 $key never reuses a preceding pass identity digest', () { + final input = _input(beats: 2300), d = Ed2bStreamDiagnostics(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); final first = d.fullListDigests; + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _rr(input)[89] += 9; _exact(input, d); + expect(d.fullListDigests, greaterThan(first)); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0); + }); + } + + test('E10 RSA bins cross 30-minute edges; sequential PRSA is batch-toleranced', () { + final input = _input(beats: 4000, gaps: true); + final d = Ed2bStreamDiagnostics(); + List? actualBins; + d.onRespWindows = (rates) { actualBins = rates; }; + _exact(_prefix(input, 1861), d); _exact(input, d); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = , List)>{}; + for (var i = 0; i < c.nn.length; i++) { + final key = ((c.nnTimesMs[i] - c.nnTimesMs.first) / 1800000).floor(); + final b = bins.putIfAbsent(key, () => ([], [])); + b.$1.add(c.nn[i]); b.$2.add(c.nnTimesMs[i]); + } + final rates = []; + for (final b in bins.values) { + if (b.$1.length < 60) continue; + final stream = ana.RsaRespStream(); + for (var i = 0; i < b.$1.length; i += 79) { + final end = math.min(i + 79, b.$1.length); + stream.fold(b.$1.sublist(i, end), b.$2.sublist(i, end)); + } + final batch = ana.rsaRespRate(b.$1, b.$2, artifactFraction: 0); + expect(_wire(stream.snapshot(artifactFraction: 0).toJson((v) => v.toJson())), + _wire(batch.toJson((v) => v.toJson()))); + final rate = batch.present ? batch.value!.brpm : null; + if (rate != null) rates.add(rate); + } + expect(_wire(actualBins), _wire(rates)); + expect(bins.length, greaterThanOrEqualTo(3)); + expect(d.slicesProcessed('rsa_bins'), greaterThanOrEqualTo(3)); + final before = d.processedBeats('rsa_bins'); _exact(input, d); + expect(d.processedBeats('rsa_bins'), before); + expect(d.retainedStreamBeats('rsa_bins'), greaterThan(0)); + expect(d.retainedStreamBeats('prsa'), greaterThan(0)); + }); + + test('E11 shared long night lists are hashed once each per pass and released', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + _exact(input, d); + expect(d.fullListDigests, greaterThan(0), reason: 'real dependency proofs must be measured'); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0, reason: 'identity keys must not pin worker arrays between passes'); + final before = d.fullListDigests; + // Reuse the SAME input identities, mutate content between passes. A digest + // surviving endPass would conceal the changed old prefix. + _rr(input)[100] += 11; _exact(input, d); + expect(d.fullListDigests, greaterThan(before)); + expect(d.maximumListDigestsPerIdentity, 1); expect(d.identityMemoEntries, 0); + }); + + test('guard: diagnostics enabled or disabled never changes bundle bytes', () { + final input = _input(gaps: true, artifacts: true); + final d = Ed2bStreamDiagnostics()..disabled = true; + _exact(input, d); d.disabled = false; _exact(input, d); + }); + + test('guard: empty and thin inputs preserve absence and no identity retention', () { + for (final n in [0, 1, 19, 29, 59]) { + final input = _input(beats: n); final d = Ed2bStreamDiagnostics(); + final got = deriveDayBundle(input, streamDiagnostics: d); + expect(_wire(got), _wire(deriveDayBundle(input))); + expect(d.identityMemoEntries, 0); + if (n == 0) expect(d.processedNightBeats, 0); + } + }); + test('E10 closed RSA bins reapply the night artifact gate after append and JSON restart', () { + final input = _input(beats: 6200); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = NightRespBins(); + bins.fold(c.nn.sublist(0, 4000), c.nnTimesMs.sublist(0, 4000)); + List batch(double artifacts) => respPerWindow(c.nn, c.nnTimesMs, + artifactFraction: artifacts); + // A positive control ensures the artifact gate, rather than absent signal, + // is the only reason a high-artifact snapshot has no rates. + expect(batch(0), isNotEmpty); + bins.fold(c.nn.sublist(4000), c.nnTimesMs.sublist(4000)); + expect(bins.snapshot(const [], const [], artifactFraction: 0), batch(0)); + expect(batch(.5), isEmpty); + expect(bins.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + final saved = (jsonDecode(jsonEncode(bins.toJson())) as Map).cast(); + final restored = NightRespBins.fromJson(saved, c.nn.length); + expect(restored.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + expect(restored.snapshot(const [], const [], artifactFraction: 0), batch(0)); + }); + +} + +// Assigned immediately before each synchronous derive; used only to inspect +// the callback's actual corrected arrays without adding them to output JSON. +Map current = {}; diff --git a/test/fixtures/two_device_day_expected.json b/test/fixtures/two_device_day_expected.json index db883f8d2..a682b3071 100644 --- a/test/fixtures/two_device_day_expected.json +++ b/test/fixtures/two_device_day_expected.json @@ -3895,7 +3895,7 @@ "skin_temp_adc": 30005.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 14.14213555551533, + "sdnn": 14.142135555515845, "sdnn_window": 14.164940954653554, "prsa_dc": 15.0, "prsa_dc_anchors": 2879.0, diff --git a/test/fixtures/whoop_freeze/gen4_minus0400.json b/test/fixtures/whoop_freeze/gen4_minus0400.json index 0204b2d6b..91de174c8 100644 --- a/test/fixtures/whoop_freeze/gen4_minus0400.json +++ b/test/fixtures/whoop_freeze/gen4_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.033995461954163, "skipped": 0, "partial": 0, - "payload_sha256": "b57127eaba2e220875324219ed002d3801c330506d1303b87a413572f89ec560" + "payload_sha256": "62a201619a8a80c7a58a4f8feefe51b9b9b17b71fb395a35456b5f06c2705908" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -47775,7 +47775,7 @@ "quiet_hrr": 0.174452349174085, "trimp_net": null, "calories_total": 2297.5125051784576, - "dyn_p90": 0.676695219032997, + "dyn_p90": 0.6766952190329969, "nap_min": 45.0, "hrr_bpm": 14.0, "hr_ceiling_bpm": 157.0 @@ -49591,11 +49591,11 @@ "readiness": 27.03409387728019, "skipped": 0, "partial": 0, - "payload_sha256": "3f4838375946c33e26860d81e49abfb0e7fc17b0087cbac28f3bcb1db8c3b89d" + "payload_sha256": "52604af7568a6615643226ce4d7f339e594e6e9ef256b6e9d9720ae3ae91d98f" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -71090,7 +71090,7 @@ "quiet_hrr": 0.18057477316419124, "trimp_net": null, "calories_total": 2151.991671845124, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 18.0 }, @@ -72717,7 +72717,7 @@ "readiness": 66.11544177079463, "skipped": 0, "partial": 0, - "payload_sha256": "eb9d15f24b218b7d4438e5a6504f33efc6a924f513657215fc436bae3d563fb7" + "payload_sha256": "339b824c3825df771b722599c2b712a19fb697c719ccb46cb77deef5ec91fb2b" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -72969,7 +72969,7 @@ { "date": "2026-07-14", "key": "dyn_p90", - "value": 0.676695219032997 + "value": 0.6766952190329969 }, { "date": "2026-07-14", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73179,7 +73179,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_plus0530.json b/test/fixtures/whoop_freeze/gen4_plus0530.json index 36585524a..9996743de 100644 --- a/test/fixtures/whoop_freeze/gen4_plus0530.json +++ b/test/fixtures/whoop_freeze/gen4_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03382323567632, "skipped": 0, "partial": 0, - "payload_sha256": "cda75458f39702ac4580c5d03d4bf6a6b5c5120aee516e8c32d1c6fe33383def" + "payload_sha256": "9df2d6b463c1d4668fe083d815dde535441090b667f7923c7601a80b2cd98cc0" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033601802905416, "skipped": 0, "partial": 0, - "payload_sha256": "3228493c5545d3a4ee5f2da9410317da44cf8e9947d17b049572ca698c63112d" + "payload_sha256": "2cece0333903e0338a57f753ae55610073e575f4bd1931f74f6555b982237c15" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -71090,7 +71090,7 @@ "quiet_hrr": 0.18057477316419124, "trimp_net": null, "calories_total": 2151.991671845124, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 18.0 }, @@ -72717,7 +72717,7 @@ "readiness": 66.1155535435143, "skipped": 0, "partial": 0, - "payload_sha256": "0187cd292873351886d856d95ba367289227f875a585b0bbbee6c5525617a6da" + "payload_sha256": "a29b3dd5f3759041399c9ac0681eb643c449ba95e59b96b1a1a993d6fcee53c7" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73179,7 +73179,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.0, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_utc.json b/test/fixtures/whoop_freeze/gen4_utc.json index edfbb77fe..5140cd865 100644 --- a/test/fixtures/whoop_freeze/gen4_utc.json +++ b/test/fixtures/whoop_freeze/gen4_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03355259577805, "skipped": 0, "partial": 0, - "payload_sha256": "f13bcc5af9185bb1b8384a9bc439a551a03bc4367ea2fba85c5742ef910df9d8" + "payload_sha256": "a9027226526086f68120d3401a25663926b07ff7515415e7e4f25ccd592e12ac" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033651010089166, "skipped": 0, "partial": 0, - "payload_sha256": "411649f61ac1083a68de60dd1ff589276d011a075e7d22d81f88d4a692c5346a" + "payload_sha256": "f171df77c61ba6ac83f14a9b68516fe1bf2fc433463febe2aaff4eed6531c827" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -71090,7 +71090,7 @@ "quiet_hrr": 0.18057477316419124, "trimp_net": null, "calories_total": 2151.991671845124, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 18.0 }, @@ -72717,7 +72717,7 @@ "readiness": 66.11559080104756, "skipped": 0, "partial": 0, - "payload_sha256": "26aef4d4f03d8744be1225adb85776a659769af4bd0c02091696b58efd7e48fe" + "payload_sha256": "7ce4b9c113c012b9e87932da751c13e2c985386e077c114c17caa6f7a158fdc8" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73179,7 +73179,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen5_minus0400.json b/test/fixtures/whoop_freeze/gen5_minus0400.json index f598c655d..ce571bba5 100644 --- a/test/fixtures/whoop_freeze/gen5_minus0400.json +++ b/test/fixtures/whoop_freeze/gen5_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a3f73d5823f32766fd6f8741f46e2a72ac064ea73b044321b9ebb50480b75058" + "payload_sha256": "7324cb5f4edff114920b2fa3d4ba97cb974cd712eadbbd27991ebd940299011c" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -47954,7 +47954,7 @@ "trimp_net": null, "calories_total": 1707.6018200286808, "steps": 17640.0, - "dyn_p90": 0.676695219032997, + "dyn_p90": 0.6766952190329969, "nap_min": 45.0, "hrr_bpm": 14.0, "hr_ceiling_bpm": 157.0 @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "9cc78bf91ecb2c555fe49878acefe1b2eea8f2e8b6746d742c03f8287350fcaa" + "payload_sha256": "b0b0de9fba0fe2e9ba7e6cadc2eb602ea27798154e12c66488143b0a603c3382" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -71436,7 +71436,7 @@ "trimp_net": null, "calories_total": 1595.2893200286808, "steps": 17640.0, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 16.0 }, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "e8a33133f77bf08bec2c0293cf63f699c19f63a18ad915ec4228cc244786d195" + "payload_sha256": "6c93cfdfb231c5b9c02369f29df360cbced3fda2c908355c6cd22314c53e6572" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73482,7 +73482,7 @@ { "date": "2026-07-14", "key": "dyn_p90", - "value": 0.676695219032997 + "value": 0.6766952190329969 }, { "date": "2026-07-14", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73692,7 +73692,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.808966711097082, "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581920140166515, "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_plus0530.json b/test/fixtures/whoop_freeze/gen5_plus0530.json index c6818f6d5..609a0e07c 100644 --- a/test/fixtures/whoop_freeze/gen5_plus0530.json +++ b/test/fixtures/whoop_freeze/gen5_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a6bb812440c40efd53dc915e78474d6581b354a787fd83f1d8d82274e94b052b" + "payload_sha256": "9f3dde3ed4f3d2bd7988104ed1312ffd3bb78cf2f9c0dc6c3bbd949763c917be" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "e6ee25535499c542ebd13b8a1df85566c229944efba4dbdb8a44ca980c6442dc" + "payload_sha256": "785a07813a59e1e553b90599fbc43708ad5cf6063202796c1777f11aa886aa18" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -71436,7 +71436,7 @@ "trimp_net": null, "calories_total": 1595.2893200286808, "steps": 17640.0, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 16.0 }, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "0ba224b91e74c386567f55d2b31b8346e4601bd61161202740d4bd0a7fa97f2c" + "payload_sha256": "be6cda6e9bd97c996db7f60f380433e10f15b05d35949248b39ad35d530a420b" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73692,7 +73692,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8090587513754603, "skin_temp_adc": 30406.0, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.5820748428079483, "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_utc.json b/test/fixtures/whoop_freeze/gen5_utc.json index f309a8e15..ecec09d5e 100644 --- a/test/fixtures/whoop_freeze/gen5_utc.json +++ b/test/fixtures/whoop_freeze/gen5_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "64bd6e2d3cd9fb8a3762d32513c756dbfe5d33e204538d64f6188eb88b25afff" + "payload_sha256": "1bd3caf43bd58cb8ec23f7fbdc3ece0bf0c5c16d4e8ee0281da29dfb9f53d499" }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "42d874e0691a2e1f19f3ca870dbf497fd820f6daa556da8bec1cbc7f62b661ca" + "payload_sha256": "1a383a9942303b44cfa356dd2d7c2d50c0edfa97a1eaa2b02e53d56cbcd1b1bf" }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -71436,7 +71436,7 @@ "trimp_net": null, "calories_total": 1595.2893200286808, "steps": 17640.0, - "dyn_p90": 0.016617360477042792, + "dyn_p90": 0.0166173604770428, "nap_min": 0.0, "hrr_bpm": 16.0 }, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "a63132ca185866d1519d888273fa38ecc42d26e091cdb18f4a5ba41823294b6c" + "payload_sha256": "24ce9711e808adc5712473639cba81eb6c0fe1c84a7668b00c19ea8bcafcd362" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73692,7 +73692,7 @@ { "date": "2026-07-15", "key": "dyn_p90", - "value": 0.016617360477042792 + "value": 0.0166173604770428 }, { "date": "2026-07-15", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 119, + "algo_version": 120, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8092033860986256, "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581998719140345, "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/guards/db_parts_test.dart b/test/guards/db_parts_test.dart new file mode 100644 index 000000000..86e1f6e45 --- /dev/null +++ b/test/guards/db_parts_test.dart @@ -0,0 +1,62 @@ +import 'dart:io'; + +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _ExecuteCalls extends RecursiveAstVisitor { + final arguments = []; + + @override + void visitMethodInvocation(MethodInvocation node) { + if (node.methodName.name == 'execute') { + arguments.add(node.argumentList.arguments.first); + } + super.visitMethodInvocation(node); + } +} + +void main() { + final parts = Directory('lib/data').listSync().whereType() + .where((f) => f.path.endsWith('.dart') && + f.readAsStringSync().contains("part of 'db.dart';")).toList(); + + test('DB parts contain only _ldb functions', () { + expect(parts.map((f) => f.path), contains('lib/data/db_input_revisions.dart')); + for (final file in parts) { + final unit = parseString(content: file.readAsStringSync()).unit; + expect(unit.declarations, isNotEmpty, reason: file.path); + for (final declaration in unit.declarations) { + expect(declaration, isA(), reason: file.path); + expect((declaration as FunctionDeclaration).name.lexeme, + startsWith('_ldb'), reason: file.path); + } + } + }); + + test('DB part execute calls carry literal SQL', () { + var calls = 0; + for (final file in parts) { + final visitor = _ExecuteCalls(); + parseString(content: file.readAsStringSync()).unit.accept(visitor); + calls += visitor.arguments.length; + for (final argument in visitor.arguments) { + expect(argument, isA(), reason: '${file.path}: $argument'); + } + } + expect(calls, greaterThanOrEqualTo(9)); + // Positive control: a variable SQL argument would fail the predicate. + final probe = _ExecuteCalls(); + parseString(content: 'void probe(dynamic db, String sql) { db.execute(sql); }') + .unit.accept(probe); + expect(probe.arguments.single, isNot(isA())); + }); + + test('DB parts stay below 6000 source lines', () { + for (final file in parts) { + expect(file.readAsLinesSync().length, lessThanOrEqualTo(6000), + reason: file.path); + } + }); +} diff --git a/test/import_bulk_restore_test.dart b/test/import_bulk_restore_test.dart index 989f7d478..1e592c10d 100644 --- a/test/import_bulk_restore_test.dart +++ b/test/import_bulk_restore_test.dart @@ -62,11 +62,14 @@ int _callsTouching(String table) { const _t0 = 1790000000; // 2026-09-21, an arbitrary fixed second /// Stamped with DateTime.now(): differs between any two -/// runs, so the equivalence dump leaves it out. +/// runs, so the equivalence dump leaves it out. store_identity.id is a fresh +/// random id on every committed restore by design (it keys night +/// checkpoints); the singleton row itself is still compared. const _wallClockColumns = { 'compute_freshness': {'updated_at'}, 'activity_review_meta': {'activated_at', 'alert_after_ts'}, 'substrate_archive': {'updated_at'}, + 'store_identity': {'id'}, }; Map archiveSecond(int ts, int hr) => { diff --git a/test/multidevice_coverage_derive_test.dart b/test/multidevice_coverage_derive_test.dart index 99d555102..6fb3c277c 100644 --- a/test/multidevice_coverage_derive_test.dart +++ b/test/multidevice_coverage_derive_test.dart @@ -134,10 +134,10 @@ Future>> _expectedOwnership() async { for (final sig in const [InputSignal.hr1Hz, InputSignal.rrIntervals]) { final raw = await LocalDb.signalPriority(sig); out[sig] = resolveOwnership( - coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo), + coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo + 1), priority: raw.isEmpty ? const [LocalDb.kPrimaryDeviceId] : raw, from: unionFrom, - to: unionTo, + to: unionTo + 1, signal: sig, ); } diff --git a/test/support/ed2a_recalc_fixture.dart b/test/support/ed2a_recalc_fixture.dart new file mode 100644 index 000000000..0f0dfb125 --- /dev/null +++ b/test/support/ed2a_recalc_fixture.dart @@ -0,0 +1,184 @@ +// Shared ED-2a inputs. No cache or invalidation policy lives in this harness. +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:ffi/ffi.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +const edProfile = Profile(ageYears: 35, weightKg: 75, heightCm: 178, + sex: 'male', restingHrManual: 54); +const edDay = '2020-01-02'; +final edStart = DateTime(2020, 1, 2, 10).millisecondsSinceEpoch ~/ 1000; + +void edInit() { + TestWidgetsFlutterBinding.ensureInitialized(); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; +} + +Future edDrop() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + LocalDb.sessionSensorSources = {}; + LocalDb.sessionWearableSource = null; + DerivationEngine.debugSecondHalfHook = null; + DerivationEngine.debugRunning = false; +} + +Future edFresh(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + await edDrop(); + SharedPreferences.setMockInitialValues({}); + await LocalDb.instance; +} + +Future edRows(int from, int to, {int hr = 170, String id = '', + String family = 'gen4', String? source, double az = 1.3}) async { + final db = await LocalDb.instance; + final batch = db.batch(); + for (var ts = from; ts < to; ts++) { + batch.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'hr': hr, 'ax': 0.0, 'ay': 0.0, + 'az': az, 'device_family': family, 'source': source}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future edSession({String id = 's1', String status = 'done', + int? start, int? end}) => LocalDb.putSession({'id': id, 'type': 'run', + 'status': status, 'start_ts': start ?? edStart, + 'end_ts': end ?? edStart + 1100, 'source': 'manual', + 'created_at': edStart * 1000}); + +// The committed two_device_day.json region table and formulas, also used by +// two_device_fixture_test.dart. Keep the fixture intact for exactness laws. +Future edExistingFixture() async { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + const start = 1577923200; + int offset(String time) { + final c = time.split(':').map(int.parse).toList(); + return c[0] * 3600 + c[1] * 60 + c[2]; + } + final db = await LocalDb.instance; + final b = db.batch(); + for (final region in spec['regions'] as List) { + final r = region as Map; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = start + i; + for (final family in r['devices'] as List) { + final primary = family == 'gen4'; + final id = primary ? '' : 'oura-a1b2c3d4'; + b.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'device_family': family, + 'hr': primary ? 60 + ts % 7 : null, + 'ax': primary ? 0.0 : null, 'ay': primary ? 0.0 : null, + 'az': primary ? 1.0 : null, + 'skin_temp_raw': primary ? 30000 + ts % 11 : null, + 'skin_temp_c': primary ? null : 33.0 + ts % 9 / 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + if (primary) { + b.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'rr_ts_ms': ts * 1000, 'beat_index': 0, 'rr_ms': 1000 - ts % 5 * 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + } + } + } + await b.commit(noResult: true); +} + +Map edBundle(PreparedDerivationDay day) { + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput(date: day.date, + dayTsSec: d.tsSec, dayHr: d.hr, dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, sleepJson: day.sleepJson, + hypnoStages: day.hypnoStages, sleepOnsetSec: day.sleepOnsetSec, + sleepOffsetSec: day.sleepOffsetSec, profile: edProfile.toMap(), + dayConfidence: day.confidence, dayFlags: day.flags, + deviceFamily: d.deviceFamily, sleepSource: day.sleepSource).toJson()); +} + +Future edPreparedOutput(DerivationEngine engine) async { + final prepared = await engine.debugPrepareDay(edDay); + expect(prepared, isNotNull); + expect(prepared!.daySub.isEmpty, isFalse, reason: 'fixture must reach compute'); + return jsonEncode({'prepared': prepared.toJson(), 'bundle': edBundle(prepared), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +// Strip only operational timestamps, never metric values, absence, notes, +// confidence, profile state or version. JSON property order stays observable. +Future edStored(String day) async { + final db = await LocalDb.instance; + final rows = await db.query('day_result', where: 'day_id = ?', + whereArgs: [day], orderBy: 'algo_version'); + expect(rows, isNotEmpty, reason: 'the real derive must have persisted a day'); + return jsonEncode({'day_result': [for (final row in rows) { + for (final e in row.entries) + if (e.key == 'payload_json') e.key: jsonEncode( + (jsonDecode(e.value as String) as Map)..remove('computed_at')) + else if (e.key != 'computed_at') e.key: e.value, + }], 'metric_series': await db.query('metric_series', where: 'date = ?', + whereArgs: [day], orderBy: 'key'), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +Future edOwnership({String id = 'other', + List signals = const ['hr1Hz', 'rrIntervals']}) async { + final db = await LocalDb.instance; + for (final signal in signals) { + await db.insert('device_coverage', {'device_id': id, 'signal': signal, + 'start_ts': edStart, 'end_ts': edStart + 600}, + conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': id, + 'rank': 0, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': '', + 'rank': 1, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +String edLabel(int second) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(second * 1000)); +Substrate edSmallSub({int? start, int count = 1200}) => Substrate.fromJson({ + 'ts_sec': [for (var i = 0; i < count; i++) (start ?? edStart) + i], + 'hr': List.filled(count, 60), + 'ax': List.filled(count, 0), 'ay': List.filled(count, 0), + 'az': List.filled(count, 1), 'device_family': 'gen4', +}); + +// Same libc seam used by the fork's zone regressions. No process spawning or +// Dart clock substitution: production DateTime resolves each instant's zone. +typedef _Setenv = Int32 Function(Pointer, Pointer, Int32); +typedef _SetenvDart = int Function(Pointer, Pointer, int); +void edSetZone(String zone) { + final libc = DynamicLibrary.process(); + final key = 'TZ'.toNativeUtf8(), value = zone.toNativeUtf8(); + try { + expect(libc.lookupFunction<_Setenv, _SetenvDart>('setenv')(key, value, 1), 0); + libc.lookupFunction('tzset')(); + } finally { calloc.free(key); calloc.free(value); } +} + +Future edPriorityOrder(List ids) async { + final db = await LocalDb.instance; + for (var rank = 0; rank < ids.length; rank++) { + await db.insert('signal_priority', {'signal': 'hr1Hz', 'device_id': ids[rank], + 'rank': rank, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +}