From 60a205284454379f5cedcd86540a6175c8dc05c2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 19:22:18 +0530 Subject: [PATCH 01/26] whoop export times were read in the phone's zone, not the export's the export writes wall-clock times with no offset and puts the offset in a separate "Cycle timezone" column, which the importer never read. a night or workout recorded in another zone was stored, and written to Apple Health / Health Connect, at the wrong instant (a Paris night imported in Los Angeles landed nine hours late). offset-less times are now read in the row's zone, falling back to the phone's zone only when the column is missing or unreadable, and the day is filed under the wake date in that zone. --- lib/import/whoop_import.dart | 80 ++++++++++++++--- test/whoop_import_timezone_test.dart | 123 +++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 test/whoop_import_timezone_test.dart diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index 78fc4858b..fc5c9a6a0 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -15,6 +15,8 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter/foundation.dart' show visibleForTesting; + import '../compute/derivation_engine.dart' show kAlgoVersion, DerivationEngine; import '../compute/profile.dart'; import '../compute/substrate.dart' show localDateLabel; @@ -219,11 +221,18 @@ class WhoopImporter { // sleeps.csv lists naps as their own rows, filed under the same date as // the night that ended that morning. A nap is not the day's sleep. if (get(['nap']).toLowerCase() == 'true') return null; - final wakeTs = _parseTs(get(['wake onset', 'sleep onset', 'cycle start time'])); - final cycleStart = _parseTs(get(['cycle start time', 'sleep onset'])); + final zone = get(_zoneCols); + final wakeTs = parseExportTs( + get(['wake onset', 'sleep onset', 'cycle start time']), zone: zone); + final cycleStart = + parseExportTs(get(['cycle start time', 'sleep onset']), zone: zone); final anchor = wakeTs ?? cycleStart; if (anchor == null) return null; - final date = localDateLabel(anchor); + // File the day under the date the user lived it in the export's own zone, + // not under whatever zone the phone is in at import time: a night slept in + // Paris and imported in Los Angeles still belongs to the Paris morning. + final off = zoneOffset(zone); + final date = off == null ? localDateLabel(anchor) : _wallDateLabel(anchor, off); num? n(List names) => double.tryParse(get(names)); return (date, { @@ -244,8 +253,8 @@ class WhoopImporter { // need, a different number; _pick takes the first column that exists, // so listing it here stored performance as efficiency. 'effPct': n(['sleep efficiency %']), - 'sleepOnset': _parseTs(get(['sleep onset'])), - 'sleepWake': _parseTs(get(['wake onset'])), + 'sleepOnset': parseExportTs(get(['sleep onset']), zone: zone), + 'sleepWake': parseExportTs(get(['wake onset']), zone: zone), }); } @@ -389,8 +398,10 @@ class WhoopImporter { static Future _writeWorkout(_Row row) async { String get(List names) => row.get(names); - final start = _parseTs(get(['workout start time', 'start time'])); - final end = _parseTs(get(['workout end time', 'end time'])); + final zone = get(_zoneCols); + final start = + parseExportTs(get(['workout start time', 'start time']), zone: zone); + final end = parseExportTs(get(['workout end time', 'end time']), zone: zone); if (start == null) return false; num? n(List names) => double.tryParse(get(names)); final id = 'whoop_$start'; @@ -429,23 +440,66 @@ class WhoopImporter { return _Kind.unknown; } + /// The export writes wall-clock times with no offset ("2024-03-09 23:10:00") + /// and puts the offset they were recorded in in a separate column. + static const List _zoneCols = ['cycle timezone', 'timezone']; + + /// "UTC+01:00" / "UTC-0800" / "GMT+5:30" / "+01:00" / "UTC" → its offset + /// from UTC; null for anything else (blank, an IANA name we can't resolve + /// without a tz database), which falls back to the phone's own zone. + @visibleForTesting + static Duration? zoneOffset(String tz) { + final t = tz.trim().toUpperCase(); + if (t == 'UTC' || t == 'GMT' || t == 'Z') return Duration.zero; + final m = RegExp(r'^(?:UTC|GMT)?\s*([+-])(\d{1,2})(?::?(\d{2}))?$').firstMatch(t); + if (m == null) return null; + final h = int.parse(m.group(2)!); + final min = int.parse(m.group(3) ?? '0'); + if (h > 14 || min > 59) return null; + final d = Duration(hours: h, minutes: min); + return m.group(1) == '-' ? -d : d; + } + + static String _wallDateLabel(int epochSec, Duration off) { + final w = DateTime.fromMillisecondsSinceEpoch(epochSec * 1000, isUtc: true) + .add(off); + String two(int x) => x.toString().padLeft(2, '0'); + return '${w.year.toString().padLeft(4, '0')}-${two(w.month)}-${two(w.day)}'; + } + /// Lenient timestamp → unix seconds. Handles ISO ("2024-01-15T06:30:12Z" / /// "+00:00"), the export's "2024-01-15 06:30:12", and "+0000" offsets. - static int? _parseTs(String s) { + /// + /// An offset-less time is read in [zone] (the row's "Cycle timezone") when + /// that parses. Reading it in the phone's current zone instead put every + /// night or workout recorded elsewhere at the wrong instant — a Paris night + /// imported in Los Angeles landed in Apple Health nine hours late. + @visibleForTesting + static int? parseExportTs(String s, {String zone = ''}) { if (s.isEmpty) return null; var t = s.trim(); // Normalise "+0000" → "+00:00" so DateTime.parse accepts it. final m = RegExp(r'([+-]\d{2})(\d{2})$').firstMatch(t); if (m != null) t = '${t.substring(0, m.start)}${m.group(1)}:${m.group(2)}'; + final off = zoneOffset(zone); + final hasOwnOffset = RegExp(r'(Z|[+-]\d{2}:\d{2})$').hasMatch(t); + if (off != null && !hasOwnOffset) { + final asUtc = DateTime.tryParse('${t}Z'); + if (asUtc != null) return asUtc.subtract(off).millisecondsSinceEpoch ~/ 1000; + } final dt = DateTime.tryParse(t); if (dt != null) return dt.millisecondsSinceEpoch ~/ 1000; - // Fallback: pull a yyyy-mm-dd and treat as local midnight. + // Fallback: pull a yyyy-mm-dd and treat as midnight in the row's zone + // (local midnight when it has none). final d = RegExp(r'(\d{4})-(\d{2})-(\d{2})').firstMatch(t); if (d != null) { - return DateTime(int.parse(d.group(1)!), int.parse(d.group(2)!), - int.parse(d.group(3)!)) - .millisecondsSinceEpoch ~/ - 1000; + final y = int.parse(d.group(1)!); + final mo = int.parse(d.group(2)!); + final da = int.parse(d.group(3)!); + final mid = off != null + ? DateTime.utc(y, mo, da).subtract(off) + : DateTime(y, mo, da); + return mid.millisecondsSinceEpoch ~/ 1000; } return null; } diff --git a/test/whoop_import_timezone_test.dart b/test/whoop_import_timezone_test.dart new file mode 100644 index 000000000..570286dc4 --- /dev/null +++ b/test/whoop_import_timezone_test.dart @@ -0,0 +1,123 @@ +// WHOOP exports write wall-clock times with no offset and carry the offset in +// a separate "Cycle timezone" column. Reading those times in the phone's +// current zone put every night or workout recorded somewhere else at the wrong +// instant (and wrote it to Apple Health / Health Connect that way). These +// assertions are absolute epochs, so they hold whatever zone the test runs in. +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; + +int _utc(int y, int mo, int d, int h, [int mi = 0]) => + DateTime.utc(y, mo, d, h, mi).millisecondsSinceEpoch ~/ 1000; + +void main() { + group('zoneOffset', () { + test('reads the export formats', () { + expect(WhoopImporter.zoneOffset('UTC+01:00'), const Duration(hours: 1)); + expect(WhoopImporter.zoneOffset('UTC-08:00'), const Duration(hours: -8)); + expect(WhoopImporter.zoneOffset('UTC+0530'), + const Duration(hours: 5, minutes: 30)); + expect(WhoopImporter.zoneOffset('GMT+5:30'), + const Duration(hours: 5, minutes: 30)); + expect(WhoopImporter.zoneOffset('-03:00'), const Duration(hours: -3)); + expect(WhoopImporter.zoneOffset('UTC'), Duration.zero); + }); + + test('anything unreadable falls back to the phone zone', () { + expect(WhoopImporter.zoneOffset(''), isNull); + expect(WhoopImporter.zoneOffset('Europe/Paris'), isNull); + expect(WhoopImporter.zoneOffset('UTC+25:00'), isNull); + }); + }); + + group('parseExportTs', () { + test('an offset-less time is read in the row zone', () { + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00', zone: 'UTC+01:00'), + _utc(2024, 3, 9, 22, 10)); + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00', zone: 'UTC-08:00'), + _utc(2024, 3, 10, 7, 10)); + }); + + test('a time carrying its own offset ignores the zone column', () { + expect( + WhoopImporter.parseExportTs('2024-03-09T23:10:00Z', + zone: 'UTC+01:00'), + _utc(2024, 3, 9, 23, 10)); + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00+0200', + zone: 'UTC+01:00'), + _utc(2024, 3, 9, 21, 10)); + }); + + test('no zone column keeps the phone-local reading', () { + expect(WhoopImporter.parseExportTs('2024-03-09 23:10:00'), + DateTime(2024, 3, 9, 23, 10).millisecondsSinceEpoch ~/ 1000); + expect(WhoopImporter.parseExportTs(''), isNull); + }); + }); + + group('import', () { + late Directory tmp; + + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_whoop_tz_test.db'; + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + tmp = Directory.systemTemp.createTempSync('openstrap_whoop_tz'); + }); + + tearDownAll(() async { + await LocalDb.close(); + try { + tmp.deleteSync(recursive: true); + } catch (_) {} + }); + + test('a Paris night lands at its real instant and Paris wake date', + () async { + final f = File(p.join(tmp.path, 'sleeps.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Sleep onset,' + 'Wake onset,Asleep duration (min),In bed duration (min),Nap\n' + '2024-03-09 23:00:00,2024-03-10 23:00:00,UTC+01:00,' + '2024-03-09 23:10:00,2024-03-10 07:00:00,440,470,false\n', + ); + final res = await WhoopImporter.importFiles([f.path]); + expect(res.days, 1); + final row = await LocalDb.dayResult('2024-03-10'); + expect(row, isNotNull, reason: 'filed under the Paris wake date'); + final win = jsonDecode(row!['window_json'] as String) as Map; + expect(win['onset_ms'], _utc(2024, 3, 9, 22, 10) * 1000); + expect(win['offset_ms'], _utc(2024, 3, 10, 6) * 1000); + }); + + test('a Paris workout is stored at its real instant', () async { + final f = File(p.join(tmp.path, 'workouts.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-03-10 07:00:00,2024-03-11 07:00:00,UTC+01:00,' + '2024-03-10 18:00:00,2024-03-10 18:45:00,Running,11.2\n', + ); + final res = await WhoopImporter.importFiles([f.path]); + expect(res.workouts, 1); + final start = _utc(2024, 3, 10, 17); + final db = await LocalDb.instance; + final rows = await db.query('sessions', + where: 'id = ?', whereArgs: ['whoop_$start']); + expect(rows, hasLength(1)); + expect((rows.first['start_ts'] as num).toInt(), start); + expect((rows.first['end_ts'] as num).toInt(), _utc(2024, 3, 10, 17, 45)); + }); + }); +} From e13f902410939d93d37d0ac9e8845cc73fdc7ce2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 12:12:02 +0530 Subject: [PATCH 02/26] a failing sleep write held every other metric out of health connect on android the newest night's sleep is written first, and when that write failed (sleep left unticked in health connect, so it fails every pass) the rest of the export never ran: no heart rate, rhr, hrv, energy or workouts, and sync now reported 0 days. the priority day also sat behind its own escalating backoff, which returned 0 without exporting anything, and a new night reset it every morning. a failed or thrown sleep write now still runs bulk, told nothing was written, so the day retries its sleep with the rest of its records and stays out of the cursor until it all lands. the priority day uses the same gate as bulk, and when it is not due the other days still export. --- lib/health/health_export.dart | 102 +++++++++++++---------------- test/health_sleep_export_test.dart | 70 +++++++++++++------- 2 files changed, 91 insertions(+), 81 deletions(-) diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 7134aae85..32f29edd9 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -352,12 +352,29 @@ Future exportPrioritySleepBeforeBulk({ required Future Function(Map) write, required Future Function(String? androidSleepAlreadyWritten) exportBulk, }) async { - final priorityResult = await exportNewestPrioritySleep( - newestFirstDays: newestFirstDays, - write: write, - ); - if (!priorityResult.succeeded) return priorityResult; - await exportBulk(priorityResult.date); + PrioritySleepExportResult priorityResult; + try { + priorityResult = await exportNewestPrioritySleep( + newestFirstDays: newestFirstDays, + write: write, + ); + } catch (e) { + debugPrint('[health] write priority Android sleep session: $e'); + priorityResult = PrioritySleepExportResult( + date: newestFirstDays + .where((d) => normalizeHealthSleepSession(d.value) != null) + .firstOrNull + ?.key, + succeeded: false, + ); + } + // A failed sleep write must not hold the rest of the export. Health Connect + // grants each type separately, so a user who leaves Sleep off (another app + // owns it) fails this write on every pass; holding bulk behind it meant no + // heart rate, RHR, HRV, energy or workouts ever reached the store. Bulk is + // told nothing was written, so it retries the sleep as part of that day and + // the day stays out of the cursor until the whole day lands. + await exportBulk(priorityResult.succeeded ? priorityResult.date : null); return priorityResult; } @@ -898,7 +915,10 @@ class HealthExporter { lastAttemptMs = null; } final nowMs = DateTime.now().millisecondsSinceEpoch; - final shouldAttempt = shouldAttemptHealthExport( + // The same gate bulk applies to this day, so a day that is due here + // is due there too: when the sleep write fails, bulk retries the + // whole day and its retry entry is the one accounting for it. + final shouldAttempt = shouldAttemptHealthBulkExport( attempts: attempts, maxAttempts: _kMaxExportAttempts, now: DateTime.fromMillisecondsSinceEpoch(nowMs), @@ -906,6 +926,7 @@ class HealthExporter { ? null : DateTime.fromMillisecondsSinceEpoch(lastAttemptMs), backoff: _backoffFor(attempts), + prioritySleepAlreadyWritten: false, lastSuccess: okMs == null ? null : DateTime.fromMillisecondsSinceEpoch(okMs), @@ -914,58 +935,25 @@ class HealthExporter { : _kNonFinalizedRewriteInterval, force: forceRetry, ); - if (!shouldAttempt) { - // The priority day is held back by the CAP or by the success-side - // rewrite throttle (it exported cleanly <30 min ago) — in either - // case its sleep session is already in the store and is not what's - // blocking, so still export every OTHER pending day. Only a - // genuine retry backoff (recent FAILURE, still under the cap) holds - // bulk, matching the pre-throttle behaviour. A day with `ok_ms` set - // carries no pending failure (the success branch clears - // attempts/last_ms), so the two conditions are mutually exclusive. - final throttledBySuccess = okMs != null && - !pendingPriorityDay.finalized && - nowMs - okMs < _kNonFinalizedRewriteInterval.inMilliseconds; - if (attempts >= _kMaxExportAttempts || throttledBySuccess) { - return exportBulk(null); - } - return 0; - } - Future recordPriorityFailure() async { - // A timed-out (unavailable) store isn't a failed export; don't - // spend an attempt on it. - if (_storeHung) return; - retryState[priorityDay!.key] = { - 'attempts': attempts + 1, - 'last_ms': nowMs, - 'finalized': pendingPriorityDay.finalized, - }; - await LocalDb.setCursor(_kRetryCursor, jsonEncode(retryState)); - } + // Not due (backoff, cap, or it exported cleanly moments ago): the + // other pending days still go out. Holding them behind this one + // day's backoff is what kept every metric out of Health Connect + // whenever the sleep write alone was failing. + if (!shouldAttempt) return exportBulk(null); var bulkDone = 0; - try { - final priorityResult = await exportPrioritySleepBeforeBulk( - // `exportNewestPrioritySleep` returns the first day carrying a - // sleep session, and the scan above already found exactly that - // day — so one entry is equivalent to the whole list, without - // holding 400 decoded bundles to re-derive the same answer. - newestFirstDays: [priorityDay], - write: _androidSleep.replace, - exportBulk: (androidSleepAlreadyWritten) async { - bulkDone = await exportBulk(androidSleepAlreadyWritten); - }, - ); - if (!priorityResult.succeeded) { - await recordPriorityFailure(); - return 0; - } - return bulkDone; - } catch (e) { - debugPrint('[health] write priority Android sleep session: $e'); - await recordPriorityFailure(); - return 0; - } + await exportPrioritySleepBeforeBulk( + // `exportNewestPrioritySleep` returns the first day carrying a + // sleep session, and the scan above already found exactly that + // day — so one entry is equivalent to the whole list, without + // holding 400 decoded bundles to re-derive the same answer. + newestFirstDays: [priorityDay], + write: _androidSleep.replace, + exportBulk: (androidSleepAlreadyWritten) async { + bulkDone = await exportBulk(androidSleepAlreadyWritten); + }, + ); + return bulkDone; } } return exportBulk(null); diff --git a/test/health_sleep_export_test.dart b/test/health_sleep_export_test.dart index 0e7d578fe..c0fc83a9c 100644 --- a/test/health_sleep_export_test.dart +++ b/test/health_sleep_export_test.dart @@ -135,49 +135,71 @@ void main() { expect(writes, 1); }); - test('failed priority sleep never invokes the bulk callback', () async { - var priorityWrites = 0; - var bulkWrites = 0; + test( + 'failed priority sleep still runs bulk without claiming the night', + () async { + // Health Connect grants Sleep separately: with it denied this write + // fails every pass, and holding bulk behind it kept heart rate, RHR, + // HRV, energy and workouts out of the store indefinitely. + var priorityWrites = 0; + final bulkPriorityDates = []; + + final result = await exportPrioritySleepBeforeBulk( + newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], + write: (bundle) async { + priorityWrites++; + return false; + }, + exportBulk: (androidSleepAlreadyWritten) async { + bulkPriorityDates.add(androidSleepAlreadyWritten); + }, + ); + + expect(result.date, '2026-08-05'); + expect(result.succeeded, isFalse); + expect(priorityWrites, 1); + expect( + bulkPriorityDates, + [null], + reason: 'bulk must retry the sleep itself, not skip it as written', + ); + }, + ); + + test('thrown priority sleep still runs bulk without claiming the night', + () async { + final bulkPriorityDates = []; final result = await exportPrioritySleepBeforeBulk( newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], - write: (bundle) async { - priorityWrites++; - return false; - }, + write: (bundle) async => throw StateError('priority write failed'), exportBulk: (androidSleepAlreadyWritten) async { - bulkWrites++; + bulkPriorityDates.add(androidSleepAlreadyWritten); }, ); expect(result.date, '2026-08-05'); expect(result.succeeded, isFalse); - expect(priorityWrites, 1); - expect(bulkWrites, 0); + expect(bulkPriorityDates, [null]); }); - test('thrown priority sleep never invokes the bulk callback', () async { - var bulkWrites = 0; - - await expectLater( - exportPrioritySleepBeforeBulk( - newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], - write: (bundle) async => throw StateError('priority write failed'), - exportBulk: (androidSleepAlreadyWritten) async { - bulkWrites++; - }, - ), - throwsA(isA()), + test('the priority day never holds bulk behind its own backoff', () { + final source = File('lib/health/health_export.dart').readAsStringSync(); + final priorityPath = source.substring( + source.indexOf('Future exportPriorityOrBulk() async {'), + source.indexOf('exportBulk = (String? androidSleepAlreadyWritten)'), ); - expect(bulkWrites, 0); + expect(priorityPath, isNot(contains('return 0;'))); + expect(priorityPath, contains('shouldAttemptHealthBulkExport(')); + expect(priorityPath, isNot(contains('shouldAttemptHealthExport('))); }); test('priority success leaves shared retry state for the bulk export', () { final source = File('lib/health/health_export.dart').readAsStringSync(); final priorityCallback = source.substring( source.indexOf( - 'final priorityResult = await exportPrioritySleepBeforeBulk(', + 'await exportPrioritySleepBeforeBulk(', ), source.indexOf('exportBulk = (String? androidSleepAlreadyWritten)'), ); From f061ade57fc433d00cbe754b6e953ec30d4282cd Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 12:32:16 +0530 Subject: [PATCH 03/26] whoop import filed the day's total burn as active energy the cycle's 'energy burned' is whoop's whole-day figure, resting metabolism included, but I stored it as calories (active energy over the wake span). health export then wrote the full ~2,500 kcal as active energy for every imported day. it is calories_total now; whoop exports no active/basal split, so active stays absent and nothing is written to health for that day's energy. --- lib/import/whoop_import.dart | 13 +++++++++---- test/import_data_safety_test.dart | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index fc5c9a6a0..8665caf27 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -240,7 +240,12 @@ class WhoopImporter { 'rhr': n(['resting heart rate (bpm)', 'resting heart rate']), 'rmssd': n(['heart rate variability (ms)', 'heart rate variability (rmssd) (ms)']), 'strain': n(['day strain', 'strain']), - 'calories': _kcal(get(_energyCols), row.header(_energyCols)), + // The cycle's 'Energy burned' is WHOOP's whole-day expenditure, resting + // metabolism included. That is `calories_total` (TDEE) here, not + // `calories` (active energy over the wake span): filed as active, Health + // export wrote the full ~2,500 kcal as ACTIVE_ENERGY_BURNED every day. + // WHOOP's export has no active/basal split, so active stays absent. + 'caloriesTotal': _kcal(get(_energyCols), row.header(_energyCols)), 'resp': n(['respiratory rate (rpm)', 'respiratory rate']), 'spo2': _pct(n(['blood oxygen %', 'blood oxygen']), 70), 'asleepMin': n(['asleep duration (min)', 'asleep duration (minutes)']), @@ -275,7 +280,7 @@ class WhoopImporter { final rhr = f['rhr'] as num?; final rmssd = f['rmssd'] as num?; final strain = f['strain'] as num?; - final calories = f['calories'] as num?; + final caloriesTotal = f['caloriesTotal'] as num?; final resp = f['resp'] as num?; final spo2 = f['spo2'] as num?; final asleepMin = f['asleepMin'] as num?; @@ -347,7 +352,7 @@ class WhoopImporter { 'readiness': recovery, 'strain': strain, 'resp_rate': resp, - 'calories': calories, + 'calories_total': caloriesTotal, 'spo2': spo2, // no skin_temp_z: WHOOP exports absolute °C and every reader of that // key treats it as SDs off your own baseline (33 °C showed as +33 SD). @@ -384,7 +389,7 @@ class WhoopImporter { 'readiness': d(recovery), 'strain': d(strain), 'resp_rate': d(resp), - 'calories': d(calories), + 'calories_total': d(caloriesTotal), 'spo2': d(spo2), 'tst_min': d(asleepMin), 'rem_min': d(remMin), diff --git a/test/import_data_safety_test.dart b/test/import_data_safety_test.dart index 4a8b179ff..5014a1d40 100644 --- a/test/import_data_safety_test.dart +++ b/test/import_data_safety_test.dart @@ -262,7 +262,7 @@ void main() { '$wake,$wake,50,60,60,5,$value,400\n', ); await WhoopImporter.importFiles([f.path]); - return _metric(day, 'calories'); + return _metric(day, 'calories_total'); } test('a real 4,500 kcal ultra day is NOT rewritten as kJ', () async { @@ -285,6 +285,30 @@ void main() { }); }); + test('WHOOP daily energy is filed as total burn, never as active energy', + () async { + // The cycle's 'Energy burned' is WHOOP's whole-day figure, resting + // metabolism included. Filed as `calories` (active energy over the wake + // span) it reached Health as ~2,600 kcal of ACTIVE_ENERGY_BURNED on an + // ordinary rest day. + const wake = '2026-04-05 07:00:00'; + final day = + localDateLabel(DateTime.parse(wake).millisecondsSinceEpoch ~/ 1000); + final f = File(p.join(tmp.path, 'energy_total.csv')); + f.writeAsStringSync( + 'Cycle start time,Wake onset,Day Strain,Energy burned (cal),' + 'Asleep duration (min)\n' + '$wake,$wake,4.1,2600,400\n', + ); + expect((await WhoopImporter.importFiles([f.path])).days, 1); + final scalars = (jsonDecode((await _row(day))!['payload_json'] as String) + as Map)['scalars'] as Map; + expect(scalars['calories_total'], 2600); + expect(scalars['calories'], isNull); + expect(await _metric(day, 'calories_total'), 2600.0); + expect(await _metric(day, 'calories'), isNull); + }); + test('WHOOP absolute skin temp °C is not filed as skin_temp_z', () async { const wake = '2026-04-10 07:00:00'; final day = From a15a5c91ed21a1eb373e7d37bbc9f172b1157213 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 12:34:42 +0530 Subject: [PATCH 04/26] a non-utf-8 csv aborted the whole import instead of reaching the vendor importer File.readAsString reports undecodable bytes as a FileSystemException, but the import router only falls back to the lenient vendor importer on FormatException. A journal or lab CSV re-saved from a spreadsheet as cp1252 therefore threw out of runImport, skipped every remaining file and reported already-imported files as a failure. Both importers now decode the bytes with utf8.decode, which throws FormatException before anything is written. --- lib/import/journal_csv_import.dart | 13 ++++++++++++- lib/import/lab_csv_import.dart | 6 ++---- test/journal_csv_import_test.dart | 13 +++++++++++++ test/lab_csv_import_test.dart | 13 +++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/lib/import/journal_csv_import.dart b/lib/import/journal_csv_import.dart index d3bfa97fc..19c4f4c12 100644 --- a/lib/import/journal_csv_import.dart +++ b/lib/import/journal_csv_import.dart @@ -252,6 +252,17 @@ class JournalImportResult { final List rejected; } +/// Strict UTF-8 read of a CSV that a caller may want to hand to a lenient +/// importer when it is not UTF-8. +/// +/// `File.readAsString` reports bad bytes as a `FileSystemException`, so a +/// cp1252 file saved out of a spreadsheet escaped the `on FormatException` +/// fallbacks in the import router and aborted the whole selection. Decoding +/// the bytes here throws a `FormatException` instead, before anything is +/// written. +Future readCsvText(String path) async => + utf8.decode(await File(path).readAsBytes()); + /// Read [path], validate it, and write the accepted rows. /// /// REPLACES the journal for each date it carries — the file is the corrected @@ -262,7 +273,7 @@ Future importJournalCsvFile( String path, { DateTime? today, }) async { - final text = await File(path).readAsString(); + final text = await readCsvText(path); final parsed = parseJournalCsv(text, today: today); for (final row in parsed.rows) { await LocalDb.putJournal(row.date, jsonEncode(row.tags), row.note); diff --git a/lib/import/lab_csv_import.dart b/lib/import/lab_csv_import.dart index 371bb659c..542e8088c 100644 --- a/lib/import/lab_csv_import.dart +++ b/lib/import/lab_csv_import.dart @@ -17,14 +17,12 @@ // marker's canonical unit, and silently reinterpreting one unit as another is // a fabrication. -import 'dart:io'; - import 'package:flutter/foundation.dart'; import '../data/db.dart'; import '../data/lab_catalogue.dart'; import 'journal_csv_import.dart' - show RejectedRow, isValidDayLabel, parseCsv; + show RejectedRow, isValidDayLabel, parseCsv, readCsvText; /// Exactly the columns `csv_export.dart` emits for the labs set. const List kLabCsvHeader = ['taken_on', 'marker', 'value', 'unit', 'note']; @@ -177,7 +175,7 @@ class LabImportResult { /// (marker, taken_on) pair the file carries, matching `putLabResult`'s own /// upsert semantics. Future importLabCsvFile(String path, {DateTime? today}) async { - final text = await File(path).readAsString(); + final text = await readCsvText(path); final parsed = parseLabCsv(text, today: today); final customKeys = {for (final d in await LocalDb.labMarkerDefs()) d['key'].toString()}; diff --git a/test/journal_csv_import_test.dart b/test/journal_csv_import_test.dart index 44b9082a1..9c64b54c2 100644 --- a/test/journal_csv_import_test.dart +++ b/test/journal_csv_import_test.dart @@ -1,3 +1,6 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/data/csv_export.dart'; import 'package:openstrap_edge/import/journal_csv_import.dart'; @@ -131,4 +134,14 @@ void main() { final p = parse("date,tags,note\n2026-01-01,a,'twas fine\n"); expect(p.rows.single.note, "'twas fine"); }); + test('a cp1252 file out of a spreadsheet fails as a FormatException, before ' + 'any write', () async { + // The import router falls back to the lenient vendor importer only on + // FormatException; File.readAsString would throw FileSystemException here. + final dir = await Directory.systemTemp.createTemp('csv_cp1252_'); + addTearDown(() => dir.delete(recursive: true)); + final path = '${dir.path}/export.csv'; + await File(path).writeAsBytes(latin1.encode('date,tags,note\n2026-10-01,alcohol,caf\u00e9\n')); + await expectLater(importJournalCsvFile(path), throwsA(isA())); + }); } diff --git a/test/lab_csv_import_test.dart b/test/lab_csv_import_test.dart index c74f1b9e0..9379c4e69 100644 --- a/test/lab_csv_import_test.dart +++ b/test/lab_csv_import_test.dart @@ -1,3 +1,6 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/data/csv_export.dart'; import 'package:openstrap_edge/import/lab_csv_import.dart'; @@ -132,4 +135,14 @@ void main() { final p = parse("taken_on,marker,value,unit,note\n2026-01-01,ferritin,42,ng/mL,'twas fine\n"); expect(p.rows.single.note, "'twas fine"); }); + test('a cp1252 file out of a spreadsheet fails as a FormatException, before ' + 'any write', () async { + // The import router falls back to the lenient vendor importer only on + // FormatException; File.readAsString would throw FileSystemException here. + final dir = await Directory.systemTemp.createTemp('csv_cp1252_'); + addTearDown(() => dir.delete(recursive: true)); + final path = '${dir.path}/export.csv'; + await File(path).writeAsBytes(latin1.encode('taken_on,marker,value,unit,note\n2026-10-01,ferritin,42,ng/mL,caf\u00e9\n')); + await expectLater(importLabCsvFile(path), throwsA(isA())); + }); } From 414583e05ac2ecb020f4eb844dd63ccfb9e547dc Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 19:25:39 +0530 Subject: [PATCH 05/26] stop a no-night day's apple sleep delete from wiping the next night a day with no night cleared our sleep samples over the whole calendar day. healthkit deletes anything that overlaps the window, and the next day's night usually starts before midnight, so the clear took out that night's in-bed sample and its pre-midnight stages. once the next day was finalized and exported nothing wrote them back. apple now clears the same noon-to-noon window android does, starting after the previous night's wake when that one woke after noon, and skips the native call when nothing is left to clear. --- lib/health/health_export.dart | 25 ++++---- lib/health/health_sleep_session.dart | 55 +++++++++++++---- test/health_sleep_export_test.dart | 91 ++++++++++++++++++++++++---- 3 files changed, 134 insertions(+), 37 deletions(-) diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 32f29edd9..a31049637 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -1133,22 +1133,23 @@ class HealthExporter { // Sleep is the smallest, highest-value write. Native replace owns cleanup // on both stores (Health Connect SleepSessionRecord; HealthKit inBed+Core). + // No night today means a clear, which must stop short of yesterday's + // night if that one woke after noon. + Future previousWake() async { + if (normalizeHealthSleepSession(b) != null) return null; + final prev = DateTime(dayStart.year, dayStart.month, dayStart.day - 1); + final pb = _decode( + (await LocalDb.dayResult(dayLabelOf(prev)))?['payload_json'], + ); + return pb == null ? null : normalizeHealthSleepSession(pb)?.end; + } + if (Platform.isAndroid && !androidSleepAlreadyWritten) { try { - // No night today means a clear, which must stop short of yesterday's - // night if that one woke after noon. - DateTime? previousWake; - if (normalizeHealthSleepSession(b) == null) { - final prev = DateTime(dayStart.year, dayStart.month, dayStart.day - 1); - final pb = _decode( - (await LocalDb.dayResult(dayLabelOf(prev)))?['payload_json'], - ); - if (pb != null) previousWake = normalizeHealthSleepSession(pb)?.end; - } if (!await _androidSleep.replace( b, dayStart: dayStart, - previousWake: previousWake, + previousWake: await previousWake(), )) { debugPrint('[health] write Android sleep session returned false'); success = false; @@ -1163,7 +1164,7 @@ class HealthExporter { if (!await _appleSleep.replace( bundle: b, dayStart: dayStart, - dayEnd: dayEnd, + previousWake: await previousWake(), )) { debugPrint('[health] write Apple sleep session returned false'); success = false; diff --git a/lib/health/health_sleep_session.dart b/lib/health/health_sleep_session.dart index f838fdfd2..c13b5ea29 100644 --- a/lib/health/health_sleep_session.dart +++ b/lib/health/health_sleep_session.dart @@ -258,15 +258,42 @@ HealthSleepStage? healthSleepStageOf(String? stage) { /// night; the native predicate matches on overlap, so it deleted the whole /// previous envelope sample — which is behind `health_export_through` and /// never gets rewritten. Irrecoverable deletion is not a cleanup. -({DateTime start, DateTime end}) sleepCleanupWindow({ +/// +/// With no night it is [noNightCleanupRange], null when that is empty. +({DateTime start, DateTime end})? sleepCleanupWindow({ required DateTime dayStart, - required DateTime dayEnd, HealthSleepSession? night, + DateTime? previousWake, }) { - if (night == null) return (start: dayStart, end: dayEnd); + if (night == null) { + return noNightCleanupRange(dayStart: dayStart, previousWake: previousWake); + } return sleepSessionCleanupRange(night); } +/// What a day with no night clears: the noon-to-noon window its night would +/// have sat in, started after the previous night's wake when that night woke +/// after noon ([previousWake]). Null when nothing is left to clear. +/// +/// Not the calendar day. HealthKit's delete matches samples that OVERLAP the +/// window, and the next day's night usually starts before midnight, so a +/// [dayStart, dayEnd) sweep deleted that night's in-bed envelope and its +/// pre-midnight stages. Once the next day is finalized and exported nothing +/// writes them back. The same goes for the previous night when it woke after +/// noon, which is what the [previousWake] clamp is for. +({DateTime start, DateTime end})? noNightCleanupRange({ + required DateTime dayStart, + DateTime? previousWake, +}) { + var from = DateTime(dayStart.year, dayStart.month, dayStart.day - 1, 12); + final to = DateTime(dayStart.year, dayStart.month, dayStart.day, 12); + if (previousWake != null && !previousWake.isBefore(from)) { + from = previousWake.add(const Duration(seconds: 1)); + } + if (!from.isBefore(to)) return null; + return (start: from, end: to); +} + abstract interface class HealthConnectSleepSessionWriter { Future replace(HealthSleepSession session); @@ -329,13 +356,12 @@ class HealthConnectSleepSessionExporter { // is behind the export cursor nothing writes it back. Start after it. if (session == null) { if (dayStart == null) return true; - var from = DateTime(dayStart.year, dayStart.month, dayStart.day - 1, 12); - final to = DateTime(dayStart.year, dayStart.month, dayStart.day, 12); - if (previousWake != null && !previousWake.isBefore(from)) { - from = previousWake.add(const Duration(seconds: 1)); - } - if (!from.isBefore(to)) return true; - return writer.clear(from, to); + final range = noNightCleanupRange( + dayStart: dayStart, + previousWake: previousWake, + ); + if (range == null) return true; + return writer.clear(range.start, range.end); } // A window WITH NO STAGES is the same kind of "nothing to write", and has // to report the same way. It used to return false, and the caller treats @@ -426,14 +452,17 @@ class HealthKitSleepSessionExporter { Future replace({ required Map bundle, required DateTime dayStart, - required DateTime dayEnd, - }) { + DateTime? previousWake, + }) async { final session = normalizeHealthSleepSession(bundle); final window = sleepCleanupWindow( dayStart: dayStart, - dayEnd: dayEnd, night: session, + previousWake: previousWake, ); + // No night and an empty window: nothing to write or delete. The native + // side rejects an empty cleanup range as a failure. + if (window == null) return true; return writer.replace( cleanupStart: window.start, cleanupEnd: window.end, diff --git a/test/health_sleep_export_test.dart b/test/health_sleep_export_test.dart index c0fc83a9c..94f4bc301 100644 --- a/test/health_sleep_export_test.dart +++ b/test/health_sleep_export_test.dart @@ -292,23 +292,92 @@ void main() { // copy, which is the truncation and the duplicate bars both. expect(night.start.isBefore(dayStart), isTrue); - final window = sleepCleanupWindow( - dayStart: dayStart, - dayEnd: dayEnd, - night: night, - ); + final window = sleepCleanupWindow(dayStart: dayStart, night: night)!; expect(window.start, DateTime(2026, 8, 4, 12)); expect(window.end, DateTime(2026, 8, 5, 12), reason: 'noon-to-noon only — the old calendar-day union reached into the ' 'previous night and deleted samples nothing ever rewrites'); - // No night to write — nothing to widen for, and the day window still has - // to be swept so stale samples from an earlier export go. - final none = sleepCleanupWindow(dayStart: dayStart, dayEnd: dayEnd); - expect(none.start, dayStart); - expect(none.end, dayEnd); + // No night to write — stale samples from an earlier export still have + // to go, but only from the noon-to-noon window this day's night owns. + // HealthKit deletes on overlap, and tomorrow's night starts this + // evening: a calendar-day sweep deleted its in-bed and pre-midnight + // stages. + final none = sleepCleanupWindow(dayStart: dayStart)!; + expect(none.start, DateTime(2026, 8, 4, 12)); + expect(none.end, DateTime(2026, 8, 5, 12)); + final tomorrowsNightStart = DateTime(2026, 8, 5, 23, 10); + expect(none.end.isBefore(tomorrowsNightStart), isTrue); + expect(none.end.isBefore(dayEnd), isTrue); + + // Yesterday's night woke after noon: start after it. + final clamped = sleepCleanupWindow( + dayStart: dayStart, + previousWake: DateTime(2026, 8, 4, 22), + )!; + expect(clamped.start, DateTime(2026, 8, 4, 22, 0, 1)); + expect( + sleepCleanupWindow( + dayStart: dayStart, + previousWake: DateTime(2026, 8, 5, 13), + ), + isNull, + ); }); + test( + 'Apple replace on a no-night day leaves the next night alone', + () async { + const channel = MethodChannel('openstrap/test_healthkit_sleep_none'); + final calls = []; + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + calls.add(call); + return true; + }); + addTearDown(() { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, null); + }); + final exporter = HealthKitSleepSessionExporter( + writer: MethodChannelHealthKitSleepSessionWriter(channel: channel), + ); + final noWindow = _overnightBundle()..remove('sleep'); + + expect( + await exporter.replace( + bundle: noWindow, + dayStart: DateTime(2026, 8, 5), + previousWake: DateTime(2026, 8, 4, 7), + ), + isTrue, + ); + final args = (calls.single.arguments as Map).cast(); + expect( + args['cleanupStartTime'], + DateTime(2026, 8, 4, 12).millisecondsSinceEpoch, + ); + expect( + args['cleanupEndTime'], + DateTime(2026, 8, 5, 12).millisecondsSinceEpoch, + reason: 'a calendar-day sweep overlaps tomorrow\'s night, which ' + 'starts this evening, and HealthKit deletes on overlap', + ); + + // Nothing left to clear: no native call, and not a failure. + calls.clear(); + expect( + await exporter.replace( + bundle: noWindow, + dayStart: DateTime(2026, 8, 5), + previousWake: DateTime(2026, 8, 5, 13), + ), + isTrue, + ); + expect(calls, isEmpty); + }, + ); + test('Apple and Android share one hypnogram stage vocabulary', () { expect(healthSleepStageOf('wake'), HealthSleepStage.awake); expect(healthSleepStageOf('awake'), HealthSleepStage.awake); @@ -934,7 +1003,6 @@ void main() { await exporter.replace( bundle: bundle, dayStart: DateTime(2026, 8, 5), - dayEnd: DateTime(2026, 8, 6), ), isTrue, ); @@ -971,7 +1039,6 @@ void main() { await exporter.replace( bundle: _overnightBundle(), dayStart: DateTime(2026, 8, 5), - dayEnd: DateTime(2026, 8, 6), ), isTrue, ); From c086a078a56f063de598d6a2dc235ce34fceb848 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 17 Oct 2026 12:34:00 +0530 Subject: [PATCH 06/26] health import said the store was empty when the read failed both health importers caught any read error and returned zero, so a health connect quota or permission exception showed up as "holds no workouts" (or "nothing came back" for readings) styled as a normal result, and the failed branch on those screens could never fire. the read and the measurement write now let the error through, so the import button shows it as a failure. the auto workout pass reports a thrown read as skipped rather than ran, and keeps its hourly stamp so a refusing store isn't hammered. (cherry picked from commit 318b17e5aa602d41378968de3b93f674e7f11ab0) --- lib/health/auto_workout_import.dart | 3 ++ lib/health/health_measurement_import.dart | 24 +++++------ lib/health/health_workout_import.dart | 43 +++++++++---------- test/health_measurement_import_test.dart | 23 ++++++++++ test/health_workout_import_sync_test.dart | 51 +++++++++++++++++++++++ 5 files changed, 107 insertions(+), 37 deletions(-) diff --git a/lib/health/auto_workout_import.dart b/lib/health/auto_workout_import.dart index 1d1a1be32..05814bc85 100644 --- a/lib/health/auto_workout_import.dart +++ b/lib/health/auto_workout_import.dart @@ -113,6 +113,9 @@ class AutoWorkoutImport { } return AutoImportOutcome.ran; } catch (_) { + // A read that threw is not a pass that ran. The stamp above still holds + // it off for the interval, so a store refusing on quota is not hammered + // on every foreground. return AutoImportOutcome.skipped; } } diff --git a/lib/health/health_measurement_import.dart b/lib/health/health_measurement_import.dart index fb1e89245..ea105c38f 100644 --- a/lib/health/health_measurement_import.dart +++ b/lib/health/health_measurement_import.dart @@ -128,8 +128,9 @@ class ImportedMeasurementImporter { } /// Read the recent window and store what is there. Returns how many rows - /// landed; 0 covers an empty store, a denied permission and a locked device - /// alike, none of which is an error the user needs a dialog for. + /// landed; 0 means the store gave back nothing. A read or write that throws + /// propagates, so the caller reports a failure instead of "nothing to + /// import" for a store it never managed to read. Future sync({DateTime? now}) async { final end = now ?? DateTime.now(); // Health Connect caps third-party reads at 30 days without @@ -139,17 +140,12 @@ class ImportedMeasurementImporter { final start = _isApple ? DateTime(end.year - 1, end.month, end.day) : end.subtract(const Duration(days: 30)); - try { - await _health.configure(); - final points = await _health.getHealthDataFromTypes( - types: types, - startTime: start, - endTime: end, - ); - return LocalDb.putImportedMeasurements(rowsFrom(points)); - } catch (e) { - debugPrint('[imported_measurement] read: $e'); - return 0; - } + await _health.configure(); + final points = await _health.getHealthDataFromTypes( + types: types, + startTime: start, + endTime: end, + ); + return LocalDb.putImportedMeasurements(rowsFrom(points)); } } diff --git a/lib/health/health_workout_import.dart b/lib/health/health_workout_import.dart index e49c8641b..addbdadb4 100644 --- a/lib/health/health_workout_import.dart +++ b/lib/health/health_workout_import.dart @@ -320,9 +320,11 @@ class HealthWorkoutImporter { /// Read the window, store the workouts, then fetch each one's route on Apple. /// - /// Never throws: an empty store, a denied permission and a locked device all - /// come back as a zero result, which the caller reports as "nothing to - /// import" rather than as a failure. + /// An empty store (and, on Apple, a denied read, which HealthKit hands back + /// as an empty list) is a zero result. A read that THROWS — a Health Connect + /// quota or SecurityException, the provider updating mid-read — propagates: + /// swallowing it into the same zero told the user the store held no + /// workouts when it was never read at all. /// /// [prompt] is true only from the manual Import tap: it lets the route /// fetch ask for the separate workoutRoute grant. The auto path leaves it @@ -332,28 +334,23 @@ class HealthWorkoutImporter { final start = end.subtract(Duration( days: _isApple ? kImportWindowDaysApple : kImportWindowDaysAndroid, )); - List rows; + final List rows; final ownSources = {}, ownUuids = {}; + await _health.configure(); + final points = await _health.getHealthDataFromTypes( + types: types, + startTime: start, + endTime: end, + ); + String? ownApp; try { - await _health.configure(); - final points = await _health.getHealthDataFromTypes( - types: types, - startTime: start, - endTime: end, - ); - String? ownApp; - try { - ownApp = (await PackageInfo.fromPlatform()).packageName; - } catch (_) {} - rows = workoutsFrom(points, ownApp: ownApp); - for (final p in points) { - if (p.type != HealthDataType.WORKOUT || !_isOwn(p, ownApp)) continue; - ownSources.add(p.sourceName); - ownUuids.add(p.uuid); - } - } catch (e) { - debugPrint('[imported_workout] read: $e'); - return WorkoutImportResult(routesSupported: routesSupported); + ownApp = (await PackageInfo.fromPlatform()).packageName; + } catch (_) {} + rows = workoutsFrom(points, ownApp: ownApp); + for (final p in points) { + if (p.type != HealthDataType.WORKOUT || !_isOwn(p, ownApp)) continue; + ownSources.add(p.sourceName); + ownUuids.add(p.uuid); } // Imports from before the own-app filter stored one copy of every band // session per export pass, each under a uuid the store no longer has, so diff --git a/test/health_measurement_import_test.dart b/test/health_measurement_import_test.dart index 531c808ac..8b8016865 100644 --- a/test/health_measurement_import_test.dart +++ b/test/health_measurement_import_test.dart @@ -1,3 +1,4 @@ +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:health/health.dart'; import 'package:openstrap_edge/health/health_measurement_import.dart'; @@ -21,6 +22,20 @@ HealthDataPoint _p( sourceName: source, ); +class _ThrowingHealth extends Health { + @override + Future configure() async {} + + @override + Future> getHealthDataFromTypes({ + required List types, + required DateTime startTime, + required DateTime endTime, + List recordingMethodsToFilter = const [], + }) async => + throw PlatformException(code: 'RemoteException', message: 'quota'); +} + void main() { test('a cuff reading keeps the cuff\'s name', () { final rows = rowsFrom([_p(HealthDataType.BLOOD_PRESSURE_SYSTOLIC, 128)]); @@ -81,4 +96,12 @@ void main() { ]); expect(rows.single['source'], 'Unknown app'); }); + + test('a read that throws surfaces as an error, not as "nothing came back"', + () async { + TestWidgetsFlutterBinding.ensureInitialized(); + final importer = + ImportedMeasurementImporter(health: _ThrowingHealth(), isApple: false); + await expectLater(importer.sync(), throwsA(isA())); + }); } diff --git a/test/health_workout_import_sync_test.dart b/test/health_workout_import_sync_test.dart index a299c46ce..96f685719 100644 --- a/test/health_workout_import_sync_test.dart +++ b/test/health_workout_import_sync_test.dart @@ -8,6 +8,7 @@ import 'dart:io'; +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:health/health.dart'; import 'package:package_info_plus/package_info_plus.dart'; @@ -16,6 +17,7 @@ import 'package:shared_preferences/shared_preferences.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/health/auto_workout_import.dart'; import 'package:openstrap_edge/health/health_workout_import.dart'; HealthDataPoint _w( @@ -56,6 +58,35 @@ class _FakeHealth extends Health { points; } +/// A store that refuses the read after the permission check passed, the way +/// Health Connect does on a quota limit or a revoked companion grant. +class _ThrowingHealth extends Health { + int reads = 0; + + @override + Future configure() async {} + + @override + Future hasPermissions( + List types, { + List? permissions, + }) async => + true; + + @override + Future> getHealthDataFromTypes({ + required List types, + required DateTime startTime, + required DateTime endTime, + List recordingMethodsToFilter = const [], + }) async { + reads++; + throw PlatformException( + code: 'SecurityException', + message: "Caller doesn't have android.permission.health.READ_DISTANCE"); + } +} + /// Records what the permission request asked for. class _PermHealth extends Health { List? asked; @@ -246,4 +277,24 @@ void main() { HealthDataType.STEPS, ])); }); + + test('a read that throws surfaces as an error, not as an empty store', + () async { + // Returning zero here sent the Import button down "holds no workouts", + // styled as a normal result, for a store that was never read. + final importer = + HealthWorkoutImporter(health: _ThrowingHealth(), isApple: false); + await expectLater(importer.sync(), throwsA(isA())); + }); + + test('the auto pass does not report a thrown read as a pass that ran', + () async { + final health = _ThrowingHealth(); + final out = await AutoWorkoutImport.maybeRun( + importer: HealthWorkoutImporter(health: health, isApple: false), + nowFn: () => DateTime(2026, 10, 1, 12), + ); + expect(health.reads, 1); + expect(out, AutoImportOutcome.skipped); + }); } From 13062538b746001c9b360faa597fc7a60e6f6dbf Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 22 Oct 2026 16:12:37 +0530 Subject: [PATCH 07/26] apple health active energy was missing every workout's calories on ios i took each exported workout's kcal off the day's hourly active energy samples, assuming the workout's totalEnergyBurned covered it. it doesn't: the health plugin saves a bare HKWorkout with no energy samples attached, and the Active Energy total and Move ring only sum activeEnergyBurned samples. a 900 kcal day with a 500 kcal run showed up as 400. the day's full active energy is written now, the same as on android. (cherry picked from commit c50f05fcaa1820809d9e2aca569b55b1ec5c6c25) --- lib/health/health_export.dart | 54 ++++--------------- ...ealth_workout_export_delete_gate_test.dart | 16 ------ 2 files changed, 10 insertions(+), 60 deletions(-) diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index a31049637..98c6f2f9d 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -99,8 +99,9 @@ List healthDeleteTypes({required bool isApplePlatform}) { /// /// Health Connect stores a workout's energy as a separate /// TotalCaloriesBurnedRecord inserted in the same atomic batch, so without that -/// write permission the whole workout write fails. On Android the session's -/// kcal stays in the day's hourly active energy instead. +/// write permission the whole workout write fails. On both platforms the +/// session's kcal also stays in the day's hourly active energy, which is what +/// the store's daily active-energy total is built from. @visibleForTesting int? healthWorkoutEnergyKcal( Map r, { @@ -1323,29 +1324,13 @@ class HealthExporter { ); final bucketCount = bucketBounds.length - 1; - // Active energy: chunked into hourly buckets over the day. - // We subtract workout calories to prevent double-counting, because workouts - // are exported separately (their totalEnergyBurned already covers it, on - // Apple only — see healthWorkoutEnergyKcal). - // Upper bound is exclusive (dayEnd - 1s): sessionsInRange is inclusive on - // both ends, so a workout starting exactly at midnight would otherwise be - // double-subtracted from both this day and the next. - var cal = sc('calories')?.toDouble() ?? 0.0; - try { - final rows = await LocalDb.sessionsInRange( - dayStart.millisecondsSinceEpoch ~/ 1000, - (dayEnd.millisecondsSinceEpoch ~/ 1000) - 1, - ); - final workoutCal = - healthWorkoutCaloriesToSubtract(rows, isApplePlatform: isApple); - cal = (cal > workoutCal) ? cal - workoutCal : 0.0; - } catch (e) { - // Unknown whether cal is workout-adjusted — still write our best guess - // below (idempotent re-export corrects it once this query succeeds), - // but flag the day so it isn't marked done on this pass. - debugPrint('[health] workout-calorie query: $e'); - success = false; - } + // Active energy: chunked into hourly buckets over the day, workouts + // included. The WORKOUT sample's totalEnergyBurned is not counted by + // HealthKit's Active Energy total or the Move ring (those sum + // activeEnergyBurned samples only, and the plugin saves a bare HKWorkout + // with none attached), so taking workout kcal out of these buckets made + // them vanish from the day instead of preventing a double count. + final cal = sc('calories')?.toDouble() ?? 0.0; if (cal > 0 && bucketCount > 0) { final calPerHour = cal / bucketCount; @@ -1652,25 +1637,6 @@ String? healthWorkoutTitleForType(String? type) { .join(' '); } -/// Calories of the day's sessions that get their own WORKOUT sample, and so -/// come off the day's active energy. A session [HealthExporter] skips (live, -/// fabricated end, private) keeps its calories in the active-energy total — -/// subtracting them too would make them vanish from the day entirely. Only -/// the energy the WORKOUT sample itself carries comes off -/// ([healthWorkoutEnergyKcal]: none on Android). -@visibleForTesting -double healthWorkoutCaloriesToSubtract(List> rows, - {bool isApplePlatform = true}) { - var total = 0.0; - for (final r in rows) { - if ((r['status']?.toString() ?? '') == 'live') continue; - if ((r['end_ts_fabricated'] as num?)?.toInt() == 1) continue; - if ((r['private'] as num?)?.toInt() == 1) continue; - total += healthWorkoutEnergyKcal(r, isApplePlatform: isApplePlatform) ?? 0; - } - return total; -} - /// Parameterised by [ios] rather than reading `Platform` directly so a unit /// test can exercise BOTH platform branches on a host VM (where `Platform.isIOS` /// and `Platform.isAndroid` are both false) — see diff --git a/test/health_workout_export_delete_gate_test.dart b/test/health_workout_export_delete_gate_test.dart index e86bbfb14..c7e6b0105 100644 --- a/test/health_workout_export_delete_gate_test.dart +++ b/test/health_workout_export_delete_gate_test.dart @@ -164,22 +164,6 @@ void main() { }); }); - group('healthWorkoutCaloriesToSubtract', () { - test('a private session stays in the day\'s active energy', () { - // Its WORKOUT sample is never written, so subtracting it too would drop - // its calories out of Health entirely. - expect( - healthWorkoutCaloriesToSubtract([ - {..._session(), 'calories': 400, 'private': 1}, - {..._session(), 'calories': 100}, - {..._session(), 'calories': 50, 'status': 'live'}, - {..._session(), 'calories': 30, 'end_ts_fabricated': 1}, - ]), - 100, - ); - }); - }); - group('deleteWorkoutWindow (retime cleanup)', () { // setWorkoutWindow only has the OLD [start,end] before it overwrites the // row — this is what it calls to clear that range so a narrowed/moved From 2f5e101e65f6e8a7e78cfac42a574033af6689a2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 22 Oct 2026 16:13:11 +0530 Subject: [PATCH 08/26] keep already-banked phone steps for an hour that turns uncovered after an android reboot the native step counter records a gap from the last sensor delivery, and any hour touching it comes back as not covered even though its earlier bins still hold real counts. the day sync skipped that hour and then replaced the whole day's phone rows, so steps an earlier sync had already banked for that hour were deleted and never came back. an uncovered hour now carries over the windows already banked for it. (cherry picked from commit 0b56b31f57cf48172f5bc220385b6152ff4ca97a) --- lib/data/db.dart | 21 +++++++++++++++++++++ lib/health/phone_pedometer.dart | 23 +++++++++++++++++++++-- test/phone_pedometer_hour_walk_test.dart | 22 ++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index e9d182631..8ccf68c58 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -3695,6 +3695,27 @@ class LocalDb { return (r.first['s'] as num?)?.toInt() ?? 0; } + /// [day]'s banked phone windows, so a re-sync can keep the ones for an hour + /// the sensor no longer covers (see `PhonePedometer.syncDay`). + static Future> + phoneCoverageForDay(String day) async { + final db = await instance; + final rows = await db.query( + 'live_coverage', + columns: ['start_ts', 'end_ts', 'steps'], + where: 'day = ? AND source = ?', + whereArgs: [day, kStepSourcePhone], + ); + return [ + for (final r in rows) + ( + startTs: (r['start_ts'] as num).toInt(), + endTs: (r['end_ts'] as num).toInt(), + steps: (r['steps'] as num).toInt(), + ), + ]; + } + /// Drop every phone-sourced coverage row (the user turned phone steps off). /// Band rows are untouched, so days fall back to the band count. static Future clearPhoneCoverage() async { diff --git a/lib/health/phone_pedometer.dart b/lib/health/phone_pedometer.dart index c7198e59c..15721c29d 100644 --- a/lib/health/phone_pedometer.dart +++ b/lib/health/phone_pedometer.dart @@ -197,6 +197,7 @@ class PhonePedometer { final windows = <({int startTs, int endTs, int steps})>[]; var total = 0; var anyRead = false; + List<({int startTs, int endTs, int steps})>? banked; // CALENDAR-AWARE hour walk. `Duration` arithmetic on a local DateTime is // ABSOLUTE, so `dayStartLocal.add(Duration(hours: h))` over a fixed 24 @@ -243,9 +244,27 @@ class PhonePedometer { // persist a partial one over a good previous sync. if (n == null) return null; // UNCOVERED, not failed and not zero: the sensor holds no record of - // this hour. It banks nothing and it does not count as read, so a day + // this hour. It reads nothing new and it does not count as read, so a day // that is entirely uncovered still comes back "unknown" below. - if (n == intervalNotCovered) continue; + if (n == intervalNotCovered) { + // KEEP WHAT AN EARLIER SYNC ALREADY BANKED FOR THIS HOUR. An hour + // can turn uncovered after the fact: an Android reboot records a gap + // from the last sensor delivery, and the native query refuses any + // interval touching it even though the bins before the gap still + // hold real counts. The replace below is delete-then-insert, so + // skipping the hour here would silently erase steps that were + // already on the day. Not counted as read: a day whose only content + // is carried-over windows is left exactly as it was. + banked ??= await LocalDb.phoneCoverageForDay(dayId); + final fromSec = from.millisecondsSinceEpoch ~/ 1000; + final toSec = to.millisecondsSinceEpoch ~/ 1000; + for (final w in banked) { + if (w.startTs < fromSec || w.startTs >= toSec) continue; + windows.add(w); + if (w.steps > 0) total += w.steps; + } + continue; + } anyRead = true; if (n < 0) continue; // A CONFIRMED ZERO is banked too, not just a nonzero count: it is the diff --git a/test/phone_pedometer_hour_walk_test.dart b/test/phone_pedometer_hour_walk_test.dart index 411d3ad06..55dc4e209 100644 --- a/test/phone_pedometer_hour_walk_test.dart +++ b/test/phone_pedometer_hour_walk_test.dart @@ -174,6 +174,28 @@ void main() { expect(await LocalDb.liveStepsForDay(dayId), total); }); + test('an hour that turns UNCOVERED later keeps what was already banked', + () async { + final day = yesterday(); + final dayId = _label(day); + + // 1. A good sync banks the whole day. + final good = PhonePedometer(stepReader: (from, to) async => 100); + final fullTotal = await good.syncDay(day); + expect(fullTotal, isNotNull); + expect(await LocalDb.liveStepsForDay(dayId), fullTotal); + + // 2. The phone reboots: the native side records a gap from the last + // delivery, and every later query touching that hour comes back + // uncovered even though its earlier counts were real. The replace is + // delete-then-insert, so this hour must be carried over, not dropped. + var h = 0; + final rebooted = PhonePedometer(stepReader: (from, to) async => + h++ == 11 ? PhonePedometer.intervalNotCovered : 100); + expect(await rebooted.syncDay(day), fullTotal); + expect(await LocalDb.liveStepsForDay(dayId), fullTotal); + }); + test('a day with NO covered hour is unknown, not zero', () async { final day = yesterday(); final dayId = _label(day); From 511485eae23b8502961bb4e0f1538d3bd1b96eab Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 22 Oct 2026 16:13:52 +0530 Subject: [PATCH 09/26] scoring an imported workout no longer exports a second copy to health scoring an imported health workout with band heart rate skipped the immediate export, but the scored session was an ordinary manual row, so the next day export wrote it anyway and the workout showed up twice in apple health / health connect. the scored session is now stamped with its own source, and both the day export and the single-workout export skip it. (cherry picked from commit 32420390f3c1c234cebc69de2c6c4d3ded4ddebe) --- lib/data/local_repository.dart | 2 ++ lib/data/local_repository_impl.dart | 2 ++ lib/health/health_export.dart | 9 ++++++ lib/ui2/screens/log_workout.dart | 15 ++++++++-- ...ealth_workout_export_delete_gate_test.dart | 29 +++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/lib/data/local_repository.dart b/lib/data/local_repository.dart index c5345dd5f..212fcf96d 100644 --- a/lib/data/local_repository.dart +++ b/lib/data/local_repository.dart @@ -267,10 +267,12 @@ abstract class LocalRepository { /// detected, or detected too narrowly. Scored from the 1 Hz substrate over /// [startTs, endTs] (epoch SECONDS); a window with no substrate is still /// saved, just unscored. Returns `{workout_id, unscored, hr_samples}`. + /// [source] is stamped on a NEW row only; an update keeps the row's own. Future> logManualWorkout({ required int startTs, required int endTs, required String type, + String source = 'manual', }) => throw UnimplementedError('re-layer: logManualWorkout'); /// Retime an existing session and re-score it over the new window. Used to diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 504f64f5c..4a8be0f5d 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2916,6 +2916,7 @@ class LocalRepositoryImpl extends LocalRepository { required int startTs, required int endTs, required String type, + String source = 'manual', }) async { // A manual row's id is derived from its start second, so re-logging the // IDENTICAL window is an update of that row, not a collision with it. A @@ -2951,6 +2952,7 @@ class LocalRepositoryImpl extends LocalRepository { existing: same ? prior : null, sessionId: moved ? '$id:${DateTime.now().millisecondsSinceEpoch}' : null, + source: source, validateAgainstId: same ? prior['id'] as String : (prior == null && !moved ? id : null), diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 98c6f2f9d..9307c66b2 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -1485,6 +1485,9 @@ class HealthExporter { if ((r['private'] as num?)?.toInt() == 1) { return null; // skip, not a failure } + if (r['source'] == kScoredImportSessionSource) { + return null; // skip, not a failure + } final st = (r['start_ts'] as num?)?.toInt(); final en = (r['end_ts'] as num?)?.toInt(); if (st == null || en == null || en <= st) { @@ -1534,6 +1537,7 @@ class HealthExporter { // exported workout, and deleting it here — only to have _writeOneWorkout // refuse to write the replacement — would erase it for nothing. if ((session['end_ts_fabricated'] as num?)?.toInt() == 1) return false; + if (session['source'] == kScoredImportSessionSource) return false; final st = (session['start_ts'] as num?)?.toInt(); final en = (session['end_ts'] as num?)?.toInt(); if (st == null || en == null || en <= st) return false; @@ -1637,6 +1641,11 @@ String? healthWorkoutTitleForType(String? type) { .join(' '); } +/// `sessions.source` of a session scored from an imported workout's window. +/// The import came from the health store and the original is still there, so +/// writing this one as well would list the same workout twice. +const String kScoredImportSessionSource = 'scored_import'; + /// Parameterised by [ios] rather than reading `Platform` directly so a unit /// test can exercise BOTH platform branches on a host VM (where `Platform.isIOS` /// and `Platform.isAndroid` are both false) — see diff --git a/lib/ui2/screens/log_workout.dart b/lib/ui2/screens/log_workout.dart index f56309725..0bab2f642 100644 --- a/lib/ui2/screens/log_workout.dart +++ b/lib/ui2/screens/log_workout.dart @@ -306,7 +306,15 @@ class _LogWorkoutState extends State { } final r = widget.sessionId == null ? await repo.logManualWorkout( - startTs: _startSec, endTs: _endSec, type: _activity.typeKey) + startTs: _startSec, + endTs: _endSec, + type: _activity.typeKey, + // Marks it as never-exported: the day export writes every + // ordinary session, and the import's original is already in + // the health store. + source: widget.importedUuid != null + ? kScoredImportSessionSource + : 'manual') : await repo.setWorkoutWindow(widget.sessionId!, startTs: _startSec, endTs: _endSec); if (widget.importedUuid case final uuid?) { @@ -576,8 +584,9 @@ Future replaceImportWithScored( // next import pass delete an import that is still the only copy. The other // order's failure only re-imports it beside the scored session. await LocalDb.supersedeImportedWorkout(uuid, id); - // The original already sits in the health store; exporting ours too would - // put the same workout there twice. + // Not exported here, and the `kScoredImportSessionSource` stamp keeps the + // day export from writing it later: the original already sits in the health + // store, and ours as well would put the same workout there twice. await rememberDeletedUuid(uuid); return true; } diff --git a/test/health_workout_export_delete_gate_test.dart b/test/health_workout_export_delete_gate_test.dart index c7e6b0105..f4123438c 100644 --- a/test/health_workout_export_delete_gate_test.dart +++ b/test/health_workout_export_delete_gate_test.dart @@ -164,6 +164,35 @@ void main() { }); }); + group('sessions scored from an imported workout', () { + // The import's original is already in the health store; the day export + // writing the scored copy as well listed the workout twice. + late _FakeHealthStore store; + + tearDown(() => store.remove()); + + test('are never written to the health store', () async { + store = _FakeHealthStore(deleteResult: true)..install(); + + final ok = await HealthExporter().exportWorkout( + {..._session(), 'source': kScoredImportSessionSource}); + + expect(ok, isFalse); + expect(store.calls, isNot(contains('writeWorkoutData'))); + expect(store.calls, isNot(contains('delete'))); + }); + + test('an ordinary manual session still is', () async { + store = _FakeHealthStore(deleteResult: true)..install(); + + final ok = await HealthExporter() + .exportWorkout({..._session(), 'source': 'manual'}); + + expect(ok, isTrue); + expect(store.calls, contains('writeWorkoutData')); + }); + }); + group('deleteWorkoutWindow (retime cleanup)', () { // setWorkoutWindow only has the OLD [start,end] before it overwrites the // row — this is what it calls to clear that range so a narrowed/moved From 02d4fe9ff13df6a8706ce14908ffa89ed41d0b27 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 22 Oct 2026 16:18:51 +0530 Subject: [PATCH 10/26] re-importing a whoop export reverted workout edits and brought back deleted workouts the importer wrote every workout row back with insert-or-replace under its deterministic id, so a newer export reset corrected types, wiped rpe, cleared the private flag (and re-exported that workout to the health store), and recreated workouts i had deleted. the importer now leaves an existing row alone, and deleting an imported workout records its id so later imports skip it. (cherry picked from commit 9055525267c486d56c0aa7d4209ad79ffd01050b) --- lib/data/local_repository_impl.dart | 10 +++- lib/import/whoop_import.dart | 48 +++++++++++++++-- test/whoop_reimport_edits_test.dart | 81 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 test/whoop_reimport_edits_test.dart diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 4a8be0f5d..3e32607c7 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -35,6 +35,7 @@ import 'db.dart'; import 'activity_store.dart'; import '../models/activity_suggestion.dart'; import '../health/health_export.dart'; +import '../import/whoop_import.dart' show WhoopImporter; import 'journal_fields.dart'; import '../ble/adapters/_registry.dart' show kBandRegistry; import 'local_repository.dart'; @@ -2833,7 +2834,14 @@ class LocalRepositoryImpl extends LocalRepository { int? _profileAge() => (getProfileMap()?['age'] as num?)?.round(); @override - Future deleteWorkout(String id) async => LocalDb.deleteSession(id); + Future deleteWorkout(String id) async { + // Tombstone first: a failed delete then only leaves a row the importer + // already skips, while the other order's failure brings it back. + if (id.startsWith(WhoopImporter.workoutIdPrefix)) { + await WhoopImporter.rememberDeletedWorkout(id); + } + await LocalDb.deleteSession(id); + } @override Future setWorkoutPrivate(String id, bool private) async { diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index 8665caf27..15ae72902 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -16,6 +16,7 @@ import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:shared_preferences/shared_preferences.dart'; import '../compute/derivation_engine.dart' show kAlgoVersion, DerivationEngine; import '../compute/profile.dart'; @@ -58,7 +59,32 @@ class _Row { } } +const String _kDeletedWorkoutsPref = 'whoop_import_deleted_workouts'; + class WhoopImporter { + /// Every imported workout's session id starts with this, followed by its + /// start epoch — the same CSV row always lands on the same id. + static const String workoutIdPrefix = 'whoop_'; + + /// Imported workout ids the user has DELETED here. The export still has + /// them, so without this list the next import would bring each one back. + static Future> deletedWorkoutIds() async { + final prefs = await SharedPreferences.getInstance(); + return {...(prefs.getStringList(_kDeletedWorkoutsPref) ?? const [])}; + } + + // Serialized: each write replaces the whole list, so two overlapping calls + // would each drop the other's id. + static Future _deletedWrites = Future.value(); + + static Future rememberDeletedWorkout(String id) => _deletedWrites = + _deletedWrites.catchError((_) {}).then((_) async { + final prefs = await SharedPreferences.getInstance(); + final set = await deletedWorkoutIds(); + if (set.contains(id)) return; + await prefs.setStringList(_kDeletedWorkoutsPref, [...set, id]); + }); + /// Column aliases for the energy field. The unit is read from whichever of /// these actually matched — never inferred from the value (see [_kcal]). static const List _energyCols = [ @@ -134,6 +160,12 @@ class WhoopImporter { // file's data. Merge is "new non-null value wins, else keep prior" so the // outcome doesn't depend on file order. final pendingDays = >{}; + Set deletedWorkouts; + try { + deletedWorkouts = await deletedWorkoutIds(); + } catch (_) { + deletedWorkouts = const {}; + } for (final path in csvPaths) { final rows = await _readCsv(path); if (rows.isEmpty) continue; @@ -160,7 +192,7 @@ class WhoopImporter { final row = _Row(col, f); if (kind == _Kind.workout) { - if (await _writeWorkout(row)) workouts++; + if (await _writeWorkout(row, deletedWorkouts)) workouts++; } else if (kind == _Kind.day) { final extracted = _extractDayFields(row); if (extracted == null) continue; @@ -401,7 +433,7 @@ class WhoopImporter { return _DayWrite.written; } - static Future _writeWorkout(_Row row) async { + static Future _writeWorkout(_Row row, Set deleted) async { String get(List names) => row.get(names); final zone = get(_zoneCols); final start = @@ -409,7 +441,17 @@ class WhoopImporter { final end = parseExportTs(get(['workout end time', 'end time']), zone: zone); if (start == null) return false; num? n(List names) => double.tryParse(get(names)); - final id = 'whoop_$start'; + final id = '$workoutIdPrefix$start'; + // A re-import (a newer export picking up recent days) carries every old + // workout again. putSession is INSERT-OR-REPLACE over the whole row, so + // rewriting one already here would revert the user's type correction, + // wipe their RPE and clear its private flag — and the export below would + // then write a workout they had made private into the health store. What + // the export says about a past workout does not change; keep the row. + // A deleted one stays deleted. + if (deleted.contains(id) || await LocalDb.session(id) != null) { + return false; + } await LocalDb.putSession({ 'id': id, 'start_ts': start, diff --git a/test/whoop_reimport_edits_test.dart b/test/whoop_reimport_edits_test.dart new file mode 100644 index 000000000..069650154 --- /dev/null +++ b/test/whoop_reimport_edits_test.dart @@ -0,0 +1,81 @@ +// Importing a newer WHOOP export carries every old workout again under the +// same deterministic id. Writing those rows back over the existing ones +// reverted the user's type correction, wiped their RPE, cleared the private +// flag (and re-exported the workout to the health store), and brought back +// workouts they had deleted. +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; + +int _utc(int y, int mo, int d, int h, [int mi = 0]) => + DateTime.utc(y, mo, d, h, mi).millisecondsSinceEpoch ~/ 1000; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory tmp; + + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_whoop_reimport_test.db'; + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + tmp = Directory.systemTemp.createTempSync('openstrap_whoop_reimport'); + }); + + setUp(() => SharedPreferences.setMockInitialValues({})); + + tearDownAll(() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + try { + tmp.deleteSync(recursive: true); + } catch (_) {} + }); + + test('a re-import keeps edits, the private flag and deletions', () async { + final f = File(p.join(tmp.path, 'workouts.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-03-10 07:00:00,2024-03-11 07:00:00,UTC,' + '2024-03-10 09:00:00,2024-03-10 10:00:00,Functional Fitness,9.1\n' + '2024-03-11 07:00:00,2024-03-12 07:00:00,UTC,' + '2024-03-11 09:00:00,2024-03-11 10:00:00,Yoga,4.0\n' + '2024-03-12 07:00:00,2024-03-13 07:00:00,UTC,' + '2024-03-12 09:00:00,2024-03-12 10:00:00,Running,11.2\n', + ); + final first = await WhoopImporter.importFiles([f.path]); + expect(first.workouts, 3); + + final edited = 'whoop_${_utc(2024, 3, 10, 9)}'; + final private = 'whoop_${_utc(2024, 3, 11, 9)}'; + final deleted = 'whoop_${_utc(2024, 3, 12, 9)}'; + expect(await LocalDb.session(deleted), isNotNull); + + await LocalDb.setSessionType(edited, 'strength_trainer'); + await LocalDb.setSessionRpe(edited, 7); + await LocalDb.setSessionPrivate(private, true); + await LocalRepositoryImpl(getProfileMap: () => const {}) + .deleteWorkout(deleted); + expect(await LocalDb.session(deleted), isNull); + + final second = await WhoopImporter.importFiles([f.path]); + expect(second.workouts, 0); + + final e = (await LocalDb.session(edited))!; + expect(e['type'], 'strength_trainer'); + expect((e['rpe'] as num?)?.toDouble(), 7.0); + expect((await LocalDb.session(private))!['private'], 1); + expect(await LocalDb.session(deleted), isNull, + reason: 'a deleted import stays deleted'); + }); +} From 1418b8cf120aedfcacf685bb718a0a4ad9e6a5e9 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:54:28 +0530 Subject: [PATCH 11/26] imported noop rr beats were reshuffled within each second noop stamps beats with whole seconds, so most of the time-sort that builds the substrate is ties, and List.sort is not stable on lists that long. beats that shared a second came out in arbitrary order, which made up successive differences and skewed rmssd, pnn50, ectopic rejection and prsa on imported nights. ties now keep their arrival order, which is the strap's detection order. also fixed the backup importer comment that said order within a day did not matter. (cherry picked from commit 636981d69376bca47c6e0783d79c87534de57272) --- lib/import/noop_backup_import.dart | 8 +++++--- lib/import/noop_ingest.dart | 19 ++++++++++++++++--- test/noop_rr_order_test.dart | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 test/noop_rr_order_test.dart diff --git a/lib/import/noop_backup_import.dart b/lib/import/noop_backup_import.dart index 1a1618439..95489f1e2 100644 --- a/lib/import/noop_backup_import.dart +++ b/lib/import/noop_backup_import.dart @@ -298,9 +298,11 @@ class NoopBackupImporter { final from = dayStart.millisecondsSinceEpoch ~/ 1000; final to = next.millisecondsSinceEpoch ~/ 1000; - // Order within a day does not matter — [NoopIngest] rebuilds the Substrate - // sorted by timestamp — but the DAYS must arrive in ascending order, since - // the high-water date is what closes out and derives the previous one. + // [NoopIngest] re-sorts by timestamp but keeps rows that share a second in + // arrival order, so beats within a second must come in detection order + // (the `ts, rowid` order above). The DAYS must also arrive in ascending + // order, since the high-water date is what closes out and derives the + // previous one. await _read(src, tables, columns, ordered, corrupt, 'hrSample', from, to, (r) async { final ts = _int(r['ts']), v = _int(r['bpm']); diff --git a/lib/import/noop_ingest.dart b/lib/import/noop_ingest.dart index fc234f53d..56135509f 100644 --- a/lib/import/noop_ingest.dart +++ b/lib/import/noop_ingest.dart @@ -290,9 +290,7 @@ class NoopIngest { skinTemp[i] = fTemp; } - // RR beats sorted by time. - final order = List.generate(rrMs.length, (i) => i) - ..sort((a, b) => rrTs[a].compareTo(rrTs[b])); + final order = rrOrder(rrTs); return Substrate( tsSec: tsSec, hr: hr, @@ -308,6 +306,21 @@ class NoopIngest { ); } + /// Indices of the buffered RR beats in time order. + /// + /// NOOP stamps beats with whole seconds, so several beats routinely share one + /// timestamp and the sort is mostly ties. Those beats already arrive in the + /// order the strap detected them, and that order is the interval series: + /// swapping two of them invents successive differences the heart never made + /// (RMSSD, pNN50, ectopic rejection, PRSA). `List.sort` is not stable past a + /// few dozen elements, so ties fall back to buffer position explicitly. + static List rrOrder(List rrTs) => + List.generate(rrTs.length, (i) => i) + ..sort((a, b) { + final c = rrTs[a].compareTo(rrTs[b]); + return c != 0 ? c : a.compareTo(b); + }); + /// Maximum gap (seconds) between consecutive step-counter samples that is /// still treated as one continuous run. Both sources sample every second but /// drop out whenever the band is off-wrist or unsynced; a gap wider than this diff --git a/test/noop_rr_order_test.dart b/test/noop_rr_order_test.dart new file mode 100644 index 000000000..c3b8aa6a4 --- /dev/null +++ b/test/noop_rr_order_test.dart @@ -0,0 +1,24 @@ +// NOOP stamps RR beats with whole seconds, so beats sharing a second are the +// normal case. Their arrival order is the strap's detection order; the +// time-sort that builds the Substrate must keep it rather than reshuffle it. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/import/noop_ingest.dart'; + +void main() { + test('beats sharing a second keep their arrival order', () { + // Long enough that List.sort leaves insertion sort for quicksort. + final rrTs = []; + for (var s = 0; s < 500; s++) { + rrTs + ..add(s * 1000.0) + ..add(s * 1000.0); + } + expect(NoopIngest.rrOrder(rrTs), List.generate(rrTs.length, (i) => i)); + }); + + test('out-of-order seconds are still sorted, ties stay in arrival order', () { + final rrTs = [2000, 1000, 2000, 1000, 0]; + expect(NoopIngest.rrOrder(rrTs), [4, 1, 3, 0, 2]); + }); +} From b24975053fa391796cc436991b41280ad17b45aa Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:55:11 +0530 Subject: [PATCH 12/26] one bad unix_s row in a noop csv could strand or blank the rest of the import the csv path fed every timestamp straight into the day walk. a millisecond- scaled row opened a date thousands of years out and stranded every real day after it, an rtc-unset 1970 row stretched the next day's substrate past the 400-day guard so it derived nothing, and a microsecond-scaled one threw a RangeError that aborted the whole import. the plausible-timestamp range the backup importer already used now lives in NoopIngest.offer, so both sources skip and count those rows. (cherry picked from commit bc3a4e8d65a37139794fb6908ca7be993300e706) --- lib/import/noop_backup_import.dart | 10 ++--- lib/import/noop_ingest.dart | 21 +++++++++- test/noop_schema_drift_test.dart | 62 ++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/lib/import/noop_backup_import.dart b/lib/import/noop_backup_import.dart index 95489f1e2..730f035e0 100644 --- a/lib/import/noop_backup_import.dart +++ b/lib/import/noop_backup_import.dart @@ -59,11 +59,11 @@ const List _kSqliteMagic = [ @visibleForTesting int kNoopBackupPageRows = 20000; -/// Plausible unix seconds. A timestamp outside this range is corrupt or -/// millisecond-scaled, and letting one into the day walk would march it across -/// decades one day at a time. -const int _kMinPlausibleTs = 1000000000; // 2001-09-09 -const int _kMaxPlausibleTs = 4102444800; // 2100-01-01 +/// Plausible unix seconds (shared with the CSV path through [NoopIngest]). A +/// timestamp outside this range is corrupt or millisecond-scaled, and letting +/// one into the day walk would march it across decades one day at a time. +const int _kMinPlausibleTs = kNoopMinPlausibleTs; +const int _kMaxPlausibleTs = kNoopMaxPlausibleTs; /// Every table read, and the columns wanted from it. ONE definition: the probe /// and the reads both key off this, so a name can no longer be spelled one way diff --git a/lib/import/noop_ingest.dart b/lib/import/noop_ingest.dart index 56135509f..ee18f4503 100644 --- a/lib/import/noop_ingest.dart +++ b/lib/import/noop_ingest.dart @@ -39,6 +39,16 @@ class StepRun { int get hashCode => Object.hash(startSec, endSec, steps); } +/// Plausible unix seconds. A timestamp outside this range is corrupt, RTC-unset +/// (the band ships with its clock unset) or milli/microsecond-scaled. One such +/// row reaching [NoopIngest.offer] either opens a date decades away — every +/// later real date is then "older" than the high-water date and strands — or +/// buffers a 1970 sample that stretches the next Substrate past the +/// 400-day `calendarDays` guard, so the real day silently derives nothing. +/// A microsecond value overflows `DateTime` outright. +const int kNoopMinPlausibleTs = 1000000000; // 2001-09-09 +const int kNoopMaxPlausibleTs = 4102444800; // 2100-01-01 + /// What to do with an incoming row given the import's high-water date. /// [advance] closes out the previous date; [buffer] folds the row into the /// current rolling window; [late] means its day was already derived + pruned, @@ -86,6 +96,10 @@ class NoopIngest { /// in, so they are counted rather than silently dropped. int lateRows = 0; + /// Samples whose timestamp fell outside [kNoopMinPlausibleTs].. + /// [kNoopMaxPlausibleTs] and were skipped. + int implausibleRows = 0; + /// Dates that arrived AFTER a later date had already opened, so they were /// folded in as context but never derived in their own right. Non-empty means /// the source was not time-ordered and those days are missing from the @@ -95,8 +109,13 @@ class NoopIngest { Set.unmodifiable(_strandedDates.difference(_derived)); /// Offer a sample at [ts] (epoch seconds). Returns false when the sample is - /// too late to use — the caller must then skip it entirely. + /// too late to use, or its timestamp is implausible — the caller must then + /// skip it entirely. Future offer(int ts) async { + if (ts < kNoopMinPlausibleTs || ts > kNoopMaxPlausibleTs) { + implausibleRows++; + return false; + } final date = localDateLabel(ts); switch (decideRow(date, _curDate, _derived)) { case RowOrder.advance: diff --git a/test/noop_schema_drift_test.dart b/test/noop_schema_drift_test.dart index 945931260..1aa3c57f5 100644 --- a/test/noop_schema_drift_test.dart +++ b/test/noop_schema_drift_test.dart @@ -387,5 +387,67 @@ void main() { f.path, const Profile(), DerivationEngine()); expect(res.days, greaterThan(0)); }, timeout: const Timeout(Duration(minutes: 5))); + + /// Two adjacent days of 300 s each, with [bad] raw lines spliced in + /// between them (written verbatim: a µs-scaled unix_s would overflow the + /// ISO column `_row` builds). + File twoDaysWith(String name, int t0, List bad) { + final b = StringBuffer()..writeln(_header); + void day(int start) { + for (var i = 0; i < 300; i++) { + b.writeln(_row(start + i, 'hr', hr: '65')); + b.writeln( + _row(start + i, 'gravity', gx: '0.1', gy: '0.2', gz: '0.97')); + } + } + + day(t0); + bad.forEach(b.writeln); + day(t0 + 86400); + return File(p.join(tmp.path, name))..writeAsStringSync(b.toString()); + } + + test('a millisecond-scaled unix_s does not strand the rest of the file', + () async { + // One ms-scaled row used to open a date ~53,000 years out; every real + // day after it was then "older" than the high-water date and stranded. + const t0 = 1789030800; // 2026-09-10T09:00:00Z + final f = twoDaysWith('ms.csv', t0, [ + '1700000000000,x,event,,,,,,,,,,,,,,BATTERY_LEVEL(3),', + '1700000000001,x,hr,70,,,,,,,,,,,,,,', + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.strandedDates, isEmpty); + expect(res.days, 2); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('an RTC-unset unix_s does not blank the day it lands in', () async { + // A 1970 sample buffered mid-file stretched the next Substrate past the + // 400-day calendarDays guard, so the real day derived nothing. + const t0 = 1789203600; // 2026-09-12T09:00:00Z + final f = twoDaysWith('rtc.csv', t0, [ + _row(250000, 'hr', hr: '70'), + _row(250001, 'gravity', gx: '0.1', gy: '0.2', gz: '0.97'), + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.strandedDates, isEmpty); + expect(res.days, 2); + final db = await LocalDb.instance; + final rows = await db.rawQuery( + "SELECT 1 FROM day_result WHERE day_id LIKE '1970-%' LIMIT 1"); + expect(rows, isEmpty); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('a microsecond-scaled unix_s is skipped, not a RangeError', () async { + const t0 = 1789376400; // 2026-09-14T09:00:00Z + final f = twoDaysWith('us.csv', t0, [ + '1700000000000000,x,hr,70,,,,,,,,,,,,,,', + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.days, 2); + }, timeout: const Timeout(Duration(minutes: 5))); }); } From 8bf13d6f3bdcecdf9db42d2cdb5510a95724ddd2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:55:36 +0530 Subject: [PATCH 13/26] noop imports skipped the hr/rr plausibility gate a noop csv or .noopbak row went into the substrate as-is, so an off-wrist hr of 255, a 1 bpm glitch or a 9000 ms rr gap from a dropped detection skewed max hr, resting hr and hrv for imported days only. the ingest now applies the same 25-230 bpm and 250-2400 ms bounds as the band decode: implausible hr becomes 0 (second kept) and implausible rr is dropped. (cherry picked from commit c7e4c50a4a569c53de8388bfca97c2aa3a7065c9) --- lib/import/noop_ingest.dart | 14 ++++++- test/noop_ingest_plausibility_test.dart | 55 +++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 test/noop_ingest_plausibility_test.dart diff --git a/lib/import/noop_ingest.dart b/lib/import/noop_ingest.dart index ee18f4503..4d510116a 100644 --- a/lib/import/noop_ingest.dart +++ b/lib/import/noop_ingest.dart @@ -154,7 +154,12 @@ class NoopIngest { return true; } - void hr(int ts, int bpm) => (_secs[ts] ??= _Sec()).hr = bpm; + // Same physiological gate as the band decode. A NOOP row is otherwise taken + // at face value, so an off-wrist sentinel (255) or a 1 bpm glitch would land + // in max HR, strain and the resting-HR window for imported days only. 0 keeps + // the second (and its accel / SpO2 context) while saying "no usable HR". + void hr(int ts, int bpm) => + (_secs[ts] ??= _Sec()).hr = plausibleHrOrNull(bpm) ?? 0; void rr(int ts, double ms) { // FINITE and positive. `ms <= 0` alone lets NaN through (it fails every @@ -163,8 +168,13 @@ class NoopIngest { // costing a single interval. Both are reachable: a CSV column can spell // `NaN`, and a SQLite REAL can hold either outright. if (!ms.isFinite || ms <= 0) return; + // Then the band path's 250-2400 ms bound: outside it the interval is a + // missed or doubled detection, not a rhythm, and one of them is enough to + // move RMSSD / SDNN / pNNx for the whole imported night. + final rr = plausibleRrOrNull(ms); + if (rr == null) return; _rrTs.add(ts * 1000.0); - _rrMs.add(ms); + _rrMs.add(rr); } void gravity(int ts, double? x, double? y, double? z) { diff --git a/test/noop_ingest_plausibility_test.dart b/test/noop_ingest_plausibility_test.dart new file mode 100644 index 000000000..ca0c07a9d --- /dev/null +++ b/test/noop_ingest_plausibility_test.dart @@ -0,0 +1,55 @@ +// A NOOP import must gate HR and RR the same way the band decode does, or the +// same night scores differently depending on where it came from. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/import/noop_ingest.dart'; + +/// Captures the Substrate the ingest hands over instead of deriving it. +class _CapturingEngine extends DerivationEngine { + final List subs = []; + + @override + Future deriveImportedDays( + Substrate sub, + Profile profile, + Set dates, { + void Function(String day)? onDayDone, + }) async { + subs.add(sub); + return dates.length; + } + + @override + Future finalizeImport(Profile profile) async {} +} + +void main() { + test('implausible HR becomes 0 and implausible RR is dropped', () async { + final engine = _CapturingEngine(); + final ingest = NoopIngest(const Profile(), engine); + // Midday, so every sample sits on the same local date. + final base = DateTime(2026, 3, 10, 12).millisecondsSinceEpoch ~/ 1000; + + final hrs = [60, 255, 1, 230, 25, 24, 231]; + for (var i = 0; i < hrs.length; i++) { + expect(await ingest.offer(base + i), isTrue); + ingest.hr(base + i, hrs[i]); + } + final rrs = [800.0, 30.0, 9000.0, 250.0, 2400.0, 249.0, 2401.0]; + for (var i = 0; i < rrs.length; i++) { + expect(await ingest.offer(base + i), isTrue); + ingest.rr(base + i, rrs[i]); + } + await ingest.finish(); + + expect(engine.subs, hasLength(1)); + final sub = engine.subs.single; + // Every second is kept, so its other channels still have a home. + expect(sub.tsSec, [for (var i = 0; i < hrs.length; i++) base + i]); + expect(sub.hr, [60, 0, 0, 230, 25, 0, 0]); + expect(sub.rrMs, [800.0, 250.0, 2400.0]); + }); +} From 22b5da0e426f5843e9cf6c05252bfd5478061165 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:55:59 +0530 Subject: [PATCH 14/26] workouts deleted in apple health or health connect stayed in the history the import only ever upserted, so a workout removed in the health app (say the duplicate copy of a run recorded by two watches) kept showing in history and counting toward this week. after a read that returned workouts, any imported row starting inside the read window whose uuid the store no longer returns is now dropped along with its route. an empty read still deletes nothing, since that is what a denied healthkit read looks like. (cherry picked from commit 4331a8cb2efee6b267b2ce74a39955c134ba8b98) --- lib/data/db.dart | 35 ++++++++++++ lib/health/health_workout_import.dart | 13 +++++ test/health_workout_import_sync_test.dart | 69 +++++++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/lib/data/db.dart b/lib/data/db.dart index 8ccf68c58..aa2a583d9 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -3114,6 +3114,41 @@ class LocalDb { }); } + /// Drop every imported workout that starts inside [[startSec], [endSec]] + /// but whose uuid is not in [present], AND its route, like + /// [deleteImportedWorkout]. Returns how many went. + /// + /// The window is the one the store was just read over: a uuid it no longer + /// returns there was deleted in the health app, and the upsert alone would + /// keep listing it (and counting it toward the week) forever. Rows outside + /// the window are left alone, since the read says nothing about them. + static Future deleteImportedWorkoutsMissing( + int startSec, + int endSec, + Set present, + ) async { + final db = await instance; + return db.transaction((txn) async { + final rows = await txn.query( + 'imported_workout', + columns: ['uuid'], + where: 'start_ts >= ? AND start_ts <= ?', + whereArgs: [startSec, endSec], + ); + final gone = [ + for (final r in rows) + if (!present.contains(r['uuid'])) r['uuid'] as String, + ]; + for (final u in gone) { + await txn.delete('imported_workout', + where: 'uuid = ?', whereArgs: [u]); + await txn.delete('workout_route', + where: 'session_id = ?', whereArgs: [u]); + } + return gone.length; + }); + } + // ── EXTERNAL HEART-RATE SENSOR (0x180D) ───────────────────────────────────── /// What a standard Bluetooth heart-rate sensor delivered during ONE workout. /// diff --git a/lib/health/health_workout_import.dart b/lib/health/health_workout_import.dart index addbdadb4..71b583c5f 100644 --- a/lib/health/health_workout_import.dart +++ b/lib/health/health_workout_import.dart @@ -368,6 +368,19 @@ class HealthWorkoutImporter { if (!tombstones.contains(r.uuid)) r, ]; await LocalDb.putImportedWorkouts([for (final r in alive) r.toRow()]); + // A workout deleted in the health app simply stops coming back, so the + // upsert above never removes it. Anything stored in this window that the + // read no longer returns goes. Only reached on a read that returned + // workouts: a denied HealthKit read is an empty list, and must not wipe + // the history. + await LocalDb.deleteImportedWorkoutsMissing( + start.millisecondsSinceEpoch ~/ 1000, + end.millisecondsSinceEpoch ~/ 1000, + { + for (final p in points) + if (p.type == HealthDataType.WORKOUT) p.uuid, + }, + ); final withRoutes = await _importRoutes(start, end, skip: {...tombstones, ...ownUuids}, prompt: prompt); // Re-read: a workout scored or deleted while this pass was in flight, or diff --git a/test/health_workout_import_sync_test.dart b/test/health_workout_import_sync_test.dart index 96f685719..f25445db4 100644 --- a/test/health_workout_import_sync_test.dart +++ b/test/health_workout_import_sync_test.dart @@ -297,4 +297,73 @@ void main() { expect(health.reads, 1); expect(out, AutoImportOutcome.skipped); }); + + group('a workout deleted in the health app leaves the history', () { + // Android window is 30 days back from `now`; _w's workouts start + // 2026-08-01 09:00, inside it. + final now = DateTime(2026, 8, 10, 12); + int sec(DateTime d) => d.millisecondsSinceEpoch ~/ 1000; + Map row(String uuid, DateTime at) => ImportedWorkoutRow( + uuid: uuid, + startTs: sec(at), + endTs: sec(at) + 3600, + kind: 'running', + source: 'Garmin Connect', + ).toRow(); + Map pt(String sid) => { + 'session_id': sid, + 'seq': 0, + 'ts_ms': 1, + 'lat': 51.5, + 'lng': -0.1, + }; + + test('a uuid the store no longer returns is dropped with its route', + () async { + // Two watches recorded the same run; both were imported, then the + // Garmin copy was deleted in the Fitness app. + await LocalDb.putImportedWorkouts([ + row('watch', DateTime(2026, 8, 1, 9)), + row('garmin', DateTime(2026, 8, 1, 9)), + ]); + await LocalDb.appendRoutePoints('garmin', [pt('garmin')]); + + await HealthWorkoutImporter( + health: _FakeHealth([_w('watch')]), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']), ['watch']); + expect(await LocalDb.routePoints('garmin'), isEmpty); + }); + + test('rows outside the read window are left alone', () async { + await LocalDb.putImportedWorkouts([ + row('old', DateTime(2026, 6, 1, 9)), + ]); + + await HealthWorkoutImporter( + health: _FakeHealth([_w('watch')]), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']).toSet(), {'watch', 'old'}); + }); + + test('an empty read (a denied HealthKit read) deletes nothing', () async { + await LocalDb.putImportedWorkouts([ + row('garmin', DateTime(2026, 8, 1, 9)), + ]); + + await HealthWorkoutImporter( + health: _FakeHealth(const []), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']), ['garmin']); + }); + }); } From 8fc92bac4151e8e1d00c2c7ef6cbca5e9cc658b1 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:56:24 +0530 Subject: [PATCH 15/26] whoop-imported workouts reached health as other the importer stored whoop's activity name lower-cased and nothing else, so 'mountain biking' kept its space and 'weightlifting' matched no strength key. both fell through to OTHER in apple health and health connect, and the weightlifting rows found no catalogue entry in the workout list. the importer now stores catalogue keys (spaces as underscores, plus aliases for whoop's own names), and the health mapping normalises spaces and knows the old spellings for rows that were imported before this. (cherry picked from commit 0fbee51f0f59de654d0f2ddaf85abeec1c8d9b24) --- lib/health/health_export.dart | 11 ++++- lib/import/whoop_import.dart | 25 +++++++++-- test/whoop_activity_type_test.dart | 66 ++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 test/whoop_activity_type_test.dart diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 9307c66b2..8720d93f6 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -1674,7 +1674,10 @@ HealthWorkoutActivityType healthActivityForType( String? type, { required bool ios, }) { - switch ((type ?? '').toLowerCase()) { + // Spaces as underscores: rows imported from a WHOOP export before the + // importer normalised its activity names are stored as 'mountain biking', + // and the importer never rewrites a row it already holds. + switch ((type ?? '').trim().toLowerCase().replaceAll(RegExp(r'\s+'), '_')) { case 'run': case 'running': case 'treadmill': @@ -1686,6 +1689,7 @@ HealthWorkoutActivityType healthActivityForType( case 'biking': case 'indoor_bike': case 'mountain_biking': + case 'spin': // WHOOP's name, on rows imported before the alias existed return HealthWorkoutActivityType.BIKING; case 'walk': case 'walking': @@ -1704,6 +1708,9 @@ HealthWorkoutActivityType healthActivityForType( case 'weight_training': case 'powerlifting': case 'kettlebell': + // WHOOP's names, on rows imported before the importer mapped them. + case 'weightlifting': + case 'strength_trainer': return ios ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING : HealthWorkoutActivityType.STRENGTH_TRAINING; @@ -1718,9 +1725,11 @@ HealthWorkoutActivityType healthActivityForType( return HealthWorkoutActivityType.ROWING; case 'hike': case 'hiking': + case 'hiking/rucking': return HealthWorkoutActivityType.HIKING; case 'climb': case 'climbing': + case 'rock_climbing': // Bare `CLIMBING` is iOS-only; `ROCK_CLIMBING` is the one spelling both // stores accept, which is the #184 lesson applied ahead of the bug. return HealthWorkoutActivityType.ROCK_CLIMBING; diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index 15ae72902..d3ee690f5 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -456,7 +456,7 @@ class WhoopImporter { 'id': id, 'start_ts': start, 'end_ts': end, - 'type': _slug(get(['activity name', 'activity'])), + 'type': activityTypeKey(get(['activity name', 'activity'])), 'status': 'done', 'source': 'whoop', 'calories': _kcal(get(_energyCols), row.header(_energyCols))?.toDouble(), @@ -570,9 +570,26 @@ class WhoopImporter { return null; } - static String _slug(String s) { - final t = s.trim().toLowerCase(); - return t.isEmpty ? 'other' : t; + /// WHOOP activity names spelt as this app's stored `sessions.type` keys + /// (`Activity.typeKey`: lower case, spaces as underscores). Storing WHOOP's + /// raw name kept the space in 'mountain biking' and WHOOP's own words for + /// things the catalogue names differently, so the health export fell + /// through to OTHER for every weightlifting session (#184 again, through + /// the importer) and the workout list found no catalogue entry for them. + static const Map _whoopActivityAliases = { + 'weightlifting': 'weight_training', + 'strength_trainer': 'weight_training', + 'functional_fitness': 'functional', + 'spin': 'indoor_bike', + 'hiking/rucking': 'hiking', + 'rock_climbing': 'climbing', + }; + + @visibleForTesting + static String activityTypeKey(String s) { + final t = s.trim().toLowerCase().replaceAll(RegExp(r'\s+'), '_'); + if (t.isEmpty) return 'other'; + return _whoopActivityAliases[t] ?? t; } /// Read a CSV via the repo's one RFC 4180 parser ([parseCsv], diff --git a/test/whoop_activity_type_test.dart b/test/whoop_activity_type_test.dart new file mode 100644 index 000000000..21960dd37 --- /dev/null +++ b/test/whoop_activity_type_test.dart @@ -0,0 +1,66 @@ +// A WHOOP export names its activities in WHOOP's own words ("Weightlifting", +// "Mountain Biking"). Stored as-is, those rows reached Apple Health / Health +// Connect as OTHER and found no catalogue entry in the workout list. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:health/health.dart'; +import 'package:openstrap_edge/health/health_export.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; +import 'package:openstrap_edge/ui2/activity/catalogue.dart'; + +void main() { + test('WHOOP names are stored as catalogue keys', () { + const cases = { + 'Weightlifting': 'weight_training', + 'Strength Trainer': 'weight_training', + 'Functional Fitness': 'functional', + 'Mountain Biking': 'mountain_biking', + 'Spin': 'indoor_bike', + 'Hiking/Rucking': 'hiking', + 'Rock Climbing': 'climbing', + ' Running ': 'running', + }; + cases.forEach((whoop, key) { + final stored = WhoopImporter.activityTypeKey(whoop); + expect(stored, key, reason: whoop); + expect(activityByName(stored)?.typeKey, key, reason: whoop); + }); + expect(WhoopImporter.activityTypeKey(' '), 'other'); + }); + + test('imported strength and cycling reach the health store typed', () { + for (final ios in [true, false]) { + final strength = ios + ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING + : HealthWorkoutActivityType.STRENGTH_TRAINING; + expect( + healthActivityForType( + WhoopImporter.activityTypeKey('Weightlifting'), ios: ios), + strength); + expect( + healthActivityForType( + WhoopImporter.activityTypeKey('Mountain Biking'), ios: ios), + HealthWorkoutActivityType.BIKING); + } + }); + + test('rows stored with the raw WHOOP name still map', () { + // The importer never rewrites a session it already holds, so rows from + // an earlier import keep the old spelling. + for (final ios in [true, false]) { + final strength = ios + ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING + : HealthWorkoutActivityType.STRENGTH_TRAINING; + expect(healthActivityForType('weightlifting', ios: ios), strength); + expect(healthActivityForType('strength trainer', ios: ios), strength); + expect(healthActivityForType('mountain biking', ios: ios), + HealthWorkoutActivityType.BIKING); + expect(healthActivityForType('spin', ios: ios), + HealthWorkoutActivityType.BIKING); + expect(healthActivityForType('hiking/rucking', ios: ios), + HealthWorkoutActivityType.HIKING); + expect(healthActivityForType('rock climbing', ios: ios), + HealthWorkoutActivityType.ROCK_CLIMBING); + } + }); +} From c0f98c2f4969bc1af0cae062a37982052c718f6e Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:56:15 +0530 Subject: [PATCH 16/26] apple health route import stored gps tracks for workouts it had dropped the route channel returns the route of every workout in the window, and only tombstones and our own exports were skipped. a workout the importer filtered out (over 24 h, or ending before it starts) still had its whole gps track written to workout_route under a uuid with no imported_workout row, so it was never shown and no delete path could reach it. routes are now stored only for the workouts this pass actually imported. (cherry picked from commit 617899731b30455ee61eba8cf143282fcea2c852) --- lib/health/health_workout_import.dart | 42 +++++++++++++++------- test/health_workout_import_test.dart | 50 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lib/health/health_workout_import.dart b/lib/health/health_workout_import.dart index 71b583c5f..f4d049762 100644 --- a/lib/health/health_workout_import.dart +++ b/lib/health/health_workout_import.dart @@ -193,6 +193,31 @@ List> routeRowsFrom(Object? payload) { return out; } +/// The routes from one channel payload worth storing, keyed by workout uuid. +/// +/// Only uuids in [keep] — the workouts this pass actually stored — get a route. +/// The Swift side returns the route of EVERY workout in the window, including +/// ones `workoutsFrom` dropped (over 24 h, ending before they start), our own +/// exports and tombstoned deletes. A route stored for any of those sits under a +/// session_id with no `imported_workout` row: never shown, and out of reach of +/// every delete path, which all start from that row. That is precise location +/// history kept for a workout the app says it did not import. +@visibleForTesting +List>>> routesToStore( + List payload, Set keep) { + final out = >>>[]; + for (final entry in payload) { + if (entry is! Map) continue; + final uuid = entry['uuid']; + if (uuid is! String || uuid.isEmpty) continue; + if (!keep.contains(uuid)) continue; + final rows = routeRowsFrom(entry); + if (rows.isEmpty) continue; + out.add(MapEntry(uuid, rows)); + } + return out; +} + /// "RUNNING" → "Running". The health store's own enum name, title-cased and /// de-underscored. /// @@ -335,7 +360,7 @@ class HealthWorkoutImporter { days: _isApple ? kImportWindowDaysApple : kImportWindowDaysAndroid, )); final List rows; - final ownSources = {}, ownUuids = {}; + final ownSources = {}; await _health.configure(); final points = await _health.getHealthDataFromTypes( types: types, @@ -350,7 +375,6 @@ class HealthWorkoutImporter { for (final p in points) { if (p.type != HealthDataType.WORKOUT || !_isOwn(p, ownApp)) continue; ownSources.add(p.sourceName); - ownUuids.add(p.uuid); } // Imports from before the own-app filter stored one copy of every band // session per export pass, each under a uuid the store no longer has, so @@ -382,7 +406,7 @@ class HealthWorkoutImporter { }, ); final withRoutes = await _importRoutes(start, end, - skip: {...tombstones, ...ownUuids}, prompt: prompt); + keep: {for (final r in alive) r.uuid}, prompt: prompt); // Re-read: a workout scored or deleted while this pass was in flight, or // one an interrupted pass tombstoned without removing, must not stay // listed (or keep a route) beside the session that replaced it. @@ -404,7 +428,7 @@ class HealthWorkoutImporter { /// async round trip, and 90 days of running is a lot of them to serialise /// across the channel one at a time. Future _importRoutes(DateTime start, DateTime end, - {Set skip = const {}, bool prompt = false}) async { + {required Set keep, bool prompt = false}) async { if (!routesSupported) return 0; List payload; try { @@ -424,14 +448,8 @@ class HealthWorkoutImporter { return 0; } var n = 0; - for (final entry in payload) { - if (entry is! Map) continue; - final uuid = entry['uuid']; - if (uuid is! String || uuid.isEmpty) continue; - if (skip.contains(uuid)) continue; - final rows = routeRowsFrom(entry); - if (rows.isEmpty) continue; - await LocalDb.appendRoutePoints(uuid, rows); + for (final route in routesToStore(payload, keep)) { + await LocalDb.appendRoutePoints(route.key, route.value); n++; } return n; diff --git a/test/health_workout_import_test.dart b/test/health_workout_import_test.dart index 61d5bc49f..1ccbb9839 100644 --- a/test/health_workout_import_test.dart +++ b/test/health_workout_import_test.dart @@ -173,4 +173,54 @@ void main() { expect(rows.single['alt'], isNull); }); }); + + group('routesToStore', () { + List pt() => [ + [51.5, -0.12, 12.0, 1000], + ]; + + test('a dropped workout\'s route is not stored', () { + // HealthKit hands back the route of every workout in the window. A 26 h + // ultra is dropped by workoutsFrom, so its GPS track would land under a + // session_id no imported_workout row points at, and nothing deletes it. + final ultra = _w( + uuid: 'ultra', + from: DateTime(2026, 8, 1, 6), + to: DateTime(2026, 8, 2, 8), + ); + final kept = workoutsFrom([ultra, _w(uuid: 'run')]); + final keep = {for (final r in kept) r.uuid}; + expect(keep, {'run'}); + + final routes = routesToStore([ + {'uuid': 'ultra', 'points': pt()}, + {'uuid': 'run', 'points': pt()}, + ], keep); + expect(routes.map((e) => e.key), ['run']); + expect(routes.single.value.single['session_id'], 'run'); + }); + + test('routes for workouts outside the kept set are dropped', () { + // Our own exports and tombstoned deletes never reach the kept set. + expect( + routesToStore([ + {'uuid': 'own', 'points': pt()}, + {'uuid': 'deleted', 'points': pt()}, + ], {'run'}), + isEmpty, + ); + }); + + test('malformed entries and empty routes are skipped', () { + expect( + routesToStore([ + null, + 'x', + {'uuid': '', 'points': pt()}, + {'uuid': 'run', 'points': const []}, + ], {'run', ''}), + isEmpty, + ); + }); + }); } From fca5e104a409d8b3cf84a77fcb5a37089a268cc4 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 14:56:19 +0530 Subject: [PATCH 17/26] close the picked zip when the importer refuses it picking a zip with no csv inside (or one with too many entries, or too big to unpack) threw before the try/finally that closes the archive stream, so every wrong pick left a file descriptor open on the picked file. the refusals now sit inside the same try/finally as the extraction loop. (cherry picked from commit ce9415ca92b1af8d1e2505313bbbaf4245eb6d0b) --- lib/import/import_container.dart | 69 +++++++++++++++++--------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/lib/import/import_container.dart b/lib/import/import_container.dart index 2f12c948d..8075847b4 100644 --- a/lib/import/import_container.dart +++ b/lib/import/import_container.dart @@ -683,43 +683,46 @@ Future> _extractCsvMembers( ); } - if (archive.files.length > _kMaxArchiveMembers) { - throw ImportFormatException( - '“$name” holds ${archive.files.length} entries, which is far more than ' - 'any $flavor export — refusing to unpack it.', - ); - } + // Every exit from here on closes `input`, including the refusals: an + // ordinary wrong pick (a ZIP of photos) otherwise left a descriptor open on + // the picked file for the life of the process, one per retry. + try { + if (archive.files.length > _kMaxArchiveMembers) { + throw ImportFormatException( + '“$name” holds ${archive.files.length} entries, which is far more than ' + 'any $flavor export — refusing to unpack it.', + ); + } - final csvFiles = [ - for (final f in archive.files) - if (f.isFile && _isCsvMember(f.name)) f, - ]; + final csvFiles = [ + for (final f in archive.files) + if (f.isFile && _isCsvMember(f.name)) f, + ]; - final declaredBytes = - csvFiles.fold(0, (sum, f) => sum + (f.size > 0 ? f.size : 0)); - if (declaredBytes > _kMaxUncompressedBytes) { - throw ImportFormatException( - '“$name” unpacks to more than ' - '${(declaredBytes / (1024 * 1024 * 1024)).toStringAsFixed(1)} GB of ' - 'CSV, which is not something we can import.', - ); - } + final declaredBytes = + csvFiles.fold(0, (sum, f) => sum + (f.size > 0 ? f.size : 0)); + if (declaredBytes > _kMaxUncompressedBytes) { + throw ImportFormatException( + '“$name” unpacks to more than ' + '${(declaredBytes / (1024 * 1024 * 1024)).toStringAsFixed(1)} GB of ' + 'CSV, which is not something we can import.', + ); + } - if (csvFiles.isEmpty) { - throw ImportFormatException( - '“$name” is an archive with no CSV files inside ' - '(${archive.files.length} entr${archive.files.length == 1 ? 'y' : 'ies'}). ' - 'Pick the $flavor CSV export instead.', - ); - } + if (csvFiles.isEmpty) { + throw ImportFormatException( + '“$name” is an archive with no CSV files inside ' + '(${archive.files.length} entr${archive.files.length == 1 ? 'y' : 'ies'}). ' + 'Pick the $flavor CSV export instead.', + ); + } - final out = []; - final used = {}; - try { + final out = []; + final used = {}; for (final f in csvFiles) { - // Members can share a basename (`daily/data.csv`, `workouts/data.csv`). - // Flattening them onto one destination silently dropped one file and - // parsed the survivor twice. + // Members can share a basename (`daily/data.csv`, `workouts/data.csv`). + // Flattening them onto one destination silently dropped one file and + // parsed the survivor twice. var base = p.basename(f.name); if (!used.add(base)) { final stem = p.basenameWithoutExtension(base); @@ -740,8 +743,8 @@ Future> _extractCsvMembers( } out.add(destPath); } + return out; } finally { await input.close(); } - return out; } From 68ef04c79b882475261da53fcda4a79f9141edc4 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 15:00:53 +0530 Subject: [PATCH 18/26] apple health sleep export deleted the previous night when it woke after noon a day with a night got the plain noon-to-noon cleanup window and ignored yesterday's wake. healthkit deletes on overlap, so a night that ended after local noon (sleep-in, shift work) lost its in-bed envelope and last stages when the next day was exported, and that day is already behind the export cursor so nothing wrote it back. the window now starts after yesterday's wake when that wake falls inside it and before tonight's onset, and the exporter always looks the previous wake up. (cherry picked from commit 0fe78faabc059b9a8e4cb9b816e072ad5a571ed6) --- lib/health/health_export.dart | 7 +++-- lib/health/health_sleep_session.dart | 19 +++++++++++-- test/health_sleep_export_test.dart | 42 ++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 8720d93f6..aa7b6b811 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -1134,10 +1134,11 @@ class HealthExporter { // Sleep is the smallest, highest-value write. Native replace owns cleanup // on both stores (Health Connect SleepSessionRecord; HealthKit inBed+Core). - // No night today means a clear, which must stop short of yesterday's - // night if that one woke after noon. + // Both the clear on a no-night day and HealthKit's replace on a day with + // a night must stop short of yesterday's night if that one woke after + // noon. Health Connect only uses it for the clear (its delete matches on + // start time; the native replace computes its own window). Future previousWake() async { - if (normalizeHealthSleepSession(b) != null) return null; final prev = DateTime(dayStart.year, dayStart.month, dayStart.day - 1); final pb = _decode( (await LocalDb.dayResult(dayLabelOf(prev)))?['payload_json'], diff --git a/lib/health/health_sleep_session.dart b/lib/health/health_sleep_session.dart index c13b5ea29..4ed122b84 100644 --- a/lib/health/health_sleep_session.dart +++ b/lib/health/health_sleep_session.dart @@ -251,7 +251,7 @@ HealthSleepStage? healthSleepStageOf(String? stage) { /// The span a day's SLEEP-type delete has to cover. /// -/// Just [sleepSessionCleanupRange] — the noon-to-noon window around the +/// [sleepSessionCleanupRange] — the noon-to-noon window around the /// night's wake, same as Android. This used to UNION that with the whole /// calendar day, and for a night waking after local noon the two windows /// concatenated into one contiguous span reaching back into the PREVIOUS @@ -260,6 +260,12 @@ HealthSleepStage? healthSleepStageOf(String? stage) { /// never gets rewritten. Irrecoverable deletion is not a cleanup. /// /// With no night it is [noNightCleanupRange], null when that is empty. +/// +/// Either way it starts after [previousWake] when the day before's night woke +/// inside the window: a night ending after local noon overlaps this day's +/// noon-to-noon span, and the overlap delete took its whole in-bed envelope +/// and last stages with it. That day is behind the export cursor, so nothing +/// wrote them back. The clamp never reaches past this night's own onset. ({DateTime start, DateTime end})? sleepCleanupWindow({ required DateTime dayStart, HealthSleepSession? night, @@ -268,7 +274,16 @@ HealthSleepStage? healthSleepStageOf(String? stage) { if (night == null) { return noNightCleanupRange(dayStart: dayStart, previousWake: previousWake); } - return sleepSessionCleanupRange(night); + final range = sleepSessionCleanupRange(night); + if (previousWake != null && + !previousWake.isBefore(range.start) && + previousWake.isBefore(night.start)) { + return ( + start: previousWake.add(const Duration(seconds: 1)), + end: range.end, + ); + } + return range; } /// What a day with no night clears: the noon-to-noon window its night would diff --git a/test/health_sleep_export_test.dart b/test/health_sleep_export_test.dart index 94f4bc301..e51407d78 100644 --- a/test/health_sleep_export_test.dart +++ b/test/health_sleep_export_test.dart @@ -325,6 +325,48 @@ void main() { ); }); + test("a night's HealthKit window stops short of yesterday's after-noon wake", () { + // Saturday's night ran 03:00-13:00. Sunday's runs Sat 23:30-Sun 07:30, + // so its noon-to-noon window starts Sat 12:00 and overlaps Saturday's + // in-bed envelope by an hour. HealthKit deletes on overlap, and + // Saturday is already behind the export cursor. + final dayStart = DateTime(2026, 8, 9); + final night = HealthSleepSession( + start: DateTime(2026, 8, 8, 23, 30), + end: DateTime(2026, 8, 9, 7, 30), + stages: const [], + ); + final saturdayWake = DateTime(2026, 8, 8, 13); + + final window = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: saturdayWake, + )!; + expect(window.start, DateTime(2026, 8, 8, 13, 0, 1)); + expect(window.end, DateTime(2026, 8, 9, 12)); + expect(window.start.isAfter(saturdayWake), isTrue); + expect(window.start.isAfter(night.start), isFalse); + + // A morning wake yesterday is outside the window: unchanged. + final normal = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: DateTime(2026, 8, 8, 7), + )!; + expect(normal.start, DateTime(2026, 8, 8, 12)); + expect(normal.end, DateTime(2026, 8, 9, 12)); + + // A "previous wake" at or after this night's onset is not a separate + // night ending before this one; never clamp past the onset. + final overlapping = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: DateTime(2026, 8, 9, 1), + )!; + expect(overlapping.start, DateTime(2026, 8, 8, 12)); + }); + test( 'Apple replace on a no-night day leaves the next night alone', () async { From 3fbf527575139e55a337b536707b66109d8ee40a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 15:04:03 +0530 Subject: [PATCH 19/26] re-importing a whoop export duplicated workouts recorded in another zone an earlier import read the export's offset-less times in the phone's zone, so a workout recorded elsewhere was stored at the wrong instant under that instant's id. now that the export's zone is honoured, a re-import computed a different id, inserted a second copy at the right instant and left the old one in place, so the workout showed twice in the app and in health. the importer now looks for the row the old reading would have produced. if its window is untouched it is moved onto the corrected instant under its old id, keeping the user's edits, and its health sample is retimed; if it was deleted, the workout stays deleted. (cherry picked from commit 6599ec0566dc09416e1bbc21499068c6dd036685) --- lib/data/db.dart | 20 ++++++++ lib/import/whoop_import.dart | 36 +++++++++++++- test/whoop_import_timezone_test.dart | 71 ++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index aa2a583d9..27a2e2885 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -12418,6 +12418,26 @@ class LocalDb { }); } + /// Move a session's window without touching any other column. Not + /// `putSession`: the row's type correction, RPE and private flag must + /// survive. Returns the number of rows changed. + static Future setSessionWindow(String id, int startTs, int? endTs) async { + final db = await instance; + return db.transaction((tx) async { + final n = await tx.update( + 'sessions', + {'start_ts': startTs, 'end_ts': endTs}, + where: 'id = ?', + whereArgs: [id], + ); + if (n > 0) { + await ActivityStore.supersede(tx, ActivityKind.workout, startTs, + endTs ?? DateTime.now().millisecondsSinceEpoch ~/ 1000); + } + return n; + }); + } + /// Update ONLY a session's derived score columns. /// /// Deliberately not `putSession`: that is INSERT-OR-REPLACE over the whole diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index d3ee690f5..fb229c2ca 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -63,7 +63,9 @@ const String _kDeletedWorkoutsPref = 'whoop_import_deleted_workouts'; class WhoopImporter { /// Every imported workout's session id starts with this, followed by its - /// start epoch — the same CSV row always lands on the same id. + /// start epoch — the same CSV row always lands on the same id. A row first + /// imported before the export's zone column was honoured keeps the id of + /// the instant it was read at then, after its window is corrected. static const String workoutIdPrefix = 'whoop_'; /// Imported workout ids the user has DELETED here. The export still has @@ -452,6 +454,38 @@ class WhoopImporter { if (deleted.contains(id) || await LocalDb.session(id) != null) { return false; } + // Imports before the export's own zone was honoured read these times in + // the phone's zone, so a workout recorded elsewhere is already here under + // that wrong instant's id. Inserting at the corrected id would show it + // twice (and write it to Health twice); move the old row onto the right + // instant instead, under its old id so its edits, tombstone and child rows + // stay attached. Only when its window is still exactly what that import + // wrote — a row the user retimed, or one healed by an earlier re-import, + // is left alone. + final legacyStart = + parseExportTs(get(['workout start time', 'start time'])); + if (legacyStart != null && legacyStart != start) { + final legacyId = '$workoutIdPrefix$legacyStart'; + if (deleted.contains(legacyId)) return false; + final legacy = await LocalDb.session(legacyId); + if (legacy != null) { + final legacyEnd = parseExportTs(get(['workout end time', 'end time'])); + final oldStart = (legacy['start_ts'] as num?)?.toInt(); + final oldEnd = (legacy['end_ts'] as num?)?.toInt(); + if (oldStart == legacyStart && + oldEnd == legacyEnd && + legacy['source'] == 'whoop' && + await LocalDb.setSessionWindow(legacyId, start, end) > 0) { + unawaited(() async { + if (oldEnd != null) { + await HealthExporter.deleteWorkoutWindow(legacyStart, oldEnd); + } + await HealthExporter.exportWorkoutId(legacyId); + }()); + } + return false; + } + } await LocalDb.putSession({ 'id': id, 'start_ts': start, diff --git a/test/whoop_import_timezone_test.dart b/test/whoop_import_timezone_test.dart index 570286dc4..b51e0b472 100644 --- a/test/whoop_import_timezone_test.dart +++ b/test/whoop_import_timezone_test.dart @@ -8,6 +8,7 @@ import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:openstrap_edge/data/db.dart'; import 'package:openstrap_edge/import/whoop_import.dart'; @@ -119,5 +120,75 @@ void main() { expect((rows.first['start_ts'] as num).toInt(), start); expect((rows.first['end_ts'] as num).toInt(), _utc(2024, 3, 10, 17, 45)); }); + + // An import made before the zone column was honoured stored this workout + // at its wall time read in the phone's zone, under that instant's id. + // +05:45 so the test host's own zone can't make the two readings agree. + const wallStart = '2024-04-02 06:00:00'; + const wallEnd = '2024-04-02 07:10:00'; + String workoutCsv(String name) => (File(p.join(tmp.path, name)) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-04-02 04:00:00,2024-04-03 04:00:00,UTC+05:45,' + '$wallStart,$wallEnd,Running,9.4\n', + )).path; + + Future>> sessionsAround(int ts) async { + final db = await LocalDb.instance; + return db.query('sessions', + where: "id LIKE 'whoop%' AND start_ts BETWEEN ? AND ?", + whereArgs: [ts - 86400, ts + 86400]); + } + + test('a re-import moves a phone-zone workout instead of duplicating it', + () async { + SharedPreferences.setMockInitialValues({}); + final legacyStart = WhoopImporter.parseExportTs(wallStart)!; + final legacyEnd = WhoopImporter.parseExportTs(wallEnd)!; + final start = _utc(2024, 4, 2, 0, 15); + final end = _utc(2024, 4, 2, 1, 25); + if (legacyStart == start) return; // host is on +05:45 itself + final legacyId = 'whoop_$legacyStart'; + await LocalDb.putSession({ + 'id': legacyId, + 'start_ts': legacyStart, + 'end_ts': legacyEnd, + 'type': 'cycling', // the user's correction + 'status': 'done', + 'source': 'whoop', + 'strain': 9.4, + 'duration_min': 70, + 'created_at': 0, + }); + await LocalDb.setSessionPrivate(legacyId, true); + + for (var i = 0; i < 2; i++) { + final res = await WhoopImporter.importFiles([workoutCsv('w$i.csv')]); + expect(res.workouts, 0); + final rows = await sessionsAround(start); + expect(rows, hasLength(1), reason: 'import ${i + 1}'); + final r = rows.single; + expect(r['id'], legacyId); + expect((r['start_ts'] as num).toInt(), start); + expect((r['end_ts'] as num).toInt(), end); + expect(r['type'], 'cycling'); + expect(r['private'], 1); + } + final db = await LocalDb.instance; + await db.delete('sessions', where: 'id = ?', whereArgs: [legacyId]); + }); + + test('a deleted phone-zone workout stays deleted on re-import', () async { + final legacyStart = WhoopImporter.parseExportTs(wallStart)!; + final start = _utc(2024, 4, 2, 0, 15); + if (legacyStart == start) return; + SharedPreferences.setMockInitialValues({ + 'whoop_import_deleted_workouts': ['whoop_$legacyStart'], + }); + final res = await WhoopImporter.importFiles([workoutCsv('wd.csv')]); + expect(res.workouts, 0); + expect(await sessionsAround(start), isEmpty); + }); }); } From 898354e2f81d2ba827fbcf71fb08d48ed74b3059 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 29 Oct 2026 19:16:24 +0530 Subject: [PATCH 20/26] noop plausibility test: match deriveImportedDays' reset generation parameter --- test/noop_ingest_plausibility_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/noop_ingest_plausibility_test.dart b/test/noop_ingest_plausibility_test.dart index ca0c07a9d..8ced348fe 100644 --- a/test/noop_ingest_plausibility_test.dart +++ b/test/noop_ingest_plausibility_test.dart @@ -17,6 +17,7 @@ class _CapturingEngine extends DerivationEngine { Profile profile, Set dates, { void Function(String day)? onDayDone, + int? resetGeneration, }) async { subs.add(sub); return dates.length; From e4530d60de9eea25fac44e2032c2116a0a380931 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 15:49:08 +0530 Subject: [PATCH 21/26] skip reading yesterday's bundle on android days that have a night health connect only uses the previous wake for the no-night clear, so a long first export decoded the day before's whole bundle for nothing. --- lib/health/health_export.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index aa7b6b811..88c0eff0c 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -1151,7 +1151,11 @@ class HealthExporter { if (!await _androidSleep.replace( b, dayStart: dayStart, - previousWake: await previousWake(), + // Only the no-night clear reads it; skip decoding yesterday's + // bundle on every other day of a long first export. + previousWake: normalizeHealthSleepSession(b) == null + ? await previousWake() + : null, )) { debugPrint('[health] write Android sleep session returned false'); success = false; From a507a0f430e646e455786efa68f286c8da366c9f Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 15:49:08 +0530 Subject: [PATCH 22/26] name a journal or lab export that isn't utf-8 instead of dropping it a cp1252 re-save of our own export fell through to the vendor importer, which ignored it without a word, so a mixed selection reported success. --- lib/ui2/onboarding/welcome.dart | 38 +++++++++++++++++++++++++++++++-- test/import_routing_test.dart | 25 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/lib/ui2/onboarding/welcome.dart b/lib/ui2/onboarding/welcome.dart index a48fdeb0e..53a1b2f5e 100644 --- a/lib/ui2/onboarding/welcome.dart +++ b/lib/ui2/onboarding/welcome.dart @@ -9,10 +9,12 @@ // [ImportOutcome] carries what the source cost us and [WelcomeView] renders it. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'package:file_picker/file_picker.dart'; +import 'package:flutter/foundation.dart' show listEquals; import 'package:flutter/material.dart'; import 'package:lucide_icons_flutter/lucide_icons.dart'; import 'package:path_provider/path_provider.dart'; @@ -449,7 +451,13 @@ Future runImport( } on FormatException { // Text, but not UTF-8 — a latin1/cp1252 CSV out of a spreadsheet. The // sniff above cannot see that, and the vendor importer decodes leniently. - vendor.add(p); + // Our own journal or lab export is not something the vendor importer + // reads, though: it would drop it without a word, so name it instead. + if (await _isOwnCsvExport(p)) { + rejected.add(_notUtf8(p)); + } else { + vendor.add(p); + } continue; } try { @@ -461,7 +469,11 @@ Future runImport( } on LabCsvFormatException { vendor.add(p); } on FormatException { - vendor.add(p); + if (await _isOwnCsvExport(p)) { + rejected.add(_notUtf8(p)); + } else { + vendor.add(p); + } } } @@ -506,6 +518,28 @@ Future runImport( ); } +/// Whether [path]'s header row is the one our journal or lab-results export +/// writes, read leniently so a file re-saved in a non-UTF-8 encoding still +/// matches. +Future _isOwnCsvExport(String path) async { + try { + final text = latin1.decode(await File(path).readAsBytes()); + final first = text.split(RegExp(r'\r?\n')).first.replaceAll('\u00ef\u00bb\u00bf', ''); + final header = [ + for (final h in first.split(',')) + h.trim().replaceAll('"', '').toLowerCase(), + ]; + return listEquals(header, kJournalCsvHeader) || + listEquals(header, kLabCsvHeader); + } catch (_) { + return false; + } +} + +String _notUtf8(String path) => + '${path.split(Platform.pathSeparator).last}: not saved as UTF-8 — ' + 'save it as "CSV UTF-8" and import it again'; + /// True when [path] starts with the encrypted-backup magic. Four bytes, so a /// mis-picked file costs one read and not a key derivation. Future isEncryptedBackup(String path) async { diff --git a/test/import_routing_test.dart b/test/import_routing_test.dart index 69a4ba736..55d29b6c0 100644 --- a/test/import_routing_test.dart +++ b/test/import_routing_test.dart @@ -137,6 +137,31 @@ void main() { expect(app.vendor, isEmpty); }); + test('a journal export re-saved as cp1252 is named, not handed to the ' + 'vendor importer', () async { + final app = _RoutingSpy(); + final journal = await write('journal.csv', + latin1.encode('date,tags,note\n2026-08-01,,caf\u00e9\n')); + final whoopPath = await write('whoop.zip', _zipOf(_whoopZipMembers)); + + final out = await runImport(app, [journal, whoopPath]); + + expect(app.vendor, [whoopPath]); + expect(out.rejectedRows.single, contains('journal.csv')); + expect(out.rejectedRows.single, contains('UTF-8')); + }); + + test('a cp1252 CSV that is not ours still goes to the vendor importer', + () async { + final app = _RoutingSpy(); + final path = await write('cycles.csv', + latin1.encode('Cycle start time,Recovery score %\ncaf\u00e9,1\n')); + + await runImport(app, [path]); + + expect(app.vendor, [path]); + }); + test('a mixed selection reaches both importers', () async { final app = _RoutingSpy(); final noopPath = await write('noop-export.csv', utf8.encode(_noopCsv)); From e74472e0e55b8826fd03e955208697e865da0a8b Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 15:49:08 +0530 Subject: [PATCH 23/26] whoop re-import: clear the old calories row, don't duplicate a moved workout earlier imports filed the daily total as calories, and that row survived a re-import. a workout moved onto its right instant under its old id was inserted again once the phone's zone matched the export's, and moving a window now drops the trace and heart rate measured over the old one. --- lib/data/db.dart | 28 ++++++++++++++++++---- lib/import/whoop_import.dart | 7 ++++++ test/spo2_import_surface_test.dart | 21 +++++++++++++++++ test/whoop_import_timezone_test.dart | 35 ++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index edb5def6c..95b395a93 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -12502,15 +12502,24 @@ class LocalDb { }); } - /// Move a session's window without touching any other column. Not - /// `putSession`: the row's type correction, RPE and private flag must - /// survive. Returns the number of rows changed. + /// Move a session's window. Not `putSession`: the row's type correction, + /// RPE and private flag must survive. What was measured over the OLD window + /// (the frozen HR trace, average HR, zone minutes) belongs to other seconds, + /// so it goes; a re-score fills it again if the new window is still on + /// record. Returns the number of rows changed. static Future setSessionWindow(String id, int startTs, int? endTs) async { final db = await instance; return db.transaction((tx) async { final n = await tx.update( 'sessions', - {'start_ts': startTs, 'end_ts': endTs}, + { + 'start_ts': startTs, + 'end_ts': endTs, + 'trace_json': null, + 'trace_samples': null, + 'avg_hr': null, + 'zone_min_json': null, + }, where: 'id = ?', whereArgs: [id], ); @@ -12563,6 +12572,17 @@ class LocalDb { ); } + /// Whether a [source] session already starts at [startTs], under any id. + static Future hasSessionStartingAt(String source, int startTs) async { + final db = await instance; + final rows = await db.query('sessions', + columns: ['id'], + where: 'source = ? AND start_ts = ?', + whereArgs: [source, startTs], + limit: 1); + return rows.isNotEmpty; + } + static Future?> session(String id) async { final db = await instance; final rows = await db.query( diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index fb229c2ca..2048275c0 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -424,6 +424,9 @@ class WhoopImporter { 'strain': d(strain), 'resp_rate': d(resp), 'calories_total': d(caloriesTotal), + // Earlier imports filed this total under `calories` (active energy); + // the series row is only replaced for keys written, so clear it. + 'calories': null, 'spo2': d(spo2), 'tst_min': d(asleepMin), 'rem_min': d(remMin), @@ -486,6 +489,10 @@ class WhoopImporter { return false; } } + // A row moved onto this instant by an earlier re-import keeps its old id, + // so the id check above misses it once the phone's zone matches the + // export's. + if (await LocalDb.hasSessionStartingAt('whoop', start)) return false; await LocalDb.putSession({ 'id': id, 'start_ts': start, diff --git a/test/spo2_import_surface_test.dart b/test/spo2_import_surface_test.dart index 89749d071..0b196e5ce 100644 --- a/test/spo2_import_surface_test.dart +++ b/test/spo2_import_surface_test.dart @@ -60,6 +60,27 @@ void main() { (payload['flags'] as List).contains('IMPORTED_WHOOP_BETA'), isTrue); }); + test('a re-import clears a calories row an older import filed as active', + () async { + final dir = await Directory.systemTemp.createTemp('calories_reimport'); + addTearDown(() => dir.delete(recursive: true)); + const wake = '2026-03-13 07:15:00'; + final day = localDateLabel( + DateTime.parse(wake).millisecondsSinceEpoch ~/ 1000); + final db = await LocalDb.instance; + await db.insert('metric_series', {'date': day, 'key': 'calories', 'value': 2400.0}, + conflictAlgorithm: ConflictAlgorithm.replace); + final f = File('${dir.path}/day.csv'); + f.writeAsStringSync( + 'Cycle start time,Wake onset,Sleep onset,Energy burned (cal)\n' + '$wake,$wake,2026-03-12 23:10:00,2400\n', + ); + await WhoopImporter.importFiles([f.path]); + final rows = await db.query('metric_series', + where: 'date = ? AND key = ?', whereArgs: [day, 'calories']); + expect(rows.where((r) => r['value'] != null), isEmpty); + }); + test('an out-of-range blood oxygen cell is dropped, not charted', () async { final dir = await Directory.systemTemp.createTemp('spo2_range'); addTearDown(() => dir.delete(recursive: true)); diff --git a/test/whoop_import_timezone_test.dart b/test/whoop_import_timezone_test.dart index b51e0b472..ab209dac2 100644 --- a/test/whoop_import_timezone_test.dart +++ b/test/whoop_import_timezone_test.dart @@ -162,6 +162,13 @@ void main() { 'created_at': 0, }); await LocalDb.setSessionPrivate(legacyId, true); + // Measured over the wrong seconds before the move. + final db0 = await LocalDb.instance; + await db0.update( + 'sessions', + {'trace_json': '{"hr":[]}', 'trace_samples': 10, 'avg_hr': 140}, + where: 'id = ?', + whereArgs: [legacyId]); for (var i = 0; i < 2; i++) { final res = await WhoopImporter.importFiles([workoutCsv('w$i.csv')]); @@ -174,11 +181,39 @@ void main() { expect((r['end_ts'] as num).toInt(), end); expect(r['type'], 'cycling'); expect(r['private'], 1); + expect(r['trace_json'], isNull); + expect(r['avg_hr'], isNull); + expect(r['strain'], 9.4); } final db = await LocalDb.instance; await db.delete('sessions', where: 'id = ?', whereArgs: [legacyId]); }); + test('a workout already moved under its old id is not inserted again', + () async { + SharedPreferences.setMockInitialValues({}); + final start = _utc(2024, 4, 2, 0, 15); + // Any id other than the one this import would compute: the row an + // earlier re-import moved onto the right instant. + const movedId = 'whoop_1'; + await LocalDb.putSession({ + 'id': movedId, + 'start_ts': start, + 'end_ts': _utc(2024, 4, 2, 1, 25), + 'type': 'running', + 'status': 'done', + 'source': 'whoop', + 'duration_min': 70, + 'created_at': 0, + }); + final res = await WhoopImporter.importFiles([workoutCsv('wm.csv')]); + expect(res.workouts, 0); + final rows = await sessionsAround(start); + expect(rows.map((r) => r['id']), [movedId]); + final db = await LocalDb.instance; + await db.delete('sessions', where: 'id = ?', whereArgs: [movedId]); + }); + test('a deleted phone-zone workout stays deleted on re-import', () async { final legacyStart = WhoopImporter.parseExportTs(wallStart)!; final start = _utc(2024, 4, 2, 0, 15); From fb0a8d2407e3315710df8739e623b10db64b2bfa Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 15:49:08 +0530 Subject: [PATCH 24/26] noop import: refuse timestamps more than two days ahead one corrupt 2099 row passed the old 2100 bound and set both the backup span's end and the ingest's newest day. --- lib/import/noop_backup_import.dart | 3 +-- lib/import/noop_ingest.dart | 14 +++++++++++--- test/noop_ingest_plausibility_test.dart | 13 +++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/import/noop_backup_import.dart b/lib/import/noop_backup_import.dart index 730f035e0..8f6157795 100644 --- a/lib/import/noop_backup_import.dart +++ b/lib/import/noop_backup_import.dart @@ -63,7 +63,6 @@ int kNoopBackupPageRows = 20000; /// timestamp outside this range is corrupt or millisecond-scaled, and letting /// one into the day walk would march it across decades one day at a time. const int _kMinPlausibleTs = kNoopMinPlausibleTs; -const int _kMaxPlausibleTs = kNoopMaxPlausibleTs; /// Every table read, and the columns wanted from it. ONE definition: the probe /// and the reads both key off this, so a name can no longer be spelled one way @@ -534,7 +533,7 @@ class NoopBackupImporter { try { r = await src.rawQuery( 'SELECT MIN(ts) AS lo, MAX(ts) AS hi FROM $t WHERE ts >= ? AND ts <= ?', - [_kMinPlausibleTs, _kMaxPlausibleTs], + [_kMinPlausibleTs, noopMaxPlausibleTs()], ); } catch (e) { if (!_isCorruptPageError(e)) rethrow; diff --git a/lib/import/noop_ingest.dart b/lib/import/noop_ingest.dart index 3a7347a9f..ae80dd2f2 100644 --- a/lib/import/noop_ingest.dart +++ b/lib/import/noop_ingest.dart @@ -48,7 +48,13 @@ class StepRun { /// 400-day `calendarDays` guard, so the real day silently derives nothing. /// A microsecond value overflows `DateTime` outright. const int kNoopMinPlausibleTs = 1000000000; // 2001-09-09 -const int kNoopMaxPlausibleTs = 4102444800; // 2100-01-01 + +/// The upper bound: now, plus two days of clock skew. A fixed far-future bound +/// let one corrupt 2099 row through, and that row alone set the backup span's +/// end (every empty day in between was then read) and the ingest's high-water +/// date (every real row after it stranded). +int noopMaxPlausibleTs() => + DateTime.now().millisecondsSinceEpoch ~/ 1000 + 2 * 86400; /// What to do with an incoming row given the import's high-water date. /// [advance] closes out the previous date; [buffer] folds the row into the @@ -76,6 +82,8 @@ class NoopIngest { /// belongs to the deleted store, so the import stops at its next write /// instead of banking it into the fresh one. final int _resetGen = ResetGate.generation; + /// Fixed for the length of one import, so a long run cannot move it. + final int _maxTs = noopMaxPlausibleTs(); void _stopIfReset() { if (ResetGate.changedSince(_resetGen)) { @@ -110,7 +118,7 @@ class NoopIngest { int lateRows = 0; /// Samples whose timestamp fell outside [kNoopMinPlausibleTs].. - /// [kNoopMaxPlausibleTs] and were skipped. + /// [noopMaxPlausibleTs] and were skipped. int implausibleRows = 0; /// Dates that arrived AFTER a later date had already opened, so they were @@ -126,7 +134,7 @@ class NoopIngest { /// skip it entirely. Future offer(int ts) async { _stopIfReset(); - if (ts < kNoopMinPlausibleTs || ts > kNoopMaxPlausibleTs) { + if (ts < kNoopMinPlausibleTs || ts > _maxTs) { implausibleRows++; return false; } diff --git a/test/noop_ingest_plausibility_test.dart b/test/noop_ingest_plausibility_test.dart index 8ced348fe..00644f32e 100644 --- a/test/noop_ingest_plausibility_test.dart +++ b/test/noop_ingest_plausibility_test.dart @@ -53,4 +53,17 @@ void main() { expect(sub.hr, [60, 0, 0, 230, 25, 0, 0]); expect(sub.rrMs, [800.0, 250.0, 2400.0]); }); + + test('a far-future timestamp is refused, not taken as the newest day', + () async { + final engine = _CapturingEngine(); + final ingest = NoopIngest(const Profile(), engine); + final now = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final y2099 = DateTime(2099, 6, 1, 12).millisecondsSinceEpoch ~/ 1000; + expect(await ingest.offer(y2099), isFalse); + expect(ingest.implausibleRows, 1); + expect(await ingest.offer(now - 3600), isTrue); + expect(await ingest.offer(now + 86400), isTrue, + reason: 'a day of clock skew is still a real row'); + }); } From 53cc28f8eaad224e82b78284e80fdaae14a4c6c2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 15:49:08 +0530 Subject: [PATCH 25/26] stop rewinding the day export when a workout is made private the day's active energy no longer has workouts taken out, so the rewind only rewrote that day and every later one. --- lib/data/local_repository_impl.dart | 8 -------- test/manual_workout_repo_test.dart | 18 +++++++----------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 3e32607c7..9fc03661d 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2849,14 +2849,6 @@ class LocalRepositoryImpl extends LocalRepository { // The workout was already exported when it stopped, before this flag // landed. Re-export: a private row clears its window and writes nothing. unawaited(HealthExporter.exportWorkoutId(id)); - // The day's active energy has exported workouts' calories taken off, so - // it changes too, and a day behind the export cursor is never rewritten. - final start = ((await LocalDb.session(id))?['start_ts'] as num?)?.toInt(); - if (start != null) { - unawaited(HealthExporter.reexportFrom( - dayLabelOf(DateTime.fromMillisecondsSinceEpoch(start * 1000)), - )); - } } @override diff --git a/test/manual_workout_repo_test.dart b/test/manual_workout_repo_test.dart index a253f71b4..6153d9cd0 100644 --- a/test/manual_workout_repo_test.dart +++ b/test/manual_workout_repo_test.dart @@ -563,13 +563,14 @@ void main() { }); test( - 'marking an exported workout private re-exports its day', + 'marking a workout private does not rewind the day export', () async { - // Its calories come back into the day's active energy, and a day - // behind the export cursor is otherwise never written again. + // The day's active energy is the full daily total, workouts included, + // so it does not change with the flag; the workout's own re-export + // clears its window. Rewinding the cursor would only rewrite this day + // and every day after it for nothing. SharedPreferences.setMockInitialValues({}); final start = sessionStart - 20 * 86400; - final day = dayLabelOf(DateTime.fromMillisecondsSinceEpoch(start * 1000)); await LocalDb.putSession({ 'id': 'w-private-old', 'start_ts': start, @@ -585,13 +586,8 @@ void main() { addTearDown(() => LocalDb.setCursor('health_export_through', '')); await repo.setWorkoutPrivate('w-private-old', true); - String? cursor; - for (var i = 0; i < 50; i++) { - cursor = await LocalDb.getCursor('health_export_through'); - if (cursor != through) break; - await Future.delayed(const Duration(milliseconds: 20)); - } - expect(cursor!.compareTo(day), lessThan(0)); + await Future.delayed(const Duration(milliseconds: 200)); + expect(await LocalDb.getCursor('health_export_through'), through); }, ); From b55ec8c748e38da3fed8f2a9fa8700a3a0a59d51 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Fri, 30 Oct 2026 16:07:10 +0530 Subject: [PATCH 26/26] keep a moved whoop workout deleted, and keep the encoding note on a failed import deleting a workout that an earlier re-import moved onto its right instant only remembered its old id, so an import from a phone in the export's zone brought it back under the new one. a padded, quoted header on our own journal export was missed, and a vendor error replaced the not-utf-8 note when nothing else landed. --- lib/data/local_repository_impl.dart | 3 ++- lib/import/whoop_import.dart | 16 +++++++++---- lib/ui2/onboarding/welcome.dart | 12 ++++++++-- test/import_routing_test.dart | 34 ++++++++++++++++++++++++++++ test/whoop_import_timezone_test.dart | 9 ++++++++ 5 files changed, 67 insertions(+), 7 deletions(-) diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 9fc03661d..e57f87d56 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2838,7 +2838,8 @@ class LocalRepositoryImpl extends LocalRepository { // Tombstone first: a failed delete then only leaves a row the importer // already skips, while the other order's failure brings it back. if (id.startsWith(WhoopImporter.workoutIdPrefix)) { - await WhoopImporter.rememberDeletedWorkout(id); + await WhoopImporter.rememberDeletedWorkout(id, + startTs: ((await LocalDb.session(id))?['start_ts'] as num?)?.toInt()); } await LocalDb.deleteSession(id); } diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index 2048275c0..b60d39799 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -79,12 +79,20 @@ class WhoopImporter { // would each drop the other's id. static Future _deletedWrites = Future.value(); - static Future rememberDeletedWorkout(String id) => _deletedWrites = - _deletedWrites.catchError((_) {}).then((_) async { + /// [startTs] is the row's start as stored. A row an earlier re-import moved + /// onto its right instant kept its old id, and once the phone's zone matches + /// the export's the import computes the id from that instant instead, so + /// both go on the list. + static Future rememberDeletedWorkout(String id, {int? startTs}) => + _deletedWrites = _deletedWrites.catchError((_) {}).then((_) async { final prefs = await SharedPreferences.getInstance(); final set = await deletedWorkoutIds(); - if (set.contains(id)) return; - await prefs.setStringList(_kDeletedWorkoutsPref, [...set, id]); + final add = { + id, + if (startTs != null) '$workoutIdPrefix$startTs', + }.difference(set); + if (add.isEmpty) return; + await prefs.setStringList(_kDeletedWorkoutsPref, [...set, ...add]); }); /// Column aliases for the energy field. The unit is read from whichever of diff --git a/lib/ui2/onboarding/welcome.dart b/lib/ui2/onboarding/welcome.dart index 53a1b2f5e..97c3e5aaa 100644 --- a/lib/ui2/onboarding/welcome.dart +++ b/lib/ui2/onboarding/welcome.dart @@ -493,7 +493,15 @@ Future runImport( skipped += r.skippedExistingDays; } } catch (e) { - if (db.isEmpty && raw.isEmpty && journalRows == 0 && labRows == 0) rethrow; + // Something already named (a journal export that isn't UTF-8) keeps its + // note: a rethrow would replace it with the vendor error alone. + if (db.isEmpty && + raw.isEmpty && + journalRows == 0 && + labRows == 0 && + rejected.isEmpty) { + rethrow; + } readError = '$e'; } } @@ -527,7 +535,7 @@ Future _isOwnCsvExport(String path) async { final first = text.split(RegExp(r'\r?\n')).first.replaceAll('\u00ef\u00bb\u00bf', ''); final header = [ for (final h in first.split(',')) - h.trim().replaceAll('"', '').toLowerCase(), + h.replaceAll('"', '').trim().toLowerCase(), ]; return listEquals(header, kJournalCsvHeader) || listEquals(header, kLabCsvHeader); diff --git a/test/import_routing_test.dart b/test/import_routing_test.dart index 55d29b6c0..69194a107 100644 --- a/test/import_routing_test.dart +++ b/test/import_routing_test.dart @@ -44,6 +44,15 @@ class _RoutingSpy extends AppState { } } +class _ThrowingVendor extends _RoutingSpy { + @override + Future importWhoopCsvs(List paths, + {void Function(int days)? onProgress}) async { + vendor.addAll(paths); + throw const FormatException('no recognised files'); + } +} + List _zipOf(Map members) { final a = Archive(); members.forEach((name, body) { @@ -151,6 +160,31 @@ void main() { expect(out.rejectedRows.single, contains('UTF-8')); }); + test('a quoted, padded header is still recognised as our journal export', + () async { + final app = _RoutingSpy(); + final journal = await write('journal.csv', + latin1.encode('date," tags ",note\n2026-08-01,,caf\u00e9\n')); + + final out = await runImport(app, [journal]); + + expect(app.vendor, isEmpty); + expect(out.rejectedRows.single, contains('UTF-8')); + }); + + test('the encoding note survives a vendor file nothing could read', + () async { + final app = _ThrowingVendor(); + final journal = await write('journal.csv', + latin1.encode('date,tags,note\n2026-08-01,,caf\u00e9\n')); + final junk = await write('junk.csv', utf8.encode('a,b\n1,2\n')); + + final out = await runImport(app, [journal, junk]); + + expect(out.rejectedRows.single, contains('UTF-8')); + expect(out.readError, contains('no recognised files')); + }); + test('a cp1252 CSV that is not ours still goes to the vendor importer', () async { final app = _RoutingSpy(); diff --git a/test/whoop_import_timezone_test.dart b/test/whoop_import_timezone_test.dart index ab209dac2..33dd01601 100644 --- a/test/whoop_import_timezone_test.dart +++ b/test/whoop_import_timezone_test.dart @@ -214,6 +214,15 @@ void main() { await db.delete('sessions', where: 'id = ?', whereArgs: [movedId]); }); + test('a moved workout deleted under its old id stays deleted', () async { + SharedPreferences.setMockInitialValues({}); + final start = _utc(2024, 4, 2, 0, 15); + await WhoopImporter.rememberDeletedWorkout('whoop_1', startTs: start); + final res = await WhoopImporter.importFiles([workoutCsv('wdm.csv')]); + expect(res.workouts, 0); + expect(await sessionsAround(start), isEmpty); + }); + test('a deleted phone-zone workout stays deleted on re-import', () async { final legacyStart = WhoopImporter.parseExportTs(wallStart)!; final start = _utc(2024, 4, 2, 0, 15);