diff --git a/lib/data/db.dart b/lib/data/db.dart index 08604659c..95b395a93 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -3161,6 +3161,41 @@ class LocalDb { }); } + /// Drop every imported workout that starts inside [[startSec], [endSec]] + /// but whose uuid is not in [present], AND its route, like + /// [deleteImportedWorkout]. Returns how many went. + /// + /// The window is the one the store was just read over: a uuid it no longer + /// returns there was deleted in the health app, and the upsert alone would + /// keep listing it (and counting it toward the week) forever. Rows outside + /// the window are left alone, since the read says nothing about them. + static Future deleteImportedWorkoutsMissing( + int startSec, + int endSec, + Set present, + ) async { + final db = await instance; + return db.transaction((txn) async { + final rows = await txn.query( + 'imported_workout', + columns: ['uuid'], + where: 'start_ts >= ? AND start_ts <= ?', + whereArgs: [startSec, endSec], + ); + final gone = [ + for (final r in rows) + if (!present.contains(r['uuid'])) r['uuid'] as String, + ]; + for (final u in gone) { + await txn.delete('imported_workout', + where: 'uuid = ?', whereArgs: [u]); + await txn.delete('workout_route', + where: 'session_id = ?', whereArgs: [u]); + } + return gone.length; + }); + } + // ── EXTERNAL HEART-RATE SENSOR (0x180D) ───────────────────────────────────── /// What a standard Bluetooth heart-rate sensor delivered during ONE workout. /// @@ -3742,6 +3777,27 @@ class LocalDb { return (r.first['s'] as num?)?.toInt() ?? 0; } + /// [day]'s banked phone windows, so a re-sync can keep the ones for an hour + /// the sensor no longer covers (see `PhonePedometer.syncDay`). + static Future> + phoneCoverageForDay(String day) async { + final db = await instance; + final rows = await db.query( + 'live_coverage', + columns: ['start_ts', 'end_ts', 'steps'], + where: 'day = ? AND source = ?', + whereArgs: [day, kStepSourcePhone], + ); + return [ + for (final r in rows) + ( + startTs: (r['start_ts'] as num).toInt(), + endTs: (r['end_ts'] as num).toInt(), + steps: (r['steps'] as num).toInt(), + ), + ]; + } + /// Drop every phone-sourced coverage row (the user turned phone steps off). /// Band rows are untouched, so days fall back to the band count. static Future clearPhoneCoverage() async { @@ -12446,6 +12502,35 @@ class LocalDb { }); } + /// Move a session's window. Not `putSession`: the row's type correction, + /// RPE and private flag must survive. What was measured over the OLD window + /// (the frozen HR trace, average HR, zone minutes) belongs to other seconds, + /// so it goes; a re-score fills it again if the new window is still on + /// record. Returns the number of rows changed. + static Future setSessionWindow(String id, int startTs, int? endTs) async { + final db = await instance; + return db.transaction((tx) async { + final n = await tx.update( + 'sessions', + { + 'start_ts': startTs, + 'end_ts': endTs, + 'trace_json': null, + 'trace_samples': null, + 'avg_hr': null, + 'zone_min_json': null, + }, + where: 'id = ?', + whereArgs: [id], + ); + if (n > 0) { + await ActivityStore.supersede(tx, ActivityKind.workout, startTs, + endTs ?? DateTime.now().millisecondsSinceEpoch ~/ 1000); + } + return n; + }); + } + /// Update ONLY a session's derived score columns. /// /// Deliberately not `putSession`: that is INSERT-OR-REPLACE over the whole @@ -12487,6 +12572,17 @@ class LocalDb { ); } + /// Whether a [source] session already starts at [startTs], under any id. + static Future hasSessionStartingAt(String source, int startTs) async { + final db = await instance; + final rows = await db.query('sessions', + columns: ['id'], + where: 'source = ? AND start_ts = ?', + whereArgs: [source, startTs], + limit: 1); + return rows.isNotEmpty; + } + static Future?> session(String id) async { final db = await instance; final rows = await db.query( diff --git a/lib/data/local_repository.dart b/lib/data/local_repository.dart index c5345dd5f..212fcf96d 100644 --- a/lib/data/local_repository.dart +++ b/lib/data/local_repository.dart @@ -267,10 +267,12 @@ abstract class LocalRepository { /// detected, or detected too narrowly. Scored from the 1 Hz substrate over /// [startTs, endTs] (epoch SECONDS); a window with no substrate is still /// saved, just unscored. Returns `{workout_id, unscored, hr_samples}`. + /// [source] is stamped on a NEW row only; an update keeps the row's own. Future> logManualWorkout({ required int startTs, required int endTs, required String type, + String source = 'manual', }) => throw UnimplementedError('re-layer: logManualWorkout'); /// Retime an existing session and re-score it over the new window. Used to diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index 504f64f5c..e57f87d56 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -35,6 +35,7 @@ import 'db.dart'; import 'activity_store.dart'; import '../models/activity_suggestion.dart'; import '../health/health_export.dart'; +import '../import/whoop_import.dart' show WhoopImporter; import 'journal_fields.dart'; import '../ble/adapters/_registry.dart' show kBandRegistry; import 'local_repository.dart'; @@ -2833,7 +2834,15 @@ class LocalRepositoryImpl extends LocalRepository { int? _profileAge() => (getProfileMap()?['age'] as num?)?.round(); @override - Future deleteWorkout(String id) async => LocalDb.deleteSession(id); + Future deleteWorkout(String id) async { + // Tombstone first: a failed delete then only leaves a row the importer + // already skips, while the other order's failure brings it back. + if (id.startsWith(WhoopImporter.workoutIdPrefix)) { + await WhoopImporter.rememberDeletedWorkout(id, + startTs: ((await LocalDb.session(id))?['start_ts'] as num?)?.toInt()); + } + await LocalDb.deleteSession(id); + } @override Future setWorkoutPrivate(String id, bool private) async { @@ -2841,14 +2850,6 @@ class LocalRepositoryImpl extends LocalRepository { // The workout was already exported when it stopped, before this flag // landed. Re-export: a private row clears its window and writes nothing. unawaited(HealthExporter.exportWorkoutId(id)); - // The day's active energy has exported workouts' calories taken off, so - // it changes too, and a day behind the export cursor is never rewritten. - final start = ((await LocalDb.session(id))?['start_ts'] as num?)?.toInt(); - if (start != null) { - unawaited(HealthExporter.reexportFrom( - dayLabelOf(DateTime.fromMillisecondsSinceEpoch(start * 1000)), - )); - } } @override @@ -2916,6 +2917,7 @@ class LocalRepositoryImpl extends LocalRepository { required int startTs, required int endTs, required String type, + String source = 'manual', }) async { // A manual row's id is derived from its start second, so re-logging the // IDENTICAL window is an update of that row, not a collision with it. A @@ -2951,6 +2953,7 @@ class LocalRepositoryImpl extends LocalRepository { existing: same ? prior : null, sessionId: moved ? '$id:${DateTime.now().millisecondsSinceEpoch}' : null, + source: source, validateAgainstId: same ? prior['id'] as String : (prior == null && !moved ? id : null), diff --git a/lib/health/auto_workout_import.dart b/lib/health/auto_workout_import.dart index 1d1a1be32..05814bc85 100644 --- a/lib/health/auto_workout_import.dart +++ b/lib/health/auto_workout_import.dart @@ -113,6 +113,9 @@ class AutoWorkoutImport { } return AutoImportOutcome.ran; } catch (_) { + // A read that threw is not a pass that ran. The stamp above still holds + // it off for the interval, so a store refusing on quota is not hammered + // on every foreground. return AutoImportOutcome.skipped; } } diff --git a/lib/health/health_export.dart b/lib/health/health_export.dart index 7134aae85..88c0eff0c 100644 --- a/lib/health/health_export.dart +++ b/lib/health/health_export.dart @@ -99,8 +99,9 @@ List healthDeleteTypes({required bool isApplePlatform}) { /// /// Health Connect stores a workout's energy as a separate /// TotalCaloriesBurnedRecord inserted in the same atomic batch, so without that -/// write permission the whole workout write fails. On Android the session's -/// kcal stays in the day's hourly active energy instead. +/// write permission the whole workout write fails. On both platforms the +/// session's kcal also stays in the day's hourly active energy, which is what +/// the store's daily active-energy total is built from. @visibleForTesting int? healthWorkoutEnergyKcal( Map r, { @@ -352,12 +353,29 @@ Future exportPrioritySleepBeforeBulk({ required Future Function(Map) write, required Future Function(String? androidSleepAlreadyWritten) exportBulk, }) async { - final priorityResult = await exportNewestPrioritySleep( - newestFirstDays: newestFirstDays, - write: write, - ); - if (!priorityResult.succeeded) return priorityResult; - await exportBulk(priorityResult.date); + PrioritySleepExportResult priorityResult; + try { + priorityResult = await exportNewestPrioritySleep( + newestFirstDays: newestFirstDays, + write: write, + ); + } catch (e) { + debugPrint('[health] write priority Android sleep session: $e'); + priorityResult = PrioritySleepExportResult( + date: newestFirstDays + .where((d) => normalizeHealthSleepSession(d.value) != null) + .firstOrNull + ?.key, + succeeded: false, + ); + } + // A failed sleep write must not hold the rest of the export. Health Connect + // grants each type separately, so a user who leaves Sleep off (another app + // owns it) fails this write on every pass; holding bulk behind it meant no + // heart rate, RHR, HRV, energy or workouts ever reached the store. Bulk is + // told nothing was written, so it retries the sleep as part of that day and + // the day stays out of the cursor until the whole day lands. + await exportBulk(priorityResult.succeeded ? priorityResult.date : null); return priorityResult; } @@ -898,7 +916,10 @@ class HealthExporter { lastAttemptMs = null; } final nowMs = DateTime.now().millisecondsSinceEpoch; - final shouldAttempt = shouldAttemptHealthExport( + // The same gate bulk applies to this day, so a day that is due here + // is due there too: when the sleep write fails, bulk retries the + // whole day and its retry entry is the one accounting for it. + final shouldAttempt = shouldAttemptHealthBulkExport( attempts: attempts, maxAttempts: _kMaxExportAttempts, now: DateTime.fromMillisecondsSinceEpoch(nowMs), @@ -906,6 +927,7 @@ class HealthExporter { ? null : DateTime.fromMillisecondsSinceEpoch(lastAttemptMs), backoff: _backoffFor(attempts), + prioritySleepAlreadyWritten: false, lastSuccess: okMs == null ? null : DateTime.fromMillisecondsSinceEpoch(okMs), @@ -914,58 +936,25 @@ class HealthExporter { : _kNonFinalizedRewriteInterval, force: forceRetry, ); - if (!shouldAttempt) { - // The priority day is held back by the CAP or by the success-side - // rewrite throttle (it exported cleanly <30 min ago) — in either - // case its sleep session is already in the store and is not what's - // blocking, so still export every OTHER pending day. Only a - // genuine retry backoff (recent FAILURE, still under the cap) holds - // bulk, matching the pre-throttle behaviour. A day with `ok_ms` set - // carries no pending failure (the success branch clears - // attempts/last_ms), so the two conditions are mutually exclusive. - final throttledBySuccess = okMs != null && - !pendingPriorityDay.finalized && - nowMs - okMs < _kNonFinalizedRewriteInterval.inMilliseconds; - if (attempts >= _kMaxExportAttempts || throttledBySuccess) { - return exportBulk(null); - } - return 0; - } - Future recordPriorityFailure() async { - // A timed-out (unavailable) store isn't a failed export; don't - // spend an attempt on it. - if (_storeHung) return; - retryState[priorityDay!.key] = { - 'attempts': attempts + 1, - 'last_ms': nowMs, - 'finalized': pendingPriorityDay.finalized, - }; - await LocalDb.setCursor(_kRetryCursor, jsonEncode(retryState)); - } + // Not due (backoff, cap, or it exported cleanly moments ago): the + // other pending days still go out. Holding them behind this one + // day's backoff is what kept every metric out of Health Connect + // whenever the sleep write alone was failing. + if (!shouldAttempt) return exportBulk(null); var bulkDone = 0; - try { - final priorityResult = await exportPrioritySleepBeforeBulk( - // `exportNewestPrioritySleep` returns the first day carrying a - // sleep session, and the scan above already found exactly that - // day — so one entry is equivalent to the whole list, without - // holding 400 decoded bundles to re-derive the same answer. - newestFirstDays: [priorityDay], - write: _androidSleep.replace, - exportBulk: (androidSleepAlreadyWritten) async { - bulkDone = await exportBulk(androidSleepAlreadyWritten); - }, - ); - if (!priorityResult.succeeded) { - await recordPriorityFailure(); - return 0; - } - return bulkDone; - } catch (e) { - debugPrint('[health] write priority Android sleep session: $e'); - await recordPriorityFailure(); - return 0; - } + await exportPrioritySleepBeforeBulk( + // `exportNewestPrioritySleep` returns the first day carrying a + // sleep session, and the scan above already found exactly that + // day — so one entry is equivalent to the whole list, without + // holding 400 decoded bundles to re-derive the same answer. + newestFirstDays: [priorityDay], + write: _androidSleep.replace, + exportBulk: (androidSleepAlreadyWritten) async { + bulkDone = await exportBulk(androidSleepAlreadyWritten); + }, + ); + return bulkDone; } } return exportBulk(null); @@ -1145,22 +1134,28 @@ class HealthExporter { // Sleep is the smallest, highest-value write. Native replace owns cleanup // on both stores (Health Connect SleepSessionRecord; HealthKit inBed+Core). + // Both the clear on a no-night day and HealthKit's replace on a day with + // a night must stop short of yesterday's night if that one woke after + // noon. Health Connect only uses it for the clear (its delete matches on + // start time; the native replace computes its own window). + Future previousWake() async { + final prev = DateTime(dayStart.year, dayStart.month, dayStart.day - 1); + final pb = _decode( + (await LocalDb.dayResult(dayLabelOf(prev)))?['payload_json'], + ); + return pb == null ? null : normalizeHealthSleepSession(pb)?.end; + } + if (Platform.isAndroid && !androidSleepAlreadyWritten) { try { - // No night today means a clear, which must stop short of yesterday's - // night if that one woke after noon. - DateTime? previousWake; - if (normalizeHealthSleepSession(b) == null) { - final prev = DateTime(dayStart.year, dayStart.month, dayStart.day - 1); - final pb = _decode( - (await LocalDb.dayResult(dayLabelOf(prev)))?['payload_json'], - ); - if (pb != null) previousWake = normalizeHealthSleepSession(pb)?.end; - } if (!await _androidSleep.replace( b, dayStart: dayStart, - previousWake: previousWake, + // Only the no-night clear reads it; skip decoding yesterday's + // bundle on every other day of a long first export. + previousWake: normalizeHealthSleepSession(b) == null + ? await previousWake() + : null, )) { debugPrint('[health] write Android sleep session returned false'); success = false; @@ -1175,7 +1170,7 @@ class HealthExporter { if (!await _appleSleep.replace( bundle: b, dayStart: dayStart, - dayEnd: dayEnd, + previousWake: await previousWake(), )) { debugPrint('[health] write Apple sleep session returned false'); success = false; @@ -1334,29 +1329,13 @@ class HealthExporter { ); final bucketCount = bucketBounds.length - 1; - // Active energy: chunked into hourly buckets over the day. - // We subtract workout calories to prevent double-counting, because workouts - // are exported separately (their totalEnergyBurned already covers it, on - // Apple only — see healthWorkoutEnergyKcal). - // Upper bound is exclusive (dayEnd - 1s): sessionsInRange is inclusive on - // both ends, so a workout starting exactly at midnight would otherwise be - // double-subtracted from both this day and the next. - var cal = sc('calories')?.toDouble() ?? 0.0; - try { - final rows = await LocalDb.sessionsInRange( - dayStart.millisecondsSinceEpoch ~/ 1000, - (dayEnd.millisecondsSinceEpoch ~/ 1000) - 1, - ); - final workoutCal = - healthWorkoutCaloriesToSubtract(rows, isApplePlatform: isApple); - cal = (cal > workoutCal) ? cal - workoutCal : 0.0; - } catch (e) { - // Unknown whether cal is workout-adjusted — still write our best guess - // below (idempotent re-export corrects it once this query succeeds), - // but flag the day so it isn't marked done on this pass. - debugPrint('[health] workout-calorie query: $e'); - success = false; - } + // Active energy: chunked into hourly buckets over the day, workouts + // included. The WORKOUT sample's totalEnergyBurned is not counted by + // HealthKit's Active Energy total or the Move ring (those sum + // activeEnergyBurned samples only, and the plugin saves a bare HKWorkout + // with none attached), so taking workout kcal out of these buckets made + // them vanish from the day instead of preventing a double count. + final cal = sc('calories')?.toDouble() ?? 0.0; if (cal > 0 && bucketCount > 0) { final calPerHour = cal / bucketCount; @@ -1511,6 +1490,9 @@ class HealthExporter { if ((r['private'] as num?)?.toInt() == 1) { return null; // skip, not a failure } + if (r['source'] == kScoredImportSessionSource) { + return null; // skip, not a failure + } final st = (r['start_ts'] as num?)?.toInt(); final en = (r['end_ts'] as num?)?.toInt(); if (st == null || en == null || en <= st) { @@ -1560,6 +1542,7 @@ class HealthExporter { // exported workout, and deleting it here — only to have _writeOneWorkout // refuse to write the replacement — would erase it for nothing. if ((session['end_ts_fabricated'] as num?)?.toInt() == 1) return false; + if (session['source'] == kScoredImportSessionSource) return false; final st = (session['start_ts'] as num?)?.toInt(); final en = (session['end_ts'] as num?)?.toInt(); if (st == null || en == null || en <= st) return false; @@ -1663,24 +1646,10 @@ String? healthWorkoutTitleForType(String? type) { .join(' '); } -/// Calories of the day's sessions that get their own WORKOUT sample, and so -/// come off the day's active energy. A session [HealthExporter] skips (live, -/// fabricated end, private) keeps its calories in the active-energy total — -/// subtracting them too would make them vanish from the day entirely. Only -/// the energy the WORKOUT sample itself carries comes off -/// ([healthWorkoutEnergyKcal]: none on Android). -@visibleForTesting -double healthWorkoutCaloriesToSubtract(List> rows, - {bool isApplePlatform = true}) { - var total = 0.0; - for (final r in rows) { - if ((r['status']?.toString() ?? '') == 'live') continue; - if ((r['end_ts_fabricated'] as num?)?.toInt() == 1) continue; - if ((r['private'] as num?)?.toInt() == 1) continue; - total += healthWorkoutEnergyKcal(r, isApplePlatform: isApplePlatform) ?? 0; - } - return total; -} +/// `sessions.source` of a session scored from an imported workout's window. +/// The import came from the health store and the original is still there, so +/// writing this one as well would list the same workout twice. +const String kScoredImportSessionSource = 'scored_import'; /// Parameterised by [ios] rather than reading `Platform` directly so a unit /// test can exercise BOTH platform branches on a host VM (where `Platform.isIOS` @@ -1710,7 +1679,10 @@ HealthWorkoutActivityType healthActivityForType( String? type, { required bool ios, }) { - switch ((type ?? '').toLowerCase()) { + // Spaces as underscores: rows imported from a WHOOP export before the + // importer normalised its activity names are stored as 'mountain biking', + // and the importer never rewrites a row it already holds. + switch ((type ?? '').trim().toLowerCase().replaceAll(RegExp(r'\s+'), '_')) { case 'run': case 'running': case 'treadmill': @@ -1722,6 +1694,7 @@ HealthWorkoutActivityType healthActivityForType( case 'biking': case 'indoor_bike': case 'mountain_biking': + case 'spin': // WHOOP's name, on rows imported before the alias existed return HealthWorkoutActivityType.BIKING; case 'walk': case 'walking': @@ -1740,6 +1713,9 @@ HealthWorkoutActivityType healthActivityForType( case 'weight_training': case 'powerlifting': case 'kettlebell': + // WHOOP's names, on rows imported before the importer mapped them. + case 'weightlifting': + case 'strength_trainer': return ios ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING : HealthWorkoutActivityType.STRENGTH_TRAINING; @@ -1754,9 +1730,11 @@ HealthWorkoutActivityType healthActivityForType( return HealthWorkoutActivityType.ROWING; case 'hike': case 'hiking': + case 'hiking/rucking': return HealthWorkoutActivityType.HIKING; case 'climb': case 'climbing': + case 'rock_climbing': // Bare `CLIMBING` is iOS-only; `ROCK_CLIMBING` is the one spelling both // stores accept, which is the #184 lesson applied ahead of the bug. return HealthWorkoutActivityType.ROCK_CLIMBING; diff --git a/lib/health/health_measurement_import.dart b/lib/health/health_measurement_import.dart index fb1e89245..ea105c38f 100644 --- a/lib/health/health_measurement_import.dart +++ b/lib/health/health_measurement_import.dart @@ -128,8 +128,9 @@ class ImportedMeasurementImporter { } /// Read the recent window and store what is there. Returns how many rows - /// landed; 0 covers an empty store, a denied permission and a locked device - /// alike, none of which is an error the user needs a dialog for. + /// landed; 0 means the store gave back nothing. A read or write that throws + /// propagates, so the caller reports a failure instead of "nothing to + /// import" for a store it never managed to read. Future sync({DateTime? now}) async { final end = now ?? DateTime.now(); // Health Connect caps third-party reads at 30 days without @@ -139,17 +140,12 @@ class ImportedMeasurementImporter { final start = _isApple ? DateTime(end.year - 1, end.month, end.day) : end.subtract(const Duration(days: 30)); - try { - await _health.configure(); - final points = await _health.getHealthDataFromTypes( - types: types, - startTime: start, - endTime: end, - ); - return LocalDb.putImportedMeasurements(rowsFrom(points)); - } catch (e) { - debugPrint('[imported_measurement] read: $e'); - return 0; - } + await _health.configure(); + final points = await _health.getHealthDataFromTypes( + types: types, + startTime: start, + endTime: end, + ); + return LocalDb.putImportedMeasurements(rowsFrom(points)); } } diff --git a/lib/health/health_sleep_session.dart b/lib/health/health_sleep_session.dart index f838fdfd2..4ed122b84 100644 --- a/lib/health/health_sleep_session.dart +++ b/lib/health/health_sleep_session.dart @@ -251,20 +251,62 @@ HealthSleepStage? healthSleepStageOf(String? stage) { /// The span a day's SLEEP-type delete has to cover. /// -/// Just [sleepSessionCleanupRange] — the noon-to-noon window around the +/// [sleepSessionCleanupRange] — the noon-to-noon window around the /// night's wake, same as Android. This used to UNION that with the whole /// calendar day, and for a night waking after local noon the two windows /// concatenated into one contiguous span reaching back into the PREVIOUS /// night; the native predicate matches on overlap, so it deleted the whole /// previous envelope sample — which is behind `health_export_through` and /// never gets rewritten. Irrecoverable deletion is not a cleanup. -({DateTime start, DateTime end}) sleepCleanupWindow({ +/// +/// With no night it is [noNightCleanupRange], null when that is empty. +/// +/// Either way it starts after [previousWake] when the day before's night woke +/// inside the window: a night ending after local noon overlaps this day's +/// noon-to-noon span, and the overlap delete took its whole in-bed envelope +/// and last stages with it. That day is behind the export cursor, so nothing +/// wrote them back. The clamp never reaches past this night's own onset. +({DateTime start, DateTime end})? sleepCleanupWindow({ required DateTime dayStart, - required DateTime dayEnd, HealthSleepSession? night, + DateTime? previousWake, +}) { + if (night == null) { + return noNightCleanupRange(dayStart: dayStart, previousWake: previousWake); + } + final range = sleepSessionCleanupRange(night); + if (previousWake != null && + !previousWake.isBefore(range.start) && + previousWake.isBefore(night.start)) { + return ( + start: previousWake.add(const Duration(seconds: 1)), + end: range.end, + ); + } + return range; +} + +/// What a day with no night clears: the noon-to-noon window its night would +/// have sat in, started after the previous night's wake when that night woke +/// after noon ([previousWake]). Null when nothing is left to clear. +/// +/// Not the calendar day. HealthKit's delete matches samples that OVERLAP the +/// window, and the next day's night usually starts before midnight, so a +/// [dayStart, dayEnd) sweep deleted that night's in-bed envelope and its +/// pre-midnight stages. Once the next day is finalized and exported nothing +/// writes them back. The same goes for the previous night when it woke after +/// noon, which is what the [previousWake] clamp is for. +({DateTime start, DateTime end})? noNightCleanupRange({ + required DateTime dayStart, + DateTime? previousWake, }) { - if (night == null) return (start: dayStart, end: dayEnd); - return sleepSessionCleanupRange(night); + var from = DateTime(dayStart.year, dayStart.month, dayStart.day - 1, 12); + final to = DateTime(dayStart.year, dayStart.month, dayStart.day, 12); + if (previousWake != null && !previousWake.isBefore(from)) { + from = previousWake.add(const Duration(seconds: 1)); + } + if (!from.isBefore(to)) return null; + return (start: from, end: to); } abstract interface class HealthConnectSleepSessionWriter { @@ -329,13 +371,12 @@ class HealthConnectSleepSessionExporter { // is behind the export cursor nothing writes it back. Start after it. if (session == null) { if (dayStart == null) return true; - var from = DateTime(dayStart.year, dayStart.month, dayStart.day - 1, 12); - final to = DateTime(dayStart.year, dayStart.month, dayStart.day, 12); - if (previousWake != null && !previousWake.isBefore(from)) { - from = previousWake.add(const Duration(seconds: 1)); - } - if (!from.isBefore(to)) return true; - return writer.clear(from, to); + final range = noNightCleanupRange( + dayStart: dayStart, + previousWake: previousWake, + ); + if (range == null) return true; + return writer.clear(range.start, range.end); } // A window WITH NO STAGES is the same kind of "nothing to write", and has // to report the same way. It used to return false, and the caller treats @@ -426,14 +467,17 @@ class HealthKitSleepSessionExporter { Future replace({ required Map bundle, required DateTime dayStart, - required DateTime dayEnd, - }) { + DateTime? previousWake, + }) async { final session = normalizeHealthSleepSession(bundle); final window = sleepCleanupWindow( dayStart: dayStart, - dayEnd: dayEnd, night: session, + previousWake: previousWake, ); + // No night and an empty window: nothing to write or delete. The native + // side rejects an empty cleanup range as a failure. + if (window == null) return true; return writer.replace( cleanupStart: window.start, cleanupEnd: window.end, diff --git a/lib/health/health_workout_import.dart b/lib/health/health_workout_import.dart index e49c8641b..f4d049762 100644 --- a/lib/health/health_workout_import.dart +++ b/lib/health/health_workout_import.dart @@ -193,6 +193,31 @@ List> routeRowsFrom(Object? payload) { return out; } +/// The routes from one channel payload worth storing, keyed by workout uuid. +/// +/// Only uuids in [keep] — the workouts this pass actually stored — get a route. +/// The Swift side returns the route of EVERY workout in the window, including +/// ones `workoutsFrom` dropped (over 24 h, ending before they start), our own +/// exports and tombstoned deletes. A route stored for any of those sits under a +/// session_id with no `imported_workout` row: never shown, and out of reach of +/// every delete path, which all start from that row. That is precise location +/// history kept for a workout the app says it did not import. +@visibleForTesting +List>>> routesToStore( + List payload, Set keep) { + final out = >>>[]; + for (final entry in payload) { + if (entry is! Map) continue; + final uuid = entry['uuid']; + if (uuid is! String || uuid.isEmpty) continue; + if (!keep.contains(uuid)) continue; + final rows = routeRowsFrom(entry); + if (rows.isEmpty) continue; + out.add(MapEntry(uuid, rows)); + } + return out; +} + /// "RUNNING" → "Running". The health store's own enum name, title-cased and /// de-underscored. /// @@ -320,9 +345,11 @@ class HealthWorkoutImporter { /// Read the window, store the workouts, then fetch each one's route on Apple. /// - /// Never throws: an empty store, a denied permission and a locked device all - /// come back as a zero result, which the caller reports as "nothing to - /// import" rather than as a failure. + /// An empty store (and, on Apple, a denied read, which HealthKit hands back + /// as an empty list) is a zero result. A read that THROWS — a Health Connect + /// quota or SecurityException, the provider updating mid-read — propagates: + /// swallowing it into the same zero told the user the store held no + /// workouts when it was never read at all. /// /// [prompt] is true only from the manual Import tap: it lets the route /// fetch ask for the separate workoutRoute grant. The auto path leaves it @@ -332,28 +359,22 @@ class HealthWorkoutImporter { final start = end.subtract(Duration( days: _isApple ? kImportWindowDaysApple : kImportWindowDaysAndroid, )); - List rows; - final ownSources = {}, ownUuids = {}; + final List rows; + final ownSources = {}; + await _health.configure(); + final points = await _health.getHealthDataFromTypes( + types: types, + startTime: start, + endTime: end, + ); + String? ownApp; try { - await _health.configure(); - final points = await _health.getHealthDataFromTypes( - types: types, - startTime: start, - endTime: end, - ); - String? ownApp; - try { - ownApp = (await PackageInfo.fromPlatform()).packageName; - } catch (_) {} - rows = workoutsFrom(points, ownApp: ownApp); - for (final p in points) { - if (p.type != HealthDataType.WORKOUT || !_isOwn(p, ownApp)) continue; - ownSources.add(p.sourceName); - ownUuids.add(p.uuid); - } - } catch (e) { - debugPrint('[imported_workout] read: $e'); - return WorkoutImportResult(routesSupported: routesSupported); + ownApp = (await PackageInfo.fromPlatform()).packageName; + } catch (_) {} + rows = workoutsFrom(points, ownApp: ownApp); + for (final p in points) { + if (p.type != HealthDataType.WORKOUT || !_isOwn(p, ownApp)) continue; + ownSources.add(p.sourceName); } // Imports from before the own-app filter stored one copy of every band // session per export pass, each under a uuid the store no longer has, so @@ -371,8 +392,21 @@ class HealthWorkoutImporter { if (!tombstones.contains(r.uuid)) r, ]; await LocalDb.putImportedWorkouts([for (final r in alive) r.toRow()]); + // A workout deleted in the health app simply stops coming back, so the + // upsert above never removes it. Anything stored in this window that the + // read no longer returns goes. Only reached on a read that returned + // workouts: a denied HealthKit read is an empty list, and must not wipe + // the history. + await LocalDb.deleteImportedWorkoutsMissing( + start.millisecondsSinceEpoch ~/ 1000, + end.millisecondsSinceEpoch ~/ 1000, + { + for (final p in points) + if (p.type == HealthDataType.WORKOUT) p.uuid, + }, + ); final withRoutes = await _importRoutes(start, end, - skip: {...tombstones, ...ownUuids}, prompt: prompt); + keep: {for (final r in alive) r.uuid}, prompt: prompt); // Re-read: a workout scored or deleted while this pass was in flight, or // one an interrupted pass tombstoned without removing, must not stay // listed (or keep a route) beside the session that replaced it. @@ -394,7 +428,7 @@ class HealthWorkoutImporter { /// async round trip, and 90 days of running is a lot of them to serialise /// across the channel one at a time. Future _importRoutes(DateTime start, DateTime end, - {Set skip = const {}, bool prompt = false}) async { + {required Set keep, bool prompt = false}) async { if (!routesSupported) return 0; List payload; try { @@ -414,14 +448,8 @@ class HealthWorkoutImporter { return 0; } var n = 0; - for (final entry in payload) { - if (entry is! Map) continue; - final uuid = entry['uuid']; - if (uuid is! String || uuid.isEmpty) continue; - if (skip.contains(uuid)) continue; - final rows = routeRowsFrom(entry); - if (rows.isEmpty) continue; - await LocalDb.appendRoutePoints(uuid, rows); + for (final route in routesToStore(payload, keep)) { + await LocalDb.appendRoutePoints(route.key, route.value); n++; } return n; diff --git a/lib/health/phone_pedometer.dart b/lib/health/phone_pedometer.dart index c7198e59c..15721c29d 100644 --- a/lib/health/phone_pedometer.dart +++ b/lib/health/phone_pedometer.dart @@ -197,6 +197,7 @@ class PhonePedometer { final windows = <({int startTs, int endTs, int steps})>[]; var total = 0; var anyRead = false; + List<({int startTs, int endTs, int steps})>? banked; // CALENDAR-AWARE hour walk. `Duration` arithmetic on a local DateTime is // ABSOLUTE, so `dayStartLocal.add(Duration(hours: h))` over a fixed 24 @@ -243,9 +244,27 @@ class PhonePedometer { // persist a partial one over a good previous sync. if (n == null) return null; // UNCOVERED, not failed and not zero: the sensor holds no record of - // this hour. It banks nothing and it does not count as read, so a day + // this hour. It reads nothing new and it does not count as read, so a day // that is entirely uncovered still comes back "unknown" below. - if (n == intervalNotCovered) continue; + if (n == intervalNotCovered) { + // KEEP WHAT AN EARLIER SYNC ALREADY BANKED FOR THIS HOUR. An hour + // can turn uncovered after the fact: an Android reboot records a gap + // from the last sensor delivery, and the native query refuses any + // interval touching it even though the bins before the gap still + // hold real counts. The replace below is delete-then-insert, so + // skipping the hour here would silently erase steps that were + // already on the day. Not counted as read: a day whose only content + // is carried-over windows is left exactly as it was. + banked ??= await LocalDb.phoneCoverageForDay(dayId); + final fromSec = from.millisecondsSinceEpoch ~/ 1000; + final toSec = to.millisecondsSinceEpoch ~/ 1000; + for (final w in banked) { + if (w.startTs < fromSec || w.startTs >= toSec) continue; + windows.add(w); + if (w.steps > 0) total += w.steps; + } + continue; + } anyRead = true; if (n < 0) continue; // A CONFIRMED ZERO is banked too, not just a nonzero count: it is the diff --git a/lib/import/import_container.dart b/lib/import/import_container.dart index 2f12c948d..8075847b4 100644 --- a/lib/import/import_container.dart +++ b/lib/import/import_container.dart @@ -683,43 +683,46 @@ Future> _extractCsvMembers( ); } - if (archive.files.length > _kMaxArchiveMembers) { - throw ImportFormatException( - '“$name” holds ${archive.files.length} entries, which is far more than ' - 'any $flavor export — refusing to unpack it.', - ); - } + // Every exit from here on closes `input`, including the refusals: an + // ordinary wrong pick (a ZIP of photos) otherwise left a descriptor open on + // the picked file for the life of the process, one per retry. + try { + if (archive.files.length > _kMaxArchiveMembers) { + throw ImportFormatException( + '“$name” holds ${archive.files.length} entries, which is far more than ' + 'any $flavor export — refusing to unpack it.', + ); + } - final csvFiles = [ - for (final f in archive.files) - if (f.isFile && _isCsvMember(f.name)) f, - ]; + final csvFiles = [ + for (final f in archive.files) + if (f.isFile && _isCsvMember(f.name)) f, + ]; - final declaredBytes = - csvFiles.fold(0, (sum, f) => sum + (f.size > 0 ? f.size : 0)); - if (declaredBytes > _kMaxUncompressedBytes) { - throw ImportFormatException( - '“$name” unpacks to more than ' - '${(declaredBytes / (1024 * 1024 * 1024)).toStringAsFixed(1)} GB of ' - 'CSV, which is not something we can import.', - ); - } + final declaredBytes = + csvFiles.fold(0, (sum, f) => sum + (f.size > 0 ? f.size : 0)); + if (declaredBytes > _kMaxUncompressedBytes) { + throw ImportFormatException( + '“$name” unpacks to more than ' + '${(declaredBytes / (1024 * 1024 * 1024)).toStringAsFixed(1)} GB of ' + 'CSV, which is not something we can import.', + ); + } - if (csvFiles.isEmpty) { - throw ImportFormatException( - '“$name” is an archive with no CSV files inside ' - '(${archive.files.length} entr${archive.files.length == 1 ? 'y' : 'ies'}). ' - 'Pick the $flavor CSV export instead.', - ); - } + if (csvFiles.isEmpty) { + throw ImportFormatException( + '“$name” is an archive with no CSV files inside ' + '(${archive.files.length} entr${archive.files.length == 1 ? 'y' : 'ies'}). ' + 'Pick the $flavor CSV export instead.', + ); + } - final out = []; - final used = {}; - try { + final out = []; + final used = {}; for (final f in csvFiles) { - // Members can share a basename (`daily/data.csv`, `workouts/data.csv`). - // Flattening them onto one destination silently dropped one file and - // parsed the survivor twice. + // Members can share a basename (`daily/data.csv`, `workouts/data.csv`). + // Flattening them onto one destination silently dropped one file and + // parsed the survivor twice. var base = p.basename(f.name); if (!used.add(base)) { final stem = p.basenameWithoutExtension(base); @@ -740,8 +743,8 @@ Future> _extractCsvMembers( } out.add(destPath); } + return out; } finally { await input.close(); } - return out; } diff --git a/lib/import/journal_csv_import.dart b/lib/import/journal_csv_import.dart index d3bfa97fc..19c4f4c12 100644 --- a/lib/import/journal_csv_import.dart +++ b/lib/import/journal_csv_import.dart @@ -252,6 +252,17 @@ class JournalImportResult { final List rejected; } +/// Strict UTF-8 read of a CSV that a caller may want to hand to a lenient +/// importer when it is not UTF-8. +/// +/// `File.readAsString` reports bad bytes as a `FileSystemException`, so a +/// cp1252 file saved out of a spreadsheet escaped the `on FormatException` +/// fallbacks in the import router and aborted the whole selection. Decoding +/// the bytes here throws a `FormatException` instead, before anything is +/// written. +Future readCsvText(String path) async => + utf8.decode(await File(path).readAsBytes()); + /// Read [path], validate it, and write the accepted rows. /// /// REPLACES the journal for each date it carries — the file is the corrected @@ -262,7 +273,7 @@ Future importJournalCsvFile( String path, { DateTime? today, }) async { - final text = await File(path).readAsString(); + final text = await readCsvText(path); final parsed = parseJournalCsv(text, today: today); for (final row in parsed.rows) { await LocalDb.putJournal(row.date, jsonEncode(row.tags), row.note); diff --git a/lib/import/lab_csv_import.dart b/lib/import/lab_csv_import.dart index 371bb659c..542e8088c 100644 --- a/lib/import/lab_csv_import.dart +++ b/lib/import/lab_csv_import.dart @@ -17,14 +17,12 @@ // marker's canonical unit, and silently reinterpreting one unit as another is // a fabrication. -import 'dart:io'; - import 'package:flutter/foundation.dart'; import '../data/db.dart'; import '../data/lab_catalogue.dart'; import 'journal_csv_import.dart' - show RejectedRow, isValidDayLabel, parseCsv; + show RejectedRow, isValidDayLabel, parseCsv, readCsvText; /// Exactly the columns `csv_export.dart` emits for the labs set. const List kLabCsvHeader = ['taken_on', 'marker', 'value', 'unit', 'note']; @@ -177,7 +175,7 @@ class LabImportResult { /// (marker, taken_on) pair the file carries, matching `putLabResult`'s own /// upsert semantics. Future importLabCsvFile(String path, {DateTime? today}) async { - final text = await File(path).readAsString(); + final text = await readCsvText(path); final parsed = parseLabCsv(text, today: today); final customKeys = {for (final d in await LocalDb.labMarkerDefs()) d['key'].toString()}; diff --git a/lib/import/noop_backup_import.dart b/lib/import/noop_backup_import.dart index 1a1618439..8f6157795 100644 --- a/lib/import/noop_backup_import.dart +++ b/lib/import/noop_backup_import.dart @@ -59,11 +59,10 @@ const List _kSqliteMagic = [ @visibleForTesting int kNoopBackupPageRows = 20000; -/// Plausible unix seconds. A timestamp outside this range is corrupt or -/// millisecond-scaled, and letting one into the day walk would march it across -/// decades one day at a time. -const int _kMinPlausibleTs = 1000000000; // 2001-09-09 -const int _kMaxPlausibleTs = 4102444800; // 2100-01-01 +/// Plausible unix seconds (shared with the CSV path through [NoopIngest]). A +/// timestamp outside this range is corrupt or millisecond-scaled, and letting +/// one into the day walk would march it across decades one day at a time. +const int _kMinPlausibleTs = kNoopMinPlausibleTs; /// Every table read, and the columns wanted from it. ONE definition: the probe /// and the reads both key off this, so a name can no longer be spelled one way @@ -298,9 +297,11 @@ class NoopBackupImporter { final from = dayStart.millisecondsSinceEpoch ~/ 1000; final to = next.millisecondsSinceEpoch ~/ 1000; - // Order within a day does not matter — [NoopIngest] rebuilds the Substrate - // sorted by timestamp — but the DAYS must arrive in ascending order, since - // the high-water date is what closes out and derives the previous one. + // [NoopIngest] re-sorts by timestamp but keeps rows that share a second in + // arrival order, so beats within a second must come in detection order + // (the `ts, rowid` order above). The DAYS must also arrive in ascending + // order, since the high-water date is what closes out and derives the + // previous one. await _read(src, tables, columns, ordered, corrupt, 'hrSample', from, to, (r) async { final ts = _int(r['ts']), v = _int(r['bpm']); @@ -532,7 +533,7 @@ class NoopBackupImporter { try { r = await src.rawQuery( 'SELECT MIN(ts) AS lo, MAX(ts) AS hi FROM $t WHERE ts >= ? AND ts <= ?', - [_kMinPlausibleTs, _kMaxPlausibleTs], + [_kMinPlausibleTs, noopMaxPlausibleTs()], ); } catch (e) { if (!_isCorruptPageError(e)) rethrow; diff --git a/lib/import/noop_ingest.dart b/lib/import/noop_ingest.dart index f8defdb46..ae80dd2f2 100644 --- a/lib/import/noop_ingest.dart +++ b/lib/import/noop_ingest.dart @@ -40,6 +40,22 @@ class StepRun { int get hashCode => Object.hash(startSec, endSec, steps); } +/// Plausible unix seconds. A timestamp outside this range is corrupt, RTC-unset +/// (the band ships with its clock unset) or milli/microsecond-scaled. One such +/// row reaching [NoopIngest.offer] either opens a date decades away — every +/// later real date is then "older" than the high-water date and strands — or +/// buffers a 1970 sample that stretches the next Substrate past the +/// 400-day `calendarDays` guard, so the real day silently derives nothing. +/// A microsecond value overflows `DateTime` outright. +const int kNoopMinPlausibleTs = 1000000000; // 2001-09-09 + +/// The upper bound: now, plus two days of clock skew. A fixed far-future bound +/// let one corrupt 2099 row through, and that row alone set the backup span's +/// end (every empty day in between was then read) and the ingest's high-water +/// date (every real row after it stranded). +int noopMaxPlausibleTs() => + DateTime.now().millisecondsSinceEpoch ~/ 1000 + 2 * 86400; + /// What to do with an incoming row given the import's high-water date. /// [advance] closes out the previous date; [buffer] folds the row into the /// current rolling window; [late] means its day was already derived + pruned, @@ -66,6 +82,8 @@ class NoopIngest { /// belongs to the deleted store, so the import stops at its next write /// instead of banking it into the fresh one. final int _resetGen = ResetGate.generation; + /// Fixed for the length of one import, so a long run cannot move it. + final int _maxTs = noopMaxPlausibleTs(); void _stopIfReset() { if (ResetGate.changedSince(_resetGen)) { @@ -99,6 +117,10 @@ class NoopIngest { /// in, so they are counted rather than silently dropped. int lateRows = 0; + /// Samples whose timestamp fell outside [kNoopMinPlausibleTs].. + /// [noopMaxPlausibleTs] and were skipped. + int implausibleRows = 0; + /// Dates that arrived AFTER a later date had already opened, so they were /// folded in as context but never derived in their own right. Non-empty means /// the source was not time-ordered and those days are missing from the @@ -108,9 +130,14 @@ class NoopIngest { Set.unmodifiable(_strandedDates.difference(_derived)); /// Offer a sample at [ts] (epoch seconds). Returns false when the sample is - /// too late to use — the caller must then skip it entirely. + /// too late to use, or its timestamp is implausible — the caller must then + /// skip it entirely. Future offer(int ts) async { _stopIfReset(); + if (ts < kNoopMinPlausibleTs || ts > _maxTs) { + implausibleRows++; + return false; + } final date = localDateLabel(ts); switch (decideRow(date, _curDate, _derived)) { case RowOrder.advance: @@ -149,7 +176,12 @@ class NoopIngest { return true; } - void hr(int ts, int bpm) => (_secs[ts] ??= _Sec()).hr = bpm; + // Same physiological gate as the band decode. A NOOP row is otherwise taken + // at face value, so an off-wrist sentinel (255) or a 1 bpm glitch would land + // in max HR, strain and the resting-HR window for imported days only. 0 keeps + // the second (and its accel / SpO2 context) while saying "no usable HR". + void hr(int ts, int bpm) => + (_secs[ts] ??= _Sec()).hr = plausibleHrOrNull(bpm) ?? 0; void rr(int ts, double ms) { // FINITE and positive. `ms <= 0` alone lets NaN through (it fails every @@ -158,8 +190,13 @@ class NoopIngest { // costing a single interval. Both are reachable: a CSV column can spell // `NaN`, and a SQLite REAL can hold either outright. if (!ms.isFinite || ms <= 0) return; + // Then the band path's 250-2400 ms bound: outside it the interval is a + // missed or doubled detection, not a rhythm, and one of them is enough to + // move RMSSD / SDNN / pNNx for the whole imported night. + final rr = plausibleRrOrNull(ms); + if (rr == null) return; _rrTs.add(ts * 1000.0); - _rrMs.add(ms); + _rrMs.add(rr); } void gravity(int ts, double? x, double? y, double? z) { @@ -310,9 +347,7 @@ class NoopIngest { skinTemp[i] = fTemp; } - // RR beats sorted by time. - final order = List.generate(rrMs.length, (i) => i) - ..sort((a, b) => rrTs[a].compareTo(rrTs[b])); + final order = rrOrder(rrTs); return Substrate( tsSec: tsSec, hr: hr, @@ -328,6 +363,21 @@ class NoopIngest { ); } + /// Indices of the buffered RR beats in time order. + /// + /// NOOP stamps beats with whole seconds, so several beats routinely share one + /// timestamp and the sort is mostly ties. Those beats already arrive in the + /// order the strap detected them, and that order is the interval series: + /// swapping two of them invents successive differences the heart never made + /// (RMSSD, pNN50, ectopic rejection, PRSA). `List.sort` is not stable past a + /// few dozen elements, so ties fall back to buffer position explicitly. + static List rrOrder(List rrTs) => + List.generate(rrTs.length, (i) => i) + ..sort((a, b) { + final c = rrTs[a].compareTo(rrTs[b]); + return c != 0 ? c : a.compareTo(b); + }); + /// Maximum gap (seconds) between consecutive step-counter samples that is /// still treated as one continuous run. Both sources sample every second but /// drop out whenever the band is off-wrist or unsynced; a gap wider than this diff --git a/lib/import/whoop_import.dart b/lib/import/whoop_import.dart index 78fc4858b..b60d39799 100644 --- a/lib/import/whoop_import.dart +++ b/lib/import/whoop_import.dart @@ -15,6 +15,9 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:shared_preferences/shared_preferences.dart'; + import '../compute/derivation_engine.dart' show kAlgoVersion, DerivationEngine; import '../compute/profile.dart'; import '../compute/substrate.dart' show localDateLabel; @@ -56,7 +59,42 @@ class _Row { } } +const String _kDeletedWorkoutsPref = 'whoop_import_deleted_workouts'; + class WhoopImporter { + /// Every imported workout's session id starts with this, followed by its + /// start epoch — the same CSV row always lands on the same id. A row first + /// imported before the export's zone column was honoured keeps the id of + /// the instant it was read at then, after its window is corrected. + static const String workoutIdPrefix = 'whoop_'; + + /// Imported workout ids the user has DELETED here. The export still has + /// them, so without this list the next import would bring each one back. + static Future> deletedWorkoutIds() async { + final prefs = await SharedPreferences.getInstance(); + return {...(prefs.getStringList(_kDeletedWorkoutsPref) ?? const [])}; + } + + // Serialized: each write replaces the whole list, so two overlapping calls + // would each drop the other's id. + static Future _deletedWrites = Future.value(); + + /// [startTs] is the row's start as stored. A row an earlier re-import moved + /// onto its right instant kept its old id, and once the phone's zone matches + /// the export's the import computes the id from that instant instead, so + /// both go on the list. + static Future rememberDeletedWorkout(String id, {int? startTs}) => + _deletedWrites = _deletedWrites.catchError((_) {}).then((_) async { + final prefs = await SharedPreferences.getInstance(); + final set = await deletedWorkoutIds(); + final add = { + id, + if (startTs != null) '$workoutIdPrefix$startTs', + }.difference(set); + if (add.isEmpty) return; + await prefs.setStringList(_kDeletedWorkoutsPref, [...set, ...add]); + }); + /// Column aliases for the energy field. The unit is read from whichever of /// these actually matched — never inferred from the value (see [_kcal]). static const List _energyCols = [ @@ -132,6 +170,12 @@ class WhoopImporter { // file's data. Merge is "new non-null value wins, else keep prior" so the // outcome doesn't depend on file order. final pendingDays = >{}; + Set deletedWorkouts; + try { + deletedWorkouts = await deletedWorkoutIds(); + } catch (_) { + deletedWorkouts = const {}; + } for (final path in csvPaths) { final rows = await _readCsv(path); if (rows.isEmpty) continue; @@ -158,7 +202,7 @@ class WhoopImporter { final row = _Row(col, f); if (kind == _Kind.workout) { - if (await _writeWorkout(row)) workouts++; + if (await _writeWorkout(row, deletedWorkouts)) workouts++; } else if (kind == _Kind.day) { final extracted = _extractDayFields(row); if (extracted == null) continue; @@ -219,11 +263,18 @@ class WhoopImporter { // sleeps.csv lists naps as their own rows, filed under the same date as // the night that ended that morning. A nap is not the day's sleep. if (get(['nap']).toLowerCase() == 'true') return null; - final wakeTs = _parseTs(get(['wake onset', 'sleep onset', 'cycle start time'])); - final cycleStart = _parseTs(get(['cycle start time', 'sleep onset'])); + final zone = get(_zoneCols); + final wakeTs = parseExportTs( + get(['wake onset', 'sleep onset', 'cycle start time']), zone: zone); + final cycleStart = + parseExportTs(get(['cycle start time', 'sleep onset']), zone: zone); final anchor = wakeTs ?? cycleStart; if (anchor == null) return null; - final date = localDateLabel(anchor); + // File the day under the date the user lived it in the export's own zone, + // not under whatever zone the phone is in at import time: a night slept in + // Paris and imported in Los Angeles still belongs to the Paris morning. + final off = zoneOffset(zone); + final date = off == null ? localDateLabel(anchor) : _wallDateLabel(anchor, off); num? n(List names) => double.tryParse(get(names)); return (date, { @@ -231,7 +282,12 @@ class WhoopImporter { 'rhr': n(['resting heart rate (bpm)', 'resting heart rate']), 'rmssd': n(['heart rate variability (ms)', 'heart rate variability (rmssd) (ms)']), 'strain': n(['day strain', 'strain']), - 'calories': _kcal(get(_energyCols), row.header(_energyCols)), + // The cycle's 'Energy burned' is WHOOP's whole-day expenditure, resting + // metabolism included. That is `calories_total` (TDEE) here, not + // `calories` (active energy over the wake span): filed as active, Health + // export wrote the full ~2,500 kcal as ACTIVE_ENERGY_BURNED every day. + // WHOOP's export has no active/basal split, so active stays absent. + 'caloriesTotal': _kcal(get(_energyCols), row.header(_energyCols)), 'resp': n(['respiratory rate (rpm)', 'respiratory rate']), 'spo2': _pct(n(['blood oxygen %', 'blood oxygen']), 70), 'asleepMin': n(['asleep duration (min)', 'asleep duration (minutes)']), @@ -244,8 +300,8 @@ class WhoopImporter { // need, a different number; _pick takes the first column that exists, // so listing it here stored performance as efficiency. 'effPct': n(['sleep efficiency %']), - 'sleepOnset': _parseTs(get(['sleep onset'])), - 'sleepWake': _parseTs(get(['wake onset'])), + 'sleepOnset': parseExportTs(get(['sleep onset']), zone: zone), + 'sleepWake': parseExportTs(get(['wake onset']), zone: zone), }); } @@ -266,7 +322,7 @@ class WhoopImporter { final rhr = f['rhr'] as num?; final rmssd = f['rmssd'] as num?; final strain = f['strain'] as num?; - final calories = f['calories'] as num?; + final caloriesTotal = f['caloriesTotal'] as num?; final resp = f['resp'] as num?; final spo2 = f['spo2'] as num?; final asleepMin = f['asleepMin'] as num?; @@ -338,7 +394,7 @@ class WhoopImporter { 'readiness': recovery, 'strain': strain, 'resp_rate': resp, - 'calories': calories, + 'calories_total': caloriesTotal, 'spo2': spo2, // no skin_temp_z: WHOOP exports absolute °C and every reader of that // key treats it as SDs off your own baseline (33 °C showed as +33 SD). @@ -375,7 +431,10 @@ class WhoopImporter { 'readiness': d(recovery), 'strain': d(strain), 'resp_rate': d(resp), - 'calories': d(calories), + 'calories_total': d(caloriesTotal), + // Earlier imports filed this total under `calories` (active energy); + // the series row is only replaced for keys written, so clear it. + 'calories': null, 'spo2': d(spo2), 'tst_min': d(asleepMin), 'rem_min': d(remMin), @@ -387,18 +446,66 @@ class WhoopImporter { return _DayWrite.written; } - static Future _writeWorkout(_Row row) async { + static Future _writeWorkout(_Row row, Set deleted) async { String get(List names) => row.get(names); - final start = _parseTs(get(['workout start time', 'start time'])); - final end = _parseTs(get(['workout end time', 'end time'])); + final zone = get(_zoneCols); + final start = + parseExportTs(get(['workout start time', 'start time']), zone: zone); + final end = parseExportTs(get(['workout end time', 'end time']), zone: zone); if (start == null) return false; num? n(List names) => double.tryParse(get(names)); - final id = 'whoop_$start'; + final id = '$workoutIdPrefix$start'; + // A re-import (a newer export picking up recent days) carries every old + // workout again. putSession is INSERT-OR-REPLACE over the whole row, so + // rewriting one already here would revert the user's type correction, + // wipe their RPE and clear its private flag — and the export below would + // then write a workout they had made private into the health store. What + // the export says about a past workout does not change; keep the row. + // A deleted one stays deleted. + if (deleted.contains(id) || await LocalDb.session(id) != null) { + return false; + } + // Imports before the export's own zone was honoured read these times in + // the phone's zone, so a workout recorded elsewhere is already here under + // that wrong instant's id. Inserting at the corrected id would show it + // twice (and write it to Health twice); move the old row onto the right + // instant instead, under its old id so its edits, tombstone and child rows + // stay attached. Only when its window is still exactly what that import + // wrote — a row the user retimed, or one healed by an earlier re-import, + // is left alone. + final legacyStart = + parseExportTs(get(['workout start time', 'start time'])); + if (legacyStart != null && legacyStart != start) { + final legacyId = '$workoutIdPrefix$legacyStart'; + if (deleted.contains(legacyId)) return false; + final legacy = await LocalDb.session(legacyId); + if (legacy != null) { + final legacyEnd = parseExportTs(get(['workout end time', 'end time'])); + final oldStart = (legacy['start_ts'] as num?)?.toInt(); + final oldEnd = (legacy['end_ts'] as num?)?.toInt(); + if (oldStart == legacyStart && + oldEnd == legacyEnd && + legacy['source'] == 'whoop' && + await LocalDb.setSessionWindow(legacyId, start, end) > 0) { + unawaited(() async { + if (oldEnd != null) { + await HealthExporter.deleteWorkoutWindow(legacyStart, oldEnd); + } + await HealthExporter.exportWorkoutId(legacyId); + }()); + } + return false; + } + } + // A row moved onto this instant by an earlier re-import keeps its old id, + // so the id check above misses it once the phone's zone matches the + // export's. + if (await LocalDb.hasSessionStartingAt('whoop', start)) return false; await LocalDb.putSession({ 'id': id, 'start_ts': start, 'end_ts': end, - 'type': _slug(get(['activity name', 'activity'])), + 'type': activityTypeKey(get(['activity name', 'activity'])), 'status': 'done', 'source': 'whoop', 'calories': _kcal(get(_energyCols), row.header(_energyCols))?.toDouble(), @@ -429,23 +536,66 @@ class WhoopImporter { return _Kind.unknown; } + /// The export writes wall-clock times with no offset ("2024-03-09 23:10:00") + /// and puts the offset they were recorded in in a separate column. + static const List _zoneCols = ['cycle timezone', 'timezone']; + + /// "UTC+01:00" / "UTC-0800" / "GMT+5:30" / "+01:00" / "UTC" → its offset + /// from UTC; null for anything else (blank, an IANA name we can't resolve + /// without a tz database), which falls back to the phone's own zone. + @visibleForTesting + static Duration? zoneOffset(String tz) { + final t = tz.trim().toUpperCase(); + if (t == 'UTC' || t == 'GMT' || t == 'Z') return Duration.zero; + final m = RegExp(r'^(?:UTC|GMT)?\s*([+-])(\d{1,2})(?::?(\d{2}))?$').firstMatch(t); + if (m == null) return null; + final h = int.parse(m.group(2)!); + final min = int.parse(m.group(3) ?? '0'); + if (h > 14 || min > 59) return null; + final d = Duration(hours: h, minutes: min); + return m.group(1) == '-' ? -d : d; + } + + static String _wallDateLabel(int epochSec, Duration off) { + final w = DateTime.fromMillisecondsSinceEpoch(epochSec * 1000, isUtc: true) + .add(off); + String two(int x) => x.toString().padLeft(2, '0'); + return '${w.year.toString().padLeft(4, '0')}-${two(w.month)}-${two(w.day)}'; + } + /// Lenient timestamp → unix seconds. Handles ISO ("2024-01-15T06:30:12Z" / /// "+00:00"), the export's "2024-01-15 06:30:12", and "+0000" offsets. - static int? _parseTs(String s) { + /// + /// An offset-less time is read in [zone] (the row's "Cycle timezone") when + /// that parses. Reading it in the phone's current zone instead put every + /// night or workout recorded elsewhere at the wrong instant — a Paris night + /// imported in Los Angeles landed in Apple Health nine hours late. + @visibleForTesting + static int? parseExportTs(String s, {String zone = ''}) { if (s.isEmpty) return null; var t = s.trim(); // Normalise "+0000" → "+00:00" so DateTime.parse accepts it. final m = RegExp(r'([+-]\d{2})(\d{2})$').firstMatch(t); if (m != null) t = '${t.substring(0, m.start)}${m.group(1)}:${m.group(2)}'; + final off = zoneOffset(zone); + final hasOwnOffset = RegExp(r'(Z|[+-]\d{2}:\d{2})$').hasMatch(t); + if (off != null && !hasOwnOffset) { + final asUtc = DateTime.tryParse('${t}Z'); + if (asUtc != null) return asUtc.subtract(off).millisecondsSinceEpoch ~/ 1000; + } final dt = DateTime.tryParse(t); if (dt != null) return dt.millisecondsSinceEpoch ~/ 1000; - // Fallback: pull a yyyy-mm-dd and treat as local midnight. + // Fallback: pull a yyyy-mm-dd and treat as midnight in the row's zone + // (local midnight when it has none). final d = RegExp(r'(\d{4})-(\d{2})-(\d{2})').firstMatch(t); if (d != null) { - return DateTime(int.parse(d.group(1)!), int.parse(d.group(2)!), - int.parse(d.group(3)!)) - .millisecondsSinceEpoch ~/ - 1000; + final y = int.parse(d.group(1)!); + final mo = int.parse(d.group(2)!); + final da = int.parse(d.group(3)!); + final mid = off != null + ? DateTime.utc(y, mo, da).subtract(off) + : DateTime(y, mo, da); + return mid.millisecondsSinceEpoch ~/ 1000; } return null; } @@ -469,9 +619,26 @@ class WhoopImporter { return null; } - static String _slug(String s) { - final t = s.trim().toLowerCase(); - return t.isEmpty ? 'other' : t; + /// WHOOP activity names spelt as this app's stored `sessions.type` keys + /// (`Activity.typeKey`: lower case, spaces as underscores). Storing WHOOP's + /// raw name kept the space in 'mountain biking' and WHOOP's own words for + /// things the catalogue names differently, so the health export fell + /// through to OTHER for every weightlifting session (#184 again, through + /// the importer) and the workout list found no catalogue entry for them. + static const Map _whoopActivityAliases = { + 'weightlifting': 'weight_training', + 'strength_trainer': 'weight_training', + 'functional_fitness': 'functional', + 'spin': 'indoor_bike', + 'hiking/rucking': 'hiking', + 'rock_climbing': 'climbing', + }; + + @visibleForTesting + static String activityTypeKey(String s) { + final t = s.trim().toLowerCase().replaceAll(RegExp(r'\s+'), '_'); + if (t.isEmpty) return 'other'; + return _whoopActivityAliases[t] ?? t; } /// Read a CSV via the repo's one RFC 4180 parser ([parseCsv], diff --git a/lib/ui2/onboarding/welcome.dart b/lib/ui2/onboarding/welcome.dart index a48fdeb0e..97c3e5aaa 100644 --- a/lib/ui2/onboarding/welcome.dart +++ b/lib/ui2/onboarding/welcome.dart @@ -9,10 +9,12 @@ // [ImportOutcome] carries what the source cost us and [WelcomeView] renders it. import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'package:file_picker/file_picker.dart'; +import 'package:flutter/foundation.dart' show listEquals; import 'package:flutter/material.dart'; import 'package:lucide_icons_flutter/lucide_icons.dart'; import 'package:path_provider/path_provider.dart'; @@ -449,7 +451,13 @@ Future runImport( } on FormatException { // Text, but not UTF-8 — a latin1/cp1252 CSV out of a spreadsheet. The // sniff above cannot see that, and the vendor importer decodes leniently. - vendor.add(p); + // Our own journal or lab export is not something the vendor importer + // reads, though: it would drop it without a word, so name it instead. + if (await _isOwnCsvExport(p)) { + rejected.add(_notUtf8(p)); + } else { + vendor.add(p); + } continue; } try { @@ -461,7 +469,11 @@ Future runImport( } on LabCsvFormatException { vendor.add(p); } on FormatException { - vendor.add(p); + if (await _isOwnCsvExport(p)) { + rejected.add(_notUtf8(p)); + } else { + vendor.add(p); + } } } @@ -481,7 +493,15 @@ Future runImport( skipped += r.skippedExistingDays; } } catch (e) { - if (db.isEmpty && raw.isEmpty && journalRows == 0 && labRows == 0) rethrow; + // Something already named (a journal export that isn't UTF-8) keeps its + // note: a rethrow would replace it with the vendor error alone. + if (db.isEmpty && + raw.isEmpty && + journalRows == 0 && + labRows == 0 && + rejected.isEmpty) { + rethrow; + } readError = '$e'; } } @@ -506,6 +526,28 @@ Future runImport( ); } +/// Whether [path]'s header row is the one our journal or lab-results export +/// writes, read leniently so a file re-saved in a non-UTF-8 encoding still +/// matches. +Future _isOwnCsvExport(String path) async { + try { + final text = latin1.decode(await File(path).readAsBytes()); + final first = text.split(RegExp(r'\r?\n')).first.replaceAll('\u00ef\u00bb\u00bf', ''); + final header = [ + for (final h in first.split(',')) + h.replaceAll('"', '').trim().toLowerCase(), + ]; + return listEquals(header, kJournalCsvHeader) || + listEquals(header, kLabCsvHeader); + } catch (_) { + return false; + } +} + +String _notUtf8(String path) => + '${path.split(Platform.pathSeparator).last}: not saved as UTF-8 — ' + 'save it as "CSV UTF-8" and import it again'; + /// True when [path] starts with the encrypted-backup magic. Four bytes, so a /// mis-picked file costs one read and not a key derivation. Future isEncryptedBackup(String path) async { diff --git a/lib/ui2/screens/log_workout.dart b/lib/ui2/screens/log_workout.dart index f56309725..0bab2f642 100644 --- a/lib/ui2/screens/log_workout.dart +++ b/lib/ui2/screens/log_workout.dart @@ -306,7 +306,15 @@ class _LogWorkoutState extends State { } final r = widget.sessionId == null ? await repo.logManualWorkout( - startTs: _startSec, endTs: _endSec, type: _activity.typeKey) + startTs: _startSec, + endTs: _endSec, + type: _activity.typeKey, + // Marks it as never-exported: the day export writes every + // ordinary session, and the import's original is already in + // the health store. + source: widget.importedUuid != null + ? kScoredImportSessionSource + : 'manual') : await repo.setWorkoutWindow(widget.sessionId!, startTs: _startSec, endTs: _endSec); if (widget.importedUuid case final uuid?) { @@ -576,8 +584,9 @@ Future replaceImportWithScored( // next import pass delete an import that is still the only copy. The other // order's failure only re-imports it beside the scored session. await LocalDb.supersedeImportedWorkout(uuid, id); - // The original already sits in the health store; exporting ours too would - // put the same workout there twice. + // Not exported here, and the `kScoredImportSessionSource` stamp keeps the + // day export from writing it later: the original already sits in the health + // store, and ours as well would put the same workout there twice. await rememberDeletedUuid(uuid); return true; } diff --git a/test/health_measurement_import_test.dart b/test/health_measurement_import_test.dart index 531c808ac..8b8016865 100644 --- a/test/health_measurement_import_test.dart +++ b/test/health_measurement_import_test.dart @@ -1,3 +1,4 @@ +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:health/health.dart'; import 'package:openstrap_edge/health/health_measurement_import.dart'; @@ -21,6 +22,20 @@ HealthDataPoint _p( sourceName: source, ); +class _ThrowingHealth extends Health { + @override + Future configure() async {} + + @override + Future> getHealthDataFromTypes({ + required List types, + required DateTime startTime, + required DateTime endTime, + List recordingMethodsToFilter = const [], + }) async => + throw PlatformException(code: 'RemoteException', message: 'quota'); +} + void main() { test('a cuff reading keeps the cuff\'s name', () { final rows = rowsFrom([_p(HealthDataType.BLOOD_PRESSURE_SYSTOLIC, 128)]); @@ -81,4 +96,12 @@ void main() { ]); expect(rows.single['source'], 'Unknown app'); }); + + test('a read that throws surfaces as an error, not as "nothing came back"', + () async { + TestWidgetsFlutterBinding.ensureInitialized(); + final importer = + ImportedMeasurementImporter(health: _ThrowingHealth(), isApple: false); + await expectLater(importer.sync(), throwsA(isA())); + }); } diff --git a/test/health_sleep_export_test.dart b/test/health_sleep_export_test.dart index 0e7d578fe..e51407d78 100644 --- a/test/health_sleep_export_test.dart +++ b/test/health_sleep_export_test.dart @@ -135,49 +135,71 @@ void main() { expect(writes, 1); }); - test('failed priority sleep never invokes the bulk callback', () async { - var priorityWrites = 0; - var bulkWrites = 0; + test( + 'failed priority sleep still runs bulk without claiming the night', + () async { + // Health Connect grants Sleep separately: with it denied this write + // fails every pass, and holding bulk behind it kept heart rate, RHR, + // HRV, energy and workouts out of the store indefinitely. + var priorityWrites = 0; + final bulkPriorityDates = []; + + final result = await exportPrioritySleepBeforeBulk( + newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], + write: (bundle) async { + priorityWrites++; + return false; + }, + exportBulk: (androidSleepAlreadyWritten) async { + bulkPriorityDates.add(androidSleepAlreadyWritten); + }, + ); + + expect(result.date, '2026-08-05'); + expect(result.succeeded, isFalse); + expect(priorityWrites, 1); + expect( + bulkPriorityDates, + [null], + reason: 'bulk must retry the sleep itself, not skip it as written', + ); + }, + ); + + test('thrown priority sleep still runs bulk without claiming the night', + () async { + final bulkPriorityDates = []; final result = await exportPrioritySleepBeforeBulk( newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], - write: (bundle) async { - priorityWrites++; - return false; - }, + write: (bundle) async => throw StateError('priority write failed'), exportBulk: (androidSleepAlreadyWritten) async { - bulkWrites++; + bulkPriorityDates.add(androidSleepAlreadyWritten); }, ); expect(result.date, '2026-08-05'); expect(result.succeeded, isFalse); - expect(priorityWrites, 1); - expect(bulkWrites, 0); + expect(bulkPriorityDates, [null]); }); - test('thrown priority sleep never invokes the bulk callback', () async { - var bulkWrites = 0; - - await expectLater( - exportPrioritySleepBeforeBulk( - newestFirstDays: [MapEntry('2026-08-05', _overnightBundle())], - write: (bundle) async => throw StateError('priority write failed'), - exportBulk: (androidSleepAlreadyWritten) async { - bulkWrites++; - }, - ), - throwsA(isA()), + test('the priority day never holds bulk behind its own backoff', () { + final source = File('lib/health/health_export.dart').readAsStringSync(); + final priorityPath = source.substring( + source.indexOf('Future exportPriorityOrBulk() async {'), + source.indexOf('exportBulk = (String? androidSleepAlreadyWritten)'), ); - expect(bulkWrites, 0); + expect(priorityPath, isNot(contains('return 0;'))); + expect(priorityPath, contains('shouldAttemptHealthBulkExport(')); + expect(priorityPath, isNot(contains('shouldAttemptHealthExport('))); }); test('priority success leaves shared retry state for the bulk export', () { final source = File('lib/health/health_export.dart').readAsStringSync(); final priorityCallback = source.substring( source.indexOf( - 'final priorityResult = await exportPrioritySleepBeforeBulk(', + 'await exportPrioritySleepBeforeBulk(', ), source.indexOf('exportBulk = (String? androidSleepAlreadyWritten)'), ); @@ -270,23 +292,134 @@ void main() { // copy, which is the truncation and the duplicate bars both. expect(night.start.isBefore(dayStart), isTrue); - final window = sleepCleanupWindow( - dayStart: dayStart, - dayEnd: dayEnd, - night: night, - ); + final window = sleepCleanupWindow(dayStart: dayStart, night: night)!; expect(window.start, DateTime(2026, 8, 4, 12)); expect(window.end, DateTime(2026, 8, 5, 12), reason: 'noon-to-noon only — the old calendar-day union reached into the ' 'previous night and deleted samples nothing ever rewrites'); - // No night to write — nothing to widen for, and the day window still has - // to be swept so stale samples from an earlier export go. - final none = sleepCleanupWindow(dayStart: dayStart, dayEnd: dayEnd); - expect(none.start, dayStart); - expect(none.end, dayEnd); + // No night to write — stale samples from an earlier export still have + // to go, but only from the noon-to-noon window this day's night owns. + // HealthKit deletes on overlap, and tomorrow's night starts this + // evening: a calendar-day sweep deleted its in-bed and pre-midnight + // stages. + final none = sleepCleanupWindow(dayStart: dayStart)!; + expect(none.start, DateTime(2026, 8, 4, 12)); + expect(none.end, DateTime(2026, 8, 5, 12)); + final tomorrowsNightStart = DateTime(2026, 8, 5, 23, 10); + expect(none.end.isBefore(tomorrowsNightStart), isTrue); + expect(none.end.isBefore(dayEnd), isTrue); + + // Yesterday's night woke after noon: start after it. + final clamped = sleepCleanupWindow( + dayStart: dayStart, + previousWake: DateTime(2026, 8, 4, 22), + )!; + expect(clamped.start, DateTime(2026, 8, 4, 22, 0, 1)); + expect( + sleepCleanupWindow( + dayStart: dayStart, + previousWake: DateTime(2026, 8, 5, 13), + ), + isNull, + ); + }); + + test("a night's HealthKit window stops short of yesterday's after-noon wake", () { + // Saturday's night ran 03:00-13:00. Sunday's runs Sat 23:30-Sun 07:30, + // so its noon-to-noon window starts Sat 12:00 and overlaps Saturday's + // in-bed envelope by an hour. HealthKit deletes on overlap, and + // Saturday is already behind the export cursor. + final dayStart = DateTime(2026, 8, 9); + final night = HealthSleepSession( + start: DateTime(2026, 8, 8, 23, 30), + end: DateTime(2026, 8, 9, 7, 30), + stages: const [], + ); + final saturdayWake = DateTime(2026, 8, 8, 13); + + final window = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: saturdayWake, + )!; + expect(window.start, DateTime(2026, 8, 8, 13, 0, 1)); + expect(window.end, DateTime(2026, 8, 9, 12)); + expect(window.start.isAfter(saturdayWake), isTrue); + expect(window.start.isAfter(night.start), isFalse); + + // A morning wake yesterday is outside the window: unchanged. + final normal = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: DateTime(2026, 8, 8, 7), + )!; + expect(normal.start, DateTime(2026, 8, 8, 12)); + expect(normal.end, DateTime(2026, 8, 9, 12)); + + // A "previous wake" at or after this night's onset is not a separate + // night ending before this one; never clamp past the onset. + final overlapping = sleepCleanupWindow( + dayStart: dayStart, + night: night, + previousWake: DateTime(2026, 8, 9, 1), + )!; + expect(overlapping.start, DateTime(2026, 8, 8, 12)); }); + test( + 'Apple replace on a no-night day leaves the next night alone', + () async { + const channel = MethodChannel('openstrap/test_healthkit_sleep_none'); + final calls = []; + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, (call) async { + calls.add(call); + return true; + }); + addTearDown(() { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(channel, null); + }); + final exporter = HealthKitSleepSessionExporter( + writer: MethodChannelHealthKitSleepSessionWriter(channel: channel), + ); + final noWindow = _overnightBundle()..remove('sleep'); + + expect( + await exporter.replace( + bundle: noWindow, + dayStart: DateTime(2026, 8, 5), + previousWake: DateTime(2026, 8, 4, 7), + ), + isTrue, + ); + final args = (calls.single.arguments as Map).cast(); + expect( + args['cleanupStartTime'], + DateTime(2026, 8, 4, 12).millisecondsSinceEpoch, + ); + expect( + args['cleanupEndTime'], + DateTime(2026, 8, 5, 12).millisecondsSinceEpoch, + reason: 'a calendar-day sweep overlaps tomorrow\'s night, which ' + 'starts this evening, and HealthKit deletes on overlap', + ); + + // Nothing left to clear: no native call, and not a failure. + calls.clear(); + expect( + await exporter.replace( + bundle: noWindow, + dayStart: DateTime(2026, 8, 5), + previousWake: DateTime(2026, 8, 5, 13), + ), + isTrue, + ); + expect(calls, isEmpty); + }, + ); + test('Apple and Android share one hypnogram stage vocabulary', () { expect(healthSleepStageOf('wake'), HealthSleepStage.awake); expect(healthSleepStageOf('awake'), HealthSleepStage.awake); @@ -912,7 +1045,6 @@ void main() { await exporter.replace( bundle: bundle, dayStart: DateTime(2026, 8, 5), - dayEnd: DateTime(2026, 8, 6), ), isTrue, ); @@ -949,7 +1081,6 @@ void main() { await exporter.replace( bundle: _overnightBundle(), dayStart: DateTime(2026, 8, 5), - dayEnd: DateTime(2026, 8, 6), ), isTrue, ); diff --git a/test/health_workout_export_delete_gate_test.dart b/test/health_workout_export_delete_gate_test.dart index e86bbfb14..f4123438c 100644 --- a/test/health_workout_export_delete_gate_test.dart +++ b/test/health_workout_export_delete_gate_test.dart @@ -164,19 +164,32 @@ void main() { }); }); - group('healthWorkoutCaloriesToSubtract', () { - test('a private session stays in the day\'s active energy', () { - // Its WORKOUT sample is never written, so subtracting it too would drop - // its calories out of Health entirely. - expect( - healthWorkoutCaloriesToSubtract([ - {..._session(), 'calories': 400, 'private': 1}, - {..._session(), 'calories': 100}, - {..._session(), 'calories': 50, 'status': 'live'}, - {..._session(), 'calories': 30, 'end_ts_fabricated': 1}, - ]), - 100, - ); + group('sessions scored from an imported workout', () { + // The import's original is already in the health store; the day export + // writing the scored copy as well listed the workout twice. + late _FakeHealthStore store; + + tearDown(() => store.remove()); + + test('are never written to the health store', () async { + store = _FakeHealthStore(deleteResult: true)..install(); + + final ok = await HealthExporter().exportWorkout( + {..._session(), 'source': kScoredImportSessionSource}); + + expect(ok, isFalse); + expect(store.calls, isNot(contains('writeWorkoutData'))); + expect(store.calls, isNot(contains('delete'))); + }); + + test('an ordinary manual session still is', () async { + store = _FakeHealthStore(deleteResult: true)..install(); + + final ok = await HealthExporter() + .exportWorkout({..._session(), 'source': 'manual'}); + + expect(ok, isTrue); + expect(store.calls, contains('writeWorkoutData')); }); }); diff --git a/test/health_workout_import_sync_test.dart b/test/health_workout_import_sync_test.dart index a299c46ce..f25445db4 100644 --- a/test/health_workout_import_sync_test.dart +++ b/test/health_workout_import_sync_test.dart @@ -8,6 +8,7 @@ import 'dart:io'; +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:health/health.dart'; import 'package:package_info_plus/package_info_plus.dart'; @@ -16,6 +17,7 @@ import 'package:shared_preferences/shared_preferences.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/health/auto_workout_import.dart'; import 'package:openstrap_edge/health/health_workout_import.dart'; HealthDataPoint _w( @@ -56,6 +58,35 @@ class _FakeHealth extends Health { points; } +/// A store that refuses the read after the permission check passed, the way +/// Health Connect does on a quota limit or a revoked companion grant. +class _ThrowingHealth extends Health { + int reads = 0; + + @override + Future configure() async {} + + @override + Future hasPermissions( + List types, { + List? permissions, + }) async => + true; + + @override + Future> getHealthDataFromTypes({ + required List types, + required DateTime startTime, + required DateTime endTime, + List recordingMethodsToFilter = const [], + }) async { + reads++; + throw PlatformException( + code: 'SecurityException', + message: "Caller doesn't have android.permission.health.READ_DISTANCE"); + } +} + /// Records what the permission request asked for. class _PermHealth extends Health { List? asked; @@ -246,4 +277,93 @@ void main() { HealthDataType.STEPS, ])); }); + + test('a read that throws surfaces as an error, not as an empty store', + () async { + // Returning zero here sent the Import button down "holds no workouts", + // styled as a normal result, for a store that was never read. + final importer = + HealthWorkoutImporter(health: _ThrowingHealth(), isApple: false); + await expectLater(importer.sync(), throwsA(isA())); + }); + + test('the auto pass does not report a thrown read as a pass that ran', + () async { + final health = _ThrowingHealth(); + final out = await AutoWorkoutImport.maybeRun( + importer: HealthWorkoutImporter(health: health, isApple: false), + nowFn: () => DateTime(2026, 10, 1, 12), + ); + expect(health.reads, 1); + expect(out, AutoImportOutcome.skipped); + }); + + group('a workout deleted in the health app leaves the history', () { + // Android window is 30 days back from `now`; _w's workouts start + // 2026-08-01 09:00, inside it. + final now = DateTime(2026, 8, 10, 12); + int sec(DateTime d) => d.millisecondsSinceEpoch ~/ 1000; + Map row(String uuid, DateTime at) => ImportedWorkoutRow( + uuid: uuid, + startTs: sec(at), + endTs: sec(at) + 3600, + kind: 'running', + source: 'Garmin Connect', + ).toRow(); + Map pt(String sid) => { + 'session_id': sid, + 'seq': 0, + 'ts_ms': 1, + 'lat': 51.5, + 'lng': -0.1, + }; + + test('a uuid the store no longer returns is dropped with its route', + () async { + // Two watches recorded the same run; both were imported, then the + // Garmin copy was deleted in the Fitness app. + await LocalDb.putImportedWorkouts([ + row('watch', DateTime(2026, 8, 1, 9)), + row('garmin', DateTime(2026, 8, 1, 9)), + ]); + await LocalDb.appendRoutePoints('garmin', [pt('garmin')]); + + await HealthWorkoutImporter( + health: _FakeHealth([_w('watch')]), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']), ['watch']); + expect(await LocalDb.routePoints('garmin'), isEmpty); + }); + + test('rows outside the read window are left alone', () async { + await LocalDb.putImportedWorkouts([ + row('old', DateTime(2026, 6, 1, 9)), + ]); + + await HealthWorkoutImporter( + health: _FakeHealth([_w('watch')]), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']).toSet(), {'watch', 'old'}); + }); + + test('an empty read (a denied HealthKit read) deletes nothing', () async { + await LocalDb.putImportedWorkouts([ + row('garmin', DateTime(2026, 8, 1, 9)), + ]); + + await HealthWorkoutImporter( + health: _FakeHealth(const []), + isApple: false, + ).sync(now: now); + + final stored = await LocalDb.importedWorkouts(); + expect(stored.map((r) => r['uuid']), ['garmin']); + }); + }); } diff --git a/test/health_workout_import_test.dart b/test/health_workout_import_test.dart index 61d5bc49f..1ccbb9839 100644 --- a/test/health_workout_import_test.dart +++ b/test/health_workout_import_test.dart @@ -173,4 +173,54 @@ void main() { expect(rows.single['alt'], isNull); }); }); + + group('routesToStore', () { + List pt() => [ + [51.5, -0.12, 12.0, 1000], + ]; + + test('a dropped workout\'s route is not stored', () { + // HealthKit hands back the route of every workout in the window. A 26 h + // ultra is dropped by workoutsFrom, so its GPS track would land under a + // session_id no imported_workout row points at, and nothing deletes it. + final ultra = _w( + uuid: 'ultra', + from: DateTime(2026, 8, 1, 6), + to: DateTime(2026, 8, 2, 8), + ); + final kept = workoutsFrom([ultra, _w(uuid: 'run')]); + final keep = {for (final r in kept) r.uuid}; + expect(keep, {'run'}); + + final routes = routesToStore([ + {'uuid': 'ultra', 'points': pt()}, + {'uuid': 'run', 'points': pt()}, + ], keep); + expect(routes.map((e) => e.key), ['run']); + expect(routes.single.value.single['session_id'], 'run'); + }); + + test('routes for workouts outside the kept set are dropped', () { + // Our own exports and tombstoned deletes never reach the kept set. + expect( + routesToStore([ + {'uuid': 'own', 'points': pt()}, + {'uuid': 'deleted', 'points': pt()}, + ], {'run'}), + isEmpty, + ); + }); + + test('malformed entries and empty routes are skipped', () { + expect( + routesToStore([ + null, + 'x', + {'uuid': '', 'points': pt()}, + {'uuid': 'run', 'points': const []}, + ], {'run', ''}), + isEmpty, + ); + }); + }); } diff --git a/test/import_data_safety_test.dart b/test/import_data_safety_test.dart index 4a8b179ff..5014a1d40 100644 --- a/test/import_data_safety_test.dart +++ b/test/import_data_safety_test.dart @@ -262,7 +262,7 @@ void main() { '$wake,$wake,50,60,60,5,$value,400\n', ); await WhoopImporter.importFiles([f.path]); - return _metric(day, 'calories'); + return _metric(day, 'calories_total'); } test('a real 4,500 kcal ultra day is NOT rewritten as kJ', () async { @@ -285,6 +285,30 @@ void main() { }); }); + test('WHOOP daily energy is filed as total burn, never as active energy', + () async { + // The cycle's 'Energy burned' is WHOOP's whole-day figure, resting + // metabolism included. Filed as `calories` (active energy over the wake + // span) it reached Health as ~2,600 kcal of ACTIVE_ENERGY_BURNED on an + // ordinary rest day. + const wake = '2026-04-05 07:00:00'; + final day = + localDateLabel(DateTime.parse(wake).millisecondsSinceEpoch ~/ 1000); + final f = File(p.join(tmp.path, 'energy_total.csv')); + f.writeAsStringSync( + 'Cycle start time,Wake onset,Day Strain,Energy burned (cal),' + 'Asleep duration (min)\n' + '$wake,$wake,4.1,2600,400\n', + ); + expect((await WhoopImporter.importFiles([f.path])).days, 1); + final scalars = (jsonDecode((await _row(day))!['payload_json'] as String) + as Map)['scalars'] as Map; + expect(scalars['calories_total'], 2600); + expect(scalars['calories'], isNull); + expect(await _metric(day, 'calories_total'), 2600.0); + expect(await _metric(day, 'calories'), isNull); + }); + test('WHOOP absolute skin temp °C is not filed as skin_temp_z', () async { const wake = '2026-04-10 07:00:00'; final day = diff --git a/test/import_routing_test.dart b/test/import_routing_test.dart index 69a4ba736..69194a107 100644 --- a/test/import_routing_test.dart +++ b/test/import_routing_test.dart @@ -44,6 +44,15 @@ class _RoutingSpy extends AppState { } } +class _ThrowingVendor extends _RoutingSpy { + @override + Future importWhoopCsvs(List paths, + {void Function(int days)? onProgress}) async { + vendor.addAll(paths); + throw const FormatException('no recognised files'); + } +} + List _zipOf(Map members) { final a = Archive(); members.forEach((name, body) { @@ -137,6 +146,56 @@ void main() { expect(app.vendor, isEmpty); }); + test('a journal export re-saved as cp1252 is named, not handed to the ' + 'vendor importer', () async { + final app = _RoutingSpy(); + final journal = await write('journal.csv', + latin1.encode('date,tags,note\n2026-08-01,,caf\u00e9\n')); + final whoopPath = await write('whoop.zip', _zipOf(_whoopZipMembers)); + + final out = await runImport(app, [journal, whoopPath]); + + expect(app.vendor, [whoopPath]); + expect(out.rejectedRows.single, contains('journal.csv')); + expect(out.rejectedRows.single, contains('UTF-8')); + }); + + test('a quoted, padded header is still recognised as our journal export', + () async { + final app = _RoutingSpy(); + final journal = await write('journal.csv', + latin1.encode('date," tags ",note\n2026-08-01,,caf\u00e9\n')); + + final out = await runImport(app, [journal]); + + expect(app.vendor, isEmpty); + expect(out.rejectedRows.single, contains('UTF-8')); + }); + + test('the encoding note survives a vendor file nothing could read', + () async { + final app = _ThrowingVendor(); + final journal = await write('journal.csv', + latin1.encode('date,tags,note\n2026-08-01,,caf\u00e9\n')); + final junk = await write('junk.csv', utf8.encode('a,b\n1,2\n')); + + final out = await runImport(app, [journal, junk]); + + expect(out.rejectedRows.single, contains('UTF-8')); + expect(out.readError, contains('no recognised files')); + }); + + test('a cp1252 CSV that is not ours still goes to the vendor importer', + () async { + final app = _RoutingSpy(); + final path = await write('cycles.csv', + latin1.encode('Cycle start time,Recovery score %\ncaf\u00e9,1\n')); + + await runImport(app, [path]); + + expect(app.vendor, [path]); + }); + test('a mixed selection reaches both importers', () async { final app = _RoutingSpy(); final noopPath = await write('noop-export.csv', utf8.encode(_noopCsv)); diff --git a/test/journal_csv_import_test.dart b/test/journal_csv_import_test.dart index 44b9082a1..9c64b54c2 100644 --- a/test/journal_csv_import_test.dart +++ b/test/journal_csv_import_test.dart @@ -1,3 +1,6 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/data/csv_export.dart'; import 'package:openstrap_edge/import/journal_csv_import.dart'; @@ -131,4 +134,14 @@ void main() { final p = parse("date,tags,note\n2026-01-01,a,'twas fine\n"); expect(p.rows.single.note, "'twas fine"); }); + test('a cp1252 file out of a spreadsheet fails as a FormatException, before ' + 'any write', () async { + // The import router falls back to the lenient vendor importer only on + // FormatException; File.readAsString would throw FileSystemException here. + final dir = await Directory.systemTemp.createTemp('csv_cp1252_'); + addTearDown(() => dir.delete(recursive: true)); + final path = '${dir.path}/export.csv'; + await File(path).writeAsBytes(latin1.encode('date,tags,note\n2026-10-01,alcohol,caf\u00e9\n')); + await expectLater(importJournalCsvFile(path), throwsA(isA())); + }); } diff --git a/test/lab_csv_import_test.dart b/test/lab_csv_import_test.dart index c74f1b9e0..9379c4e69 100644 --- a/test/lab_csv_import_test.dart +++ b/test/lab_csv_import_test.dart @@ -1,3 +1,6 @@ +import 'dart:convert'; +import 'dart:io'; + import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/data/csv_export.dart'; import 'package:openstrap_edge/import/lab_csv_import.dart'; @@ -132,4 +135,14 @@ void main() { final p = parse("taken_on,marker,value,unit,note\n2026-01-01,ferritin,42,ng/mL,'twas fine\n"); expect(p.rows.single.note, "'twas fine"); }); + test('a cp1252 file out of a spreadsheet fails as a FormatException, before ' + 'any write', () async { + // The import router falls back to the lenient vendor importer only on + // FormatException; File.readAsString would throw FileSystemException here. + final dir = await Directory.systemTemp.createTemp('csv_cp1252_'); + addTearDown(() => dir.delete(recursive: true)); + final path = '${dir.path}/export.csv'; + await File(path).writeAsBytes(latin1.encode('taken_on,marker,value,unit,note\n2026-10-01,ferritin,42,ng/mL,caf\u00e9\n')); + await expectLater(importLabCsvFile(path), throwsA(isA())); + }); } diff --git a/test/manual_workout_repo_test.dart b/test/manual_workout_repo_test.dart index a253f71b4..6153d9cd0 100644 --- a/test/manual_workout_repo_test.dart +++ b/test/manual_workout_repo_test.dart @@ -563,13 +563,14 @@ void main() { }); test( - 'marking an exported workout private re-exports its day', + 'marking a workout private does not rewind the day export', () async { - // Its calories come back into the day's active energy, and a day - // behind the export cursor is otherwise never written again. + // The day's active energy is the full daily total, workouts included, + // so it does not change with the flag; the workout's own re-export + // clears its window. Rewinding the cursor would only rewrite this day + // and every day after it for nothing. SharedPreferences.setMockInitialValues({}); final start = sessionStart - 20 * 86400; - final day = dayLabelOf(DateTime.fromMillisecondsSinceEpoch(start * 1000)); await LocalDb.putSession({ 'id': 'w-private-old', 'start_ts': start, @@ -585,13 +586,8 @@ void main() { addTearDown(() => LocalDb.setCursor('health_export_through', '')); await repo.setWorkoutPrivate('w-private-old', true); - String? cursor; - for (var i = 0; i < 50; i++) { - cursor = await LocalDb.getCursor('health_export_through'); - if (cursor != through) break; - await Future.delayed(const Duration(milliseconds: 20)); - } - expect(cursor!.compareTo(day), lessThan(0)); + await Future.delayed(const Duration(milliseconds: 200)); + expect(await LocalDb.getCursor('health_export_through'), through); }, ); diff --git a/test/noop_ingest_plausibility_test.dart b/test/noop_ingest_plausibility_test.dart new file mode 100644 index 000000000..00644f32e --- /dev/null +++ b/test/noop_ingest_plausibility_test.dart @@ -0,0 +1,69 @@ +// A NOOP import must gate HR and RR the same way the band decode does, or the +// same night scores differently depending on where it came from. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/import/noop_ingest.dart'; + +/// Captures the Substrate the ingest hands over instead of deriving it. +class _CapturingEngine extends DerivationEngine { + final List subs = []; + + @override + Future deriveImportedDays( + Substrate sub, + Profile profile, + Set dates, { + void Function(String day)? onDayDone, + int? resetGeneration, + }) async { + subs.add(sub); + return dates.length; + } + + @override + Future finalizeImport(Profile profile) async {} +} + +void main() { + test('implausible HR becomes 0 and implausible RR is dropped', () async { + final engine = _CapturingEngine(); + final ingest = NoopIngest(const Profile(), engine); + // Midday, so every sample sits on the same local date. + final base = DateTime(2026, 3, 10, 12).millisecondsSinceEpoch ~/ 1000; + + final hrs = [60, 255, 1, 230, 25, 24, 231]; + for (var i = 0; i < hrs.length; i++) { + expect(await ingest.offer(base + i), isTrue); + ingest.hr(base + i, hrs[i]); + } + final rrs = [800.0, 30.0, 9000.0, 250.0, 2400.0, 249.0, 2401.0]; + for (var i = 0; i < rrs.length; i++) { + expect(await ingest.offer(base + i), isTrue); + ingest.rr(base + i, rrs[i]); + } + await ingest.finish(); + + expect(engine.subs, hasLength(1)); + final sub = engine.subs.single; + // Every second is kept, so its other channels still have a home. + expect(sub.tsSec, [for (var i = 0; i < hrs.length; i++) base + i]); + expect(sub.hr, [60, 0, 0, 230, 25, 0, 0]); + expect(sub.rrMs, [800.0, 250.0, 2400.0]); + }); + + test('a far-future timestamp is refused, not taken as the newest day', + () async { + final engine = _CapturingEngine(); + final ingest = NoopIngest(const Profile(), engine); + final now = DateTime.now().millisecondsSinceEpoch ~/ 1000; + final y2099 = DateTime(2099, 6, 1, 12).millisecondsSinceEpoch ~/ 1000; + expect(await ingest.offer(y2099), isFalse); + expect(ingest.implausibleRows, 1); + expect(await ingest.offer(now - 3600), isTrue); + expect(await ingest.offer(now + 86400), isTrue, + reason: 'a day of clock skew is still a real row'); + }); +} diff --git a/test/noop_rr_order_test.dart b/test/noop_rr_order_test.dart new file mode 100644 index 000000000..c3b8aa6a4 --- /dev/null +++ b/test/noop_rr_order_test.dart @@ -0,0 +1,24 @@ +// NOOP stamps RR beats with whole seconds, so beats sharing a second are the +// normal case. Their arrival order is the strap's detection order; the +// time-sort that builds the Substrate must keep it rather than reshuffle it. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/import/noop_ingest.dart'; + +void main() { + test('beats sharing a second keep their arrival order', () { + // Long enough that List.sort leaves insertion sort for quicksort. + final rrTs = []; + for (var s = 0; s < 500; s++) { + rrTs + ..add(s * 1000.0) + ..add(s * 1000.0); + } + expect(NoopIngest.rrOrder(rrTs), List.generate(rrTs.length, (i) => i)); + }); + + test('out-of-order seconds are still sorted, ties stay in arrival order', () { + final rrTs = [2000, 1000, 2000, 1000, 0]; + expect(NoopIngest.rrOrder(rrTs), [4, 1, 3, 0, 2]); + }); +} diff --git a/test/noop_schema_drift_test.dart b/test/noop_schema_drift_test.dart index 945931260..1aa3c57f5 100644 --- a/test/noop_schema_drift_test.dart +++ b/test/noop_schema_drift_test.dart @@ -387,5 +387,67 @@ void main() { f.path, const Profile(), DerivationEngine()); expect(res.days, greaterThan(0)); }, timeout: const Timeout(Duration(minutes: 5))); + + /// Two adjacent days of 300 s each, with [bad] raw lines spliced in + /// between them (written verbatim: a µs-scaled unix_s would overflow the + /// ISO column `_row` builds). + File twoDaysWith(String name, int t0, List bad) { + final b = StringBuffer()..writeln(_header); + void day(int start) { + for (var i = 0; i < 300; i++) { + b.writeln(_row(start + i, 'hr', hr: '65')); + b.writeln( + _row(start + i, 'gravity', gx: '0.1', gy: '0.2', gz: '0.97')); + } + } + + day(t0); + bad.forEach(b.writeln); + day(t0 + 86400); + return File(p.join(tmp.path, name))..writeAsStringSync(b.toString()); + } + + test('a millisecond-scaled unix_s does not strand the rest of the file', + () async { + // One ms-scaled row used to open a date ~53,000 years out; every real + // day after it was then "older" than the high-water date and stranded. + const t0 = 1789030800; // 2026-09-10T09:00:00Z + final f = twoDaysWith('ms.csv', t0, [ + '1700000000000,x,event,,,,,,,,,,,,,,BATTERY_LEVEL(3),', + '1700000000001,x,hr,70,,,,,,,,,,,,,,', + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.strandedDates, isEmpty); + expect(res.days, 2); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('an RTC-unset unix_s does not blank the day it lands in', () async { + // A 1970 sample buffered mid-file stretched the next Substrate past the + // 400-day calendarDays guard, so the real day derived nothing. + const t0 = 1789203600; // 2026-09-12T09:00:00Z + final f = twoDaysWith('rtc.csv', t0, [ + _row(250000, 'hr', hr: '70'), + _row(250001, 'gravity', gx: '0.1', gy: '0.2', gz: '0.97'), + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.strandedDates, isEmpty); + expect(res.days, 2); + final db = await LocalDb.instance; + final rows = await db.rawQuery( + "SELECT 1 FROM day_result WHERE day_id LIKE '1970-%' LIMIT 1"); + expect(rows, isEmpty); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('a microsecond-scaled unix_s is skipped, not a RangeError', () async { + const t0 = 1789376400; // 2026-09-14T09:00:00Z + final f = twoDaysWith('us.csv', t0, [ + '1700000000000000,x,hr,70,,,,,,,,,,,,,,', + ]); + final res = await NoopImporter.importFile( + f.path, const Profile(), DerivationEngine()); + expect(res.days, 2); + }, timeout: const Timeout(Duration(minutes: 5))); }); } diff --git a/test/phone_pedometer_hour_walk_test.dart b/test/phone_pedometer_hour_walk_test.dart index 411d3ad06..55dc4e209 100644 --- a/test/phone_pedometer_hour_walk_test.dart +++ b/test/phone_pedometer_hour_walk_test.dart @@ -174,6 +174,28 @@ void main() { expect(await LocalDb.liveStepsForDay(dayId), total); }); + test('an hour that turns UNCOVERED later keeps what was already banked', + () async { + final day = yesterday(); + final dayId = _label(day); + + // 1. A good sync banks the whole day. + final good = PhonePedometer(stepReader: (from, to) async => 100); + final fullTotal = await good.syncDay(day); + expect(fullTotal, isNotNull); + expect(await LocalDb.liveStepsForDay(dayId), fullTotal); + + // 2. The phone reboots: the native side records a gap from the last + // delivery, and every later query touching that hour comes back + // uncovered even though its earlier counts were real. The replace is + // delete-then-insert, so this hour must be carried over, not dropped. + var h = 0; + final rebooted = PhonePedometer(stepReader: (from, to) async => + h++ == 11 ? PhonePedometer.intervalNotCovered : 100); + expect(await rebooted.syncDay(day), fullTotal); + expect(await LocalDb.liveStepsForDay(dayId), fullTotal); + }); + test('a day with NO covered hour is unknown, not zero', () async { final day = yesterday(); final dayId = _label(day); diff --git a/test/spo2_import_surface_test.dart b/test/spo2_import_surface_test.dart index 89749d071..0b196e5ce 100644 --- a/test/spo2_import_surface_test.dart +++ b/test/spo2_import_surface_test.dart @@ -60,6 +60,27 @@ void main() { (payload['flags'] as List).contains('IMPORTED_WHOOP_BETA'), isTrue); }); + test('a re-import clears a calories row an older import filed as active', + () async { + final dir = await Directory.systemTemp.createTemp('calories_reimport'); + addTearDown(() => dir.delete(recursive: true)); + const wake = '2026-03-13 07:15:00'; + final day = localDateLabel( + DateTime.parse(wake).millisecondsSinceEpoch ~/ 1000); + final db = await LocalDb.instance; + await db.insert('metric_series', {'date': day, 'key': 'calories', 'value': 2400.0}, + conflictAlgorithm: ConflictAlgorithm.replace); + final f = File('${dir.path}/day.csv'); + f.writeAsStringSync( + 'Cycle start time,Wake onset,Sleep onset,Energy burned (cal)\n' + '$wake,$wake,2026-03-12 23:10:00,2400\n', + ); + await WhoopImporter.importFiles([f.path]); + final rows = await db.query('metric_series', + where: 'date = ? AND key = ?', whereArgs: [day, 'calories']); + expect(rows.where((r) => r['value'] != null), isEmpty); + }); + test('an out-of-range blood oxygen cell is dropped, not charted', () async { final dir = await Directory.systemTemp.createTemp('spo2_range'); addTearDown(() => dir.delete(recursive: true)); diff --git a/test/whoop_activity_type_test.dart b/test/whoop_activity_type_test.dart new file mode 100644 index 000000000..21960dd37 --- /dev/null +++ b/test/whoop_activity_type_test.dart @@ -0,0 +1,66 @@ +// A WHOOP export names its activities in WHOOP's own words ("Weightlifting", +// "Mountain Biking"). Stored as-is, those rows reached Apple Health / Health +// Connect as OTHER and found no catalogue entry in the workout list. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:health/health.dart'; +import 'package:openstrap_edge/health/health_export.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; +import 'package:openstrap_edge/ui2/activity/catalogue.dart'; + +void main() { + test('WHOOP names are stored as catalogue keys', () { + const cases = { + 'Weightlifting': 'weight_training', + 'Strength Trainer': 'weight_training', + 'Functional Fitness': 'functional', + 'Mountain Biking': 'mountain_biking', + 'Spin': 'indoor_bike', + 'Hiking/Rucking': 'hiking', + 'Rock Climbing': 'climbing', + ' Running ': 'running', + }; + cases.forEach((whoop, key) { + final stored = WhoopImporter.activityTypeKey(whoop); + expect(stored, key, reason: whoop); + expect(activityByName(stored)?.typeKey, key, reason: whoop); + }); + expect(WhoopImporter.activityTypeKey(' '), 'other'); + }); + + test('imported strength and cycling reach the health store typed', () { + for (final ios in [true, false]) { + final strength = ios + ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING + : HealthWorkoutActivityType.STRENGTH_TRAINING; + expect( + healthActivityForType( + WhoopImporter.activityTypeKey('Weightlifting'), ios: ios), + strength); + expect( + healthActivityForType( + WhoopImporter.activityTypeKey('Mountain Biking'), ios: ios), + HealthWorkoutActivityType.BIKING); + } + }); + + test('rows stored with the raw WHOOP name still map', () { + // The importer never rewrites a session it already holds, so rows from + // an earlier import keep the old spelling. + for (final ios in [true, false]) { + final strength = ios + ? HealthWorkoutActivityType.TRADITIONAL_STRENGTH_TRAINING + : HealthWorkoutActivityType.STRENGTH_TRAINING; + expect(healthActivityForType('weightlifting', ios: ios), strength); + expect(healthActivityForType('strength trainer', ios: ios), strength); + expect(healthActivityForType('mountain biking', ios: ios), + HealthWorkoutActivityType.BIKING); + expect(healthActivityForType('spin', ios: ios), + HealthWorkoutActivityType.BIKING); + expect(healthActivityForType('hiking/rucking', ios: ios), + HealthWorkoutActivityType.HIKING); + expect(healthActivityForType('rock climbing', ios: ios), + HealthWorkoutActivityType.ROCK_CLIMBING); + } + }); +} diff --git a/test/whoop_import_timezone_test.dart b/test/whoop_import_timezone_test.dart new file mode 100644 index 000000000..33dd01601 --- /dev/null +++ b/test/whoop_import_timezone_test.dart @@ -0,0 +1,238 @@ +// WHOOP exports write wall-clock times with no offset and carry the offset in +// a separate "Cycle timezone" column. Reading those times in the phone's +// current zone put every night or workout recorded somewhere else at the wrong +// instant (and wrote it to Apple Health / Health Connect that way). These +// assertions are absolute epochs, so they hold whatever zone the test runs in. +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; + +int _utc(int y, int mo, int d, int h, [int mi = 0]) => + DateTime.utc(y, mo, d, h, mi).millisecondsSinceEpoch ~/ 1000; + +void main() { + group('zoneOffset', () { + test('reads the export formats', () { + expect(WhoopImporter.zoneOffset('UTC+01:00'), const Duration(hours: 1)); + expect(WhoopImporter.zoneOffset('UTC-08:00'), const Duration(hours: -8)); + expect(WhoopImporter.zoneOffset('UTC+0530'), + const Duration(hours: 5, minutes: 30)); + expect(WhoopImporter.zoneOffset('GMT+5:30'), + const Duration(hours: 5, minutes: 30)); + expect(WhoopImporter.zoneOffset('-03:00'), const Duration(hours: -3)); + expect(WhoopImporter.zoneOffset('UTC'), Duration.zero); + }); + + test('anything unreadable falls back to the phone zone', () { + expect(WhoopImporter.zoneOffset(''), isNull); + expect(WhoopImporter.zoneOffset('Europe/Paris'), isNull); + expect(WhoopImporter.zoneOffset('UTC+25:00'), isNull); + }); + }); + + group('parseExportTs', () { + test('an offset-less time is read in the row zone', () { + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00', zone: 'UTC+01:00'), + _utc(2024, 3, 9, 22, 10)); + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00', zone: 'UTC-08:00'), + _utc(2024, 3, 10, 7, 10)); + }); + + test('a time carrying its own offset ignores the zone column', () { + expect( + WhoopImporter.parseExportTs('2024-03-09T23:10:00Z', + zone: 'UTC+01:00'), + _utc(2024, 3, 9, 23, 10)); + expect( + WhoopImporter.parseExportTs('2024-03-09 23:10:00+0200', + zone: 'UTC+01:00'), + _utc(2024, 3, 9, 21, 10)); + }); + + test('no zone column keeps the phone-local reading', () { + expect(WhoopImporter.parseExportTs('2024-03-09 23:10:00'), + DateTime(2024, 3, 9, 23, 10).millisecondsSinceEpoch ~/ 1000); + expect(WhoopImporter.parseExportTs(''), isNull); + }); + }); + + group('import', () { + late Directory tmp; + + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_whoop_tz_test.db'; + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + tmp = Directory.systemTemp.createTempSync('openstrap_whoop_tz'); + }); + + tearDownAll(() async { + await LocalDb.close(); + try { + tmp.deleteSync(recursive: true); + } catch (_) {} + }); + + test('a Paris night lands at its real instant and Paris wake date', + () async { + final f = File(p.join(tmp.path, 'sleeps.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Sleep onset,' + 'Wake onset,Asleep duration (min),In bed duration (min),Nap\n' + '2024-03-09 23:00:00,2024-03-10 23:00:00,UTC+01:00,' + '2024-03-09 23:10:00,2024-03-10 07:00:00,440,470,false\n', + ); + final res = await WhoopImporter.importFiles([f.path]); + expect(res.days, 1); + final row = await LocalDb.dayResult('2024-03-10'); + expect(row, isNotNull, reason: 'filed under the Paris wake date'); + final win = jsonDecode(row!['window_json'] as String) as Map; + expect(win['onset_ms'], _utc(2024, 3, 9, 22, 10) * 1000); + expect(win['offset_ms'], _utc(2024, 3, 10, 6) * 1000); + }); + + test('a Paris workout is stored at its real instant', () async { + final f = File(p.join(tmp.path, 'workouts.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-03-10 07:00:00,2024-03-11 07:00:00,UTC+01:00,' + '2024-03-10 18:00:00,2024-03-10 18:45:00,Running,11.2\n', + ); + final res = await WhoopImporter.importFiles([f.path]); + expect(res.workouts, 1); + final start = _utc(2024, 3, 10, 17); + final db = await LocalDb.instance; + final rows = await db.query('sessions', + where: 'id = ?', whereArgs: ['whoop_$start']); + expect(rows, hasLength(1)); + expect((rows.first['start_ts'] as num).toInt(), start); + expect((rows.first['end_ts'] as num).toInt(), _utc(2024, 3, 10, 17, 45)); + }); + + // An import made before the zone column was honoured stored this workout + // at its wall time read in the phone's zone, under that instant's id. + // +05:45 so the test host's own zone can't make the two readings agree. + const wallStart = '2024-04-02 06:00:00'; + const wallEnd = '2024-04-02 07:10:00'; + String workoutCsv(String name) => (File(p.join(tmp.path, name)) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-04-02 04:00:00,2024-04-03 04:00:00,UTC+05:45,' + '$wallStart,$wallEnd,Running,9.4\n', + )).path; + + Future>> sessionsAround(int ts) async { + final db = await LocalDb.instance; + return db.query('sessions', + where: "id LIKE 'whoop%' AND start_ts BETWEEN ? AND ?", + whereArgs: [ts - 86400, ts + 86400]); + } + + test('a re-import moves a phone-zone workout instead of duplicating it', + () async { + SharedPreferences.setMockInitialValues({}); + final legacyStart = WhoopImporter.parseExportTs(wallStart)!; + final legacyEnd = WhoopImporter.parseExportTs(wallEnd)!; + final start = _utc(2024, 4, 2, 0, 15); + final end = _utc(2024, 4, 2, 1, 25); + if (legacyStart == start) return; // host is on +05:45 itself + final legacyId = 'whoop_$legacyStart'; + await LocalDb.putSession({ + 'id': legacyId, + 'start_ts': legacyStart, + 'end_ts': legacyEnd, + 'type': 'cycling', // the user's correction + 'status': 'done', + 'source': 'whoop', + 'strain': 9.4, + 'duration_min': 70, + 'created_at': 0, + }); + await LocalDb.setSessionPrivate(legacyId, true); + // Measured over the wrong seconds before the move. + final db0 = await LocalDb.instance; + await db0.update( + 'sessions', + {'trace_json': '{"hr":[]}', 'trace_samples': 10, 'avg_hr': 140}, + where: 'id = ?', + whereArgs: [legacyId]); + + for (var i = 0; i < 2; i++) { + final res = await WhoopImporter.importFiles([workoutCsv('w$i.csv')]); + expect(res.workouts, 0); + final rows = await sessionsAround(start); + expect(rows, hasLength(1), reason: 'import ${i + 1}'); + final r = rows.single; + expect(r['id'], legacyId); + expect((r['start_ts'] as num).toInt(), start); + expect((r['end_ts'] as num).toInt(), end); + expect(r['type'], 'cycling'); + expect(r['private'], 1); + expect(r['trace_json'], isNull); + expect(r['avg_hr'], isNull); + expect(r['strain'], 9.4); + } + final db = await LocalDb.instance; + await db.delete('sessions', where: 'id = ?', whereArgs: [legacyId]); + }); + + test('a workout already moved under its old id is not inserted again', + () async { + SharedPreferences.setMockInitialValues({}); + final start = _utc(2024, 4, 2, 0, 15); + // Any id other than the one this import would compute: the row an + // earlier re-import moved onto the right instant. + const movedId = 'whoop_1'; + await LocalDb.putSession({ + 'id': movedId, + 'start_ts': start, + 'end_ts': _utc(2024, 4, 2, 1, 25), + 'type': 'running', + 'status': 'done', + 'source': 'whoop', + 'duration_min': 70, + 'created_at': 0, + }); + final res = await WhoopImporter.importFiles([workoutCsv('wm.csv')]); + expect(res.workouts, 0); + final rows = await sessionsAround(start); + expect(rows.map((r) => r['id']), [movedId]); + final db = await LocalDb.instance; + await db.delete('sessions', where: 'id = ?', whereArgs: [movedId]); + }); + + test('a moved workout deleted under its old id stays deleted', () async { + SharedPreferences.setMockInitialValues({}); + final start = _utc(2024, 4, 2, 0, 15); + await WhoopImporter.rememberDeletedWorkout('whoop_1', startTs: start); + final res = await WhoopImporter.importFiles([workoutCsv('wdm.csv')]); + expect(res.workouts, 0); + expect(await sessionsAround(start), isEmpty); + }); + + test('a deleted phone-zone workout stays deleted on re-import', () async { + final legacyStart = WhoopImporter.parseExportTs(wallStart)!; + final start = _utc(2024, 4, 2, 0, 15); + if (legacyStart == start) return; + SharedPreferences.setMockInitialValues({ + 'whoop_import_deleted_workouts': ['whoop_$legacyStart'], + }); + final res = await WhoopImporter.importFiles([workoutCsv('wd.csv')]); + expect(res.workouts, 0); + expect(await sessionsAround(start), isEmpty); + }); + }); +} diff --git a/test/whoop_reimport_edits_test.dart b/test/whoop_reimport_edits_test.dart new file mode 100644 index 000000000..069650154 --- /dev/null +++ b/test/whoop_reimport_edits_test.dart @@ -0,0 +1,81 @@ +// Importing a newer WHOOP export carries every old workout again under the +// same deterministic id. Writing those rows back over the existing ones +// reverted the user's type correction, wiped their RPE, cleared the private +// flag (and re-exported the workout to the health store), and brought back +// workouts they had deleted. +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:openstrap_edge/import/whoop_import.dart'; + +int _utc(int y, int mo, int d, int h, [int mi = 0]) => + DateTime.utc(y, mo, d, h, mi).millisecondsSinceEpoch ~/ 1000; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory tmp; + + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_whoop_reimport_test.db'; + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + tmp = Directory.systemTemp.createTempSync('openstrap_whoop_reimport'); + }); + + setUp(() => SharedPreferences.setMockInitialValues({})); + + tearDownAll(() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + try { + tmp.deleteSync(recursive: true); + } catch (_) {} + }); + + test('a re-import keeps edits, the private flag and deletions', () async { + final f = File(p.join(tmp.path, 'workouts.csv')) + ..writeAsStringSync( + 'Cycle start time,Cycle end time,Cycle timezone,Workout start time,' + 'Workout end time,Activity name,Activity Strain\n' + '2024-03-10 07:00:00,2024-03-11 07:00:00,UTC,' + '2024-03-10 09:00:00,2024-03-10 10:00:00,Functional Fitness,9.1\n' + '2024-03-11 07:00:00,2024-03-12 07:00:00,UTC,' + '2024-03-11 09:00:00,2024-03-11 10:00:00,Yoga,4.0\n' + '2024-03-12 07:00:00,2024-03-13 07:00:00,UTC,' + '2024-03-12 09:00:00,2024-03-12 10:00:00,Running,11.2\n', + ); + final first = await WhoopImporter.importFiles([f.path]); + expect(first.workouts, 3); + + final edited = 'whoop_${_utc(2024, 3, 10, 9)}'; + final private = 'whoop_${_utc(2024, 3, 11, 9)}'; + final deleted = 'whoop_${_utc(2024, 3, 12, 9)}'; + expect(await LocalDb.session(deleted), isNotNull); + + await LocalDb.setSessionType(edited, 'strength_trainer'); + await LocalDb.setSessionRpe(edited, 7); + await LocalDb.setSessionPrivate(private, true); + await LocalRepositoryImpl(getProfileMap: () => const {}) + .deleteWorkout(deleted); + expect(await LocalDb.session(deleted), isNull); + + final second = await WhoopImporter.importFiles([f.path]); + expect(second.workouts, 0); + + final e = (await LocalDb.session(edited))!; + expect(e['type'], 'strength_trainer'); + expect((e['rpe'] as num?)?.toDouble(), 7.0); + expect((await LocalDb.session(private))!['private'], 1); + expect(await LocalDb.session(deleted), isNull, + reason: 'a deleted import stays deleted'); + }); +}