From 32ab50236f68814ee251b7bde5e0ef792702bde7 Mon Sep 17 00:00:00 2001
From: Alex Wagner <der.wagner@tum.de>
Date: Thu, 22 Oct 2026 11:45:11 +0200
Subject: [PATCH 1/9] test: database growth + load-time soak

Simulates days of use through the real sync commit, derive-pass prune,
substrate archive and VACUUM, and records per-table size per day in
build/soak/db_growth.csv. Then ages the install to 30/180/365/730 days of
history and times the launch reads, the home load and the trend charts.

Opt-in (OPENSTRAP_SOAK=1), a simulated day costs 15-30 s; the new soak
workflow runs it nightly for gen4 and WHOOP MG and keeps the CSV.

On current main it fails on three real problems:
- after a VACUUM the -wal stays the size of the whole database (no
  journal_size_limit), doubling disk use;
- WHOOP MG grows 3.5 MB/day in steady state (thinned v20 frames), 115 MB/day
  before the prune;
- getToday goes from ~20 ms to ~300-400 ms between 30 and 730 days of
  history: recentDayResults sorts every day's full row before its LIMIT.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/soak.yml    |  63 +++++
 test/db_growth_soak_test.dart | 509 ++++++++++++++++++++++++++++++++++
 2 files changed, 572 insertions(+)
 create mode 100644 .github/workflows/soak.yml
 create mode 100644 test/db_growth_soak_test.dart

diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml
new file mode 100644
index 000000000..dea22939a
--- /dev/null
+++ b/.github/workflows/soak.yml
@@ -0,0 +1,63 @@
+name: soak
+
+# Database growth + load-time soak (test/db_growth_soak_test.dart). Too slow
+# for every PR — a simulated day costs ~15-30 s through the real ingest — so it
+# runs nightly and on demand, and the per-day size report is kept as an
+# artifact either way.
+on:
+  schedule:
+    - cron: '30 2 * * *'
+  workflow_dispatch:
+    inputs:
+      days:
+        description: Live days to simulate
+        default: '30'
+      history:
+        description: History lengths (days) to time loads at
+        default: '30,180,365,730'
+
+concurrency:
+  group: soak-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  soak:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        device: [gen4, mg]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      # Same pin as test.yml, for the same reasons.
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version: 3.41.6
+          channel: stable
+          cache: true
+
+      - name: Install dependencies
+        run: flutter pub get
+
+      - name: Soak (${{ matrix.device }})
+        env:
+          OPENSTRAP_SOAK: '1'
+          SOAK_DEVICE: ${{ matrix.device }}
+          SOAK_DAYS: ${{ inputs.days || '30' }}
+          SOAK_HISTORY: ${{ inputs.history || '30,180,365,730' }}
+        run: flutter test --concurrency=1 --reporter=expanded test/db_growth_soak_test.dart
+
+      - name: Keep the size report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: db-growth-${{ matrix.device }}
+          path: build/soak/db_growth.csv
+          if-no-files-found: ignore
diff --git a/test/db_growth_soak_test.dart b/test/db_growth_soak_test.dart
new file mode 100644
index 000000000..69560d81f
--- /dev/null
+++ b/test/db_growth_soak_test.dart
@@ -0,0 +1,509 @@
+// DATABASE GROWTH + LOAD-TIME SOAK. Opt-in: a full simulated day costs ~15 s
+// on a laptop, so this never runs in the default `flutter test`.
+//
+//   OPENSTRAP_SOAK=1 flutter test test/db_growth_soak_test.dart
+//
+// Knobs (all optional):
+//   SOAK_DAYS          live days to simulate through the real ingest (14)
+//   SOAK_DEVICE        gen4, or mg: a WHOOP MG also banks the undecodable
+//                      `undecodable_rec_v20` frames, ~45 MB/day at full rate
+//                      (measured, see LocalDb.thinRawArchiveBefore) (gen4)
+//   SOAK_HISTORY       history lengths, in days, to time loads at
+//                      (30,180,365,730)
+//   SOAK_BUDGET_MB     steady-state growth budget, MB per day (3)
+//   SOAK_MAX_RATIO     slowest-history / shortest-history load time (2)
+//
+// WHAT IS REAL AND WHAT IS NOT.
+//   * Every simulated day goes through `LocalDb.commitSyncBatch` — the same
+//     ACK-gating commit a WHOOP drain uses — from [SyntheticDay]'s 1 Hz day,
+//     so samples, decoded_onehz, decoded_rr and the coverage tables are
+//     written exactly as on a phone.
+//   * Then `DerivationEngine().run(heavy: true)`, which is what runs after a
+//     sync. Its raw prune, substrate archive and storage housekeeping are the
+//     production code paths.
+//   * Then `LocalDb.vacuumIfBloated()`, as `_maybeReclaimDiskSpace` does once
+//     per launch after a foreground heavy derive (one launch per day here).
+//   * NOT REAL: the analytics. A real derive costs ~75 s per day under
+//     `flutter test`, so ONE day is derived for real and its per-day rows
+//     (day_result, metric_series, …) are cloned onto every other day and
+//     marked finalized. That makes the engine take its "all days finalized"
+//     branch, which still prunes. A healthy install prunes at the same cutoff:
+//     its unfinalized days are the newest ones, inside the retention window.
+//     The stuck-day hold is pure and covered by raw_prune_gating_test.
+//
+// The per-day numbers land in build/soak/db_growth.csv.
+
+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:openstrap_edge/compute/derivation_engine.dart';
+import 'package:openstrap_edge/compute/profile.dart';
+import 'package:openstrap_edge/data/day_label.dart';
+import 'package:openstrap_edge/data/db.dart';
+import 'package:openstrap_edge/data/local_repository_impl.dart';
+import 'package:openstrap_edge/data/models.dart' show ArchiveRecord;
+import 'package:openstrap_edge/ui2/screens/home_screen.dart' show HomeData;
+import 'package:path/path.dart' as p;
+import 'package:sqflite_common_ffi/sqflite_ffi.dart';
+
+import 'support/app_state_derive_harness.dart';
+import 'support/synthetic_day.dart';
+
+const Profile _profile = Profile(
+  ageYears: 32,
+  weightKg: 72,
+  heightCm: 176,
+  sex: 'm',
+);
+
+const String _dbName = 'db_growth_soak_test.db';
+
+/// Tables a derive writes one generation of per day, and their day column.
+const List<(String, String)> _perDayTables = [
+  ('day_result', 'day_id'),
+  ('metric_series', 'date'),
+  ('metric_series_version', 'date'),
+  ('sleep_session_candidates', 'day_id'),
+  ('wake_day_features', 'day_id'),
+];
+
+final Map<String, String> _env = Platform.environment;
+int _int(String k, int d) => int.tryParse(_env[k] ?? '') ?? d;
+double _num(String k, double d) => double.tryParse(_env[k] ?? '') ?? d;
+
+const double _mb = 1024 * 1024;
+
+/// The real MG export banked 115,672 `undecodable_rec_v20` frames of 4,256
+/// hex chars in 11 days (see LocalDb.thinRawArchiveBefore).
+const int _mgFramesPerDay = 115672 ~/ 11;
+const int _mgFrameHexChars = 4256;
+
+/// Deterministic, incompressible-looking frame bytes, unique per [counter].
+String _mgFrameHex(int counter) {
+  final out = StringBuffer(counter.toRadixString(16).padLeft(8, '0'));
+  var x = counter * 2654435761 + 1;
+  while (out.length < _mgFrameHexChars) {
+    x = (x * 6364136223846793005 + 1442695040888963407) & 0xFFFFFFFFFFFF;
+    out.write((x >> 16).toRadixString(16).padLeft(8, '0'));
+  }
+  return out.toString().substring(0, _mgFrameHexChars);
+}
+
+/// One row of the growth report.
+class _Snap {
+  _Snap(
+    this.day,
+    this.fileBytes,
+    this.usedBytes,
+    this.freeBytes,
+    this.walBytes,
+    this.onehzRows,
+    this.tables,
+    this.dayMs,
+  );
+  final int day;
+  final int fileBytes, usedBytes, freeBytes, walBytes, onehzRows;
+  final Map<String, int> tables;
+  final int dayMs;
+}
+
+void main() {
+  final enabled = _env['OPENSTRAP_SOAK'] == '1';
+  final skip = enabled ? null : 'set OPENSTRAP_SOAK=1 to run the soak';
+
+  final liveDays = _int('SOAK_DAYS', 14);
+  final mg = _env['SOAK_DEVICE'] == 'mg';
+  var mgCounter = 0;
+  // Steady state: retention plus the night lookback plus a day of slack.
+  final warmup = rawRetentionDays + 3;
+  final today = DateTime.now();
+  // The last simulated day ends yesterday 22:00, so "today" in the app's sense
+  // is the morning after the newest synced night.
+  final first = DateTime(today.year, today.month, today.day - liveDays);
+  final snaps = <_Snap>[];
+  late String template;
+  late String path;
+  var committedTo = 0;
+
+  Future<Database> db() => LocalDb.instance;
+
+  Future<void> ingest(SyntheticDay day) async {
+    for (final (raws, samples) in day.primaryBatches()) {
+      // Consecutive synthetic days overlap 18:00–22:00; a band never sends
+      // the same second twice, so neither does this.
+      final keep = [
+        for (var j = 0; j < raws.length; j++)
+          if (raws[j].recTs! > committedTo) j,
+      ];
+      if (keep.isEmpty) continue;
+      final at = raws[keep.first].recTs! * 1000;
+      await LocalDb.commitSyncBatch(
+        [for (final j in keep) raws[j]],
+        [for (final j in keep) samples[j]],
+        archives: !mg
+            ? null
+            : [
+                for (var k = 0; k < keep.length * _mgFramesPerDay ~/ 86400; k++)
+                  ArchiveRecord(
+                    counter: ++mgCounter,
+                    hex: _mgFrameHex(mgCounter),
+                    packetType: 0x2F,
+                    capturedAt: at,
+                    reason: 'undecodable_rec_v20',
+                  ),
+              ],
+        // The substrate stays gen4 in both scenarios so the template derive
+        // is the same; the MG scenario differs only in the banked frames.
+        deviceFamily: 'gen4',
+      );
+      committedTo = raws[keep.last].recTs!;
+    }
+  }
+
+  Future<void> cloneDay(String from, String to) async {
+    final d = await db();
+    for (final (table, col) in _perDayTables) {
+      final cols = [
+        for (final r in await d.rawQuery('PRAGMA table_info($table)'))
+          r['name'] as String,
+      ];
+      final select = [for (final c in cols) c == col ? '?' : c].join(', ');
+      await d.rawInsert(
+        'INSERT OR REPLACE INTO $table (${cols.join(', ')}) '
+        'SELECT $select FROM $table WHERE $col = ?',
+        [to, from],
+      );
+    }
+  }
+
+  /// Give every day that still has substrate a finalized result at the
+  /// current algorithm version — a cloned one if it has none.
+  Future<void> finalizeRawDays() async {
+    final d = await db();
+    for (final day in (await LocalDb.decodedRecTsMaxByDay()).keys) {
+      final has = await d.query(
+        'day_result',
+        columns: ['day_id'],
+        where: 'day_id = ? AND algo_version = ?',
+        whereArgs: [day, kAlgoVersion],
+      );
+      if (has.isEmpty) await cloneDay(template, day);
+    }
+    await d.update('day_result', {'finalized': 1});
+  }
+
+  Future<Map<String, int>> tableBytes() async {
+    final d = await db();
+    try {
+      final rows = await d.rawQuery(
+        'SELECT m.tbl_name AS t, SUM(s.pgsize) AS b FROM dbstat s '
+        'JOIN sqlite_master m ON m.name = s.name GROUP BY m.tbl_name',
+      );
+      return {for (final r in rows) r['t'] as String: (r['b'] as num).toInt()};
+    } catch (_) {
+      // No dbstat in this SQLite build: the table-only estimate.
+      return {
+        for (final r in await LocalDb.tableStorageStats())
+          r['table'] as String: (r['bytes'] as num?)?.toInt() ?? 0,
+      };
+    }
+  }
+
+  Future<_Snap> snapshot(int day, int dayMs) async {
+    final d = await db();
+    Future<int> pragma(String q) async =>
+        ((await d.rawQuery('PRAGMA $q')).first.values.first as num).toInt();
+    final pageSize = await pragma('page_size');
+    final pages = await pragma('page_count');
+    final free = await pragma('freelist_count');
+    final wal = File('$path-wal');
+    final onehz =
+        (await d.rawQuery('SELECT COUNT(*) AS n FROM decoded_onehz')).first['n']
+            as int;
+    return _Snap(
+      day,
+      File(path).lengthSync(),
+      (pages - free) * pageSize,
+      free * pageSize,
+      wal.existsSync() ? wal.lengthSync() : 0,
+      onehz,
+      await tableBytes(),
+      dayMs,
+    );
+  }
+
+  void writeReport() {
+    final tables = <String>{for (final s in snaps) ...s.tables.keys}.toList()
+      ..sort((a, b) => snaps.last.tables[b]!.compareTo(snaps.last.tables[a]!));
+    final out = File('build/soak/db_growth.csv')..createSync(recursive: true);
+    out.writeAsStringSync(
+      [
+        'day,day_ms,file_mb,used_mb,freelist_mb,wal_mb,onehz_rows,'
+            '${tables.map((t) => '${t}_mb').join(',')}',
+        for (final s in snaps)
+          [
+            s.day,
+            s.dayMs,
+            for (final b in [s.fileBytes, s.usedBytes, s.freeBytes, s.walBytes])
+              (b / _mb).toStringAsFixed(2),
+            s.onehzRows,
+            for (final t in tables)
+              ((s.tables[t] ?? 0) / _mb).toStringAsFixed(2),
+          ].join(','),
+      ].join('\n'),
+    );
+    // ignore: avoid_print
+    print('[soak] report: ${out.absolute.path}');
+  }
+
+  setUpAll(() async {
+    if (!enabled) return;
+    await deriveDbSetUp(_dbName);
+    path = p.join(await databaseFactory.getDatabasesPath(), _dbName);
+    // The template: one fully-synced night, derived for real.
+    await ingest(SyntheticDay(first, seed: 1));
+    await ingest(SyntheticDay(first.add(const Duration(days: 1)), seed: 2));
+    template = dayLabelOf(first);
+    await DerivationEngine().runDays(_profile, {template}, force: true);
+    expect(
+      await LocalDb.dayResult(template),
+      isNotNull,
+      reason: 'the template day did not derive',
+    );
+  });
+
+  tearDownAll(() async {
+    if (enabled) await deriveDbTearDown(_dbName);
+  });
+
+  test(
+    'storage levels off once the retention prune is running',
+    () async {
+      expect(
+        liveDays,
+        greaterThanOrEqualTo(warmup + 4),
+        reason:
+            'SOAK_DAYS must leave a steady-state window after $warmup '
+            'warm-up days',
+      );
+      for (var i = 2; i < liveDays; i++) {
+        final sw = Stopwatch()..start();
+        await ingest(
+          SyntheticDay(
+            DateTime(first.year, first.month, first.day + i),
+            seed: 1 + i,
+          ),
+        );
+        await finalizeRawDays();
+        await DerivationEngine().run(_profile, heavy: true);
+        await LocalDb.vacuumIfBloated();
+        final s = await snapshot(i, sw.elapsedMilliseconds);
+        snaps.add(s);
+        // ignore: avoid_print
+        print(
+          '[soak] day $i: used ${(s.usedBytes / _mb).toStringAsFixed(1)} MB'
+          ', file ${(s.fileBytes / _mb).toStringAsFixed(1)} MB'
+          ', free ${(s.freeBytes / _mb).toStringAsFixed(1)} MB'
+          ', wal ${(s.walBytes / _mb).toStringAsFixed(1)} MB'
+          ', onehz ${s.onehzRows} rows, ${s.dayMs} ms',
+        );
+      }
+      writeReport();
+
+      final steady = snaps.where((s) => s.day >= warmup).toList();
+      final base = steady.first;
+      // Every check runs and every miss is reported, so one failure cannot
+      // hide another.
+      final problems = <String>[];
+      String mb(int b) => (b / _mb).toStringAsFixed(0);
+      // 1. The live substrate stops growing: it is capped at the retention
+      //    window, so its row count plateaus instead of climbing a day at a time.
+      final grown = steady.lastWhere(
+        (s) => s.onehzRows > base.onehzRows * 1.15,
+        orElse: () => base,
+      );
+      if (grown != base) {
+        problems.add(
+          'decoded_onehz still growing: ${grown.onehzRows} rows on '
+          'day ${grown.day} vs ${base.onehzRows} on day ${base.day}',
+        );
+      }
+      // 2. What the phone actually holds — the file plus its -wal — stays near
+      //    what the data needs. Freed pages are reused and a bloated file is
+      //    VACUUMed, but in WAL mode that VACUUM writes the whole database
+      //    through the -wal, and SQLite only shrinks a -wal on a clean close or
+      //    with `journal_size_limit` set.
+      final bloated = steady.where(
+        (s) => s.fileBytes + s.walBytes > s.usedBytes * 1.25 + 32 * _mb,
+      );
+      if (bloated.isNotEmpty) {
+        final s = bloated.last;
+        problems.add(
+          '${bloated.length} of ${steady.length} days hold more on '
+          'disk than the data needs; day ${s.day}: '
+          '${mb(s.fileBytes + s.walBytes)} MB (file ${mb(s.fileBytes)} + wal '
+          '${mb(s.walBytes)}) for ${mb(s.usedBytes)} MB of data',
+        );
+      }
+      // 3. What does keep growing (derived history, the compressed archive)
+      //    grows slowly. Least-squares slope of used bytes per day.
+      final n = steady.length;
+      final mx = steady.map((s) => s.day).reduce((a, b) => a + b) / n;
+      final my = steady.map((s) => s.usedBytes).reduce((a, b) => a + b) / n;
+      var sxy = 0.0, sxx = 0.0;
+      for (final s in steady) {
+        sxy += (s.day - mx) * (s.usedBytes - my);
+        sxx += (s.day - mx) * (s.day - mx);
+      }
+      final slopeMb = sxy / sxx / _mb;
+      final budget = _num('SOAK_BUDGET_MB', 3);
+      // ignore: avoid_print
+      print(
+        '[soak] steady-state growth ${slopeMb.toStringAsFixed(2)} MB/day; '
+        'one year from here: '
+        '${((snaps.last.usedBytes / _mb) + slopeMb * 365).toStringAsFixed(0)}'
+        ' MB',
+      );
+      if (slopeMb > budget) {
+        problems.add(
+          'steady-state growth ${slopeMb.toStringAsFixed(2)} MB/day '
+          'is over the ${budget.toStringAsFixed(1)} MB/day budget — '
+          'build/soak/db_growth.csv shows which table grew',
+        );
+      }
+      if (problems.isNotEmpty) fail(problems.join('\n'));
+    },
+    skip: skip,
+    timeout: const Timeout(Duration(hours: 6)),
+  );
+
+  test(
+    'launch and home load times do not grow with history',
+    () async {
+      expect(
+        snaps,
+        isNotEmpty,
+        reason: 'run the whole file: this extends the growth run\'s database',
+      );
+      final repo = LocalRepositoryImpl(
+        getProfileMap: () => {
+          'age': 32,
+          'weight_kg': 72,
+          'height_cm': 176,
+          'sex': 'm',
+        },
+      );
+      final archived = await (await db()).query(
+        'substrate_archive',
+        orderBy: 'utc_day DESC',
+        limit: 1,
+      );
+      final edgeDay = committedTo ~/ 86400;
+      var oldest = first;
+
+      // Ages the install to [days] of history: older days get a derived
+      // result and, inside the archive's keep window, an archived bucket —
+      // what a long-time user's database holds for a day with no substrate.
+      Future<void> extendHistoryTo(int days) async {
+        final d = await db();
+        final target = DateTime(today.year, today.month, today.day - days);
+        while (oldest.isAfter(target)) {
+          oldest = DateTime(oldest.year, oldest.month, oldest.day - 1);
+          final label = dayLabelOf(oldest);
+          await cloneDay(template, label);
+          final utcDay =
+              DateTime.utc(
+                oldest.year,
+                oldest.month,
+                oldest.day,
+              ).millisecondsSinceEpoch ~/
+              86400000;
+          if (archived.isNotEmpty && edgeDay - utcDay <= 365) {
+            final a = archived.first;
+            final shift = ((a['utc_day'] as int) - utcDay) * 86400;
+            await d.insert('substrate_archive', {
+              ...a,
+              'utc_day': utcDay,
+              'from_ts': (a['from_ts'] as int) - shift,
+              'to_ts': (a['to_ts'] as int) - shift,
+            }, conflictAlgorithm: ConflictAlgorithm.replace);
+          }
+        }
+        await d.update('day_result', {'finalized': 1});
+      }
+
+      Future<int> medianMs(Future<void> Function() body) async {
+        await body(); // warm the page cache like a second launch would
+        final ms = <int>[];
+        for (var i = 0; i < 5; i++) {
+          final sw = Stopwatch()..start();
+          await body();
+          ms.add(sw.elapsedMilliseconds);
+        }
+        ms.sort();
+        return ms[2];
+      }
+
+      final loads = <String, Future<void> Function()>{
+        // What AppState._initSteps reads off a cold database.
+        'launch': () async {
+          await LocalDb.close();
+          await LocalDb.instance;
+          await LocalDb.latestSample();
+          await LocalDb.refreshComputeFreshness();
+        },
+        // What the home screen awaits before it draws (HomeData.load and the
+        // day switcher), whole and by part, so a failure names its query.
+        'home': () async {
+          await HomeData.load(repo);
+          await repo.availableDays();
+        },
+        'home.today': () => repo.getToday(),
+        'home.insights': () => repo.getInsights(),
+        'home.days': () => repo.availableDays(),
+        // The trend charts read the whole metric series.
+        'trends': () async {
+          await repo.getChart('readiness');
+          await repo.getChart('hrv');
+        },
+      };
+
+      final history =
+          (_env['SOAK_HISTORY'] ?? '30,180,365,730')
+              .split(',')
+              .map(int.parse)
+              .toList()
+            ..sort();
+      final results = <int, Map<String, int>>{};
+      for (final days in history) {
+        await extendHistoryTo(days);
+        // One real pass so housekeeping (archive eviction) sees the new history.
+        await DerivationEngine().run(_profile, heavy: true);
+        results[days] = {
+          for (final e in loads.entries) e.key: await medianMs(e.value),
+        };
+        // ignore: avoid_print
+        print(
+          '[soak] $days days of history, '
+          '${(File(path).lengthSync() / _mb).toStringAsFixed(0)} MB: '
+          '${results[days]}',
+        );
+      }
+
+      final ratio = _num('SOAK_MAX_RATIO', 2);
+      // Absolute slack, so a 3 ms load going to 9 ms is not a failure.
+      const slackMs = 25;
+      final shortest = results[history.first]!,
+          longest = results[history.last]!;
+      final slow = [
+        for (final k in loads.keys)
+          if (longest[k]! > shortest[k]! * ratio + slackMs)
+            '$k: ${shortest[k]} ms at ${history.first} days of history, '
+                '${longest[k]} ms at ${history.last}',
+      ];
+      if (slow.isNotEmpty) fail(slow.join('\n'));
+    },
+    skip: skip,
+    timeout: const Timeout(Duration(hours: 2)),
+  );
+}

From e84537c9e1f196f72a45f118e5c6757d16c2348e Mon Sep 17 00:00:00 2001
From: Alex Wagner <der.wagner@tum.de>
Date: Thu, 22 Oct 2026 12:12:19 +0200
Subject: [PATCH 2/9] perf(db): read the newest N days without sorting all of
 history

recentDayResults applied its LIMIT after joining and sorting every day's
served row, payload_json (~100 KB) included, so the 30-day read behind
refreshComputeFreshness went from ~60 ms at 30 days of history to 250-600 ms
at 730. That runs on every derive and on every Home load while a night is
still building: the growth soak (#609) measured getToday at 22 -> 303 ms.

Pick the newest N served versions from the covering index first, then fetch
only those rows. The served-version subquery is factored out of
_servedDayJoin, which expands to the same SQL as before, so the ceiling stays
in one place. recentDayResultsMeta takes the same join.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 lib/data/db.dart                          | 24 +++++---
 test/db_serve_version_and_reads_test.dart | 69 +++++++++++++++++++++++
 2 files changed, 86 insertions(+), 7 deletions(-)

diff --git a/lib/data/db.dart b/lib/data/db.dart
index ed954c51b..87ee0f995 100644
--- a/lib/data/db.dart
+++ b/lib/data/db.dart
@@ -8776,11 +8776,21 @@ class LocalDb {
   /// THIS rather than writing its own `MAX(algo_version)` — the pattern had
   /// already reappeared at five readers and two views, and the ceiling has to
   /// be on all of them or it is on none of them.
-  static const String _servedDayJoin =
-      'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
-      'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id) m '
+  static const String _servedDayJoin = 'JOIN ($_servedVersions) m $_servedOn';
+  static const String _servedVersions =
+      'SELECT day_id, MAX(algo_version) AS v FROM day_result '
+      'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id';
+  static const String _servedOn =
       'ON r.day_id = m.day_id AND r.algo_version = m.v';
 
+  /// [_servedDayJoin] for the newest N days only (bind N). A LIMIT on the
+  /// outer query comes after the join and its sort, so SQLite fetched and
+  /// sorted EVERY day's row, bundle and all, to return N: a 30-day read went
+  /// from ~60 ms to 250-600 ms between 30 and 730 days of history. Choosing
+  /// the days first, from the covering index, leaves N rows to fetch.
+  static const String _servedNewestDaysJoin =
+      'JOIN ($_servedVersions ORDER BY day_id DESC LIMIT ?) m $_servedOn';
+
   /// The result row for one day_id at the highest version this build serves
   /// (see [_servedAlgoCeiling]), with a normalized `date` alias for callers.
   /// Null if absent.
@@ -8943,8 +8953,8 @@ class LocalDb {
     // For each day_id pick MAX(algo_version), then join back for the full row.
     final rows = await db.rawQuery(
       'SELECT r.* FROM day_result r '
-      '$_servedDayJoin '
-      'ORDER BY r.day_id DESC LIMIT ?',
+      '$_servedNewestDaysJoin '
+      'ORDER BY r.day_id DESC',
       [limit],
     );
     return [for (final r in rows) _withDate(r)];
@@ -8968,8 +8978,8 @@ class LocalDb {
       'SELECT r.day_id, r.algo_version, r.computed_at, r.finalized, '
       'r.skipped, r.partial, r.rhr, r.rmssd, r.readiness '
       'FROM day_result r '
-      '$_servedDayJoin '
-      'ORDER BY r.day_id DESC LIMIT ?',
+      '$_servedNewestDaysJoin '
+      'ORDER BY r.day_id DESC',
       [limit],
     );
     return [for (final r in rows) _withDate(r)];
diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart
index f160691c5..9f6eb5d13 100644
--- a/test/db_serve_version_and_reads_test.dart
+++ b/test/db_serve_version_and_reads_test.dart
@@ -231,5 +231,74 @@ void main() {
       await db.delete('decoded_onehz');
       expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
     });
+
+    test('the newest-N reads stay flat as history grows', () async {
+      // recentDayResults joined EVERY day's full row (~100 KB of bundle) and
+      // sorted them all before its LIMIT, so reading the last 30 days took
+      // 57 ms at 30 days of history and 250-600 ms at 730 — on every
+      // refreshComputeFreshness, i.e. every derive and every Home load whose
+      // night is still building.
+      final db = await LocalDb.instance;
+      await db.delete('day_result');
+      final bundle = '{"pad":"${'x' * 100000}"}';
+      final today = DateTime.now();
+      var have = 0;
+      Future<void> grow(int days) async {
+        final b = db.batch();
+        for (var i = have; i < days; i++) {
+          final day = dayLabelOf(DateTime(today.year, today.month, today.day - i));
+          for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) {
+            // Every third day stops at an older version; the rest also carry a
+            // row from a future build, which must never be served.
+            if (v == kAlgoVersion && i % 3 == 0) continue;
+            b.insert('day_result', {
+              'day_id': day,
+              'algo_version': v,
+              'payload_json': bundle,
+              'window_json': '{}',
+              'computed_at': 0,
+            });
+          }
+        }
+        await b.commit(noResult: true);
+        have = days;
+      }
+
+      Future<int> medianMs(Future<Object?> Function() f) async {
+        await f();
+        final us = <int>[];
+        for (var i = 0; i < 5; i++) {
+          final sw = Stopwatch()..start();
+          await f();
+          us.add(sw.elapsedMicroseconds);
+        }
+        us.sort();
+        return us[2] ~/ 1000;
+      }
+
+      // The query it replaced, verbatim, as the oracle for WHICH rows come back.
+      Future<List<Map<String, Object?>>> oracle(int limit) => db.rawQuery(
+            'SELECT r.* FROM day_result r '
+            'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
+            'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m '
+            'ON r.day_id = m.day_id AND r.algo_version = m.v '
+            'ORDER BY r.day_id DESC LIMIT ?',
+            [limit],
+          );
+      List<String> key(List<Map<String, Object?>> rows) =>
+          [for (final r in rows) '${r['day_id']}@${r['algo_version']}'];
+
+      await grow(30);
+      final short = await medianMs(() => LocalDb.recentDayResults(30));
+      await grow(600);
+      final long = await medianMs(() => LocalDb.recentDayResults(30));
+
+      expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30)));
+      expect(key(await LocalDb.recentDayResultsMeta(30)), key(await oracle(30)));
+      expect(long, lessThanOrEqualTo(short * 3 + 30),
+          reason: 'recentDayResults(30): $short ms at 30 days of history, '
+              '$long ms at 600');
+      await db.delete('day_result');
+    });
   });
 }

From c7492adcb1f485040b6798cdb00969110beaf618 Mon Sep 17 00:00:00 2001
From: Alex Wagner <der.wagner@tum.de>
Date: Thu, 22 Oct 2026 12:30:27 +0200
Subject: [PATCH 3/9] test: time the newest-N read against the query it
 replaced

An absolute bound (600-day read <= 3x the 30-day read + 30 ms) can fail on a
slow or loaded runner. Time the old full-history query back to back on the
same 600 days instead, so load slows both sides alike: 3x faster or better
(measured ~10x with the change, 1x without).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/db_serve_version_and_reads_test.dart | 246 ++++++++++++----------
 1 file changed, 139 insertions(+), 107 deletions(-)

diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart
index 9f6eb5d13..93e5baaa5 100644
--- a/test/db_serve_version_and_reads_test.dart
+++ b/test/db_serve_version_and_reads_test.dart
@@ -46,8 +46,10 @@ void main() {
   test('the sibling pins match the SHAs kAlgoVersion was derived against', () {
     final deps =
         (loadYaml(File('pubspec.yaml').readAsStringSync())
-            as Map)['dependencies'] as Map;
-    String ref(String pkg) => ((deps[pkg] as Map)['git'] as Map)['ref'] as String;
+                as Map)['dependencies']
+            as Map;
+    String ref(String pkg) =>
+        ((deps[pkg] as Map)['git'] as Map)['ref'] as String;
 
     const why =
         'sibling repinned without visiting kAlgoVersion. bump it and say what '
@@ -62,8 +64,8 @@ void main() {
     // comments promise this test catches, and it did not until it read this
     // file. `resolved-ref` too: `ref` alone can name a moved branch.
     final locked =
-        (loadYaml(File('pubspec.lock').readAsStringSync())
-            as Map)['packages'] as Map;
+        (loadYaml(File('pubspec.lock').readAsStringSync()) as Map)['packages']
+            as Map;
     String lockRef(String pkg, String field) =>
         ((locked[pkg] as Map)['description'] as Map)[field] as String;
 
@@ -72,10 +74,16 @@ void main() {
         'partial repin. Move all three together, or a build resolves a '
         'sibling nobody reasoned about';
     for (final field in const ['ref', 'resolved-ref']) {
-      expect(lockRef('openstrap_analytics', field), kAnalyticsPin,
-          reason: whyLock);
-      expect(lockRef('openstrap_protocol', field), kProtocolPin,
-          reason: whyLock);
+      expect(
+        lockRef('openstrap_analytics', field),
+        kAnalyticsPin,
+        reason: whyLock,
+      );
+      expect(
+        lockRef('openstrap_protocol', field),
+        kProtocolPin,
+        reason: whyLock,
+      );
     }
   });
 
@@ -137,26 +145,30 @@ void main() {
       expect(await LocalDb.dayResultDayIdsDesc(), contains(day));
     });
 
-    test('the future row is kept, not deleted — re-upgrading gets it back',
-        () async {
-      const day = '2026-08-10';
-      final db = await LocalDb.instance;
-      final all = await db.query(
-        'day_result',
-        where: 'day_id = ?',
-        whereArgs: [day],
-      );
-      expect(all, hasLength(2));
-    });
+    test(
+      'the future row is kept, not deleted — re-upgrading gets it back',
+      () async {
+        const day = '2026-08-10';
+        final db = await LocalDb.instance;
+        final all = await db.query(
+          'day_result',
+          where: 'day_id = ?',
+          whereArgs: [day],
+        );
+        expect(all, hasLength(2));
+      },
+    );
 
-    test('a day that ONLY has a future row is served as absent, not wrong',
-        () async {
-      const day = '2026-08-11';
-      await _put(day, kAlgoVersion + 3, readiness: 12);
-      expect(await LocalDb.dayResult(day), isNull);
-      expect(await LocalDb.availableDayIds(), isNot(contains(day)));
-      expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
-    });
+    test(
+      'a day that ONLY has a future row is served as absent, not wrong',
+      () async {
+        const day = '2026-08-11';
+        await _put(day, kAlgoVersion + 3, readiness: 12);
+        expect(await LocalDb.dayResult(day), isNull);
+        expect(await LocalDb.availableDayIds(), isNot(contains(day)));
+        expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
+      },
+    );
 
     test('an OLDER version is still served — that is not the bug', () async {
       const day = '2026-08-09';
@@ -168,69 +180,79 @@ void main() {
   });
 
   group('read-path hygiene', () {
-    test('the decoded counter index is gone and sessions.start_ts is indexed',
-        () async {
-      final db = await LocalDb.instance;
-      final idx = await db.rawQuery(
-        "SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
-      );
-      final names = idx.map((r) => r['name']).toSet();
-      // Never on a read path: every decoded read orders by (rec_ts, counter),
-      // which a single-column counter index cannot serve.
-      expect(names, isNot(contains('idx_decoded_onehz_counter')));
-      // The most-called uncovered query in the UI seam was a full scan + sort.
-      expect(names, contains('idx_sessions_start'));
-    });
-
-    test('decodedRecTsMaxByDay matches the strftime grouping it replaced',
-        () async {
-      final db = await LocalDb.instance;
-      await db.delete('decoded_onehz');
-      // Three local days' worth of edges, plus a second row per day so the MAX
-      // actually has to choose.
-      final days = <String>[];
-      var t = DateTime.now()
-              .subtract(const Duration(days: 3))
-              .millisecondsSinceEpoch ~/
-          1000;
-      for (var i = 0; i < 3; i++) {
-        final label = dayLabelOf(
-          DateTime.fromMillisecondsSinceEpoch(t * 1000, isUtc: true).toLocal(),
+    test(
+      'the decoded counter index is gone and sessions.start_ts is indexed',
+      () async {
+        final db = await LocalDb.instance;
+        final idx = await db.rawQuery(
+          "SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
         );
-        days.add(label);
-        final start = localDayStartSec(label)!;
-        final end = localDayEndSec(label)!;
-        for (final ts in [start + 60, end - 90]) {
-          await db.insert('decoded_onehz', {
-            // v47 key — see _createDecodedStore.
-            'ts_ms': ts * 1000,
-            'rec_ts': ts,
-            'counter': ts,
-            'hr': 60,
-          });
+        final names = idx.map((r) => r['name']).toSet();
+        // Never on a read path: every decoded read orders by (rec_ts, counter),
+        // which a single-column counter index cannot serve.
+        expect(names, isNot(contains('idx_decoded_onehz_counter')));
+        // The most-called uncovered query in the UI seam was a full scan + sort.
+        expect(names, contains('idx_sessions_start'));
+      },
+    );
+
+    test(
+      'decodedRecTsMaxByDay matches the strftime grouping it replaced',
+      () async {
+        final db = await LocalDb.instance;
+        await db.delete('decoded_onehz');
+        // Three local days' worth of edges, plus a second row per day so the MAX
+        // actually has to choose.
+        final days = <String>[];
+        var t =
+            DateTime.now()
+                .subtract(const Duration(days: 3))
+                .millisecondsSinceEpoch ~/
+            1000;
+        for (var i = 0; i < 3; i++) {
+          final label = dayLabelOf(
+            DateTime.fromMillisecondsSinceEpoch(
+              t * 1000,
+              isUtc: true,
+            ).toLocal(),
+          );
+          days.add(label);
+          final start = localDayStartSec(label)!;
+          final end = localDayEndSec(label)!;
+          for (final ts in [start + 60, end - 90]) {
+            await db.insert('decoded_onehz', {
+              // v47 key — see _createDecodedStore.
+              'ts_ms': ts * 1000,
+              'rec_ts': ts,
+              'counter': ts,
+              'hr': 60,
+            });
+          }
+          t = end + 60;
         }
-        t = end + 60;
-      }
 
-      final got = await LocalDb.decodedRecTsMaxByDay();
-      // The old implementation, verbatim, as the oracle.
-      final oracle = await db.rawQuery(
-        "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
-        'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
-      );
-      final want = {
-        for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
-      };
-      expect(got, want);
-      expect(got.keys.toSet(), days.toSet());
-    });
+        final got = await LocalDb.decodedRecTsMaxByDay();
+        // The old implementation, verbatim, as the oracle.
+        final oracle = await db.rawQuery(
+          "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
+          'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
+        );
+        final want = {
+          for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
+        };
+        expect(got, want);
+        expect(got.keys.toSet(), days.toSet());
+      },
+    );
 
-    test('decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
-        () async {
-      final db = await LocalDb.instance;
-      await db.delete('decoded_onehz');
-      expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
-    });
+    test(
+      'decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
+      () async {
+        final db = await LocalDb.instance;
+        await db.delete('decoded_onehz');
+        expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
+      },
+    );
 
     test('the newest-N reads stay flat as history grows', () async {
       // recentDayResults joined EVERY day's full row (~100 KB of bundle) and
@@ -242,11 +264,12 @@ void main() {
       await db.delete('day_result');
       final bundle = '{"pad":"${'x' * 100000}"}';
       final today = DateTime.now();
-      var have = 0;
       Future<void> grow(int days) async {
         final b = db.batch();
-        for (var i = have; i < days; i++) {
-          final day = dayLabelOf(DateTime(today.year, today.month, today.day - i));
+        for (var i = 0; i < days; i++) {
+          final day = dayLabelOf(
+            DateTime(today.year, today.month, today.day - i),
+          );
           for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) {
             // Every third day stops at an older version; the rest also carry a
             // row from a future build, which must never be served.
@@ -261,7 +284,6 @@ void main() {
           }
         }
         await b.commit(noResult: true);
-        have = days;
       }
 
       Future<int> medianMs(Future<Object?> Function() f) async {
@@ -278,26 +300,36 @@ void main() {
 
       // The query it replaced, verbatim, as the oracle for WHICH rows come back.
       Future<List<Map<String, Object?>>> oracle(int limit) => db.rawQuery(
-            'SELECT r.* FROM day_result r '
-            'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
-            'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m '
-            'ON r.day_id = m.day_id AND r.algo_version = m.v '
-            'ORDER BY r.day_id DESC LIMIT ?',
-            [limit],
-          );
-      List<String> key(List<Map<String, Object?>> rows) =>
-          [for (final r in rows) '${r['day_id']}@${r['algo_version']}'];
+        'SELECT r.* FROM day_result r '
+        'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result '
+        'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m '
+        'ON r.day_id = m.day_id AND r.algo_version = m.v '
+        'ORDER BY r.day_id DESC LIMIT ?',
+        [limit],
+      );
+      List<String> key(List<Map<String, Object?>> rows) => [
+        for (final r in rows) '${r['day_id']}@${r['algo_version']}',
+      ];
 
-      await grow(30);
-      final short = await medianMs(() => LocalDb.recentDayResults(30));
       await grow(600);
-      final long = await medianMs(() => LocalDb.recentDayResults(30));
-
       expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30)));
-      expect(key(await LocalDb.recentDayResultsMeta(30)), key(await oracle(30)));
-      expect(long, lessThanOrEqualTo(short * 3 + 30),
-          reason: 'recentDayResults(30): $short ms at 30 days of history, '
-              '$long ms at 600');
+      expect(
+        key(await LocalDb.recentDayResultsMeta(30)),
+        key(await oracle(30)),
+      );
+
+      // Against the old query, timed back to back on the same machine, so a
+      // slow or loaded runner slows both sides alike. On 600 days the old one
+      // sorts ~60 MB of bundles to return 30; measured ~10x slower.
+      final now = await medianMs(() => LocalDb.recentDayResults(30));
+      final before = await medianMs(() => oracle(30));
+      expect(
+        now * 3,
+        lessThanOrEqualTo(before),
+        reason:
+            'recentDayResults(30) took $now ms at 600 days of history, '
+            'the full-history sort it replaced $before ms',
+      );
       await db.delete('day_result');
     });
   });

From c290d90fb9d67a5f01e3ada9e2c693fbecf237ed Mon Sep 17 00:00:00 2001
From: Alex Wagner <der.wagner@tum.de>
Date: Thu, 22 Oct 2026 16:00:14 +0200
Subject: [PATCH 4/9] test(soak): address review findings

- Concurrency group per event, so a manual run cannot cancel the nightly
  one before its CSV is uploaded.
- The load test asserts Home, today and the day list come back populated,
  so a load that got fast by returning nothing is not a pass.
- SOAK_HISTORY needs two or more distinct lengths, each at least SOAK_DAYS;
  otherwise the comparison checks a history against itself or mislabels it.
- Header notes that cloned rows keep the template's timestamps and blobs;
  nothing timed decodes them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/soak.yml    |  4 +++-
 test/db_growth_soak_test.dart | 25 +++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml
index dea22939a..4d15de745 100644
--- a/.github/workflows/soak.yml
+++ b/.github/workflows/soak.yml
@@ -16,8 +16,10 @@ on:
         description: History lengths (days) to time loads at
         default: '30,180,365,730'
 
+# Per event, so a manual run never cancels the nightly one before its report
+# is uploaded.
 concurrency:
-  group: soak-${{ github.ref }}
+  group: soak-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
 permissions:
diff --git a/test/db_growth_soak_test.dart b/test/db_growth_soak_test.dart
index 69560d81f..c31b8a93e 100644
--- a/test/db_growth_soak_test.dart
+++ b/test/db_growth_soak_test.dart
@@ -30,6 +30,10 @@
 //     branch, which still prunes. A healthy install prunes at the same cutoff:
 //     its unfinalized days are the newest ones, inside the retention window.
 //     The stuck-day hold is pure and covered by raw_prune_gating_test.
+//   * NOT REAL either: the cloned rows keep the template day's timestamps
+//     inside their payloads, and the cloned substrate_archive buckets keep
+//     the template's blob. Nothing timed here decodes either; they stand in
+//     for the bytes and the row counts, which is what this measures.
 //
 // The per-day numbers land in build/soak/db_growth.csv.
 
@@ -472,13 +476,34 @@ void main() {
           (_env['SOAK_HISTORY'] ?? '30,180,365,730')
               .split(',')
               .map(int.parse)
+              .toSet()
               .toList()
             ..sort();
+      // A single length compares a history with itself, and live days beyond
+      // the shortest length would give it more history than its label says.
+      expect(
+        history.length,
+        greaterThanOrEqualTo(2),
+        reason: 'SOAK_HISTORY needs two or more distinct lengths',
+      );
+      expect(
+        history.first,
+        greaterThanOrEqualTo(liveDays),
+        reason: 'SOAK_HISTORY lengths must be at least SOAK_DAYS',
+      );
       final results = <int, Map<String, int>>{};
       for (final days in history) {
         await extendHistoryTo(days);
         // One real pass so housekeeping (archive eviction) sees the new history.
         await DerivationEngine().run(_profile, heavy: true);
+        // A load that got fast by coming back empty is not a pass.
+        expect(
+          (await HomeData.load(repo)).dayId,
+          isNotNull,
+          reason: 'Home has no day at $days days of history',
+        );
+        expect(await repo.getToday(), isNotEmpty);
+        expect((await repo.availableDays()).length, greaterThanOrEqualTo(days));
         results[days] = {
           for (final e in loads.entries) e.key: await medianMs(e.value),
         };

From b253ce5b9db7331d4a893b00da6be41628695d7b Mon Sep 17 00:00:00 2001
From: Alex Wagner <der.wagner@tum.de>
Date: Thu, 22 Oct 2026 18:21:45 +0200
Subject: [PATCH 5/9] test(soak): DST-safe template day, validate SOAK_HISTORY
 up front

The template's second day stepped by Duration(days: 1), which lands on the
same calendar date across a fall-back DST night (AGENTS.md 3.7); use
calendar arithmetic like the rest of the file. SOAK_HISTORY is now parsed
and checked in setUpAll, so a bad value fails in seconds instead of after
the growth run. The SOAK_MAX_RATIO header now states the per-load
comparison and the 25 ms slack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/db_growth_soak_test.dart | 46 +++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/test/db_growth_soak_test.dart b/test/db_growth_soak_test.dart
index c31b8a93e..4ff8c3166 100644
--- a/test/db_growth_soak_test.dart
+++ b/test/db_growth_soak_test.dart
@@ -11,7 +11,8 @@
 //   SOAK_HISTORY       history lengths, in days, to time loads at
 //                      (30,180,365,730)
 //   SOAK_BUDGET_MB     steady-state growth budget, MB per day (3)
-//   SOAK_MAX_RATIO     slowest-history / shortest-history load time (2)
+//   SOAK_MAX_RATIO     longest-history / shortest-history load time, per
+//                      load, plus 25 ms of absolute slack (2)
 //
 // WHAT IS REAL AND WHAT IS NOT.
 //   * Every simulated day goes through `LocalDb.commitSyncBatch` — the same
@@ -124,6 +125,13 @@ void main() {
   // The last simulated day ends yesterday 22:00, so "today" in the app's sense
   // is the morning after the newest synced night.
   final first = DateTime(today.year, today.month, today.day - liveDays);
+  final history =
+      (_env['SOAK_HISTORY'] ?? '30,180,365,730')
+          .split(',')
+          .map(int.parse)
+          .toSet()
+          .toList()
+        ..sort();
   final snaps = <_Snap>[];
   late String template;
   late String path;
@@ -262,11 +270,26 @@ void main() {
 
   setUpAll(() async {
     if (!enabled) return;
+    // Checked here, not after the hours-long growth run. A single length
+    // compares a history with itself, and live days beyond the shortest
+    // length would give it more history than its label says.
+    expect(
+      history.length,
+      greaterThanOrEqualTo(2),
+      reason: 'SOAK_HISTORY needs two or more distinct lengths',
+    );
+    expect(
+      history.first,
+      greaterThanOrEqualTo(liveDays),
+      reason: 'SOAK_HISTORY lengths must be at least SOAK_DAYS',
+    );
     await deriveDbSetUp(_dbName);
     path = p.join(await databaseFactory.getDatabasesPath(), _dbName);
     // The template: one fully-synced night, derived for real.
     await ingest(SyntheticDay(first, seed: 1));
-    await ingest(SyntheticDay(first.add(const Duration(days: 1)), seed: 2));
+    await ingest(
+      SyntheticDay(DateTime(first.year, first.month, first.day + 1), seed: 2),
+    );
     template = dayLabelOf(first);
     await DerivationEngine().runDays(_profile, {template}, force: true);
     expect(
@@ -472,25 +495,6 @@ void main() {
         },
       };
 
-      final history =
-          (_env['SOAK_HISTORY'] ?? '30,180,365,730')
-              .split(',')
-              .map(int.parse)
-              .toSet()
-              .toList()
-            ..sort();
-      // A single length compares a history with itself, and live days beyond
-      // the shortest length would give it more history than its label says.
-      expect(
-        history.length,
-        greaterThanOrEqualTo(2),
-        reason: 'SOAK_HISTORY needs two or more distinct lengths',
-      );
-      expect(
-        history.first,
-        greaterThanOrEqualTo(liveDays),
-        reason: 'SOAK_HISTORY lengths must be at least SOAK_DAYS',
-      );
       final results = <int, Map<String, int>>{};
       for (final days in history) {
         await extendHistoryTo(days);

From 5f0d3c7581b527a1d547b359631fd5afb2966943 Mon Sep 17 00:00:00 2001
From: abdulsaheel <abdulsaheel@users.noreply.github.com>
Date: Fri, 30 Oct 2026 14:14:03 +0530
Subject: [PATCH 6/9] keep the existing tests as they were formatted

the test file was reformatted along with the new timing test, which buried
the one real addition in a few hundred lines of reflow.
---
 test/db_serve_version_and_reads_test.dart | 194 ++++++++++------------
 1 file changed, 86 insertions(+), 108 deletions(-)

diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart
index 93e5baaa5..832d156f8 100644
--- a/test/db_serve_version_and_reads_test.dart
+++ b/test/db_serve_version_and_reads_test.dart
@@ -46,10 +46,8 @@ void main() {
   test('the sibling pins match the SHAs kAlgoVersion was derived against', () {
     final deps =
         (loadYaml(File('pubspec.yaml').readAsStringSync())
-                as Map)['dependencies']
-            as Map;
-    String ref(String pkg) =>
-        ((deps[pkg] as Map)['git'] as Map)['ref'] as String;
+            as Map)['dependencies'] as Map;
+    String ref(String pkg) => ((deps[pkg] as Map)['git'] as Map)['ref'] as String;
 
     const why =
         'sibling repinned without visiting kAlgoVersion. bump it and say what '
@@ -64,8 +62,8 @@ void main() {
     // comments promise this test catches, and it did not until it read this
     // file. `resolved-ref` too: `ref` alone can name a moved branch.
     final locked =
-        (loadYaml(File('pubspec.lock').readAsStringSync()) as Map)['packages']
-            as Map;
+        (loadYaml(File('pubspec.lock').readAsStringSync())
+            as Map)['packages'] as Map;
     String lockRef(String pkg, String field) =>
         ((locked[pkg] as Map)['description'] as Map)[field] as String;
 
@@ -74,16 +72,10 @@ void main() {
         'partial repin. Move all three together, or a build resolves a '
         'sibling nobody reasoned about';
     for (final field in const ['ref', 'resolved-ref']) {
-      expect(
-        lockRef('openstrap_analytics', field),
-        kAnalyticsPin,
-        reason: whyLock,
-      );
-      expect(
-        lockRef('openstrap_protocol', field),
-        kProtocolPin,
-        reason: whyLock,
-      );
+      expect(lockRef('openstrap_analytics', field), kAnalyticsPin,
+          reason: whyLock);
+      expect(lockRef('openstrap_protocol', field), kProtocolPin,
+          reason: whyLock);
     }
   });
 
@@ -145,30 +137,26 @@ void main() {
       expect(await LocalDb.dayResultDayIdsDesc(), contains(day));
     });
 
-    test(
-      'the future row is kept, not deleted — re-upgrading gets it back',
-      () async {
-        const day = '2026-08-10';
-        final db = await LocalDb.instance;
-        final all = await db.query(
-          'day_result',
-          where: 'day_id = ?',
-          whereArgs: [day],
-        );
-        expect(all, hasLength(2));
-      },
-    );
+    test('the future row is kept, not deleted — re-upgrading gets it back',
+        () async {
+      const day = '2026-08-10';
+      final db = await LocalDb.instance;
+      final all = await db.query(
+        'day_result',
+        where: 'day_id = ?',
+        whereArgs: [day],
+      );
+      expect(all, hasLength(2));
+    });
 
-    test(
-      'a day that ONLY has a future row is served as absent, not wrong',
-      () async {
-        const day = '2026-08-11';
-        await _put(day, kAlgoVersion + 3, readiness: 12);
-        expect(await LocalDb.dayResult(day), isNull);
-        expect(await LocalDb.availableDayIds(), isNot(contains(day)));
-        expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
-      },
-    );
+    test('a day that ONLY has a future row is served as absent, not wrong',
+        () async {
+      const day = '2026-08-11';
+      await _put(day, kAlgoVersion + 3, readiness: 12);
+      expect(await LocalDb.dayResult(day), isNull);
+      expect(await LocalDb.availableDayIds(), isNot(contains(day)));
+      expect(await LocalDb.dayResultDayIdsDesc(), isNot(contains(day)));
+    });
 
     test('an OLDER version is still served — that is not the bug', () async {
       const day = '2026-08-09';
@@ -180,79 +168,69 @@ void main() {
   });
 
   group('read-path hygiene', () {
-    test(
-      'the decoded counter index is gone and sessions.start_ts is indexed',
-      () async {
-        final db = await LocalDb.instance;
-        final idx = await db.rawQuery(
-          "SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
-        );
-        final names = idx.map((r) => r['name']).toSet();
-        // Never on a read path: every decoded read orders by (rec_ts, counter),
-        // which a single-column counter index cannot serve.
-        expect(names, isNot(contains('idx_decoded_onehz_counter')));
-        // The most-called uncovered query in the UI seam was a full scan + sort.
-        expect(names, contains('idx_sessions_start'));
-      },
-    );
+    test('the decoded counter index is gone and sessions.start_ts is indexed',
+        () async {
+      final db = await LocalDb.instance;
+      final idx = await db.rawQuery(
+        "SELECT name, tbl_name FROM sqlite_master WHERE type='index'",
+      );
+      final names = idx.map((r) => r['name']).toSet();
+      // Never on a read path: every decoded read orders by (rec_ts, counter),
+      // which a single-column counter index cannot serve.
+      expect(names, isNot(contains('idx_decoded_onehz_counter')));
+      // The most-called uncovered query in the UI seam was a full scan + sort.
+      expect(names, contains('idx_sessions_start'));
+    });
 
-    test(
-      'decodedRecTsMaxByDay matches the strftime grouping it replaced',
-      () async {
-        final db = await LocalDb.instance;
-        await db.delete('decoded_onehz');
-        // Three local days' worth of edges, plus a second row per day so the MAX
-        // actually has to choose.
-        final days = <String>[];
-        var t =
-            DateTime.now()
-                .subtract(const Duration(days: 3))
-                .millisecondsSinceEpoch ~/
-            1000;
-        for (var i = 0; i < 3; i++) {
-          final label = dayLabelOf(
-            DateTime.fromMillisecondsSinceEpoch(
-              t * 1000,
-              isUtc: true,
-            ).toLocal(),
-          );
-          days.add(label);
-          final start = localDayStartSec(label)!;
-          final end = localDayEndSec(label)!;
-          for (final ts in [start + 60, end - 90]) {
-            await db.insert('decoded_onehz', {
-              // v47 key — see _createDecodedStore.
-              'ts_ms': ts * 1000,
-              'rec_ts': ts,
-              'counter': ts,
-              'hr': 60,
-            });
-          }
-          t = end + 60;
+    test('decodedRecTsMaxByDay matches the strftime grouping it replaced',
+        () async {
+      final db = await LocalDb.instance;
+      await db.delete('decoded_onehz');
+      // Three local days' worth of edges, plus a second row per day so the MAX
+      // actually has to choose.
+      final days = <String>[];
+      var t = DateTime.now()
+              .subtract(const Duration(days: 3))
+              .millisecondsSinceEpoch ~/
+          1000;
+      for (var i = 0; i < 3; i++) {
+        final label = dayLabelOf(
+          DateTime.fromMillisecondsSinceEpoch(t * 1000, isUtc: true).toLocal(),
+        );
+        days.add(label);
+        final start = localDayStartSec(label)!;
+        final end = localDayEndSec(label)!;
+        for (final ts in [start + 60, end - 90]) {
+          await db.insert('decoded_onehz', {
+            // v47 key — see _createDecodedStore.
+            'ts_ms': ts * 1000,
+            'rec_ts': ts,
+            'counter': ts,
+            'hr': 60,
+          });
         }
+        t = end + 60;
+      }
 
-        final got = await LocalDb.decodedRecTsMaxByDay();
-        // The old implementation, verbatim, as the oracle.
-        final oracle = await db.rawQuery(
-          "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
-          'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
-        );
-        final want = {
-          for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
-        };
-        expect(got, want);
-        expect(got.keys.toSet(), days.toSet());
-      },
-    );
+      final got = await LocalDb.decodedRecTsMaxByDay();
+      // The old implementation, verbatim, as the oracle.
+      final oracle = await db.rawQuery(
+        "SELECT strftime('%Y-%m-%d', rec_ts, 'unixepoch', 'localtime') AS d, "
+        'MAX(rec_ts) AS mx FROM decoded_onehz WHERE rec_ts > 0 GROUP BY d',
+      );
+      final want = {
+        for (final r in oracle) r['d'] as String: (r['mx'] as num).toInt(),
+      };
+      expect(got, want);
+      expect(got.keys.toSet(), days.toSet());
+    });
 
-    test(
-      'decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
-      () async {
-        final db = await LocalDb.instance;
-        await db.delete('decoded_onehz');
-        expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
-      },
-    );
+    test('decodedRecTsMaxByDay on an empty store is empty, not a 1970 day',
+        () async {
+      final db = await LocalDb.instance;
+      await db.delete('decoded_onehz');
+      expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty);
+    });
 
     test('the newest-N reads stay flat as history grows', () async {
       // recentDayResults joined EVERY day's full row (~100 KB of bundle) and

From 2bbba0b65ad2607d32497f8e4b0004ce780b9ced Mon Sep 17 00:00:00 2001
From: abdulsaheel <abdulsaheel@users.noreply.github.com>
Date: Fri, 30 Oct 2026 14:14:03 +0530
Subject: [PATCH 7/9] pick the newest days first in recentDayScalars too

same sort-everything-then-limit shape as recentDayResults: every served row
was extracted and sorted to return the last N.
---
 lib/data/db.dart | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/data/db.dart b/lib/data/db.dart
index e45aa1937..886977641 100644
--- a/lib/data/db.dart
+++ b/lib/data/db.dart
@@ -9635,8 +9635,8 @@ class LocalDb {
     final rows = await db.rawQuery(
       'SELECT r.day_id${cols.isEmpty ? '' : ', ${cols.join(', ')}'} '
       'FROM day_result r '
-      '$_servedDayJoin '
-      'ORDER BY r.day_id DESC LIMIT ?',
+      '$_servedNewestDaysJoin '
+      'ORDER BY r.day_id DESC',
       [limit],
     );
     return [for (final r in rows) _withDate(r)];

From bfb3617d6f3bd9dc5a349173b886a20a71b334cc Mon Sep 17 00:00:00 2001
From: abdulsaheel <abdulsaheel@users.noreply.github.com>
Date: Sun, 1 Nov 2026 03:27:04 +0530
Subject: [PATCH 8/9] run the sibling pin guard before pub get in the soak
 workflow

---
 .github/workflows/soak.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml
index 4d15de745..4b78005db 100644
--- a/.github/workflows/soak.yml
+++ b/.github/workflows/soak.yml
@@ -45,6 +45,9 @@ jobs:
           channel: stable
           cache: true
 
+      - name: Guard — the lock pins sibling commits, and the pins aren't stale
+        run: bash .github/scripts/check_sibling_pins.sh
+
       - name: Install dependencies
         run: flutter pub get
 

From 2953cfcea78d470b2e5a18e68cda1b36250cad88 Mon Sep 17 00:00:00 2001
From: abdulsaheel <abdulsaheel@users.noreply.github.com>
Date: Sun, 1 Nov 2026 04:52:01 +0530
Subject: [PATCH 9/9] write the soak report however the run ends

---
 test/db_growth_soak_test.dart | 60 +++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/test/db_growth_soak_test.dart b/test/db_growth_soak_test.dart
index 4ff8c3166..cf1898167 100644
--- a/test/db_growth_soak_test.dart
+++ b/test/db_growth_soak_test.dart
@@ -245,8 +245,15 @@ void main() {
   }
 
   void writeReport() {
-    final tables = <String>{for (final s in snaps) ...s.tables.keys}.toList()
-      ..sort((a, b) => snaps.last.tables[b]!.compareTo(snaps.last.tables[a]!));
+    final tables = <String>{for (final s in snaps) ...s.tables.keys}.toList();
+    // A run that failed before its first snapshot still writes the header.
+    if (snaps.isEmpty) {
+      tables.sort();
+    } else {
+      tables.sort(
+        (a, b) => snaps.last.tables[b]!.compareTo(snaps.last.tables[a]!),
+      );
+    }
     final out = File('build/soak/db_growth.csv')..createSync(recursive: true);
     out.writeAsStringSync(
       [
@@ -313,29 +320,34 @@ void main() {
             'SOAK_DAYS must leave a steady-state window after $warmup '
             'warm-up days',
       );
-      for (var i = 2; i < liveDays; i++) {
-        final sw = Stopwatch()..start();
-        await ingest(
-          SyntheticDay(
-            DateTime(first.year, first.month, first.day + i),
-            seed: 1 + i,
-          ),
-        );
-        await finalizeRawDays();
-        await DerivationEngine().run(_profile, heavy: true);
-        await LocalDb.vacuumIfBloated();
-        final s = await snapshot(i, sw.elapsedMilliseconds);
-        snaps.add(s);
-        // ignore: avoid_print
-        print(
-          '[soak] day $i: used ${(s.usedBytes / _mb).toStringAsFixed(1)} MB'
-          ', file ${(s.fileBytes / _mb).toStringAsFixed(1)} MB'
-          ', free ${(s.freeBytes / _mb).toStringAsFixed(1)} MB'
-          ', wal ${(s.walBytes / _mb).toStringAsFixed(1)} MB'
-          ', onehz ${s.onehzRows} rows, ${s.dayMs} ms',
-        );
+      // The report is the artifact a failed run is read from, so it is
+      // written however the loop ends.
+      try {
+        for (var i = 2; i < liveDays; i++) {
+          final sw = Stopwatch()..start();
+          await ingest(
+            SyntheticDay(
+              DateTime(first.year, first.month, first.day + i),
+              seed: 1 + i,
+            ),
+          );
+          await finalizeRawDays();
+          await DerivationEngine().run(_profile, heavy: true);
+          await LocalDb.vacuumIfBloated();
+          final s = await snapshot(i, sw.elapsedMilliseconds);
+          snaps.add(s);
+          // ignore: avoid_print
+          print(
+            '[soak] day $i: used ${(s.usedBytes / _mb).toStringAsFixed(1)} MB'
+            ', file ${(s.fileBytes / _mb).toStringAsFixed(1)} MB'
+            ', free ${(s.freeBytes / _mb).toStringAsFixed(1)} MB'
+            ', wal ${(s.walBytes / _mb).toStringAsFixed(1)} MB'
+            ', onehz ${s.onehzRows} rows, ${s.dayMs} ms',
+          );
+        }
+      } finally {
+        writeReport();
       }
-      writeReport();
 
       final steady = snaps.where((s) => s.day >= warmup).toList();
       final base = steady.first;