diff --git a/lib/data/db.dart b/lib/data/db.dart index 51cf3d383..513bf7d2e 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -9619,11 +9619,21 @@ class LocalDb { /// THIS rather than writing its own `MAX(algo_version)` — the pattern had /// already reappeared at five readers and two views, and the ceiling has to /// be on all of them or it is on none of them. - static const String _servedDayJoin = - 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' - 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id) m ' + static const String _servedDayJoin = 'JOIN ($_servedVersions) m $_servedOn'; + static const String _servedVersions = + 'SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $_servedAlgoCeiling GROUP BY day_id'; + static const String _servedOn = 'ON r.day_id = m.day_id AND r.algo_version = m.v'; + /// [_servedDayJoin] for the newest N days only (bind N). A LIMIT on the + /// outer query comes after the join and its sort, so SQLite fetched and + /// sorted EVERY day's row, bundle and all, to return N: a 30-day read went + /// from ~60 ms to 250-600 ms between 30 and 730 days of history. Choosing + /// the days first, from the covering index, leaves N rows to fetch. + static const String _servedNewestDaysJoin = + 'JOIN ($_servedVersions ORDER BY day_id DESC LIMIT ?) m $_servedOn'; + /// The result row for one day_id at the highest version this build serves /// (see [_servedAlgoCeiling]), with a normalized `date` alias for callers. /// Null if absent. @@ -9824,8 +9834,8 @@ class LocalDb { // For each day_id pick MAX(algo_version), then join back for the full row. final rows = await db.rawQuery( 'SELECT r.* FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; @@ -9849,8 +9859,8 @@ class LocalDb { 'SELECT r.day_id, r.algo_version, r.computed_at, r.finalized, ' 'r.skipped, r.partial, r.rhr, r.rmssd, r.readiness ' 'FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; @@ -9896,8 +9906,8 @@ class LocalDb { final rows = await db.rawQuery( 'SELECT r.day_id${cols.isEmpty ? '' : ', ${cols.join(', ')}'} ' 'FROM day_result r ' - '$_servedDayJoin ' - 'ORDER BY r.day_id DESC LIMIT ?', + '$_servedNewestDaysJoin ' + 'ORDER BY r.day_id DESC', [limit], ); return [for (final r in rows) _withDate(r)]; diff --git a/test/db_serve_version_and_reads_test.dart b/test/db_serve_version_and_reads_test.dart index f160691c5..832d156f8 100644 --- a/test/db_serve_version_and_reads_test.dart +++ b/test/db_serve_version_and_reads_test.dart @@ -231,5 +231,84 @@ void main() { await db.delete('decoded_onehz'); expect(await LocalDb.decodedRecTsMaxByDay(), isEmpty); }); + + test('the newest-N reads stay flat as history grows', () async { + // recentDayResults joined EVERY day's full row (~100 KB of bundle) and + // sorted them all before its LIMIT, so reading the last 30 days took + // 57 ms at 30 days of history and 250-600 ms at 730 — on every + // refreshComputeFreshness, i.e. every derive and every Home load whose + // night is still building. + final db = await LocalDb.instance; + await db.delete('day_result'); + final bundle = '{"pad":"${'x' * 100000}"}'; + final today = DateTime.now(); + Future grow(int days) async { + final b = db.batch(); + for (var i = 0; i < days; i++) { + final day = dayLabelOf( + DateTime(today.year, today.month, today.day - i), + ); + for (final v in [kAlgoVersion - 1, kAlgoVersion, kAlgoVersion + 1]) { + // Every third day stops at an older version; the rest also carry a + // row from a future build, which must never be served. + if (v == kAlgoVersion && i % 3 == 0) continue; + b.insert('day_result', { + 'day_id': day, + 'algo_version': v, + 'payload_json': bundle, + 'window_json': '{}', + 'computed_at': 0, + }); + } + } + await b.commit(noResult: true); + } + + Future medianMs(Future Function() f) async { + await f(); + final us = []; + for (var i = 0; i < 5; i++) { + final sw = Stopwatch()..start(); + await f(); + us.add(sw.elapsedMicroseconds); + } + us.sort(); + return us[2] ~/ 1000; + } + + // The query it replaced, verbatim, as the oracle for WHICH rows come back. + Future>> oracle(int limit) => db.rawQuery( + 'SELECT r.* FROM day_result r ' + 'JOIN (SELECT day_id, MAX(algo_version) AS v FROM day_result ' + 'WHERE algo_version <= $kAlgoVersion GROUP BY day_id) m ' + 'ON r.day_id = m.day_id AND r.algo_version = m.v ' + 'ORDER BY r.day_id DESC LIMIT ?', + [limit], + ); + List key(List> rows) => [ + for (final r in rows) '${r['day_id']}@${r['algo_version']}', + ]; + + await grow(600); + expect(key(await LocalDb.recentDayResults(30)), key(await oracle(30))); + expect( + key(await LocalDb.recentDayResultsMeta(30)), + key(await oracle(30)), + ); + + // Against the old query, timed back to back on the same machine, so a + // slow or loaded runner slows both sides alike. On 600 days the old one + // sorts ~60 MB of bundles to return 30; measured ~10x slower. + final now = await medianMs(() => LocalDb.recentDayResults(30)); + final before = await medianMs(() => oracle(30)); + expect( + now * 3, + lessThanOrEqualTo(before), + reason: + 'recentDayResults(30) took $now ms at 600 days of history, ' + 'the full-history sort it replaced $before ms', + ); + await db.delete('day_result'); + }); }); }