Canary: validate cached collections and bulk SQLite writes

src/store/collection.js

@@ -5,12 +5,13 @@ import { readEncryptedJson, writeEncryptedJson } from "./crypto.js";
 import {
   countRecords,
   deleteRecord,
+  deleteRecords,
   getMeta,
-  getRecord,
   listRecords,
   replaceRecords,
   setMeta,
   upsertRecord,
+  upsertRecords,
 } from "./database.js";
 
 const MIRROR_DELAY_MS = 150;
@@ -23,6 +24,7 @@ export function flushCollectionMirrors() {
 export function createCollection({ name, legacyFile, normalize = (value) => value, validate = (value) => Boolean(value?.id) }) {
   const legacyPath = path.join(config.DATA_DIR, legacyFile);
   const migrationKey = `legacy-json-migrated:${name}`;
+  const cache = new Map();
   let loaded = false;
   let mirrorDirty = false;
   let mirrorTimer = null;
@@ -33,8 +35,13 @@ export function createCollection({ name, legacyFile, normalize = (value) => valu
     return validate(normalized) ? normalized : null;
   }
 
+  function cacheRecords(records) {
+    cache.clear();
+    for (const record of records) cache.set(record.id, record);
+  }
+
   function currentRecords() {
-    return listRecords(name).map(clean).filter(Boolean);
+    return [...cache.values()].map((record) => structuredClone(record));
   }
 
   function flushMirror() {
@@ -67,6 +74,7 @@ export function createCollection({ name, legacyFile, normalize = (value) => valu
       setMeta(migrationKey, JSON.stringify({ importedAt: new Date().toISOString(), count: records.length }));
       console.log(`[noema] Imported ${records.length} ${name} records from ${legacyFile} into SQLite.`);
     }
+    cacheRecords(listRecords(name).map(clean).filter(Boolean));
     loaded = true;
     return currentRecords();
   }
@@ -78,37 +86,65 @@ export function createCollection({ name, legacyFile, normalize = (value) => valu
 
   function get(id) {
     if (!loaded) load();
-    const value = getRecord(name, id);
-    return value ? clean(value) : null;
+    const value = cache.get(String(id));
+    return value ? structuredClone(value) : null;
   }
 
   function set(value) {
     if (!loaded) load();
     const normalized = clean(value);
     if (!normalized) throw new Error(`Invalid ${name} record.`);
     upsertRecord(name, normalized);
+    cache.set(normalized.id, normalized);
+    scheduleMirror();
+    return structuredClone(normalized);
+  }
+
+  function setMany(values) {
+    if (!loaded) load();
+    const records = (Array.isArray(values) ? values : []).map(clean).filter(Boolean);
+    if (!records.length) return [];
+    upsertRecords(name, records);
+    for (const record of records) cache.set(record.id, record);
     scheduleMirror();
-    return normalized;
+    return records.map((record) => structuredClone(record));
   }
 
   function remove(id) {
     if (!loaded) load();
-    const removed = deleteRecord(name, id);
-    if (removed) scheduleMirror();
+    const key = String(id);
+    const removed = deleteRecord(name, key);
+    if (removed) {
+      cache.delete(key);
+      scheduleMirror();
+    }
     return removed;
   }
 
+  function removeMany(ids) {
+    if (!loaded) load();
+    const values = [...new Set((Array.isArray(ids) ? ids : []).map((id) => String(id)).filter(Boolean))];
+    if (!values.length) return 0;
+    const count = deleteRecords(name, values);
+    if (count) {
+      for (const id of values) cache.delete(id);
+      scheduleMirror();
+    }
+    return count;
+  }
+
   function replace(values) {
     if (!loaded) load();
     const records = (Array.isArray(values) ? values : []).map(clean).filter(Boolean);
     replaceRecords(name, records);
+    cacheRecords(records);
     scheduleMirror();
-    return records;
+    return records.map((record) => structuredClone(record));
   }
 
   function close() {
     flushMirror();
   }
 
-  return { load, list, get, set, remove, replace, close, legacyPath };
+  return { load, list, get, set, setMany, remove, removeMany, replace, close, legacyPath };
 }

src/store/database.js

@@ -6,6 +6,7 @@ import { decryptData, encryptData } from "./crypto.js";
 
 const DATABASE_FILE = path.join(config.DATA_DIR, "noema.sqlite");
 let database = null;
+let transactionDepth = 0;
 
 function requireCollection(value) {
   const collection = String(value || "").trim();
@@ -37,6 +38,7 @@ export function openDatabase() {
     PRAGMA synchronous = NORMAL;
     PRAGMA foreign_keys = ON;
     PRAGMA busy_timeout = 5000;
+    PRAGMA temp_store = MEMORY;
     CREATE TABLE IF NOT EXISTS noema_meta (
       key TEXT PRIMARY KEY,
       value TEXT NOT NULL
@@ -61,11 +63,6 @@ export function databasePath() {
   return DATABASE_FILE;
 }
 
-/**
- * Verify the currently loaded installation key against one real encrypted
- * record before rewriting legacy master.key metadata. Empty databases are
- * valid and need no additional check.
- */
 export function assertDatabaseCryptoReadable() {
   const row = openDatabase().prepare("SELECT payload FROM noema_records LIMIT 1").get();
   if (!row) return true;
@@ -132,17 +129,73 @@ export function deleteRecord(collection, id) {
   return Number(result.changes || 0) > 0;
 }
 
-export function replaceRecords(collection, records) {
+export function deleteRecords(collection, ids) {
+  const name = requireCollection(collection);
+  const values = [...new Set((Array.isArray(ids) ? ids : []).map((id) => String(id)).filter(Boolean))];
+  if (!values.length) return 0;
+  return withImmediateTransaction((db) => {
+    const remove = db.prepare("DELETE FROM noema_records WHERE collection = ? AND id = ?");
+    let count = 0;
+    for (const id of values) count += Number(remove.run(name, id).changes || 0);
+    return count;
+  });
+}
+
+export function upsertRecords(collection, records) {
   const name = requireCollection(collection);
   const values = Array.isArray(records) ? records : [];
+  return withImmediateTransaction((db) => {
+    const upsert = db.prepare(`
+      INSERT INTO noema_records (collection, id, payload, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(collection, id) DO UPDATE SET
+        payload = excluded.payload,
+        created_at = excluded.created_at,
+        updated_at = excluded.updated_at
+    `);
+    let count = 0;
+    for (const record of values) {
+      if (!record || typeof record !== "object" || typeof record.id !== "string" || !record.id) continue;
+      const now = Date.now();
+      const createdAt = Number.isFinite(record.createdAt) ? record.createdAt : now;
+      const updatedAt = Number.isFinite(record.updatedAt) ? record.updatedAt : createdAt;
+      upsert.run(name, record.id, encodeRecord(record), createdAt, updatedAt);
+      count += 1;
+    }
+    return count;
+  });
+}
+
+export function withImmediateTransaction(callback) {
+  if (typeof callback !== "function") throw new TypeError("Transaction callback is required.");
   const db = openDatabase();
-  const remove = db.prepare("DELETE FROM noema_records WHERE collection = ?");
-  const insert = db.prepare(`
-    INSERT INTO noema_records (collection, id, payload, created_at, updated_at)
-    VALUES (?, ?, ?, ?, ?)
-  `);
+  if (transactionDepth > 0) return callback(db);
   db.exec("BEGIN IMMEDIATE");
+  transactionDepth += 1;
   try {
+    const result = callback(db);
+    if (result && typeof result.then === "function") {
+      throw new Error("SQLite transaction callback must be synchronous.");
+    }
+    db.exec("COMMIT");
+    return result;
+  } catch (error) {
+    try { db.exec("ROLLBACK"); } catch {}
+    throw error;
+  } finally {
+    transactionDepth -= 1;
+  }
+}
+
+export function replaceRecords(collection, records) {
+  const name = requireCollection(collection);
+  const values = Array.isArray(records) ? records : [];
+  return withImmediateTransaction((db) => {
+    const remove = db.prepare("DELETE FROM noema_records WHERE collection = ?");
+    const insert = db.prepare(`
+      INSERT INTO noema_records (collection, id, payload, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);
     remove.run(name);
     for (const record of values) {
       if (!record || typeof record.id !== "string" || !record.id) continue;
@@ -151,11 +204,7 @@ export function replaceRecords(collection, records) {
       const updatedAt = Number.isFinite(record.updatedAt) ? record.updatedAt : createdAt;
       insert.run(name, record.id, encodeRecord(record), createdAt, updatedAt);
     }
-    db.exec("COMMIT");
-  } catch (error) {
-    try { db.exec("ROLLBACK"); } catch {}
-    throw error;
-  }
+  });
 }
 
 export function checkpointDatabase() {
@@ -168,4 +217,5 @@ export function closeDatabase() {
   checkpointDatabase();
   database.close();
   database = null;
+  transactionDepth = 0;
 }