diff --git a/docs/d1-polling-cache-operations.md b/docs/d1-polling-cache-operations.md index 672e971..1f43d60 100644 --- a/docs/d1-polling-cache-operations.md +++ b/docs/d1-polling-cache-operations.md @@ -6,6 +6,40 @@ tables. Relevant source writes and counter changes commit or roll back together. Missing state and unsafe counter overflow reject writes rather than silently lose notifications. No application mutation caller needs an invalidation hook. +### Mentions and responses domains (migration 0032) + +`0032_mentions_change_domain.sql` adds two counters to `change_state` and six +triggers (insert, update, delete on `mentions_in` and `mentions_out`), so the +tracked set is now 36 `change_*` triggers over twelve tables (27 from 0024, three +from 0026 on `read_state`, six here): + +| Domain | Moves when | Read by | +|---|---|---| +| `mentions` | any `mentions_out` change; a `mentions_in` change to a row whose status is, or was, `verified` or `gone` | Studio mentions view (both lists) and, with `reading`, each mention's source card | +| `responses` | a `mentions_in` change to a row that is, or was, `verified` and not hidden | public HTML: the responses list under an item (`listPublicResponses`) | + +`responses` is in the public HTML cache key (`publicHtmlRevision`), so a newly +verified response, a hide or a show reaches the item page at once instead of +after the edge window (60 s plus up to 300 s stale-while-revalidate). A pending +claim, its failure and the daily prune of failed claims move neither counter: +anyone can POST a claim, and if claims rolled the key a stranger could empty +the edge cache at the receiver's rate cap. They still write the state row once, +like every tracked table, and are refused if it is missing. + +Apply 0032 before deploying a Worker that reads these columns: `/api/changes` +selects every domain by name and fails without them, which stops Studio from +loading new data, and public pages fall back to the plain cache key (the +pre-0.46.1 60-second behaviour) until the migration lands. The first deploy +after 0032 changes every public cache key once, so each page renders fresh on +its next read. + +Studio already polled the mentions view every 15 seconds, re-reading both lists +in full and every row's source card on each tick. Those reads are now gated on +the counters like every other owner read, so an unchanged tick costs one +`/api/changes` read per collection. Polls stop while the tab or installed PWA is +hidden and run at once on `visibilitychange` back to visible and on window +focus (`src/ui/polling.ts`), for every domain. + The feed uses the existing `MEDIA` R2 binding. Objects under `__cache/feed/` are internal XML artifacts, separate from attachment paths. Keys include software/ renderer version, request origin and mount. No new secret, bucket or Durable @@ -69,7 +103,11 @@ write totals separately; these changes do not establish a free-plan capacity. ## Database restores Pause requests and disable cron triggers before restoring/replacing D1. Apply any missing migrations if -the backup predates `0024_change_state.sql`. Then run: +the backup predates `0024_change_state.sql`, and in particular `0032_mentions_change_domain.sql`: +the reset script below names the `mentions` and `responses` columns and fails on a +database without them. A restore brings back mention rows as they were at the backup, +so a response verified or hidden since then is undone on the public page too; the new +epoch makes every Studio and public cache entry re-read. Then run: ```sh npx wrangler d1 execute DB --remote --file scripts/reset-change-epoch.sql diff --git a/e2e/polling-cache.spec.ts b/e2e/polling-cache.spec.ts index 5234e55..e3cfed7 100644 --- a/e2e/polling-cache.spec.ts +++ b/e2e/polling-cache.spec.ts @@ -75,3 +75,49 @@ test('a buffered query response keeps its pre-fetch revision and receives a late await expect(page.getByText(marker, { exact: false }).first()).toBeVisible(); } finally { release(); } }); + +// Delivery was never the delay: live mentions verify in seconds. What lagged +// was every view of them. A verified mention written by the receiver (here the +// fixture's stand-in for verifyMention) must reach an open mentions view within +// one poll, and a view resumed from the background must check at once. +test('a newly verified mention reaches the open mentions view in one poll, and a resumed view checks at once', async ({ page, request }) => { + test.setTimeout(60_000); + await page.clock.install(); + await page.goto('/studio/login'); await page.locator('[name=password]').fill('test-password'); + await page.getByRole('button', { name: 'log in', exact: true }).click(); await expect(page.locator('#composer-text')).toBeVisible(); + const marker = `Live mention target ${crypto.randomUUID()}`; + const item = await page.evaluate(async content_md => { + const created = await (await fetch('/api/items', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ content_md, kind: 'thread' }) })).json() as { id: string }; + await fetch(`/api/items/${created.id}/publish`, { method: 'POST' }); + return created.id; + }, marker); + await page.goto('/studio/mentions'); + await expect(page.locator('.mention-group').first()).toBeVisible(); + await page.clock.pauseAt(new Date(Date.now() + 10_000)); + const group = page.locator('.mention-group').filter({ hasText: marker }); + await expect(group).toHaveCount(0); + const id = `live${Date.now()}`; + expect((await request.post('http://127.0.0.1:8789/__fixture/verified-mention', { data: { id, item, origin: 'https://responder.example/' } })).status()).toBe(204); + await page.clock.runFor(15_000); + await expect(group).toContainText('1 responses'); + await expect(group).toContainText('Live responder'); + await expect(group.getByRole('button', { name: 'hide from page', exact: true })).toBeVisible(); + + // Backgrounded: no reads at all, however long it stays hidden. + await page.evaluate(() => { + Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => 'hidden' }); + document.dispatchEvent(new Event('visibilitychange')); + }); + const reads: string[] = []; + page.on('request', r => { if (r.method() === 'GET' && new URL(r.url()).pathname.startsWith('/api/')) reads.push(new URL(r.url()).pathname); }); + expect(await page.evaluate(async id => (await fetch(`/api/mentions/${id}`, { method: 'PATCH', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ hidden: true }) })).ok, id)).toBe(true); + await page.clock.runFor(60_000); + expect(reads).toEqual([]); + // Resumed: the check runs on the visibility change, not at the next tick. + await page.evaluate(() => { + Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => 'visible' }); + document.dispatchEvent(new Event('visibilitychange')); + }); + await expect(group.getByRole('button', { name: 'show on page', exact: true })).toBeVisible(); + expect(reads).toContain('/api/changes'); +}); diff --git a/migrations/0032_mentions_change_domain.sql b/migrations/0032_mentions_change_domain.sql new file mode 100644 index 0000000..01dcd77 --- /dev/null +++ b/migrations/0032_mentions_change_domain.sql @@ -0,0 +1,85 @@ +-- Two revision domains for Webmentions, so their views stop waiting out a +-- timer: `mentions` for the Studio (GET /mentions, both directions) and +-- `responses` for public HTML (the responses list under an item, which reads +-- mentions_in through listPublicResponses). +-- +-- They are separate because their response dependencies differ, and because +-- the public one must not move on unverified claims. Anyone can POST a claim; +-- if every pending row rolled the public cache key, a stranger could empty the +-- edge cache at the receiver's rate cap. So: +-- mentions moves on any mentions_out change, and on a mentions_in change +-- that touches a row the Studio lists (status verified or gone, +-- before or after). +-- responses moves on a mentions_in change that touches a row the public list +-- shows (status verified and not hidden, before or after): a new +-- verification, a hide, a show, a re-claim that sends it back to +-- pending, a withdrawal found on re-verification. +-- A pending claim, and its failure, move neither; they still pay the one +-- state-row write every tracked table pays, and still need the state row. +-- +-- Same shape as 0024's guards: SELECT RAISE ... WHERE for the missing row, and +-- every CASE closed inside parentheses, never by END and a semicolon, which +-- D1's remote executor takes as the end of the trigger (session 37). +ALTER TABLE change_state ADD COLUMN mentions INTEGER NOT NULL DEFAULT 0 CHECK(typeof(mentions)='integer' AND mentions BETWEEN 0 AND 9007199254740991); +ALTER TABLE change_state ADD COLUMN responses INTEGER NOT NULL DEFAULT 0 CHECK(typeof(responses)='integer' AND responses BETWEEN 0 AND 9007199254740991); + +CREATE TRIGGER change_mentions_in_insert +AFTER INSERT ON mentions_in +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(CASE WHEN NEW.status IN ('verified','gone') THEN 1 ELSE 0 END), + responses=responses+(CASE WHEN NEW.status='verified' AND NEW.hidden=0 THEN 1 ELSE 0 END) + WHERE id=1; +END; + +CREATE TRIGGER change_mentions_in_update +AFTER UPDATE ON mentions_in +WHEN OLD."attempts" IS NOT NEW."attempts" OR OLD."error" IS NOT NEW."error" OR OLD."first_seen" IS NOT NEW."first_seen" OR OLD."hidden" IS NOT NEW."hidden" OR OLD."id" IS NOT NEW."id" OR OLD."last_seen" IS NOT NEW."last_seen" OR OLD."relation" IS NOT NEW."relation" OR OLD."source" IS NOT NEW."source" OR OLD."source_author_json" IS NOT NEW."source_author_json" OR OLD."source_id" IS NOT NEW."source_id" OR OLD."source_kind" IS NOT NEW."source_kind" OR OLD."source_origin" IS NOT NEW."source_origin" OR OLD."source_page" IS NOT NEW."source_page" OR OLD."source_version" IS NOT NEW."source_version" OR OLD."status" IS NOT NEW."status" OR OLD."target" IS NOT NEW."target" OR OLD."target_item_id" IS NOT NEW."target_item_id" OR OLD."verified_at" IS NOT NEW."verified_at" +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(CASE WHEN OLD.status IN ('verified','gone') OR NEW.status IN ('verified','gone') THEN 1 ELSE 0 END), + responses=responses+(CASE WHEN (OLD.status='verified' AND OLD.hidden=0) OR (NEW.status='verified' AND NEW.hidden=0) THEN 1 ELSE 0 END) + WHERE id=1; +END; + +CREATE TRIGGER change_mentions_in_delete +AFTER DELETE ON mentions_in +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(CASE WHEN OLD.status IN ('verified','gone') THEN 1 ELSE 0 END), + responses=responses+(CASE WHEN OLD.status='verified' AND OLD.hidden=0 THEN 1 ELSE 0 END) + WHERE id=1; +END; + +-- The Studio lists every outbound row, status and retry time included, so any +-- change is a mentions change. Outbound rows never reach a public page. +CREATE TRIGGER change_mentions_out_insert +AFTER INSERT ON mentions_out +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(1) + WHERE id=1; +END; + +CREATE TRIGGER change_mentions_out_update +AFTER UPDATE ON mentions_out +WHEN OLD."attempts" IS NOT NEW."attempts" OR OLD."created" IS NOT NEW."created" OR OLD."endpoint" IS NOT NEW."endpoint" OR OLD."id" IS NOT NEW."id" OR OLD."item_id" IS NOT NEW."item_id" OR OLD."last_error" IS NOT NEW."last_error" OR OLD."next_attempt_at" IS NOT NEW."next_attempt_at" OR OLD."status" IS NOT NEW."status" OR OLD."target" IS NOT NEW."target" OR OLD."target_version" IS NOT NEW."target_version" OR OLD."version" IS NOT NEW."version" +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(1) + WHERE id=1; +END; + +CREATE TRIGGER change_mentions_out_delete +AFTER DELETE ON mentions_out +BEGIN + SELECT RAISE(ABORT, 'missing change state') WHERE NOT EXISTS(SELECT 1 FROM change_state WHERE id=1); + UPDATE change_state SET + mentions=mentions+(1) + WHERE id=1; +END; diff --git a/openapi.json b/openapi.json index 1ad20a4..5218477 100644 --- a/openapi.json +++ b/openapi.json @@ -187,6 +187,16 @@ "type": "integer", "minimum": 0, "maximum": 9007199254740991 + }, + "mentions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "responses": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "required": [ @@ -196,7 +206,9 @@ "hoppers", "signals", "settings", - "feed" + "feed", + "mentions", + "responses" ] } }, diff --git a/scripts/e2e-server.ts b/scripts/e2e-server.ts index f08d9fa..f204aed 100644 --- a/scripts/e2e-server.ts +++ b/scripts/e2e-server.ts @@ -76,6 +76,20 @@ const proxy = createServer(async (request, response) => { response.end(''); return; } + // Disposable stand-in for a sender whose claim this blyg has just verified: + // writes one verified inbound mention straight into the 8787 fixture + // database, as verifyMention's UPDATE would. Fixture-only; the Worker ships + // no such route (e2e/polling-cache.spec.ts, live mentions). + if (path === "/__fixture/verified-mention" && request.method === "POST") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const { id, item, origin } = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { id: string; item: string; origin: string }; + const now = new Date().toISOString(), source = `${origin}t/${id}/`; + await db.prepare("INSERT INTO mentions_in (id, source, target, target_item_id, status, relation, source_origin, source_id, source_kind, source_version, source_author_json, source_page, first_seen, last_seen, verified_at, hidden) VALUES (?, ?, ?, ?, 'verified', 'stub', ?, ?, 'thread', 1, ?, ?, ?, ?, ?, 0)") + .bind(id, source, `http://127.0.0.1:8787/t/${item}/`, item, origin, id, JSON.stringify({ name: "Live responder" }), source, now, now, now).run(); + response.writeHead(204); response.end(); + return; + } if (!path.startsWith("/notes/b/") && !path.startsWith("/api/")) { response.writeHead(404); response.end("outside forwarding ranges"); return; } const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.from(chunk)); diff --git a/scripts/reset-change-epoch.sql b/scripts/reset-change-epoch.sql index fb02c77..ca3e42d 100644 --- a/scripts/reset-change-epoch.sql +++ b/scripts/reset-change-epoch.sql @@ -2,4 +2,5 @@ -- automatically during ordinary upgrades: cursors should survive deployments. INSERT INTO change_state(id) VALUES(1) ON CONFLICT(id) DO UPDATE SET epoch=excluded.epoch, - items=0, reading=0, subscriptions=0, hoppers=0, signals=0, settings=0, feed=0; + items=0, reading=0, subscriptions=0, hoppers=0, signals=0, settings=0, feed=0, + mentions=0, responses=0; diff --git a/sdk/generated/types.gen.ts b/sdk/generated/types.gen.ts index 67ca1d3..443dc22 100644 --- a/sdk/generated/types.gen.ts +++ b/sdk/generated/types.gen.ts @@ -42,6 +42,8 @@ export type ChangeState = { signals: number; settings: number; feed: number; + mentions: number; + responses: number; }; }; diff --git a/sdk/generated/zod.gen.ts b/sdk/generated/zod.gen.ts index d103ace..cecaf5c 100644 --- a/sdk/generated/zod.gen.ts +++ b/sdk/generated/zod.gen.ts @@ -37,7 +37,9 @@ export const zChangeState = z.object({ hoppers: z.int().gte(0).lte(9007199254740991), signals: z.int().gte(0).lte(9007199254740991), settings: z.int().gte(0).lte(9007199254740991), - feed: z.int().gte(0).lte(9007199254740991) + feed: z.int().gte(0).lte(9007199254740991), + mentions: z.int().gte(0).lte(9007199254740991), + responses: z.int().gte(0).lte(9007199254740991) }) }); diff --git a/src/change-state.ts b/src/change-state.ts index 43f6a7d..8d561cb 100644 --- a/src/change-state.ts +++ b/src/change-state.ts @@ -1,5 +1,8 @@ /** Fixed domains shared by the API and Studio. Adding a response dependency - * requires its trigger and receiving witness, not a caller invalidation hook. */ -export const CHANGE_DOMAINS = ['items', 'reading', 'subscriptions', 'hoppers', 'signals', 'settings', 'feed'] as const; + * requires its trigger and receiving witness, not a caller invalidation hook. + * `mentions` is the Studio's view of both mention tables; `responses` is the + * narrower public one (verified, unhidden inbound rows), so unverified claims + * cannot roll the public HTML cache key (migration 0032). */ +export const CHANGE_DOMAINS = ['items', 'reading', 'subscriptions', 'hoppers', 'signals', 'settings', 'feed', 'mentions', 'responses'] as const; export type ChangeDomain = typeof CHANGE_DOMAINS[number]; export type ChangeState = { epoch: string; domains: Record }; diff --git a/src/html-cache.ts b/src/html-cache.ts index 7835268..e8bc00b 100644 --- a/src/html-cache.ts +++ b/src/html-cache.ts @@ -14,10 +14,14 @@ export function publicHtmlRequest(request: Request, mount: string): boolean { (PATH_SHAPE.test(page.slice(1)) && !RESERVED_PATHS.includes(page.slice(1))); } -// Every domain a public page can read. Reading and signals are the owner's -// private state, so moving them must not empty the cache. Inbound mentions -// have no domain yet and still wait out the 60-second window. -const PUBLIC_DOMAINS: readonly ChangeDomain[] = ['items', 'subscriptions', 'hoppers', 'settings', 'feed']; +// Every domain a public page can read. Reading, signals and mentions are the +// owner's private state, so moving them must not empty the cache. `responses` +// is the one mentions fact a page shows: the responses list under an item +// (listPublicResponses: verified and not hidden). Its triggers (0032) move it +// only when a row enters, leaves or changes inside that set, so a verified +// response or a hide/show appears at once, while a stranger's unverified claim, +// or a flood of them, cannot roll the key and empty the cache. +const PUBLIC_DOMAINS: readonly ChangeDomain[] = ['items', 'subscriptions', 'hoppers', 'settings', 'feed', 'responses']; /** A token that moves whenever a publish, withdrawal or setting can change public HTML. */ export function publicHtmlRevision(changes: ChangeState): string { diff --git a/src/mentions/store.ts b/src/mentions/store.ts index e460ff2..2bb8333 100644 --- a/src/mentions/store.ts +++ b/src/mentions/store.ts @@ -309,8 +309,10 @@ export async function pendingInboundCount(db: D1Database, now: number = Date.now */ export async function pruneFailedInbound(db: D1Database, now: number = Date.now()): Promise { const cutoff = new Date(now - FAILED_INBOUND_RETENTION_MS).toISOString(); - const res = await db.prepare("DELETE FROM mentions_in WHERE status = 'failed' AND last_seen < ?").bind(cutoff).run(); - return res.meta.changes ?? 0; + // Counted from the rows deleted, not meta.changes: D1 adds the change-state + // trigger's write to that count (0032), which keeps only its zero/nonzero meaning. + const res = await db.prepare("DELETE FROM mentions_in WHERE status = 'failed' AND last_seen < ? RETURNING id").bind(cutoff).all(); + return res.results.length; } /** diff --git a/src/ui/catalog.tsx b/src/ui/catalog.tsx index eaa1d0e..ec7af74 100644 --- a/src/ui/catalog.tsx +++ b/src/ui/catalog.tsx @@ -750,10 +750,6 @@ export function MentionsPage() { now

-

- The public page can take about a minute to reflect a change - here: it is cached at the edge. -

{rows.map((mention) => ( ))} diff --git a/src/ui/data.ts b/src/ui/data.ts index 69f66f5..f2a1f86 100644 --- a/src/ui/data.ts +++ b/src/ui/data.ts @@ -58,7 +58,7 @@ export const polling = new Polling( polling.start(window, document); function cachedQuery( context: QueryFunctionContext, - domain: ChangeDomain, + domain: ChangeDomain | readonly ChangeDomain[], schema: S, load: (context: QueryFunctionContext, generation: Generation) => Promise>, ) { @@ -718,39 +718,44 @@ async function mentions(direction: 'inbound' | 'outbound', signal: AbortSignal) client, query: { direction, offset, limit: 100 }, signal, }))); } +// Mentions views poll every tick, as before, but an unchanged `mentions` +// counter (0032) answers from cache: before it, each tick re-read both lists +// in full and every row's source card. export const inbound = createCollection(queryCollectionOptions({ schema: sourceSchemas.inbound, id: 'mentions-in', syncMode: 'on-demand', queryKey: ['mentions-in'], queryClient, getKey: (row) => row.id, - queryFn: async ({ signal }) => { - const rows = (await mentions('inbound', signal)) - .filter((row): row is Mention => 'hidden' in row); - return sourceSchemas.inbound.array().parse(rows); - }, + queryFn: (context) => + cachedQuery(context, 'mentions', sourceSchemas.inbound.array(), async ({ signal }) => + (await mentions('inbound', signal)).filter((row): row is Mention => 'hidden' in row)), + select: (response) => response.data, })); export const outbound = createCollection(queryCollectionOptions({ schema: sourceSchemas.outbound, id: 'mentions-out', syncMode: 'on-demand', queryKey: ['mentions-out'], queryClient, getKey: (row) => row.id, - queryFn: async ({ signal }) => { - const rows = (await mentions('outbound', signal)) - .filter((row): row is MentionOutRow => 'next_attempt_at' in row); - return sourceSchemas.outbound.array().parse(rows); - }, + queryFn: (context) => + cachedQuery(context, 'mentions', sourceSchemas.outbound.array(), async ({ signal }) => + (await mentions('outbound', signal)).filter((row): row is MentionOutRow => 'next_attempt_at' in row)), + select: (response) => response.data, })); +// A source card joins the mention to the Reading tables: whether an import of +// the source is held (stub back) and whether its origin is subscribed. Both +// tables advance `reading`, so the card follows mentions + reading. export const mentionSources = createCollection(queryCollectionOptions({ schema: sourceSchemas.mentionSources, id: 'mention-sources', syncMode: 'on-demand', queryKey: ['mention-sources'], queryClient, getKey: (row) => row.key, - queryFn: async (context) => { - const ids = subsetIds(context, 'key'); - if (!ids) throw new Error('Mention sources require a key predicate'); - const rows = await existing(ids, async (id) => ({ - ...(await unwrap(BlyggerApi.getMentionSource({ client, path: { id }, signal: context.signal }))), - key: id, - })); - return sourceSchemas.mentionSources.array().parse(rows); - }, + queryFn: (context) => + cachedQuery(context, ['mentions', 'reading'], sourceSchemas.mentionSources.array(), async ({ signal }) => { + const ids = subsetIds(context, 'key'); + if (!ids) throw new Error('Mention sources require a key predicate'); + return existing(ids, async (id) => ({ + ...(await unwrap(BlyggerApi.getMentionSource({ client, path: { id }, signal }))), + key: id, + })); + }), + select: (response) => response.data, })); export const mentionSource = scoped((id) => createLiveQueryCollection({ id: `mention-source-view:${id}`, diff --git a/src/ui/revision-query.ts b/src/ui/revision-query.ts index 48faa04..a7da888 100644 --- a/src/ui/revision-query.ts +++ b/src/ui/revision-query.ts @@ -8,13 +8,21 @@ export type CachedResponse = { generation: Generation; }; +/** A response that joins several domains is labelled with their sum. Counters + * only increase within an epoch, so the sum moves exactly when one of them does + * (a mention's source card reads the mention and the Reading tables). */ +export function domainRevision(changes: ChangeState, domain: ChangeDomain | readonly ChangeDomain[]): number { + if (typeof domain === 'string') return changes.domains[domain]; + return domain.reduce((sum, d) => sum + changes.domains[d], 0); +} + /** Cache the response and its pre-fetch revision together in TanStack Query. * Query Collection owns materialization, cancellation and optimistic overlays. * Each invocation checks revisions afresh, including mutation refetches. */ export async function readIfChanged( context: QueryFunctionContext, - domain: ChangeDomain, + domain: ChangeDomain | readonly ChangeDomain[], readChanges: QueryFunction, load: (context: QueryFunctionContext, generation: Generation) => T | Promise, ): Promise> { @@ -22,7 +30,7 @@ export async function readIfChanged( signal.throwIfAborted(); const changes = await readChanges(context); signal.throwIfAborted(); - const generation = { epoch: changes.epoch, revision: changes.domains[domain] }; + const generation = { epoch: changes.epoch, revision: domainRevision(changes, domain) }; const cached = context.client.getQueryData>(context.queryKey); if (cached?.generation.epoch === generation.epoch && cached.generation.revision === generation.revision) { diff --git a/test-ui/polling-cache.oracle.test.ts b/test-ui/polling-cache.oracle.test.ts index 26d6e19..fe1c725 100644 --- a/test-ui/polling-cache.oracle.test.ts +++ b/test-ui/polling-cache.oracle.test.ts @@ -17,7 +17,7 @@ import { StudioQueryReference, type OracleToken } from '../test/polling-cache-or import { campaign, atCheckpoint } from '../test/oracle-campaign.ts'; import { withOracleCleanup } from '../test/oracle-cleanup.ts'; -const token = (revision = 0, epoch = 'e'): OracleToken => ({ epoch, domains: { items: revision, reading: revision, subscriptions: revision, hoppers: revision, signals: revision, settings: revision, feed: revision } }); +const token = (revision = 0, epoch = 'e'): OracleToken => ({ epoch, domains: { items: revision, reading: revision, subscriptions: revision, hoppers: revision, signals: revision, settings: revision, feed: revision, mentions: revision, responses: revision } }); const makeClient = () => new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity } } }); function deferred() { let resolve!: () => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; } function rowsCollection(client: QueryClient, read: QueryFunction, load: QueryFunction<{ id: string; revision: number }[]>) { @@ -66,6 +66,24 @@ describe('Studio query-cache oracle', () => { atCheckpoint('Studio failed-query retry', () => { expect(collection.get('a')!.revision).toBe(1); expect(load).toHaveBeenCalledTimes(3); }); }, [() => collection.cleanup(), async () => { client.clear(); }]); }); + it('a response joining two domains reloads when either moves and not when another does', async () => { + const client = makeClient(); let source = token(); + const load = vi.fn(async () => [{ id: 'a', revision: source.domains.mentions + source.domains.reading }]); + const collection = createCollection(queryCollectionOptions({ + queryKey: ['mention-sources'], queryClient: client, getKey: (row: { id: string; revision: number }) => row.id, + queryFn: context => readIfChanged(context, ['mentions', 'reading'], async () => source, load), + select: (response: CachedResponse<{ id: string; revision: number }[]>) => response.data, + })); + const bump = (domain: keyof OracleToken['domains']) => { source = { ...source, domains: { ...source.domains, [domain]: source.domains[domain] + 1 } }; }; + await withOracleCleanup(async () => { + await collection.preload(); + for (const [domain, loads] of [['responses', 1], ['items', 1], ['mentions', 2], ['reading', 3], ['subscriptions', 3]] as const) { + bump(domain); await collection.utils.refetch({ throwOnError: true }); + atCheckpoint('Studio unchanged query work', () => expect(load, domain).toHaveBeenCalledTimes(loads)); + } + expect(collection.get('a')!.revision).toBe(2); + }, [() => collection.cleanup(), async () => { client.clear(); }]); + }); it('failed revision reads retain rows and cannot reuse a stale changes-query result', async () => { const client = makeClient(); let source = token(), fail = false; const load = vi.fn(async () => [{ id: 'a', revision: source.domains.items }]); diff --git a/test-ui/source-collections.test.ts b/test-ui/source-collections.test.ts index 257f48a..0a83eab 100644 --- a/test-ui/source-collections.test.ts +++ b/test-ui/source-collections.test.ts @@ -33,7 +33,7 @@ beforeAll(async () => { vi.stubGlobal('fetch', async (request: Request) => { const url = new URL(request.url); requests.push({ path: url.pathname, params: url.searchParams }); - if (url.pathname === '/api/changes') return json({ epoch, domains: Object.fromEntries(['items', 'reading', 'hoppers', 'subscriptions', 'signals', 'settings', 'feed'].map((domain) => [domain, revision])) }); + if (url.pathname === '/api/changes') return json({ epoch, domains: Object.fromEntries(['items', 'reading', 'hoppers', 'subscriptions', 'signals', 'settings', 'feed', 'mentions', 'responses'].map((domain) => [domain, revision])) }); if (url.pathname === '/api/items/a' && request.method === 'PATCH') { const body = await request.json(); await writeGate; @@ -64,6 +64,7 @@ beforeAll(async () => { return json({ items: entries, offset, limit: 25, selected: sub === 'invalid' ? 'all' : sub, read_state: true, read_state_clear: true, total: 30, counts: { all: 30, own: 30, subscriptions: {} } }); } if (url.pathname === '/api/mentions/m/source') return json({ holder: 's', subscription: null }); + if (url.pathname === '/api/mentions') return json({ items: [], total: 0, offset: 0, limit: 100, direction: url.searchParams.get('direction') }); throw new Error(`Unexpected request: ${request.method} ${request.url}`); }); db = await import('../src/ui/data.ts'); @@ -317,6 +318,21 @@ test('mention-source views reuse one on-demand source and requested keys', async expect(requests.filter((request) => request.path === '/api/mentions/m/source')).toHaveLength(1); }); +test('mention lists and source cards re-read only when their revisions move', async () => { + const view = db.mentionSource('m'); views.push(view); + await Promise.all([db.listViews.inbound.preload(), db.listViews.outbound.preload(), view.preload()]); + const reads = (path: string) => requests.filter((request) => request.path === path).length; + const counts = () => [reads('/api/mentions'), reads('/api/mentions/m/source')]; + expect(counts()).toEqual([2, 1]); + // An unchanged tick answers from the cached responses after one revision check each. + await Promise.all([db.inbound.utils.refetch(), db.outbound.utils.refetch(), db.mentionSources.utils.refetch()]); + expect(counts()).toEqual([2, 1]); + revision++; + await Promise.all([db.inbound.utils.refetch(), db.outbound.utils.refetch(), db.mentionSources.utils.refetch()]); + expect(counts()).toEqual([4, 2]); + await Promise.all([db.inbound.cleanup(), db.outbound.cleanup()]); +}); + test('a missing reading source keeps the requested identity for the router redirect', async () => { const view = db.readingView('invalid', 0); views.push(view); await view.preload(); diff --git a/test/html-cache.test.ts b/test/html-cache.test.ts index 38f242e..96920e8 100644 --- a/test/html-cache.test.ts +++ b/test/html-cache.test.ts @@ -3,7 +3,8 @@ import { describe, expect, it } from 'vitest'; import worker from '../src/index.ts'; import { htmlCacheResponse, publicHtmlKey, publicHtmlRequest, publicHtmlRevision } from '../src/html-cache.ts'; import { readChanges } from '../src/changes.ts'; -import { BASE, login, createAndPublish, apiJson } from './helpers.ts'; +import { BASE, login, createAndPublish, apiJson, getPublic } from './helpers.ts'; +import { markInboundUnverified, markInboundVerified, setMentionHidden, upsertInbound } from '../src/mentions/store.ts'; const EDGE = 'public, max-age=60, stale-while-revalidate=300, stale-if-error=0'; @@ -202,6 +203,67 @@ describe('cached entrypoint dispatch', () => { expect(await keyFor('/')).toBe(afterSetting); }); + it('a verified response, a hide and a show move the key; a pending claim, its failure and outbound work do not', async () => { + const keys: string[] = []; + const ctx = { exports: { PublicHtml: { fetch: (_req: Request, init: { cf: { cacheKey: string } }) => { + keys.push(init.cf.cacheKey); + return Promise.resolve(new Response('saved HTML')); + } } } } as unknown as ExecutionContext; + const keyFor = async () => { + await (await worker.fetch(new Request(`${BASE}/blyg/`), env, ctx)).arrayBuffer(); + return keys.at(-1)!; + }; + const cookie = await login(); + const item = await createAndPublish(cookie, 'Responses roll the public key'); + const found = { relation: 'stub' as const, sourceOrigin: 'https://responder.example/', sourceId: 'r1', sourceKind: 'thread', sourceVersion: 1, authorJson: null, sourcePage: 't/r1/' }; + const start = await keyFor(), changesAt = await readChanges(env.DB); + + // Anyone can make a claim. Neither the claim nor its failure is public. + const spam = await upsertInbound(env.DB, 'https://spam.example/x', `${BASE}/blyg/f/${item}/`, item); + await markInboundUnverified(env.DB, spam.id, 'failed', 'no link'); + expect(await keyFor()).toBe(start); + // The Studio lists only verified and gone rows, so its counter stays too. + expect((await readChanges(env.DB)).domains.mentions).toBe(changesAt.domains.mentions); + // Outbound sends are the Studio's business, never the public page's. + await env.DB.prepare("INSERT INTO mentions_out(id,item_id,version,target,status,created) VALUES('out-key',?,1,'https://x.example/','pending','t')").bind(item).run(); + await env.DB.prepare("UPDATE mentions_out SET status='sent',attempts=1 WHERE id='out-key'").run(); + expect(await keyFor()).toBe(start); + const outbound = await readChanges(env.DB); + expect(outbound.domains.mentions).toBeGreaterThan(changesAt.domains.mentions); + expect(outbound.domains.responses).toBe(changesAt.domains.responses); + + const claim = await upsertInbound(env.DB, 'https://responder.example/t/r1/', `${BASE}/blyg/f/${item}/`, item); + expect(await keyFor()).toBe(start); + await markInboundVerified(env.DB, claim.id, found); + const verified = await keyFor(); + expect(verified).not.toBe(start); + await setMentionHidden(env.DB, claim.id, true); + const hidden = await keyFor(); + expect(hidden).not.toBe(verified); + // Re-verifying a hidden row is invisible to readers. + await upsertInbound(env.DB, 'https://responder.example/t/r1/', `${BASE}/blyg/f/${item}/`, item); + await markInboundVerified(env.DB, claim.id, found); + expect(await keyFor()).toBe(hidden); + await setMentionHidden(env.DB, claim.id, false); + expect(await keyFor()).not.toBe(hidden); + }); + + // The test pool has no persistent edge cache, so this witnesses what the page + // renders; the key test above is what keeps a cached entry from outliving it. + it('the item page renders a newly verified response and drops it when hidden', async () => { + const cookie = await login(); + const item = await createAndPublish(cookie, 'Cached page awaiting a response'); + expect((await apiJson(cookie, 'PATCH', `/api/items/${item}`, { responses: 'show' })).status).toBe(200); + const path = `/blyg/f/${item}/`; + const first = await (await getPublic(path)).text(); + expect(first).not.toContain('responder.example'); + const claim = await upsertInbound(env.DB, 'https://responder.example/t/r2/', `${BASE}${path}`, item); + await markInboundVerified(env.DB, claim.id, { relation: 'stub', sourceOrigin: 'https://responder.example/', sourceId: 'r2', sourceKind: 'thread', sourceVersion: 1, authorJson: JSON.stringify({ name: 'Prompt responder' }), sourcePage: 't/r2/' }); + expect(await (await getPublic(path)).text()).toContain('Prompt responder'); + await setMentionHidden(env.DB, claim.id, true); + expect(await (await getPublic(path)).text()).not.toContain('Prompt responder'); + }); + it('still serves public HTML under the plain key when the revision cannot be read', async () => { let key: unknown; const ctx = { exports: { PublicHtml: { fetch: (_req: Request, init: { cf: { cacheKey: string } }) => { diff --git a/test/polling-cache-oracle-model.ts b/test/polling-cache-oracle-model.ts index b44451a..9a706dc 100644 --- a/test/polling-cache-oracle-model.ts +++ b/test/polling-cache-oracle-model.ts @@ -9,7 +9,7 @@ * Histories can serve any saved legal snapshot; SWR promises no maximum age. * A successful publisher cannot lower generation within the same epoch. */ -export const oracleDomains = ['items', 'reading', 'subscriptions', 'hoppers', 'signals', 'settings', 'feed'] as const; +export const oracleDomains = ['items', 'reading', 'subscriptions', 'hoppers', 'signals', 'settings', 'feed', 'mentions', 'responses'] as const; export type OracleDomain = typeof oracleDomains[number]; export type OracleToken = { epoch: string; domains: Record }; export function changedDomains(before: OracleToken, after: OracleToken) { diff --git a/test/polling-cache-trigger-cases.ts b/test/polling-cache-trigger-cases.ts index 7ae6b12..6d5bc4d 100644 --- a/test/polling-cache-trigger-cases.ts +++ b/test/polling-cache-trigger-cases.ts @@ -1,5 +1,6 @@ /** Handwritten receiving witnesses for the SQLite prototype's nine-table event - * envelope, plus read_state (migration 0026: Reading carries readVersion). + * envelope, plus read_state (migration 0026: Reading carries readVersion) and + * mentions_in/mentions_out (0032: the mentions and responses domains). * These expected domains come from response dependencies, not the production * trigger generator. Working-copy edits deliberately exclude Reading. */ @@ -14,6 +15,11 @@ export const triggerFamilies = [ { table: 'hopper_items', insert: "INSERT INTO hopper_items(hopper_id,subscription_id,remote_id,added_at) VALUES('matrix-hopper','matrix-sub','matrix-remote','t')", update: "UPDATE hopper_items SET added_at='later' WHERE hopper_id='matrix-hopper'", remove: "DELETE FROM hopper_items WHERE hopper_id='matrix-hopper'", domains: ['hoppers'] }, { table: 'signals', insert: "INSERT INTO signals(subscription_id,remote_id,thumb,at) VALUES('matrix-sub','matrix-remote',1,'t')", update: "UPDATE signals SET thumb=-1 WHERE subscription_id='matrix-sub'", remove: "DELETE FROM signals WHERE subscription_id='matrix-sub'", domains: ['signals'] }, { table: 'read_state', insert: "INSERT INTO read_state(subscription_id,remote_id,read_version,updated) VALUES('matrix-sub','matrix-remote',1,'t')", update: "UPDATE read_state SET read_version=2,updated='later' WHERE subscription_id='matrix-sub'", remove: "DELETE FROM read_state WHERE subscription_id='matrix-sub'", domains: ['reading'] }, + // 0032: a verified, unhidden inbound row is on both the Studio list and the + // public responses list; an outbound row is on the Studio list only. The + // status-dependent branches have their own receiving witness in the oracle. + { table: 'mentions_in', insert: "INSERT INTO mentions_in(id,source,target,target_item_id,status,first_seen,last_seen) VALUES('matrix-mention','https://m/s','https://us/f/matrix-item/','matrix-item','verified','t','t')", update: "UPDATE mentions_in SET source_author_json='{}' WHERE id='matrix-mention'", remove: "DELETE FROM mentions_in WHERE id='matrix-mention'", domains: ['mentions', 'responses'] }, + { table: 'mentions_out', insert: "INSERT INTO mentions_out(id,item_id,version,target,status,created) VALUES('matrix-out','matrix-item',1,'https://them/','pending','t')", update: "UPDATE mentions_out SET status='sent',attempts=1 WHERE id='matrix-out'", remove: "DELETE FROM mentions_out WHERE id='matrix-out'", domains: ['mentions'] }, ] as const; // Independent one-field branches that the SQLite prototype supplied in SQL @@ -26,4 +32,12 @@ export const triggerFields = [ // A clear's tombstone alone (0026) is a Reading change: it decides whether a later read applies. { family: 9, field: 'unread_at', value: 'later', domains: ['reading'] }, ...['subscription_id','remote_id','kind','page'].map(field => ({ family: 5, field, value: field === 'kind' ? 'thread' : 'changed', domains: ['reading','hoppers','feed'] })), + // Every column of a listed, public inbound row is on both lists' side of the guard. + ...['source','target','target_item_id','relation','source_origin','source_id','source_kind','source_page','source_author_json','first_seen','last_seen','verified_at','error','id'].map(field => ({ family: 10, field, value: 'changed', domains: ['mentions','responses'] })), + ...['source_version','attempts'].map(field => ({ family: 10, field, value: 7, domains: ['mentions','responses'] })), + // Leaving the public set is a public change; the Studio still lists the row. + { family: 10, field: 'hidden', value: 1, domains: ['mentions','responses'] }, + { family: 10, field: 'status', value: 'gone', domains: ['mentions','responses'] }, + ...['endpoint','last_error','next_attempt_at','target','created','id','item_id'].map(field => ({ family: 11, field, value: 'changed', domains: ['mentions'] })), + ...['version','target_version','attempts'].map(field => ({ family: 11, field, value: 7, domains: ['mentions'] })), ]; diff --git a/test/polling-cache.oracle.test.ts b/test/polling-cache.oracle.test.ts index 080afe7..6cceaa3 100644 --- a/test/polling-cache.oracle.test.ts +++ b/test/polling-cache.oracle.test.ts @@ -40,8 +40,8 @@ beforeEach(async () => { // and reset only tables owned by this oracle, before running a new history. for (const sql of triggerDefinitions) await env.DB.prepare(sql.replace('CREATE TRIGGER ', 'CREATE TRIGGER IF NOT EXISTS ')).run(); await env.DB.prepare('INSERT OR IGNORE INTO change_state(id) VALUES(1)').run(); - await env.DB.prepare("UPDATE change_state SET epoch=lower(hex(randomblob(16))),items=0,reading=0,subscriptions=0,hoppers=0,signals=0,settings=0,feed=0 WHERE id=1").run(); - await env.DB.batch(['versions','media','items','settings','hopper_items','signals','read_state','imported_items','subscriptions','hoppers'].map(table => env.DB.prepare(`DELETE FROM ${table}`))); + await env.DB.prepare("UPDATE change_state SET epoch=lower(hex(randomblob(16))),items=0,reading=0,subscriptions=0,hoppers=0,signals=0,settings=0,feed=0,mentions=0,responses=0 WHERE id=1").run(); + await env.DB.batch(['versions','media','items','settings','hopper_items','signals','read_state','imported_items','subscriptions','hoppers','mentions_in','mentions_out'].map(table => env.DB.prepare(`DELETE FROM ${table}`))); await clearArtifacts(); }); afterEach(() => vi.useRealTimers()); @@ -195,6 +195,48 @@ describe('trigger revision oracle', () => { expect(await call('GET', '/api/reading')).toEqual(cleared); expect(await changes(cookie)).toEqual(clearedChanges); }); + it('mention status transitions move exactly the domains whose responses they change', async () => { + const cookie = await login(); await seed(); + await env.DB.prepare("UPDATE items SET responses_override=1 WHERE id='cache-item'").run(); + async function read(path: string, headers: HeadersInit = { cookie }) { + const ctx = createExecutionContext(); const response = await app.fetch(new Request(BASE + path, { headers }), env, ctx); + expect(response.status).toBe(200); const body = await response.text(); await waitOnExecutionContext(ctx); return body; + } + // The Studio's inbound list, and the public page's responses section as a reader receives it. + const studio = () => read('/api/mentions?direction=inbound'); + const page = async () => (await read('/blyg/f/cache-item/', {})).match(/]*responses[\s\S]*?<\/section>/)?.[0] ?? ''; + const verify = "status='verified',relation='stub',source_origin='https://responder.example/',source_id='r',source_kind='thread',source_version=1,source_page='t/r/',source_author_json='{\"name\":\"Witness\"}',verified_at='2026-10-18T00:00:00Z'"; + // Receiver writes, in the order a claim's life takes them. Expected domains + // follow from the observed responses below, not from the trigger text. + const steps: [string, string, OracleDomain[]][] = [ + ['claim', "INSERT INTO mentions_in(id,source,target,target_item_id,status,first_seen,last_seen) VALUES('w','https://responder.example/t/r/','https://us/f/cache-item/','cache-item','pending','t','t')", []], + ['verification fails', "UPDATE mentions_in SET status='failed',error='no link' WHERE id='w'", []], + ['claimed again', "UPDATE mentions_in SET status='pending',error=NULL,attempts=1,last_seen='t2' WHERE id='w'", []], + ['verified', `UPDATE mentions_in SET ${verify} WHERE id='w'`, ['mentions', 'responses']], + ['hidden', "UPDATE mentions_in SET hidden=1 WHERE id='w'", ['mentions', 'responses']], + ['hidden, claimed again', "UPDATE mentions_in SET status='pending',attempts=2,last_seen='t3' WHERE id='w'", ['mentions']], + ['hidden, verified', `UPDATE mentions_in SET ${verify} WHERE id='w'`, ['mentions']], + ['hidden, withdrawn at source', "UPDATE mentions_in SET status='gone',error='withdrawn' WHERE id='w'", ['mentions']], + ['gone, claimed again', "UPDATE mentions_in SET status='pending',attempts=3,last_seen='t4' WHERE id='w'", ['mentions']], + ['shown while pending', "UPDATE mentions_in SET hidden=0 WHERE id='w'", []], + ['shown, verified', `UPDATE mentions_in SET ${verify} WHERE id='w'`, ['mentions', 'responses']], + ['pruned', "DELETE FROM mentions_in WHERE id='w'", ['mentions', 'responses']], + ['failed claim pruned', "INSERT INTO mentions_in(id,source,target,target_item_id,status,first_seen,last_seen) VALUES('f','https://spam.example/','https://us/f/cache-item/','cache-item','failed','t','t')", []], + ['failed claim pruned', "DELETE FROM mentions_in WHERE id='f'", []], + ]; + for (const [name, sql, domains] of steps) { + const before = { studio: await studio(), page: await page(), changes: await changes(cookie) }; + await env.DB.prepare(sql).run(); + const after = { studio: await studio(), page: await page(), changes: await changes(cookie) }; + const observed: OracleDomain[] = []; + if (after.studio !== before.studio) observed.push('mentions'); + if (after.page !== before.page) observed.push('responses'); + // A step with no observable change must not have been written as one. + expect(observed, name).toEqual(domains); + atCheckpoint('trigger exact domain effects', () => expect(changedDomains(before.changes, after.changes), name).toEqual(domains)); + } + expect(await page()).toBe(''); + }); it('the timed update-state read crosses its exact daily deadline without source changes', async () => { vi.useFakeTimers({ toFake: ['Date'] }); const start = Date.parse('2026-10-01T12:00:00Z'); vi.setSystemTime(start); const cookie = await login(); @@ -316,7 +358,7 @@ describe('trigger revision oracle', () => { expect((await env.DB.prepare("SELECT value FROM settings WHERE key='site_title'").first<{ value: string }>())!.value).toBe('A'); }); it('rejects missing, extra and stale domain answers at the public checkpoint', () => { - const baseline: OracleToken = { epoch: 'e', domains: { items: 0, reading: 0, subscriptions: 0, hoppers: 0, signals: 0, settings: 0, feed: 0 } }; + const baseline: OracleToken = { epoch: 'e', domains: { items: 0, reading: 0, subscriptions: 0, hoppers: 0, signals: 0, settings: 0, feed: 0, mentions: 0, responses: 0 } }; for (const wrong of [[], ['subscriptions', 'feed'], ['reading']]) { expect(() => atCheckpoint('trigger exact domain effects', () => expect(wrong).toEqual(changedDomains(baseline, { ...baseline, domains: { ...baseline.domains, subscriptions: 1 } })))).toThrow('trigger exact domain effects'); }