From fb97ddf810da5016f7bd344a4323b8df4834041b Mon Sep 17 00:00:00 2001 From: Aneesh Sathe Date: Sun, 18 Oct 2026 13:30:19 -0700 Subject: [PATCH] OPML subscription import (#63); backfill flag on createSubscription - createSubscription takes an optional `backfill` (default true). With false the route skips the waitUntil backfill; isDue() treats a null last_poll_at as due, so the next quarter-hour cron tick imports it. - Studio: "import OPML" on Sources and in the subscribe sheet. The file is parsed in the browser (DOMParser, folders flattened, type="rss" or no type, bare & and HTML entities repaired), previewed with already- subscribed rows marked, then imported one request at a time about a second apart, under the 120/min owner write budget. 409 counts as already subscribed; 429 waits out Retry-After. Optional "add these to my blogroll" box, off by default, uses PATCH /subscriptions/{id}. - docs/api.md: a Subscriptions section with the flag and the budgets. - Tests: worker (backfill false/default/invalid), UI-state parser and pacing runner, Playwright import flow (desktop + mobile). Co-Authored-By: Claude Opus 5.5 --- docs/api.md | 24 ++ e2e/opml-import.spec.ts | 135 ++++++++++++ openapi.json | 4 + sdk/generated/types.gen.ts | 4 + sdk/generated/zod.gen.ts | 3 +- src/contract/routes.ts | 2 +- src/importer/api.ts | 6 +- src/ui/opml-import.tsx | 263 ++++++++++++++++++++++ src/ui/opml.ts | 270 +++++++++++++++++++++++ src/ui/reading.css | 21 ++ src/ui/reading.tsx | 25 ++- test-ui/opml.test.ts | 194 ++++++++++++++++ test/importer/subscribe-backfill.test.ts | 81 +++++++ 13 files changed, 1026 insertions(+), 6 deletions(-) create mode 100644 e2e/opml-import.spec.ts create mode 100644 src/ui/opml-import.tsx create mode 100644 src/ui/opml.ts create mode 100644 test-ui/opml.test.ts create mode 100644 test/importer/subscribe-backfill.test.ts diff --git a/docs/api.md b/docs/api.md index 82ce842b..ede06593 100644 --- a/docs/api.md +++ b/docs/api.md @@ -89,6 +89,30 @@ Creation returns the item, status 201, and a `Location` header. Hopper and media creation also return 201. Subscription creation returns 200 when the owner must accept the proposed subscription, then 201 when it creates the subscription. +## Subscriptions + +`POST /subscriptions` takes `{ url, confirm?, title?, backfill? }`. +Without `confirm` it resolves the URL and returns the proposal (200); with `confirm: true` it creates the subscription (201). +A source that is already subscribed, by origin or by feed, is refused with 409. +New subscriptions are not in the public `blogroll.opml`; `PATCH /subscriptions/{id}` with `in_blogroll: true` adds one. + +`backfill` (boolean, default `true`) applies with `confirm: true`. +By default the server starts importing the source's archive as soon as it has answered. +With `backfill: false` it creates the subscription and imports nothing yet: a subscription that has never been polled is always due, so the next quarter-hour poll imports it. +Use it for bulk imports, so that many subscriptions do not each start an archive import at once: + +```ts +const result = await BlyggerApi.createSubscription({ + client, + body: { url: "https://example.com/feed.xml", confirm: true, backfill: false }, +}); +if (result.response?.status === 409) { /* already subscribed */ } +if (result.response?.status === 429) { /* wait Retry-After seconds, then retry */ } +``` + +The Studio's OPML import works this way. It reads the file in the browser, flattens folders and sends one request at a time, about one a second, which stays under the owner's write budget of 120 writes a minute (`API_WRITE_LIMIT`). +Delegated clients have less room: 120 a minute per grant, and 60 a minute shared by every grant (`API_DELEGATED_WRITE_LIMIT`), so a token-bearing importer should go slower than one a second and honour `Retry-After` on 429. + ## Partial edits PATCH changes only supplied fields. diff --git a/e2e/opml-import.spec.ts b/e2e/opml-import.spec.ts new file mode 100644 index 00000000..5889cb89 --- /dev/null +++ b/e2e/opml-import.spec.ts @@ -0,0 +1,135 @@ +import { test, expect, type Page } from './fixture'; + +// OPML import (studio#63): the file is parsed in the browser, the preview marks +// what is already subscribed, and each feed is one createSubscription with +// backfill: false, sent one at a time. The POSTs are answered here, because a +// real subscribe would reach the network and change the shared fixture +// subscriptions other tests read; the server side of `backfill` is covered by +// test/importer/subscribe-backfill.test.ts. +async function login(page: Page) { + await page.goto('/studio/login'); + await page.locator('[name=password]').fill('test-password'); + await page.getByRole('button', { name: 'log in', exact: true }).click(); + await expect(page.locator('#composer-text')).toBeVisible(); +} + +// Real-world untidiness on purpose: nested folders, a feed with no type, a +// folder outline with no xmlUrl, a double-escaped title, an HTML entity XML does +// not define and a bare & in a URL (both repaired), and the fixture's own +// legacy feed, which is already subscribed. +const OPML = ` + + Exported + + + + + + + + + + + +`; + +const subscription = (url: string) => ({ + id: `sub-${new URL(url).hostname}`, kind: 'rss', origin: url, feed_url: url, title: new URL(url).hostname, status: 'active', + last_poll_at: null, fail_count: 0, last_index_sync_at: null, created: '2026-10-18T00:00:00Z', + title_follows_source: true, in_blogroll: false, flags: [], +}); + +test('an OPML file is previewed, imported one feed at a time, and summarised', async ({ page }) => { + const errors: string[] = []; + page.on('pageerror', (error) => errors.push(error.message)); + await login(page); + await page.goto('/studio/reading?view=sources'); + + const posts: { url: string; backfill: unknown; confirm: unknown; at: number }[] = []; + let inFlight = 0, overlapped = false, gammaTries = 0; + const patches: string[] = []; + await page.route('**/api/subscriptions', async (route) => { + if (route.request().method() !== 'POST') return route.fallback(); + const body = route.request().postDataJSON() as { url: string; backfill: unknown; confirm: unknown }; + posts.push({ ...body, at: Date.now() }); + if (inFlight++) overlapped = true; + try { + const host = new URL(body.url).hostname; + if (host === 'alpha.example' || host === 'beta.example') return await route.fulfill({ status: 201, json: subscription(body.url) }); + if (host === 'gamma.example' && gammaTries++ === 0) return await route.fulfill({ status: 429, headers: { 'Retry-After': '1' }, json: { error: 'API work budget exceeded' } }); + if (host === 'gamma.example') return await route.fulfill({ status: 409, json: { error: 'already subscribed to Gamma' } }); + return await route.fulfill({ status: 422, json: { error: 'could not resolve this URL to a blyg or a feed', tried: [body.url] } }); + } finally { + inFlight--; + } + }); + await page.route('**/api/subscriptions/sub-*', async (route) => { + if (route.request().method() !== 'PATCH') return route.fallback(); + patches.push(route.request().url()); + await route.fulfill({ json: { ...subscription('https://x.example/'), in_blogroll: true } }); + }); + + await page.locator('#opml-open').click(); + const sheet = page.getByRole('dialog', { name: 'import OPML' }); + await sheet.locator('#opml-file').setInputFiles({ name: 'feeds.opml', mimeType: 'text/x-opml', buffer: Buffer.from(OPML) }); + + const list = sheet.getByRole('list', { name: 'feeds in the file' }); + await expect(list.locator('.opml-row')).toHaveCount(5); + await expect(list.locator('.opml-title')).toHaveText(['Alpha & Co', 'Café notes', 'Legacy again', 'Gamma', 'Delta']); + await expect(list.locator('.opml-url').nth(1)).toHaveText('https://beta.example/rss?a=1&b=2'); + await expect(list.locator('.opml-row').nth(2)).toContainText('already subscribed'); + await expect(sheet.getByRole('checkbox', { name: 'import Legacy again' })).toBeDisabled(); + await expect(sheet.getByRole('checkbox', { name: 'add these to my blogroll' })).not.toBeChecked(); + + // Leave Delta out; the blogroll box stays off. + await sheet.getByRole('checkbox', { name: 'import Delta' }).uncheck(); + await sheet.getByRole('button', { name: 'import 3 feeds' }).click(); + + const summary = sheet.locator('.opml-summary'); + await expect(summary).toContainText('2 added · 2 already subscribed · 0 failed', { timeout: 15_000 }); + await expect(list.locator('.opml-row').nth(0)).toContainText('added'); + await expect(list.locator('.opml-row').nth(3)).toContainText('already subscribed'); + + // Alpha, Beta, Gamma (throttled), Gamma again. Legacy was never sent, nor Delta. + expect(posts.map((post) => new URL(post.url).hostname)).toEqual(['alpha.example', 'beta.example', 'gamma.example', 'gamma.example']); + expect(posts.every((post) => post.backfill === false && post.confirm === true)).toBe(true); + expect(overlapped).toBe(false); + // Paced: about a second between writes, and the 429's Retry-After waited out. + expect(posts[1].at - posts[0].at).toBeGreaterThanOrEqual(900); + expect(posts[3].at - posts[2].at).toBeGreaterThanOrEqual(900); + expect(patches).toEqual([]); + + await sheet.getByRole('button', { name: 'done' }).click(); + await expect(sheet).toHaveCount(0); + expect(errors).toEqual([]); +}); + +test('the blogroll box adds each new subscription with PATCH, and a malformed file is refused', async ({ page }) => { + await login(page); + await page.goto('/studio/reading?view=sources'); + const patched: unknown[] = []; + await page.route('**/api/subscriptions', (route) => + route.request().method() === 'POST' + ? route.fulfill({ status: 201, json: subscription((route.request().postDataJSON() as { url: string }).url) }) + : route.fallback(), + ); + await page.route('**/api/subscriptions/sub-*', async (route) => { + if (route.request().method() !== 'PATCH') return route.fallback(); + patched.push(route.request().postDataJSON()); + await route.fulfill({ json: { ...subscription('https://one.example/feed'), in_blogroll: true } }); + }); + + await page.locator('#opml-open').click(); + const sheet = page.getByRole('dialog', { name: 'import OPML' }); + await sheet.locator('#opml-file').setInputFiles({ name: 'broken.opml', mimeType: 'text/x-opml', buffer: Buffer.from('') }); + await expect(sheet.getByRole('alert')).toContainText('not well-formed XML'); + + await sheet.locator('#opml-file').setInputFiles({ + name: 'one.opml', mimeType: 'text/x-opml', + buffer: Buffer.from(''), + }); + await sheet.getByRole('checkbox', { name: 'add these to my blogroll' }).check(); + await sheet.getByRole('button', { name: 'import 1 feed' }).click(); + await expect(sheet.locator('.opml-summary')).toContainText('1 added · 0 already subscribed · 0 failed'); + expect(patched).toEqual([{ in_blogroll: true }]); +}); diff --git a/openapi.json b/openapi.json index 1ad20a42..033fea52 100644 --- a/openapi.json +++ b/openapi.json @@ -6293,6 +6293,10 @@ }, "title": { "type": "string" + }, + "backfill": { + "type": "boolean", + "description": "With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML." } }, "required": [ diff --git a/sdk/generated/types.gen.ts b/sdk/generated/types.gen.ts index 67ca1d37..addc3859 100644 --- a/sdk/generated/types.gen.ts +++ b/sdk/generated/types.gen.ts @@ -2202,6 +2202,10 @@ export type CreateSubscriptionData = { url: string; confirm?: boolean; title?: string; + /** + * With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML. + */ + backfill?: boolean; }; path?: never; query?: never; diff --git a/sdk/generated/zod.gen.ts b/sdk/generated/zod.gen.ts index d103ace0..07c533e8 100644 --- a/sdk/generated/zod.gen.ts +++ b/sdk/generated/zod.gen.ts @@ -973,7 +973,8 @@ export const zListSubscriptionsResponse = z.object({ export const zCreateSubscriptionBody = z.object({ url: z.string(), confirm: z.boolean().optional(), - title: z.string().optional() + title: z.string().optional(), + backfill: z.boolean().optional() }).strict(); export const zCreateSubscriptionResponse = z.union([ diff --git a/src/contract/routes.ts b/src/contract/routes.ts index df153f3c..a21df4cd 100644 --- a/src/contract/routes.ts +++ b/src/contract/routes.ts @@ -89,7 +89,7 @@ export const routes = { uploadMedia: createRoute({ ...route("uploadMedia", "post", "/media", z.object({ id: z.string(), url: z.string(), mime: z.string() }), undefined, 201), request: { body: { required: true, content: { "multipart/form-data": { schema: z.object({ file: z.custom((value) => value instanceof File, "file field required (multipart)").openapi({ type: "string", format: "binary" }), item_id: z.string().optional(), alt: z.string().optional(), inline: z.enum(["true", "false"]).optional() }) } } } } }), deleteMedia: route("deleteMedia", "delete", "/media/{id}", ok.extend({ outcome: z.enum(["deleted", "detached"]) })), updateSettings: route("updateSettings", "patch", "/settings", SettingsSchema, settingsBody), - createSubscription: route("createSubscription", "post", "/subscriptions", confirmation, z.object({ url: z.string(), confirm: z.boolean().optional(), title: z.string().optional() })), + createSubscription: route("createSubscription", "post", "/subscriptions", confirmation, z.object({ url: z.string(), confirm: z.boolean().optional(), title: z.string().optional(), backfill: z.boolean().optional().describe("With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML.") })), updateSubscription: route("updateSubscription", "patch", "/subscriptions/{id}", SubscriptionSchema, z.object({ in_blogroll: z.boolean().optional(), title: z.string().trim().min(1).nullable().optional().describe("A name of your own; null hands the name back to the source, refreshed while polling."), paused: z.boolean().optional() }).strict()), pollAllSubscriptions: route("pollAllSubscriptions", "post", "/subscriptions/poll", z.object({ polling: z.number().int().nonnegative() })), resyncSubscription: route("resyncSubscription", "post", "/subscriptions/{id}/resync", ok.extend({ changed: z.number() })), diff --git a/src/importer/api.ts b/src/importer/api.ts index f0880c15..49db1f04 100644 --- a/src/importer/api.ts +++ b/src/importer/api.ts @@ -50,7 +50,7 @@ function titleFromUrl(url: string): string { * the confirm button for many seconds on a large blyg. */ importerApi.openapi(routes.createSubscription, async (c) => { - const body = await readJson<{ url: string; confirm?: boolean; title?: string }>(c); + const body = await readJson<{ url: string; confirm?: boolean; title?: string; backfill?: boolean }>(c); if (!body.url.trim()) return c.json({ error: "url required" }, 400); const result = await resolve(body.url.trim(), platformFetchFor(c.env)); @@ -96,7 +96,9 @@ importerApi.openapi(routes.createSubscription, async (c) => { // detection (§3.2) in one pass, for both kinds uniformly. If waitUntil cuts // it short, the null last_poll_at and last_index_sync_at make the next // scheduled poll due and reconcile again, so the backfill completes there. - c.executionCtx.waitUntil(pollSubscription(c.env.DB, sub, platformFetchFor(c.env)).catch(() => {})); + // `backfill: false` (bulk imports, e.g. OPML) skips it: isDue() treats a + // null last_poll_at as due, so the next quarter-hour tick imports it instead. + if (body.backfill !== false) c.executionCtx.waitUntil(pollSubscription(c.env.DB, sub, platformFetchFor(c.env)).catch(() => {})); c.header("Location", `/api/subscriptions/${sub.id}`); return c.json(subscriptionResource(sub), 201); }); diff --git a/src/ui/opml-import.tsx b/src/ui/opml-import.tsx new file mode 100644 index 00000000..e70ae1b5 --- /dev/null +++ b/src/ui/opml-import.tsx @@ -0,0 +1,263 @@ +/* + * Import OPML (studio#63): pick a file, see what it lists, subscribe to it. + * The file is parsed here, in the browser; each feed is one ordinary + * createSubscription with `backfill: false`, sent one at a time and paced + * under the owner's write budget (opml.ts), so the next quarter-hour poll + * imports the archives rather than every subscribe at once. Imported + * subscriptions stay out of the public blogroll unless the box is ticked. + */ +import { useMemo, useRef, useState } from 'react'; +import { useLiveQuery } from '@tanstack/react-db'; +import { BlyggerApi } from '../../sdk/dist/browser.js'; +import { client, subscriptions, changed } from './data.ts'; +import { Button, Failure } from './components.tsx'; +import { Sheet, toast } from './sheets.tsx'; +import { + feedKey, + parseOpml, + retryAfterMs, + runImport, + subscribedKeys, + type Attempt, + type ImportSummary, + type OpmlFeed, + type RowStatus, +} from './opml.ts'; + +type Phase = 'pick' | 'preview' | 'running' | 'done'; + +function errorText(error: unknown, status: number): string { + if (error && typeof error === 'object' && 'error' in error) return String((error as { error: unknown }).error); + return `request failed (${status})`; +} +/** One answer from the server, read for the import: created, duplicate, throttled or failed. */ +async function attempt(pending: Promise<{ data?: unknown; error?: unknown; response?: Response }>): Promise { + const result = await pending; + const response = result.response; + if (!response) return { kind: 'failed', reason: result.error instanceof Error ? result.error.message : 'network error' }; + if (response.status === 409) return { kind: 'exists', message: errorText(result.error, 409) }; + if (response.status === 429) return { kind: 'throttled', retryAfterMs: retryAfterMs(response.headers.get('Retry-After')) }; + if (!response.ok) return { kind: 'failed', reason: errorText(result.error, response.status) }; + const data = result.data; + if (data && typeof data === 'object' && 'needsConfirm' in data) return { kind: 'failed', reason: 'the server asked for confirmation' }; + return { kind: 'ok', id: data && typeof data === 'object' && 'id' in data ? String((data as { id: unknown }).id) : undefined }; +} + +function statusBadge(status: RowStatus | undefined, known: boolean) { + const state = status?.state ?? (known ? 'exists' : 'pending'); + switch (state) { + case 'working': + return adding…; + case 'added': + return added; + case 'exists': + return already subscribed; + case 'failed': + return failed; + case 'stopped': + return not imported; + default: + return null; + } +} + +export function OpmlImportSheet({ open, onClose }: { open: boolean; onClose: () => void }) { + const current = useLiveQuery({ query: (q) => q.from({ source: subscriptions }) }).data ?? []; + const known = useMemo(() => subscribedKeys(current), [current]); + const [phase, setPhase] = useState('pick'); + const [error, setError] = useState(); + const [fileName, setFileName] = useState(''); + const [feeds, setFeeds] = useState([]); + const [skipped, setSkipped] = useState(0); + // Feeds already followed when the file was read: never sent, shown as such. + const [held, setHeld] = useState>(new Set()); + const [excluded, setExcluded] = useState>(new Set()); + const [blogroll, setBlogroll] = useState(false); + const [rows, setRows] = useState>({}); + const [summary, setSummary] = useState(); + const stop = useRef(false); + const visible = useRef(open); + visible.current = open; + + const reset = () => { + setPhase('pick'); + setError(undefined); + setFileName(''); + setFeeds([]); + setSkipped(0); + setHeld(new Set()); + setExcluded(new Set()); + setRows({}); + setSummary(undefined); + setBlogroll(false); + }; + const close = () => { + // A running import keeps going behind the sheet; anything else starts over next time. + if (phase !== 'running') reset(); + onClose(); + }; + const read = async (file: File) => { + setError(undefined); + try { + const parsed = parseOpml(await file.text()); + if (!parsed.feeds.length) throw new Error('No feeds found in this file: no outline has an xmlUrl.'); + setFileName(file.name); + setFeeds(parsed.feeds); + setSkipped(parsed.skipped); + setHeld(new Set(parsed.feeds.filter((feed) => known.has(feedKey(feed.xmlUrl))).map((feed) => feed.xmlUrl))); + setExcluded(new Set()); + setRows({}); + setPhase('preview'); + } catch (error) { + setError(error); + } + }; + const chosen = feeds.filter((feed, index) => !excluded.has(index) && !held.has(feed.xmlUrl)).length; + const start = async () => { + stop.current = false; + setPhase('running'); + const indices = feeds.map((_, index) => index).filter((index) => !excluded.has(index)); + const list = indices.map((index) => feeds[index]); + const result = await runImport(list, { + known: (feed) => held.has(feed.xmlUrl), + subscribe: (feed) => + attempt(BlyggerApi.createSubscription({ client, body: { url: feed.xmlUrl, confirm: true, backfill: false } })), + afterAdd: blogroll + ? (id) => attempt(BlyggerApi.updateSubscription({ client, path: { id }, body: { in_blogroll: true } })) + : undefined, + onRow: (i, status) => setRows((prev) => ({ ...prev, [indices[i]]: status })), + shouldStop: () => stop.current, + }); + setSummary(result); + setPhase('done'); + void changed('subscriptions', 'reading'); + if (!visible.current) toast(`OPML import: ${result.added} added, ${result.exists} already subscribed, ${result.failed} failed`); + }; + + return ( + + + {phase === 'pick' ? ( + <> +

+ Choose an OPML file exported from another reader. Folders are flattened, and the + new subscriptions stay out of your public blogroll unless you say otherwise. +

+ + + ) : ( + <> +

+ {fileName} · {feeds.length} feed{feeds.length === 1 ? '' : 's'} + {held.size ? ` · ${held.size} already subscribed` : ''} + {skipped ? ` · ${skipped} outline${skipped === 1 ? '' : 's'} skipped` : ''} +

+
    + {feeds.map((feed, index) => { + const isHeld = held.has(feed.xmlUrl); + const status = rows[index]; + return ( +
  • + {phase === 'preview' ? ( + { + const next = new Set(excluded); + if (event.target.checked) next.delete(index); + else next.add(index); + setExcluded(next); + }} + /> + ) : null} + + {feed.title} + {feed.xmlUrl} + {status?.state === 'failed' ? {status.reason} : null} + {(status?.state === 'working' || status?.state === 'added') && status.note ? ( + {status.note} + ) : null} + + {statusBadge(status, isHeld)} +
    • + ); + })} +
+ {phase === 'preview' ? ( + <> + +

+ Feeds are added one at a time, about one a second. Their posts arrive with the next + scheduled poll, within a quarter of an hour. +

+
+ + +
+ + ) : null} + {phase === 'running' ? ( +
+

+ {Object.values(rows).filter((row) => row.state !== 'pending' && row.state !== 'working').length} of{' '} + {feeds.length - excluded.size} done +

+ +
+ ) : null} + {phase === 'done' && summary ? ( +
+

+ + {summary.added} added · {summary.exists} already subscribed · {summary.failed} failed + {summary.stopped ? ` · ${summary.stopped} not imported` : ''} + +

+ {summary.added ? ( +

Their posts arrive with the next scheduled poll, within a quarter of an hour.

+ ) : null} +
+ +
+
+ ) : null} + + )} + + ); +} diff --git a/src/ui/opml.ts b/src/ui/opml.ts new file mode 100644 index 00000000..508e0847 --- /dev/null +++ b/src/ui/opml.ts @@ -0,0 +1,270 @@ +/* + * OPML subscription import (studio#63): the pure half. Parsing runs in the + * browser (DOMParser) and the import walks the list through the ordinary + * createSubscription route, one request at a time, so there is no server route + * of its own. Folders are flattened (no hopper mapping) and nothing reaches the + * public blogroll unless the owner asks. + * + * Nothing here touches DOM globals at import time, so the UI-state suite can + * test it under node with its own XML parser. + */ + +/** The slice of a DOM Element the parser reads. A real Element satisfies it. */ +export interface XmlElement { + readonly localName: string; + getAttribute(name: string): string | null; + readonly children: ArrayLike; +} +/** The slice of a DOM Document the parser reads. */ +export interface XmlDocument { + readonly documentElement: XmlElement | null; + getElementsByTagName(name: string): ArrayLike; +} +export type ParseXml = (text: string) => XmlDocument; + +export interface OpmlFeed { + title: string; + xmlUrl: string; + htmlUrl?: string; +} +export interface OpmlResult { + feeds: OpmlFeed[]; + /** Outlines carrying an xmlUrl that were not taken: a non-feed type, a URL that is not http(s), or a repeat. */ + skipped: number; +} +export class OpmlError extends Error { + constructor(message: string) { + super(message); + this.name = 'OpmlError'; + } +} + +const browserParse: ParseXml = (text) => new DOMParser().parseFromString(text, 'application/xml'); + +/** Some exporters write a bare `&` inside a URL, and HTML entities XML does not define. */ +export function repairEntities(text: string): string { + return text.replace(/&(?!(?:amp|lt|gt|quot|apos|#[0-9]+|#x[0-9a-fA-F]+);)/g, '&'); +} + +const HTML_ENTITIES: Record = { + amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', hellip: '…', mdash: '—', ndash: '–', + lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™', + eacute: 'é', egrave: 'è', aacute: 'á', agrave: 'à', oacute: 'ó', iacute: 'í', uacute: 'ú', ntilde: 'ñ', + ouml: 'ö', uuml: 'ü', auml: 'ä', szlig: 'ß', ccedil: 'ç', +}; +/** + * Titles are often escaped twice (`&#8217;` in the file, `’` after + * XML parsing). Decode that second layer; leave an unknown name as written. + */ +export function decodeTitle(text: string): string { + return text + .replace(/&(#[0-9]+|#x[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (whole, name: string) => { + if (name[0] === '#') { + const code = name[1] === 'x' || name[1] === 'X' ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10); + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole; + } + return HTML_ENTITIES[name] ?? whole; + }) + .replace(/\s+/g, ' ') + .trim(); +} + +function attr(el: XmlElement, ...names: string[]): string { + for (const name of names) { + const value = el.getAttribute(name); + if (value !== null && value.trim()) return value.trim(); + } + return ''; +} +function httpUrl(raw: string): string | null { + try { + const url = new URL(raw); + return url.protocol === 'http:' || url.protocol === 'https:' ? url.toString() : null; + } catch { + return null; + } +} +const FEED_TYPES = new Set(['rss', 'atom']); +const failed = (doc: XmlDocument) => doc.getElementsByTagName('parsererror').length > 0 || !doc.documentElement; + +/** + * Read an OPML file into a flat, de-duplicated list of feeds. Outlines with an + * xmlUrl and `type="rss"` (or no type) are feeds; every outline is recursed + * into, so a folder's feeds are found whether or not the folder has a URL. + * Throws OpmlError for a file that is not OPML. + */ +export function parseOpml(text: string, parseXml: ParseXml = browserParse): OpmlResult { + const source = text.replace(/^/, ''); + if (!source.trim()) throw new OpmlError('The file is empty.'); + let doc = parseXml(source); + if (failed(doc)) doc = parseXml(repairEntities(source)); + if (failed(doc)) throw new OpmlError('The file is not well-formed XML, so it cannot be read as OPML.'); + const root = doc.documentElement!; + if (root.localName.toLowerCase() !== 'opml') throw new OpmlError(`This is not an OPML file (its root element is <${root.localName}>).`); + const body = Array.from(root.children).find((el) => el.localName.toLowerCase() === 'body'); + if (!body) throw new OpmlError('This OPML file has no , so it lists no feeds.'); + const feeds: OpmlFeed[] = []; + const seen = new Set(); + let skipped = 0; + const walk = (parent: XmlElement, depth: number) => { + if (depth > 64) return; + for (const el of Array.from(parent.children)) { + if (el.localName.toLowerCase() !== 'outline') continue; + const raw = attr(el, 'xmlUrl', 'xmlurl', 'xmlURL'); + if (raw) { + const type = attr(el, 'type').toLowerCase(); + const xmlUrl = httpUrl(raw); + const key = xmlUrl ? feedKey(xmlUrl) : ''; + if ((type && !FEED_TYPES.has(type)) || !xmlUrl || seen.has(key)) skipped++; + else { + seen.add(key); + const htmlUrl = httpUrl(attr(el, 'htmlUrl', 'htmlurl', 'htmlURL')) ?? undefined; + const title = decodeTitle(attr(el, 'title') || attr(el, 'text')) || new URL(xmlUrl).hostname; + feeds.push(htmlUrl ? { title, xmlUrl, htmlUrl } : { title, xmlUrl }); + } + } + walk(el, depth + 1); + } + }; + walk(body, 0); + return { feeds, skipped }; +} + +/** One spelling per feed: scheme, `www.`, case of the host and a trailing slash do not tell feeds apart. */ +export function feedKey(url: string): string { + try { + const u = new URL(url); + const path = u.pathname.replace(/\/+$/, ''); + return `${u.hostname.toLowerCase().replace(/^www\./, '')}${u.port ? `:${u.port}` : ''}${path}${u.search}`; + } catch { + return url.trim(); + } +} +/** The keys of what the owner already follows: each subscription's feed and origin. */ +export function subscribedKeys(subs: readonly { origin: string; feed_url: string }[]): Set { + const keys = new Set(); + for (const sub of subs) { + keys.add(feedKey(sub.feed_url)); + keys.add(feedKey(sub.origin)); + } + return keys; +} + +/* ---------------- the paced import ---------------- */ + +/** What one createSubscription (or blogroll PATCH) attempt came to. */ +export type Attempt = + | { kind: 'ok'; id?: string } + | { kind: 'exists'; message?: string } + | { kind: 'throttled'; retryAfterMs: number } + | { kind: 'failed'; reason: string }; +export type RowStatus = + | { state: 'pending' } + | { state: 'working'; note?: string } + | { state: 'added'; note?: string } + | { state: 'exists' } + | { state: 'failed'; reason: string } + | { state: 'stopped' }; +export interface ImportSummary { + added: number; + exists: number; + failed: number; + stopped: number; +} +export interface ImportOptions { + subscribe: (feed: OpmlFeed) => Promise; + /** Runs after a subscription is created (the optional blogroll PATCH). */ + afterAdd?: (id: string) => Promise; + /** Rows to settle without a request, as "already subscribed". */ + known?: (feed: OpmlFeed) => boolean; + onRow: (index: number, status: RowStatus) => void; + sleep?: (ms: number) => Promise; + /** Pause between write requests. */ + spacingMs?: number; + /** Throttled answers tolerated per request before the row fails. */ + maxRetries?: number; + /** Longest Retry-After honoured; longer ones wait this long and try again. */ + maxWaitMs?: number; + shouldStop?: () => boolean; +} + +/** + * Seconds or an HTTP date (RFC 9110 §10.2.3); a missing or unreadable value + * waits a whole budget window. + */ +export function retryAfterMs(header: string | null | undefined, now = Date.now()): number { + const fallback = 60_000; + if (!header) return fallback; + const value = header.trim(); + if (/^\d+$/.test(value)) return Number(value) * 1000; + const at = Date.parse(value); + return Number.isNaN(at) ? fallback : Math.max(0, at - now); +} + +export const IMPORT_SPACING_MS = 1000; + +/** + * Subscribe to each feed in turn: one write in flight, a pause between writes, + * and a 429 waited out for as long as its Retry-After says before the same row + * is tried again. A 409 is "already subscribed", not a failure. + */ +export async function runImport(feeds: readonly OpmlFeed[], options: ImportOptions): Promise { + const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const spacing = options.spacingMs ?? IMPORT_SPACING_MS; + const maxRetries = options.maxRetries ?? 5; + const maxWait = options.maxWaitMs ?? 120_000; + const summary: ImportSummary = { added: 0, exists: 0, failed: 0, stopped: 0 }; + let wrote = false; + const write = async (index: number, call: () => Promise): Promise => { + for (let tries = 0; ; tries++) { + if (wrote) await sleep(spacing); + wrote = true; + const result = await call(); + if (result.kind !== 'throttled') return result; + if (tries >= maxRetries) return { kind: 'failed', reason: 'rate limited: the write budget stayed exhausted' }; + const wait = Math.min(Math.max(result.retryAfterMs, spacing), maxWait); + options.onRow(index, { state: 'working', note: `rate limited, retrying in ${Math.ceil(wait / 1000)} s` }); + await sleep(wait); + } + }; + for (const [index, feed] of feeds.entries()) { + if (options.shouldStop?.()) { + options.onRow(index, { state: 'stopped' }); + summary.stopped++; + continue; + } + if (options.known?.(feed)) { + options.onRow(index, { state: 'exists' }); + summary.exists++; + continue; + } + options.onRow(index, { state: 'working' }); + let result: Attempt; + try { + result = await write(index, () => options.subscribe(feed)); + } catch (error) { + result = { kind: 'failed', reason: error instanceof Error ? error.message : String(error) }; + } + if (result.kind === 'exists') { + options.onRow(index, { state: 'exists' }); + summary.exists++; + } else if (result.kind === 'failed' || result.kind === 'throttled') { + options.onRow(index, { state: 'failed', reason: result.kind === 'failed' ? result.reason : 'rate limited' }); + summary.failed++; + } else { + let note: string | undefined; + if (options.afterAdd && result.id) { + let extra: Attempt; + try { + extra = await write(index, () => options.afterAdd!(result.id!)); + } catch (error) { + extra = { kind: 'failed', reason: error instanceof Error ? error.message : String(error) }; + } + if (extra.kind !== 'ok') note = 'not added to the blogroll'; + } + options.onRow(index, note ? { state: 'added', note } : { state: 'added' }); + summary.added++; + } + } + return summary; +} diff --git a/src/ui/reading.css b/src/ui/reading.css index 3721793e..198c803a 100644 --- a/src/ui/reading.css +++ b/src/ui/reading.css @@ -18,6 +18,8 @@ .back-link { display: inline-block; font-size: .86rem; color: var(--page-ink-soft); text-decoration: none; margin: 0 2px 2px; padding: 4px 0; } .back-link:hover { color: var(--page-ink); } .list-h a { color: inherit; text-transform: none; letter-spacing: 0; font-weight: 600; } +.list-h-actions { display: flex; gap: 12px; align-items: baseline; } +.list-h .link-btn { background: none; border: none; padding: 0; font: inherit; color: inherit; text-transform: none; letter-spacing: 0; font-weight: 600; text-decoration: underline; text-underline-offset: 2px; cursor: pointer; } /* ---------- SOURCES LIST ---------- */ .feeds { list-style: none; margin: 0 0 14px; padding: 0; background: var(--card); border: 1px solid var(--rule); border-radius: var(--radius); box-shadow: var(--shadow); overflow: hidden; } @@ -127,6 +129,25 @@ .add-sub-confirm { margin-top: 14px; border-top: 1px solid var(--rule); padding-top: 12px; } .add-sub-confirm p { margin: 0 0 6px; } .add-sub-confirm .mismatch { color: var(--warn); font-size: .86rem; } +.add-sub-more { margin-top: 10px; } + +/* ---------- OPML IMPORT ---------- */ +.opml-pick { position: relative; justify-content: center; padding: 13px; width: 100%; cursor: pointer; } +.opml-pick input { position: absolute; inset: 0; opacity: 0; cursor: pointer; } +.opml-head { margin: 0 0 8px; font-size: .88rem; color: var(--ink-soft); overflow-wrap: anywhere; } +.opml-head strong { color: var(--ink); } +.opml-list { list-style: none; margin: 0 0 12px; padding: 0; border: 1px solid var(--rule); border-radius: var(--radius); max-height: 42vh; overflow-y: auto; } +.opml-row { display: flex; align-items: center; gap: 10px; padding: 9px 12px; border-top: 1px solid var(--rule); } +.opml-row:first-child { border-top: none; } +.opml-row input { flex: 0 0 auto; accent-color: var(--pencil); } +.opml-row .badge { flex: 0 0 auto; } +.opml-text { flex: 1; min-width: 0; } +.opml-title { display: block; font-weight: 600; font-size: .92rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.opml-url { display: block; font-size: .74rem; color: var(--ink-soft); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.opml-reason { display: block; font-size: .74rem; color: var(--alert); overflow-wrap: anywhere; } +.opml-row[data-state=added] .opml-reason, .opml-row[data-state=working] .opml-reason { color: var(--warn); } +.opml-progress { margin: 0; font-size: .86rem; color: var(--ink-soft); text-align: center; } +.opml-summary p { margin: 0 0 8px; } /* ---------- HOPPERS ---------- */ .hopper-row { position: relative; } diff --git a/src/ui/reading.tsx b/src/ui/reading.tsx index ce8b1e42..32b66028 100644 --- a/src/ui/reading.tsx +++ b/src/ui/reading.tsx @@ -58,6 +58,7 @@ import { Sheet, confirm, menu, prompt, toast } from './sheets.tsx'; import { displayUrl, sourceTitleAndUrl } from '../importer/util.ts'; import { formatDateIn } from '../dates.ts'; import { AddFeedForm } from './catalog.tsx'; +import { OpmlImportSheet } from './opml-import.tsx'; import { diffText, type DiffOp } from '../word-diff.ts'; import { useSwipe } from './swipe.ts'; import type { EntryContext, StudioExtension } from './extension-api.ts'; @@ -837,9 +838,11 @@ function Inspector({ function SubscribeSheet({ open, onClose, + onImport, }: { open: boolean; onClose: () => void; + onImport: () => void; }) { return ( @@ -849,6 +852,17 @@ function SubscribeSheet({ toast('subscribed'); }} /> +
+ +
); } @@ -1064,6 +1078,7 @@ function Sources() { const actions = useReadingActions(); const { toggle } = useSubscriptionActions(actions); const [inspect, setInspect] = useState(); + const [importing, setImporting] = useState(false); // The first page of "all" carries the counts for every source; it is also // what tapping "all" shows, so it is warm when that happens. const lens = useContext(LensContext); @@ -1113,8 +1128,11 @@ function Sources() { ) : null}
subscriptions · {sources.length} - + {sources.filter((source) => source.in_blogroll).length} in blogroll +
{sources.length ? ( @@ -1150,6 +1168,7 @@ function Sources() { onClose={() => setInspect(undefined)} actions={actions} /> + setImporting(false)} /> ); } @@ -1199,6 +1218,7 @@ function ReadingHead({ children?: ReactNode; }) { const [adding, setAdding] = useState(false); + const [importing, setImporting] = useState(false); const [resyncing, setResyncing] = useState(false); // Polls run in the background on the server; refresh twice as they land. const resyncAll = async () => { @@ -1242,7 +1262,8 @@ function ReadingHead({ - setAdding(false)} /> + setAdding(false)} onImport={() => setImporting(true)} /> + setImporting(false)} /> ); } diff --git a/test-ui/opml.test.ts b/test-ui/opml.test.ts new file mode 100644 index 00000000..156d22e2 --- /dev/null +++ b/test-ui/opml.test.ts @@ -0,0 +1,194 @@ +import { expect, test } from 'vitest'; +import { XMLParser, XMLValidator } from 'fast-xml-parser'; +import { + OpmlError, + decodeTitle, + feedKey, + parseOpml, + repairEntities, + retryAfterMs, + runImport, + subscribedKeys, + type Attempt, + type OpmlFeed, + type ParseXml, + type RowStatus, + type XmlElement, +} from '../src/ui/opml.ts'; + +/* + * The browser parses with DOMParser; this suite runs under node, so it hands + * parseOpml a stand-in with DOMParser's contract: a malformed document comes + * back carrying a , and an entity XML does not define (or a bare + * `&`) makes the document malformed, as it does in the browser. The real + * DOMParser path is driven by e2e/opml-import.spec.ts. + */ +type Node = Record; +function element(node: Node): XmlElement { + const name = Object.keys(node).find((key) => key !== ':@')!; + const attrs = (node[':@'] ?? {}) as Record; + const kids = (node[name] as Node[]).filter((child) => !('#text' in child)).map(element); + return { localName: name, getAttribute: (key) => (key in attrs ? attrs[key] : null), children: kids }; +} +const BAD_ENTITY = /&(?!(?:amp|lt|gt|quot|apos|#[0-9]+|#x[0-9a-fA-F]+);)/; +const nodeParse: ParseXml = (text) => { + const broken = { documentElement: null, getElementsByTagName: (name: string) => (name === 'parsererror' ? [{}] : []) }; + if (XMLValidator.validate(text) !== true || BAD_ENTITY.test(text)) return broken; + const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '', preserveOrder: true, processEntities: true }); + const top = (parser.parse(text) as Node[]).filter((node) => !('?xml' in node) && !('#text' in node)); + return { documentElement: top[0] ? element(top[0]) : null, getElementsByTagName: () => [] }; +}; +const parse = (text: string) => parseOpml(text, nodeParse); + +const NESTED = ` + + my feeds + + + + + + + + + + +`; + +test('nested folders are flattened, in document order', () => { + const { feeds, skipped } = parse(NESTED); + expect(feeds.map((feed) => feed.title)).toEqual(['Alpha', 'Beta', 'Gamma, no type', 'Delta']); + expect(feeds[0]).toEqual({ title: 'Alpha', xmlUrl: 'https://alpha.example/feed.xml', htmlUrl: 'https://alpha.example/' }); + expect(skipped).toBe(0); +}); + +test('an outline with an xmlUrl and no type is a feed; title falls back to text, then the host', () => { + const { feeds } = parse(` + + + `); + expect(feeds).toEqual([ + { title: 'notype.example', xmlUrl: 'https://notype.example/feed' }, + { title: 'By text', xmlUrl: 'https://bytext.example/feed' }, + ]); +}); + +test('outlines without an xmlUrl are recursed into but never listed; other types and bad URLs are skipped', () => { + const { feeds, skipped } = parse(` + + + + + + + `); + expect(feeds.map((feed) => feed.title)).toEqual(['Kept']); + expect(skipped).toBe(3); +}); + +test('entities: XML ones decode, double-escaped titles decode, bare ampersands and HTML entities are repaired', () => { + const escaped = parse(` + + `); + expect(escaped.feeds[0]).toEqual({ title: 'Tom & Jerry’s ', xmlUrl: 'https://e.example/feed?a=1&b=2' }); + + const sloppy = parse(` + + `); + expect(sloppy.feeds[0]).toEqual({ title: 'Café — notes', xmlUrl: 'https://e.example/feed?a=1&b=2' }); +}); + +test('a malformed or non-OPML file is refused with a reason', () => { + expect(() => parse('')).toThrow(OpmlError); + expect(() => parse('')).toThrow(/well-formed/); + expect(() => parse('')).toThrow(/empty/); + expect(() => parse('')).toThrow(/not an OPML file/); + expect(() => parse('')).toThrow(/no /); + expect(() => parse('')).not.toThrow(); +}); + +test('helpers: repair, decode, keys', () => { + expect(repairEntities('a & b & & é')).toBe('a & b & & &eacute;'); + expect(decodeTitle(' x—y &unknown; ')).toBe('x—y &unknown;'); + expect(feedKey('https://WWW.Example.com/feed/')).toBe(feedKey('http://example.com/feed')); + expect(feedKey('https://example.com/feed?a=1')).not.toBe(feedKey('https://example.com/feed?a=2')); + const keys = subscribedKeys([{ origin: 'https://blyg.example/', feed_url: 'https://blyg.example/feed.xml' }]); + expect(keys.has(feedKey('https://blyg.example/feed.xml'))).toBe(true); + expect(keys.has(feedKey('https://blyg.example'))).toBe(true); +}); + +test('Retry-After: seconds, an HTTP date, or a whole window when absent', () => { + expect(retryAfterMs('7')).toBe(7000); + expect(retryAfterMs('Thu, 01 Jan 2026 00:01:00 GMT', Date.parse('Thu, 01 Jan 2026 00:00:30 GMT'))).toBe(30_000); + expect(retryAfterMs(null)).toBe(60_000); + expect(retryAfterMs('soon')).toBe(60_000); +}); + +const feeds: OpmlFeed[] = ['a', 'b', 'c', 'd'].map((name) => ({ title: name, xmlUrl: `https://${name}.example/feed` })); + +test('the import is sequential and paced, 409 is "already subscribed", 429 waits out Retry-After', async () => { + const log: string[] = []; + const answers: Record = { + a: [{ kind: 'ok', id: 'id-a' }], + b: [{ kind: 'throttled', retryAfterMs: 5000 }, { kind: 'ok', id: 'id-b' }], + c: [{ kind: 'exists', message: 'already subscribed to c' }], + d: [{ kind: 'failed', reason: 'could not resolve this URL to a blyg or a feed' }], + }; + let inFlight = 0; + const rows: Record = {}; + const summary = await runImport(feeds, { + subscribe: async (feed) => { + inFlight++; + expect(inFlight).toBe(1); + log.push(`post ${feed.title}`); + await Promise.resolve(); + inFlight--; + return answers[feed.title].shift()!; + }, + sleep: async (ms) => void log.push(`sleep ${ms}`), + onRow: (index, status) => (rows[index] = status), + spacingMs: 1000, + }); + expect(log).toEqual(['post a', 'sleep 1000', 'post b', 'sleep 5000', 'sleep 1000', 'post b', 'sleep 1000', 'post c', 'sleep 1000', 'post d']); + expect(summary).toEqual({ added: 2, exists: 1, failed: 1, stopped: 0 }); + expect(rows[2]).toEqual({ state: 'exists' }); + expect(rows[3]).toEqual({ state: 'failed', reason: 'could not resolve this URL to a blyg or a feed' }); +}); + +test('known rows send nothing, the blogroll PATCH is paced like any write, and stop leaves the rest', async () => { + const log: string[] = []; + let stop = false; + const summary = await runImport(feeds, { + known: (feed) => feed.title === 'a', + subscribe: async (feed) => { + log.push(`post ${feed.title}`); + if (feed.title === 'c') stop = true; + return { kind: 'ok', id: `id-${feed.title}` }; + }, + afterAdd: async (id) => { + log.push(`patch ${id}`); + return id === 'id-c' ? { kind: 'failed', reason: 'nope' } : { kind: 'ok' }; + }, + sleep: async (ms) => void log.push(`sleep ${ms}`), + onRow: () => {}, + shouldStop: () => stop, + spacingMs: 1000, + }); + expect(log).toEqual(['post b', 'sleep 1000', 'patch id-b', 'sleep 1000', 'post c', 'sleep 1000', 'patch id-c']); + expect(summary).toEqual({ added: 2, exists: 1, failed: 0, stopped: 1 }); +}); + +test('a budget that stays exhausted fails the row after the retries, and a long Retry-After is capped', async () => { + const waits: number[] = []; + const rows: Record = {}; + const summary = await runImport(feeds.slice(0, 1), { + subscribe: async () => ({ kind: 'throttled', retryAfterMs: 3_600_000 }), + sleep: async (ms) => void waits.push(ms), + onRow: (index, status) => (rows[index] = status), + maxRetries: 2, + maxWaitMs: 90_000, + }); + expect(waits.filter((ms) => ms === 90_000)).toHaveLength(2); + expect(summary.failed).toBe(1); + expect(rows[0]).toMatchObject({ state: 'failed', reason: expect.stringMatching(/rate limited/) }); +}); diff --git a/test/importer/subscribe-backfill.test.ts b/test/importer/subscribe-backfill.test.ts new file mode 100644 index 00000000..25af7311 --- /dev/null +++ b/test/importer/subscribe-backfill.test.ts @@ -0,0 +1,81 @@ +// `backfill` on createSubscription (studio#63, OPML import): false creates the +// subscription and leaves the archive to the quarter-hour cron; omitted keeps +// today's behaviour, a backfill under waitUntil. Driven through the real REST +// route with a deterministic public DNS answer and a fixture RSS feed, since +// the route's own fetch is the platform's (api.test.ts header note). +import { env, createExecutionContext, waitOnExecutionContext } from "cloudflare:test"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { Hono } from "hono"; +import { createOwnerApi } from "../../src/owner-api.ts"; +import { getSubscription, listImportedItems } from "../../src/importer/store.ts"; +import { dueSubscriptions } from "../../src/importer/schedule.ts"; + +const RSS = (host: string) => `${host}https://${host}/ + ${host}-1onehttps://${host}/1firstMon, 05 Oct 2026 10:00:00 GMT +`; + +afterEach(() => vi.restoreAllMocks()); + +/** Answers DNS with a public address, the feed URL with RSS, anything else 404; counts feed fetches. */ +function network(feedUrl: string) { + const host = new URL(feedUrl).host; + const counts = { feed: 0 }; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const url = String(input instanceof Request ? input.url : input); + if (url.startsWith("https://cloudflare-dns.com/")) return Response.json({ Status: 0, Answer: [{ type: 1, data: "93.184.216.34" }] }); + if (url === feedUrl) { + counts.feed++; + return new Response(RSS(host), { headers: { "content-type": "application/rss+xml" } }); + } + return new Response("not found", { status: 404 }); + }); + return counts; +} + +async function subscribe(body: Record) { + const app = new Hono().route("/api", createOwnerApi({ scope: ["owner:manage"], clientId: "opml-backfill", userId: "owner" })); + const ctx = createExecutionContext(); + const waitUntil = vi.spyOn(ctx, "waitUntil"); + const res = await app.fetch(new Request("https://studio.example/api/subscriptions", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }), env, ctx); + await waitOnExecutionContext(ctx); + return { status: res.status, json: (await res.json()) as any, waitUntil }; +} + +describe("createSubscription backfill flag", () => { + it("backfill: false creates the subscription and queues no backfill; the cron finds it due", async () => { + const feedUrl = "https://quiet.example/feed.xml"; + const counts = network(feedUrl); + const { status, json, waitUntil } = await subscribe({ url: feedUrl, confirm: true, backfill: false }); + expect(status).toBe(201); + expect(json).toMatchObject({ kind: "rss", feed_url: feedUrl, in_blogroll: false, last_poll_at: null }); + // Resolution reads the feed once; nothing polls it afterwards. + expect(counts.feed).toBe(1); + expect(waitUntil.mock.calls.length).toBe(0); + const row = await getSubscription(env.DB, json.id); + expect(row).toMatchObject({ last_poll_at: null, last_index_sync_at: null }); + expect(await listImportedItems(env.DB, json.id)).toHaveLength(0); + // The claim the flag rests on: a never-polled, unpaused subscription is due on the next tick. + expect(dueSubscriptions([row!], Date.now()).map((s) => s.id)).toEqual([json.id]); + }); + + it("the default still backfills after the response", async () => { + const feedUrl = "https://eager.example/feed.xml"; + const counts = network(feedUrl); + const { status, json, waitUntil } = await subscribe({ url: feedUrl, confirm: true }); + expect(status).toBe(201); + expect(waitUntil.mock.calls.length).toBeGreaterThanOrEqual(1); + expect(counts.feed).toBe(2); + const row = await getSubscription(env.DB, json.id); + expect(row?.last_poll_at).not.toBeNull(); + expect(await listImportedItems(env.DB, json.id)).toHaveLength(1); + }); + + it("backfill: true is the default spelled out, and a non-boolean is refused", async () => { + const feedUrl = "https://explicit.example/feed.xml"; + const counts = network(feedUrl); + expect((await subscribe({ url: feedUrl, confirm: true, backfill: "no" })).status).toBe(400); + const { status } = await subscribe({ url: feedUrl, confirm: true, backfill: true }); + expect(status).toBe(201); + expect(counts.feed).toBe(2); + }); +});