Import subscriptions from an OPML file (#63)

docs/api.md

@@ -89,6 +89,30 @@ Creation returns the item, status 201, and a `Location` header.
 Hopper and media creation also return 201.
 Subscription creation returns 200 when the owner must accept the proposed subscription, then 201 when it creates the subscription.
 
+## Subscriptions
+
+`POST /subscriptions` takes `{ url, confirm?, title?, backfill? }`.
+Without `confirm` it resolves the URL and returns the proposal (200); with `confirm: true` it creates the subscription (201).
+A source that is already subscribed, by origin or by feed, is refused with 409.
+New subscriptions are not in the public `blogroll.opml`; `PATCH /subscriptions/{id}` with `in_blogroll: true` adds one.
+
+`backfill` (boolean, default `true`) applies with `confirm: true`.
+By default the server starts importing the source's archive as soon as it has answered.
+With `backfill: false` it creates the subscription and imports nothing yet: a subscription that has never been polled is always due, so the next quarter-hour poll imports it.
+Use it for bulk imports, so that many subscriptions do not each start an archive import at once:
+
+```ts
+const result = await BlyggerApi.createSubscription({
+  client,
+  body: { url: "https://example.com/feed.xml", confirm: true, backfill: false },
+});
+if (result.response?.status === 409) { /* already subscribed */ }
+if (result.response?.status === 429) { /* wait Retry-After seconds, then retry */ }
+```
+
+The Studio's OPML import works this way. It reads the file in the browser, flattens folders and sends one request at a time, about one a second, which stays under the owner's write budget of 120 writes a minute (`API_WRITE_LIMIT`).
+Delegated clients have less room: 120 a minute per grant, and 60 a minute shared by every grant (`API_DELEGATED_WRITE_LIMIT`), so a token-bearing importer should go slower than one a second and honour `Retry-After` on 429.
+
 ## Partial edits
 
 PATCH changes only supplied fields.

e2e/opml-import.spec.ts

@@ -0,0 +1,135 @@
+import { test, expect, type Page } from './fixture';
+
+// OPML import (studio#63): the file is parsed in the browser, the preview marks
+// what is already subscribed, and each feed is one createSubscription with
+// backfill: false, sent one at a time. The POSTs are answered here, because a
+// real subscribe would reach the network and change the shared fixture
+// subscriptions other tests read; the server side of `backfill` is covered by
+// test/importer/subscribe-backfill.test.ts.
+async function login(page: Page) {
+  await page.goto('/studio/login');
+  await page.locator('[name=password]').fill('test-password');
+  await page.getByRole('button', { name: 'log in', exact: true }).click();
+  await expect(page.locator('#composer-text')).toBeVisible();
+}
+
+// Real-world untidiness on purpose: nested folders, a feed with no type, a
+// folder outline with no xmlUrl, a double-escaped title, an HTML entity XML does
+// not define and a bare & in a URL (both repaired), and the fixture's own
+// legacy feed, which is already subscribed.
+const OPML = `<?xml version="1.0" encoding="UTF-8"?>
+<opml version="1.0">
+  <head><title>Exported</title></head>
+  <body>
+    <outline text="News">
+      <outline type="rss" text="Alpha &amp;amp; Co" xmlUrl="https://alpha.example/feed.xml"/>
+      <outline text="Deeper">
+        <outline text="Caf&eacute; notes" xmlUrl="https://beta.example/rss?a=1&b=2"/>
+      </outline>
+    </outline>
+    <outline type="rss" text="Legacy again" xmlUrl="https://legacy.example/feed"/>
+    <outline type="rss" text="Gamma" xmlUrl="https://gamma.example/feed"/>
+    <outline type="rss" text="Delta" xmlUrl="https://delta.example/feed"/>
+  </body>
+</opml>`;
+
+const subscription = (url: string) => ({
+  id: `sub-${new URL(url).hostname}`, kind: 'rss', origin: url, feed_url: url, title: new URL(url).hostname, status: 'active',
+  last_poll_at: null, fail_count: 0, last_index_sync_at: null, created: '2026-10-18T00:00:00Z',
+  title_follows_source: true, in_blogroll: false, flags: [],
+});
+
+test('an OPML file is previewed, imported one feed at a time, and summarised', async ({ page }) => {
+  const errors: string[] = [];
+  page.on('pageerror', (error) => errors.push(error.message));
+  await login(page);
+  await page.goto('/studio/reading?view=sources');
+
+  const posts: { url: string; backfill: unknown; confirm: unknown; at: number }[] = [];
+  let inFlight = 0, overlapped = false, gammaTries = 0;
+  const patches: string[] = [];
+  await page.route('**/api/subscriptions', async (route) => {
+    if (route.request().method() !== 'POST') return route.fallback();
+    const body = route.request().postDataJSON() as { url: string; backfill: unknown; confirm: unknown };
+    posts.push({ ...body, at: Date.now() });
+    if (inFlight++) overlapped = true;
+    try {
+      const host = new URL(body.url).hostname;
+      if (host === 'alpha.example' || host === 'beta.example') return await route.fulfill({ status: 201, json: subscription(body.url) });
+      if (host === 'gamma.example' && gammaTries++ === 0) return await route.fulfill({ status: 429, headers: { 'Retry-After': '1' }, json: { error: 'API work budget exceeded' } });
+      if (host === 'gamma.example') return await route.fulfill({ status: 409, json: { error: 'already subscribed to Gamma' } });
+      return await route.fulfill({ status: 422, json: { error: 'could not resolve this URL to a blyg or a feed', tried: [body.url] } });
+    } finally {
+      inFlight--;
+    }
+  });
+  await page.route('**/api/subscriptions/sub-*', async (route) => {
+    if (route.request().method() !== 'PATCH') return route.fallback();
+    patches.push(route.request().url());
+    await route.fulfill({ json: { ...subscription('https://x.example/'), in_blogroll: true } });
+  });
+
+  await page.locator('#opml-open').click();
+  const sheet = page.getByRole('dialog', { name: 'import OPML' });
+  await sheet.locator('#opml-file').setInputFiles({ name: 'feeds.opml', mimeType: 'text/x-opml', buffer: Buffer.from(OPML) });
+
+  const list = sheet.getByRole('list', { name: 'feeds in the file' });
+  await expect(list.locator('.opml-row')).toHaveCount(5);
+  await expect(list.locator('.opml-title')).toHaveText(['Alpha & Co', 'Café notes', 'Legacy again', 'Gamma', 'Delta']);
+  await expect(list.locator('.opml-url').nth(1)).toHaveText('https://beta.example/rss?a=1&b=2');
+  await expect(list.locator('.opml-row').nth(2)).toContainText('already subscribed');
+  await expect(sheet.getByRole('checkbox', { name: 'import Legacy again' })).toBeDisabled();
+  await expect(sheet.getByRole('checkbox', { name: 'add these to my blogroll' })).not.toBeChecked();
+
+  // Leave Delta out; the blogroll box stays off.
+  await sheet.getByRole('checkbox', { name: 'import Delta' }).uncheck();
+  await sheet.getByRole('button', { name: 'import 3 feeds' }).click();
+
+  const summary = sheet.locator('.opml-summary');
+  await expect(summary).toContainText('2 added · 2 already subscribed · 0 failed', { timeout: 15_000 });
+  await expect(list.locator('.opml-row').nth(0)).toContainText('added');
+  await expect(list.locator('.opml-row').nth(3)).toContainText('already subscribed');
+
+  // Alpha, Beta, Gamma (throttled), Gamma again. Legacy was never sent, nor Delta.
+  expect(posts.map((post) => new URL(post.url).hostname)).toEqual(['alpha.example', 'beta.example', 'gamma.example', 'gamma.example']);
+  expect(posts.every((post) => post.backfill === false && post.confirm === true)).toBe(true);
+  expect(overlapped).toBe(false);
+  // Paced: about a second between writes, and the 429's Retry-After waited out.
+  expect(posts[1].at - posts[0].at).toBeGreaterThanOrEqual(900);
+  expect(posts[3].at - posts[2].at).toBeGreaterThanOrEqual(900);
+  expect(patches).toEqual([]);
+
+  await sheet.getByRole('button', { name: 'done' }).click();
+  await expect(sheet).toHaveCount(0);
+  expect(errors).toEqual([]);
+});
+
+test('the blogroll box adds each new subscription with PATCH, and a malformed file is refused', async ({ page }) => {
+  await login(page);
+  await page.goto('/studio/reading?view=sources');
+  const patched: unknown[] = [];
+  await page.route('**/api/subscriptions', (route) =>
+    route.request().method() === 'POST'
+      ? route.fulfill({ status: 201, json: subscription((route.request().postDataJSON() as { url: string }).url) })
+      : route.fallback(),
+  );
+  await page.route('**/api/subscriptions/sub-*', async (route) => {
+    if (route.request().method() !== 'PATCH') return route.fallback();
+    patched.push(route.request().postDataJSON());
+    await route.fulfill({ json: { ...subscription('https://one.example/feed'), in_blogroll: true } });
+  });
+
+  await page.locator('#opml-open').click();
+  const sheet = page.getByRole('dialog', { name: 'import OPML' });
+  await sheet.locator('#opml-file').setInputFiles({ name: 'broken.opml', mimeType: 'text/x-opml', buffer: Buffer.from('<opml><body><outline xmlUrl="https://x.example/"></body>') });
+  await expect(sheet.getByRole('alert')).toContainText('not well-formed XML');
+
+  await sheet.locator('#opml-file').setInputFiles({
+    name: 'one.opml', mimeType: 'text/x-opml',
+    buffer: Buffer.from('<opml version="2.0"><body><outline text="One" xmlUrl="https://one.example/feed"/></body></opml>'),
+  });
+  await sheet.getByRole('checkbox', { name: 'add these to my blogroll' }).check();
+  await sheet.getByRole('button', { name: 'import 1 feed' }).click();
+  await expect(sheet.locator('.opml-summary')).toContainText('1 added · 0 already subscribed · 0 failed');
+  expect(patched).toEqual([{ in_blogroll: true }]);
+});

openapi.json

@@ -6293,6 +6293,10 @@
                   },
                   "title": {
                     "type": "string"
+                  },
+                  "backfill": {
+                    "type": "boolean",
+                    "description": "With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML."
                   }
                 },
                 "required": [

sdk/generated/types.gen.ts

@@ -2202,6 +2202,10 @@ export type CreateSubscriptionData = {
         url: string;
         confirm?: boolean;
         title?: string;
+        /**
+         * With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML.
+         */
+        backfill?: boolean;
     };
     path?: never;
     query?: never;

sdk/generated/zod.gen.ts

@@ -973,7 +973,8 @@ export const zListSubscriptionsResponse = z.object({
 export const zCreateSubscriptionBody = z.object({
     url: z.string(),
     confirm: z.boolean().optional(),
-    title: z.string().optional()
+    title: z.string().optional(),
+    backfill: z.boolean().optional()
 }).strict();
 
 export const zCreateSubscriptionResponse = z.union([

src/contract/routes.ts

@@ -89,7 +89,7 @@ export const routes = {
   uploadMedia: createRoute({ ...route("uploadMedia", "post", "/media", z.object({ id: z.string(), url: z.string(), mime: z.string() }), undefined, 201), request: { body: { required: true, content: { "multipart/form-data": { schema: z.object({ file: z.custom<File>((value) => value instanceof File, "file field required (multipart)").openapi({ type: "string", format: "binary" }), item_id: z.string().optional(), alt: z.string().optional(), inline: z.enum(["true", "false"]).optional() }) } } } } }),
   deleteMedia: route("deleteMedia", "delete", "/media/{id}", ok.extend({ outcome: z.enum(["deleted", "detached"]) })),
   updateSettings: route("updateSettings", "patch", "/settings", SettingsSchema, settingsBody),
-  createSubscription: route("createSubscription", "post", "/subscriptions", confirmation, z.object({ url: z.string(), confirm: z.boolean().optional(), title: z.string().optional() })),
+  createSubscription: route("createSubscription", "post", "/subscriptions", confirmation, z.object({ url: z.string(), confirm: z.boolean().optional(), title: z.string().optional(), backfill: z.boolean().optional().describe("With confirm: start importing the archive at once (true, the default). false creates the subscription only; the next quarter-hour poll imports it, since a never-polled subscription is always due. For bulk imports such as OPML.") })),
   updateSubscription: route("updateSubscription", "patch", "/subscriptions/{id}", SubscriptionSchema, z.object({ in_blogroll: z.boolean().optional(), title: z.string().trim().min(1).nullable().optional().describe("A name of your own; null hands the name back to the source, refreshed while polling."), paused: z.boolean().optional() }).strict()),
   pollAllSubscriptions: route("pollAllSubscriptions", "post", "/subscriptions/poll", z.object({ polling: z.number().int().nonnegative() })),
   resyncSubscription: route("resyncSubscription", "post", "/subscriptions/{id}/resync", ok.extend({ changed: z.number() })),

src/importer/api.ts

@@ -50,7 +50,7 @@ function titleFromUrl(url: string): string {
  * the confirm button for many seconds on a large blyg.
  */
 importerApi.openapi(routes.createSubscription, async (c) => {
-  const body = await readJson<{ url: string; confirm?: boolean; title?: string }>(c);
+  const body = await readJson<{ url: string; confirm?: boolean; title?: string; backfill?: boolean }>(c);
   if (!body.url.trim()) return c.json({ error: "url required" }, 400);
 
   const result = await resolve(body.url.trim(), platformFetchFor(c.env));
@@ -96,7 +96,9 @@ importerApi.openapi(routes.createSubscription, async (c) => {
   // detection (§3.2) in one pass, for both kinds uniformly. If waitUntil cuts
   // it short, the null last_poll_at and last_index_sync_at make the next
   // scheduled poll due and reconcile again, so the backfill completes there.
-  c.executionCtx.waitUntil(pollSubscription(c.env.DB, sub, platformFetchFor(c.env)).catch(() => {}));
+  // `backfill: false` (bulk imports, e.g. OPML) skips it: isDue() treats a
+  // null last_poll_at as due, so the next quarter-hour tick imports it instead.
+  if (body.backfill !== false) c.executionCtx.waitUntil(pollSubscription(c.env.DB, sub, platformFetchFor(c.env)).catch(() => {}));
   c.header("Location", `/api/subscriptions/${sub.id}`);
   return c.json(subscriptionResource(sub), 201);
 });