diff --git a/.github/workflows/mirror-sth.yml b/.github/workflows/mirror-sth.yml new file mode 100644 index 00000000..7f5de15c --- /dev/null +++ b/.github/workflows/mirror-sth.yml @@ -0,0 +1,69 @@ +name: Mirror Stairway to Hell archive + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Scrape only — skip the S3 upload" + type: boolean + required: true + default: true + aws_region: + description: "AWS region for the bucket" + type: string + required: false + default: us-east-1 + +jobs: + mirror: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Scrape STH into ./mirror + run: node tools/mirror-sth.js --out mirror + + - name: Show mirror summary + run: | + echo "Top-level manifest:" + cat mirror/manifest.json + echo + echo "On-disk size:" + du -sh mirror + echo "Object count:" + find mirror -type f | wc -l + + - name: Configure AWS credentials + if: ${{ !inputs.dry_run }} + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws_region }} + + - name: Sync immutable archive blobs + if: ${{ !inputs.dry_run }} + run: | + aws s3 sync mirror/ s3://bbc.xania.org/archive/sth/ \ + --no-progress \ + --cache-control "public, max-age=31536000, immutable" \ + --metadata-directive REPLACE \ + --exclude "*manifest.json" \ + --exclude "meta/*" + + - name: Sync manifests and meta files + if: ${{ !inputs.dry_run }} + run: | + aws s3 sync mirror/ s3://bbc.xania.org/archive/sth/ \ + --no-progress \ + --cache-control "public, max-age=300" \ + --metadata-directive REPLACE \ + --exclude "*" \ + --include "*manifest.json" \ + --include "meta/*" diff --git a/package.json b/package.json index f0ca8e4f..afc856e5 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "coverage:unit": "vitest run tests/unit --coverage", "coverage:all-tests": "vitest run --coverage", "benchmark": "node src/app-bench.js", + "mirror-sth": "node tools/mirror-sth.js --out mirror", "electron": "npm run build && ELECTRON_DISABLE_SANDBOX=1 electron .", "electron:build": "electron-builder" }, diff --git a/tools/README-sth-mirror.md b/tools/README-sth-mirror.md new file mode 100644 index 00000000..1b0bd214 --- /dev/null +++ b/tools/README-sth-mirror.md @@ -0,0 +1,118 @@ +# Mirroring the Stairway to Hell archive + +`tools/mirror-sth.js` scrapes the public BBC Micro, Acorn Electron, and +sideways-ROM areas of `https://www.stairwaytohell.com/` into a local +directory tree that is sync'd into `s3://bbc.xania.org/archive/sth/` by +`.github/workflows/mirror-sth.yml`. + +The mirror exists so jsbeeb's `sth:` URLs and the in-app archive browser +keep working if the upstream site disappears. STH has been effectively +frozen since around 2008 (see `meta/bbc-disklog.txt` and +`meta/bbc-tapelog.txt` in the mirror), so this is a one-shot snapshot +rather than a continuous sync. + +## What is mirrored + +S3 paths mirror STH's upstream layout — `archive/sth//...` matches +the natural URL structure on `stairwaytohell.com`. + +| Category | Source on STH | Notes | +| ------------------------ | ---------------------------------------------------------- | ------------------------------------- | +| `diskimages/` | `bbc/archive/diskimages/reclist.php?sort=name&filter=.zip` | ~1,600 zips, ~26 MB | +| `tapeimages/` | `bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip` | ~1,500 zips, ~24 MB | +| `sthcollection/` | `bbc/sthcollection.html` | ~140 zips, magazine disk packs | +| `other/educational/` | `bbc/other/educational/reclist.php?...` | ~75 zips | +| `roms/` | `roms/homepage.html` | ~50 BBC + Electron sideways ROMs | +| `electron/uefarchive/` | `electron/uefarchive/reclist.php?...` | ~890 Electron tape image zips, ~15 MB | +| `electron/dfs/` | `electron/dfs/homepage.html` | ~230 Electron DFS disk images | +| `electron/adfs/` | `electron/adfs/homepage.html` | ~23 Electron ADFS disk images | +| `electron/multiplexing/` | `electron/multiplexing/homepage.html` | curiosity, 1 file | +| `electron/t2p3/` | `electron/t2p3/homepage.html` | curiosity, 4 files | +| `meta/bbc-disklog.txt` | `bbc/disklog.txt`, `bbc/tapelog.txt` | upstream changelogs | +| `meta/*.html` | `bbc/homepage.html`, `roms/homepage.html`, etc. | site index pages, provenance | + +Total ~4,500 zips, ~80 MB across 10 categories. + +The Electron categories are mirrored even though jsbeeb does not currently +emulate the Electron — the goal is a complete archival snapshot, not just +what the running app uses today. The `roms/` category similarly overlaps +with what jsbeeb already ships in `public/roms/`; mirroring it is purely +archival. + +## Manifest format (schemaVersion 1) + +The top-level `archive/sth/manifest.json` lists categories and points at +each per-category manifest: + +```json +{ + "schemaVersion": 1, + "name": "Stairway to Hell BBC Micro Software Archive", + "source": "https://www.stairwaytohell.com/bbc/", + "scrapedAt": "2026-05-03T17:00:00Z", + "categories": [ + { + "id": "diskimages", + "title": "Disk Images", + "manifest": "diskimages/manifest.json", + "source": "https://www.stairwaytohell.com/bbc/archive/diskimages/", + "fileCount": 1608, + "totalBytes": 27315281 + } + ] +} +``` + +Each per-category manifest is a flat list of files, sorted by path: + +```json +{ + "schemaVersion": 1, + "files": [{ "path": "Acornsoft/Elite.zip", "size": 12345, "mtime": "2003-04-28T00:00:00.000Z" }] +} +``` + +Paths are POSIX-style, relative to the category directory. + +## Running locally + +```sh +# Catalog parse only — no zip downloads. Fast (~2s) sanity check. +node tools/mirror-sth.js --out /tmp/sth-mirror --quick + +# Full mirror — ~80 MB, a few minutes depending on STH's bandwidth. +node tools/mirror-sth.js --out /tmp/sth-mirror + +# Or just one category: +node tools/mirror-sth.js --out /tmp/sth-mirror --source diskimages +``` + +The script is resumable: it skips files that are already on disk with the +expected size, so a re-run after a partial download will only fetch what's +missing. + +## Running in CI + +`.github/workflows/mirror-sth.yml` is `workflow_dispatch` only — no +schedule. The `dry_run` input (default `true`) makes the workflow scrape +into the runner's tmp dir and stop short of uploading. Re-run with +`dry_run: false` once the dry-run output looks right. + +The S3 sync is **strictly additive** — there's no `--delete`. If a file +ever needs to be removed from the mirror, do it by hand or in a follow-up +workflow guarded by an explicit `prune` input. This avoids the failure +mode where a typo on the destination URL with `--delete` could nuke the +live app served from the same bucket. + +Cache headers: + +- Zips: `public, max-age=31536000, immutable` (paths are content-stable) +- Manifests + `meta/*`: `public, max-age=300` (these are what actually + change when we re-mirror) + +## Costs + +Storage for ~80 MB of objects is well under one cent per month at S3 +standard pricing. Egress is fronted by CloudFront (already in front of +`bbc.xania.org`) and dominated by the existing app traffic; the marginal +cost of mirror traffic is negligible. diff --git a/tools/mirror-sth.js b/tools/mirror-sth.js new file mode 100644 index 00000000..f1981a05 --- /dev/null +++ b/tools/mirror-sth.js @@ -0,0 +1,436 @@ +#!/usr/bin/env node +/** + * Mirror the Stairway to Hell BBC Micro software archive into a local + * directory tree. The output is structured to be sync'd as-is into + * s3://bbc.xania.org/archive/sth/ — see tools/README-sth-mirror.md and + * .github/workflows/mirror-sth.yml. + * + * Usage: + * node tools/mirror-sth.js --out [--concurrency 8] + * [--source ] [--quick] + * + * Self-contained: relies only on Node 22 builtins (fetch, fs/promises). + */ + +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { dirname, join, posix } from "node:path"; +import { argv, exit, stderr, stdout } from "node:process"; + +const STH_ROOT = "https://www.stairwaytohell.com"; +const USER_AGENT = "jsbeeb-mirror (+https://github.com/mattgodbolt/jsbeeb)"; +const SCHEMA_VERSION = 1; + +// Each "category" is one downloadable section of the STH archive. +// id — directory name we use under archive/sth/, mirrors upstream layout +// title — human-friendly label for the top-level manifest +// indexUrl — page to fetch to enumerate files +// indexer — function that turns the index page into [{ path, size, mtime }] +// fileBase — URL prefix that .path is resolved against to GET each file +// source — URL we record in the manifest as the upstream +const CATEGORIES = [ + { + id: "diskimages", + title: "BBC Disk Images", + indexUrl: `${STH_ROOT}/bbc/archive/diskimages/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/archive/diskimages/`, + source: `${STH_ROOT}/bbc/archive/diskimages/`, + }, + { + id: "tapeimages", + title: "BBC Tape Images", + indexUrl: `${STH_ROOT}/bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/archive/tapeimages/`, + source: `${STH_ROOT}/bbc/archive/tapeimages/`, + }, + { + id: "sthcollection", + title: "STH Collection", + indexUrl: `${STH_ROOT}/bbc/sthcollection.html`, + indexer: (html) => parseHomepage(html, { stripPrefix: "sthcollection/" }), + fileBase: `${STH_ROOT}/bbc/sthcollection/`, + source: `${STH_ROOT}/bbc/sthcollection.html`, + }, + { + id: "other/educational", + title: "BBC Educational", + indexUrl: `${STH_ROOT}/bbc/other/educational/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/other/educational/`, + source: `${STH_ROOT}/bbc/other/educational/`, + }, + { + id: "roms", + title: "BBC and Electron ROMs", + indexUrl: `${STH_ROOT}/roms/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/roms/`, + source: `${STH_ROOT}/roms/homepage.html`, + }, + { + id: "electron/uefarchive", + title: "Electron Tape Images", + indexUrl: `${STH_ROOT}/electron/uefarchive/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/electron/uefarchive/`, + source: `${STH_ROOT}/electron/uefarchive/`, + }, + { + id: "electron/dfs", + title: "Electron DFS Disk Images", + indexUrl: `${STH_ROOT}/electron/dfs/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/dfs/`, + source: `${STH_ROOT}/electron/dfs/homepage.html`, + }, + { + id: "electron/adfs", + title: "Electron ADFS Disk Images", + indexUrl: `${STH_ROOT}/electron/adfs/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/adfs/`, + source: `${STH_ROOT}/electron/adfs/homepage.html`, + }, + { + id: "electron/multiplexing", + title: "Electron Multiplexing", + indexUrl: `${STH_ROOT}/electron/multiplexing/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/multiplexing/`, + source: `${STH_ROOT}/electron/multiplexing/homepage.html`, + }, + { + id: "electron/t2p3", + title: "Electron T2P3", + indexUrl: `${STH_ROOT}/electron/t2p3/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/t2p3/`, + source: `${STH_ROOT}/electron/t2p3/homepage.html`, + }, +]; + +// Small auxiliary files saved verbatim under meta/ for provenance. +const META_FILES = [ + { url: `${STH_ROOT}/bbc/disklog.txt`, dest: "meta/bbc-disklog.txt" }, + { url: `${STH_ROOT}/bbc/tapelog.txt`, dest: "meta/bbc-tapelog.txt" }, + { url: `${STH_ROOT}/bbc/homepage.html`, dest: "meta/bbc-homepage.html" }, + { url: `${STH_ROOT}/bbc/diskimages.html`, dest: "meta/bbc-diskimages.html" }, + { url: `${STH_ROOT}/bbc/tapeimages.html`, dest: "meta/bbc-tapeimages.html" }, + { url: `${STH_ROOT}/bbc/sthcollection.html`, dest: "meta/bbc-sthcollection.html" }, + { url: `${STH_ROOT}/bbc/other.html`, dest: "meta/bbc-other.html" }, + { url: `${STH_ROOT}/roms/homepage.html`, dest: "meta/roms-homepage.html" }, + { url: `${STH_ROOT}/electron/homepage.html`, dest: "meta/electron-homepage.html" }, + { url: `${STH_ROOT}/electron/other.html`, dest: "meta/electron-other.html" }, +]; + +function parseArgs(args) { + const opts = { out: null, concurrency: 8, source: null, quick: false }; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === "--out") opts.out = args[++i]; + else if (a === "--concurrency") opts.concurrency = Number(args[++i]); + else if (a === "--source") opts.source = args[++i]; + else if (a === "--quick") opts.quick = true; + else if (a === "-h" || a === "--help") { + stdout.write(`Usage: mirror-sth.js --out [--concurrency 8] [--source ] [--quick]\n`); + exit(0); + } else { + stderr.write(`Unknown argument: ${a}\n`); + exit(2); + } + } + if (!opts.out) { + stderr.write("--out is required\n"); + exit(2); + } + if (!Number.isFinite(opts.concurrency) || opts.concurrency < 1) { + stderr.write("--concurrency must be a positive integer\n"); + exit(2); + } + return opts; +} + +async function fetchWithRetry(url, init = {}, { retries = 2 } = {}) { + let lastErr; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const res = await fetch(url, { + ...init, + headers: { "user-agent": USER_AGENT, ...(init.headers ?? {}) }, + }); + if (!res.ok) { + if (res.status >= 500 && attempt < retries) { + await sleep(500 * (attempt + 1)); + continue; + } + throw new Error(`HTTP ${res.status} for ${url}`); + } + return res; + } catch (err) { + lastErr = err; + if (attempt < retries) await sleep(500 * (attempt + 1)); + } + } + throw lastErr; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Strip HTML tags from a fragment. +function stripTags(s) { + return s.replace(/<[^>]*>/g, "").trim(); +} + +// Parse a Last-Modified-style date (e.g. "Mar 31 2010 10:33:18") into ISO 8601. +// STH serves these in UTC; we treat them as UTC for stable round-tripping. +function parseStModifiedDate(s) { + const cleaned = s.replace(/ /g, " ").trim(); + const t = Date.parse(cleaned + " UTC"); + if (!Number.isFinite(t)) return null; + return new Date(t).toISOString(); +} + +// Parse a reclist.php page. Each data row is a with three s: +// size, modified-date, and an anchor pointing at the file path (relative +// to the directory the reclist is in). The third may also contain a +// "(publisher)" link we must ignore — match the FIRST in the cell, +// matching what jsbeeb's existing scraper does (src/sth.js). +function parseReclist(html) { + const rows = html.match(/]*>[\s\S]*?<\/TR>/gi) ?? []; + const files = []; + for (const row of rows) { + const tds = [...row.matchAll(/]*>([\s\S]*?)<\/TD>/gi)].map((m) => m[1]); + if (tds.length < 3) continue; + const sizeText = stripTags(tds[0]); + const dateText = stripTags(tds[1]); + if (!/^\d+$/.test(sizeText)) continue; // skip header rows + const hrefMatch = tds[2].match(/]*href="([^"]+)"/i); + if (!hrefMatch) continue; + const path = hrefMatch[1]; + if (!path.toLowerCase().endsWith(".zip")) continue; + files.push({ + path, + size: Number(sizeText), + mtime: parseStModifiedDate(dateText), + }); + } + return files; +} + +// Parse a hand-rolled HTML index page (e.g. roms/homepage.html, electron/dfs/homepage.html, +// bbc/sthcollection.html). Picks up in the same directory tree, ignoring +// external (http://...), absolute (/...), and upward (../...) references — those belong to +// other categories or unrelated content. +// +// stripPrefix: optional path prefix to remove from each href (e.g. "sthcollection/" when the +// index page sits one level above the files). Hrefs that don't start with the prefix are +// rejected — this is what differentiates real download links from incidental anchors. +// +// No size/mtime in the index, so they're filled in later via HEAD requests. +function parseHomepage(html, { stripPrefix = "" } = {}) { + const hrefs = [...html.matchAll(/href="([^"]+\.zip)"/gi)].map((m) => m[1]); + const filtered = []; + for (const href of new Set(hrefs)) { + if (/^(?:https?:|\/)/i.test(href)) continue; + if (href.startsWith("../")) continue; + if (stripPrefix) { + if (!href.startsWith(stripPrefix)) continue; + filtered.push(href.slice(stripPrefix.length)); + } else { + filtered.push(href); + } + } + return filtered.sort().map((path) => ({ path, size: null, mtime: null })); +} + +// URL-encode the path components in a STH-relative path, preserving slashes. +function encodePath(path) { + return path.split("/").map(encodeURIComponent).join("/"); +} + +async function ensureDir(dir) { + await mkdir(dir, { recursive: true }); +} + +async function fileExistsWithSize(path, expected) { + try { + const s = await stat(path); + return s.isFile() && (expected == null || s.size === expected); + } catch { + return false; + } +} + +async function downloadFile(url, dest) { + await ensureDir(dirname(dest)); + const res = await fetchWithRetry(url); + const buf = Buffer.from(await res.arrayBuffer()); + await writeFile(dest, buf); + const lastModified = res.headers.get("last-modified"); + const mtime = lastModified ? new Date(lastModified).toISOString() : null; + return { size: buf.length, mtime }; +} + +// A simple bounded-concurrency pool. Returns when every task has settled; +// throws on the first error. +async function runPool(items, concurrency, worker) { + let next = 0; + let firstError = null; + let completed = 0; + const total = items.length; + let lastReport = 0; + + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (true) { + if (firstError) return; + const i = next++; + if (i >= items.length) return; + try { + await worker(items[i], i); + } catch (err) { + if (!firstError) firstError = err; + return; + } + completed++; + const now = Date.now(); + if (now - lastReport > 2000 || completed === total) { + lastReport = now; + stderr.write(` ${completed}/${total}\n`); + } + } + }); + await Promise.all(runners); + if (firstError) throw firstError; +} + +async function indexCategory(cat) { + stderr.write(`\n[${cat.id}] indexing ${cat.indexUrl}\n`); + const res = await fetchWithRetry(cat.indexUrl); + const html = await res.text(); + const entries = cat.indexer(html); + entries.sort((a, b) => a.path.localeCompare(b.path)); + stderr.write(`[${cat.id}] ${entries.length} files\n`); + return entries; +} + +async function fillMissingMetadata(cat, entries, concurrency) { + // sthcollection's index lacks size/mtime — HEAD each file to get them. + const needs = entries.filter((e) => e.size == null || e.mtime == null); + if (needs.length === 0) return; + stderr.write(`[${cat.id}] HEADing ${needs.length} files for size/mtime\n`); + await runPool(needs, concurrency, async (entry) => { + const url = cat.fileBase + encodePath(entry.path); + const res = await fetchWithRetry(url, { method: "HEAD" }); + const len = Number(res.headers.get("content-length")); + if (Number.isFinite(len)) entry.size = len; + const lm = res.headers.get("last-modified"); + if (lm) entry.mtime = new Date(lm).toISOString(); + }); +} + +async function downloadCategory(cat, entries, outRoot, concurrency) { + const categoryRoot = join(outRoot, cat.id); + let skipped = 0; + let downloaded = 0; + await runPool(entries, concurrency, async (entry) => { + const dest = join(categoryRoot, entry.path); + if (await fileExistsWithSize(dest, entry.size)) { + skipped++; + return; + } + const url = cat.fileBase + encodePath(entry.path); + const result = await downloadFile(url, dest); + // Trust the HTTP response if it disagreed with the index — keeps the + // manifest honest even if STH's reclist gets out of sync. + if (entry.size == null) entry.size = result.size; + if (entry.mtime == null) entry.mtime = result.mtime; + downloaded++; + }); + stderr.write(`[${cat.id}] downloaded ${downloaded}, skipped ${skipped} (already present)\n`); +} + +async function writeCategoryManifest(cat, entries, outRoot) { + const dest = join(outRoot, cat.id, "manifest.json"); + await ensureDir(dirname(dest)); + const manifest = { + schemaVersion: SCHEMA_VERSION, + files: entries.map(({ path, size, mtime }) => ({ path, size, mtime })), + }; + await writeFile(dest, JSON.stringify(manifest, null, 2) + "\n"); + return manifest; +} + +async function downloadMetaFiles(outRoot, concurrency) { + stderr.write(`\n[meta] fetching ${META_FILES.length} auxiliary files\n`); + await runPool(META_FILES, concurrency, async (m) => { + const dest = join(outRoot, m.dest); + const res = await fetchWithRetry(m.url); + const buf = Buffer.from(await res.arrayBuffer()); + await ensureDir(dirname(dest)); + await writeFile(dest, buf); + }); +} + +async function writeTopLevelManifest(outRoot, perCategory) { + const manifest = { + schemaVersion: SCHEMA_VERSION, + name: "Stairway to Hell BBC Micro Software Archive", + source: `${STH_ROOT}/bbc/`, + scrapedAt: new Date().toISOString(), + categories: perCategory.map(({ cat, entries }) => ({ + id: cat.id, + title: cat.title, + manifest: posix.join(cat.id, "manifest.json"), + source: cat.source, + fileCount: entries.length, + totalBytes: entries.reduce((sum, e) => sum + (e.size ?? 0), 0), + })), + }; + const dest = join(outRoot, "manifest.json"); + await writeFile(dest, JSON.stringify(manifest, null, 2) + "\n"); + return manifest; +} + +async function main() { + const opts = parseArgs(argv.slice(2)); + const outRoot = opts.out; + await ensureDir(outRoot); + + const wantedCategories = opts.source ? CATEGORIES.filter((c) => c.id === opts.source) : CATEGORIES; + if (opts.source && wantedCategories.length === 0) { + stderr.write(`Unknown --source: ${opts.source}\n`); + stderr.write(`Known sources: ${CATEGORIES.map((c) => c.id).join(", ")}\n`); + exit(2); + } + + const perCategory = []; + for (const cat of wantedCategories) { + const entries = await indexCategory(cat); + if (!opts.quick) { + await fillMissingMetadata(cat, entries, opts.concurrency); + await downloadCategory(cat, entries, outRoot, opts.concurrency); + } + await writeCategoryManifest(cat, entries, outRoot); + perCategory.push({ cat, entries }); + } + + if (!opts.quick) await downloadMetaFiles(outRoot, opts.concurrency); + + // Only rewrite the top-level manifest when scraping the full set, so a + // single-source partial run (e.g. --source diskimages) doesn't drop the + // other categories out of the index. + if (!opts.source) { + const top = await writeTopLevelManifest(outRoot, perCategory); + stderr.write( + `\nWrote top-level manifest: ${top.categories.length} categories, ` + + `${top.categories.reduce((n, c) => n + c.fileCount, 0)} files, ` + + `${(top.categories.reduce((n, c) => n + c.totalBytes, 0) / 1024 / 1024).toFixed(1)} MB\n`, + ); + } +} + +main().catch((err) => { + stderr.write(`\nERROR: ${err.stack ?? err.message ?? err}\n`); + exit(1); +});