From d8d2620127227f11cb4153f39bf8b6137623185e Mon Sep 17 00:00:00 2001 From: mattgodbolt-molty Date: Sun, 3 May 2026 12:32:11 -0500 Subject: [PATCH 1/2] feat: mirror Stairway to Hell archive into bbc.xania.org S3 bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tooling and a manual GitHub workflow to snapshot the upstream STH BBC Micro software archive into s3://bbc.xania.org/archive/sth/. The mirror exists so jsbeeb's sth: URLs and the in-app archive browser keep working if the volunteer-run upstream site disappears. The client (src/sth.js) is intentionally not changed in this commit; that swap will follow in a separate PR after the mirror has been populated and spot-checked. * tools/mirror-sth.js scrapes four categories (disk images, tape images, sthcollection, educational) plus a handful of changelog/index files for provenance, writes per-category and top-level manifests in a versioned JSON format, and is resumable by size. * .github/workflows/mirror-sth.yml is workflow_dispatch only with a dry_run input that defaults to true. The S3 sync is strictly additive — no --delete — to make a typo on the destination URL safe against wiping the live app served from the same bucket. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/mirror-sth.yml | 69 ++++++ package.json | 1 + tools/README-sth-mirror.md | 104 +++++++++ tools/mirror-sth.js | 372 +++++++++++++++++++++++++++++++ 4 files changed, 546 insertions(+) create mode 100644 .github/workflows/mirror-sth.yml create mode 100644 tools/README-sth-mirror.md create mode 100644 tools/mirror-sth.js diff --git a/.github/workflows/mirror-sth.yml b/.github/workflows/mirror-sth.yml new file mode 100644 index 00000000..7f5de15c --- /dev/null +++ b/.github/workflows/mirror-sth.yml @@ -0,0 +1,69 @@ +name: Mirror Stairway to Hell archive + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Scrape only — skip the S3 upload" + type: boolean + required: true + default: true + aws_region: + description: "AWS region for the bucket" + type: string + required: false + default: us-east-1 + +jobs: + mirror: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Scrape STH into ./mirror + run: node tools/mirror-sth.js --out mirror + + - name: Show mirror summary + run: | + echo "Top-level manifest:" + cat mirror/manifest.json + echo + echo "On-disk size:" + du -sh mirror + echo "Object count:" + find mirror -type f | wc -l + + - name: Configure AWS credentials + if: ${{ !inputs.dry_run }} + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.aws_region }} + + - name: Sync immutable archive blobs + if: ${{ !inputs.dry_run }} + run: | + aws s3 sync mirror/ s3://bbc.xania.org/archive/sth/ \ + --no-progress \ + --cache-control "public, max-age=31536000, immutable" \ + --metadata-directive REPLACE \ + --exclude "*manifest.json" \ + --exclude "meta/*" + + - name: Sync manifests and meta files + if: ${{ !inputs.dry_run }} + run: | + aws s3 sync mirror/ s3://bbc.xania.org/archive/sth/ \ + --no-progress \ + --cache-control "public, max-age=300" \ + --metadata-directive REPLACE \ + --exclude "*" \ + --include "*manifest.json" \ + --include "meta/*" diff --git a/package.json b/package.json index f0ca8e4f..afc856e5 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "coverage:unit": "vitest run tests/unit --coverage", "coverage:all-tests": "vitest run --coverage", "benchmark": "node src/app-bench.js", + "mirror-sth": "node tools/mirror-sth.js --out mirror", "electron": "npm run build && ELECTRON_DISABLE_SANDBOX=1 electron .", "electron:build": "electron-builder" }, diff --git a/tools/README-sth-mirror.md b/tools/README-sth-mirror.md new file mode 100644 index 00000000..571026b2 --- /dev/null +++ b/tools/README-sth-mirror.md @@ -0,0 +1,104 @@ +# Mirroring the Stairway to Hell archive + +`tools/mirror-sth.js` scrapes the public BBC Micro software archive at +`https://www.stairwaytohell.com/bbc/` into a local directory tree that is +sync'd into `s3://bbc.xania.org/archive/sth/` by +`.github/workflows/mirror-sth.yml`. + +The mirror exists so jsbeeb's `sth:` URLs and the in-app archive browser +keep working if the upstream site disappears. STH has been effectively +frozen since around 2008 (see `meta/disklog.txt` and `meta/tapelog.txt` in +the mirror), so this is a one-shot snapshot rather than a continuous sync. + +## What is mirrored + +| Category | Source on STH | Notes | +| ----------------------- | ---------------------------------------------------------- | ------------------------------ | +| `diskimages/` | `bbc/archive/diskimages/reclist.php?sort=name&filter=.zip` | ~1,600 zips, ~26 MB | +| `tapeimages/` | `bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip` | ~1,500 zips, ~24 MB | +| `sthcollection/` | `bbc/sthcollection.html` | ~140 zips, magazine disk packs | +| `other/educational/` | `bbc/other/educational/reclist.php?...` | ~75 zips | +| `meta/disklog.txt` etc. | `bbc/disklog.txt`, `bbc/tapelog.txt` | upstream changelogs | +| `meta/*.html` | `bbc/homepage.html`, `bbc/diskimages.html`, etc. | site index pages, provenance | + +BBC sideways ROMs are not mirrored — STH does not host them (the linked +`bbc/../roms/*.zip` URLs already 404). jsbeeb ships its own ROMs in +`public/roms/`. + +## Manifest format (schemaVersion 1) + +The top-level `archive/sth/manifest.json` lists categories and points at +each per-category manifest: + +```json +{ + "schemaVersion": 1, + "name": "Stairway to Hell BBC Micro Software Archive", + "source": "https://www.stairwaytohell.com/bbc/", + "scrapedAt": "2026-05-03T17:00:00Z", + "categories": [ + { + "id": "diskimages", + "title": "Disk Images", + "manifest": "diskimages/manifest.json", + "source": "https://www.stairwaytohell.com/bbc/archive/diskimages/", + "fileCount": 1608, + "totalBytes": 27315281 + } + ] +} +``` + +Each per-category manifest is a flat list of files, sorted by path: + +```json +{ + "schemaVersion": 1, + "files": [{ "path": "Acornsoft/Elite.zip", "size": 12345, "mtime": "2003-04-28T00:00:00.000Z" }] +} +``` + +Paths are POSIX-style, relative to the category directory. + +## Running locally + +```sh +# Catalog parse only — no zip downloads. Fast (~2s) sanity check. +node tools/mirror-sth.js --out /tmp/sth-mirror --quick + +# Full mirror — ~80 MB, a few minutes depending on STH's bandwidth. +node tools/mirror-sth.js --out /tmp/sth-mirror + +# Or just one category: +node tools/mirror-sth.js --out /tmp/sth-mirror --source diskimages +``` + +The script is resumable: it skips files that are already on disk with the +expected size, so a re-run after a partial download will only fetch what's +missing. + +## Running in CI + +`.github/workflows/mirror-sth.yml` is `workflow_dispatch` only — no +schedule. The `dry_run` input (default `true`) makes the workflow scrape +into the runner's tmp dir and stop short of uploading. Re-run with +`dry_run: false` once the dry-run output looks right. + +The S3 sync is **strictly additive** — there's no `--delete`. If a file +ever needs to be removed from the mirror, do it by hand or in a follow-up +workflow guarded by an explicit `prune` input. This avoids the failure +mode where a typo on the destination URL with `--delete` could nuke the +live app served from the same bucket. + +Cache headers: + +- Zips: `public, max-age=31536000, immutable` (paths are content-stable) +- Manifests + `meta/*`: `public, max-age=300` (these are what actually + change when we re-mirror) + +## Costs + +Storage for ~80 MB of objects is well under one cent per month at S3 +standard pricing. Egress is fronted by CloudFront (already in front of +`bbc.xania.org`) and dominated by the existing app traffic; the marginal +cost of mirror traffic is negligible. diff --git a/tools/mirror-sth.js b/tools/mirror-sth.js new file mode 100644 index 00000000..26d24a39 --- /dev/null +++ b/tools/mirror-sth.js @@ -0,0 +1,372 @@ +#!/usr/bin/env node +/** + * Mirror the Stairway to Hell BBC Micro software archive into a local + * directory tree. The output is structured to be sync'd as-is into + * s3://bbc.xania.org/archive/sth/ — see tools/README-sth-mirror.md and + * .github/workflows/mirror-sth.yml. + * + * Usage: + * node tools/mirror-sth.js --out [--concurrency 8] + * [--source ] [--quick] + * + * Self-contained: relies only on Node 22 builtins (fetch, fs/promises). + */ + +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { dirname, join, posix } from "node:path"; +import { argv, exit, stderr, stdout } from "node:process"; + +const STH_ROOT = "https://www.stairwaytohell.com"; +const USER_AGENT = "jsbeeb-mirror (+https://github.com/mattgodbolt/jsbeeb)"; +const SCHEMA_VERSION = 1; + +// Each "category" is one downloadable section of the STH archive. +// id — directory name we use under archive/sth/ +// title — human-friendly label for the top-level manifest +// indexUrl — page to fetch to enumerate files +// indexer — function that turns the index page into [{ path, size, mtime }] +// fileBase — URL prefix that .path is resolved against to GET each file +// source — URL we record in the manifest as the upstream +const CATEGORIES = [ + { + id: "diskimages", + title: "Disk Images", + indexUrl: `${STH_ROOT}/bbc/archive/diskimages/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/archive/diskimages/`, + source: `${STH_ROOT}/bbc/archive/diskimages/`, + }, + { + id: "tapeimages", + title: "Tape Images", + indexUrl: `${STH_ROOT}/bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/archive/tapeimages/`, + source: `${STH_ROOT}/bbc/archive/tapeimages/`, + }, + { + id: "sthcollection", + title: "STH Collection", + indexUrl: `${STH_ROOT}/bbc/sthcollection.html`, + indexer: parseSthCollection, + fileBase: `${STH_ROOT}/bbc/`, + source: `${STH_ROOT}/bbc/sthcollection.html`, + }, + { + id: "other/educational", + title: "Educational", + indexUrl: `${STH_ROOT}/bbc/other/educational/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/bbc/other/educational/`, + source: `${STH_ROOT}/bbc/other/educational/`, + }, +]; + +// Small auxiliary files saved verbatim under meta/ for provenance. +const META_FILES = [ + { url: `${STH_ROOT}/bbc/disklog.txt`, dest: "meta/disklog.txt" }, + { url: `${STH_ROOT}/bbc/tapelog.txt`, dest: "meta/tapelog.txt" }, + { url: `${STH_ROOT}/bbc/homepage.html`, dest: "meta/homepage.html" }, + { url: `${STH_ROOT}/bbc/diskimages.html`, dest: "meta/diskimages.html" }, + { url: `${STH_ROOT}/bbc/tapeimages.html`, dest: "meta/tapeimages.html" }, + { url: `${STH_ROOT}/bbc/sthcollection.html`, dest: "meta/sthcollection.html" }, + { url: `${STH_ROOT}/bbc/other.html`, dest: "meta/other.html" }, +]; + +function parseArgs(args) { + const opts = { out: null, concurrency: 8, source: null, quick: false }; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === "--out") opts.out = args[++i]; + else if (a === "--concurrency") opts.concurrency = Number(args[++i]); + else if (a === "--source") opts.source = args[++i]; + else if (a === "--quick") opts.quick = true; + else if (a === "-h" || a === "--help") { + stdout.write(`Usage: mirror-sth.js --out [--concurrency 8] [--source ] [--quick]\n`); + exit(0); + } else { + stderr.write(`Unknown argument: ${a}\n`); + exit(2); + } + } + if (!opts.out) { + stderr.write("--out is required\n"); + exit(2); + } + if (!Number.isFinite(opts.concurrency) || opts.concurrency < 1) { + stderr.write("--concurrency must be a positive integer\n"); + exit(2); + } + return opts; +} + +async function fetchWithRetry(url, init = {}, { retries = 2 } = {}) { + let lastErr; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const res = await fetch(url, { + ...init, + headers: { "user-agent": USER_AGENT, ...(init.headers ?? {}) }, + }); + if (!res.ok) { + if (res.status >= 500 && attempt < retries) { + await sleep(500 * (attempt + 1)); + continue; + } + throw new Error(`HTTP ${res.status} for ${url}`); + } + return res; + } catch (err) { + lastErr = err; + if (attempt < retries) await sleep(500 * (attempt + 1)); + } + } + throw lastErr; +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Strip HTML tags from a fragment. +function stripTags(s) { + return s.replace(/<[^>]*>/g, "").trim(); +} + +// Parse a Last-Modified-style date (e.g. "Mar 31 2010 10:33:18") into ISO 8601. +// STH serves these in UTC; we treat them as UTC for stable round-tripping. +function parseStModifiedDate(s) { + const cleaned = s.replace(/ /g, " ").trim(); + const t = Date.parse(cleaned + " UTC"); + if (!Number.isFinite(t)) return null; + return new Date(t).toISOString(); +} + +// Parse a reclist.php page. Each data row is a with three s: +// size, modified-date, and an anchor pointing at the file path (relative +// to the directory the reclist is in). The third may also contain a +// "(publisher)" link we must ignore — match the FIRST in the cell, +// matching what jsbeeb's existing scraper does (src/sth.js). +function parseReclist(html) { + const rows = html.match(/]*>[\s\S]*?<\/TR>/gi) ?? []; + const files = []; + for (const row of rows) { + const tds = [...row.matchAll(/]*>([\s\S]*?)<\/TD>/gi)].map((m) => m[1]); + if (tds.length < 3) continue; + const sizeText = stripTags(tds[0]); + const dateText = stripTags(tds[1]); + if (!/^\d+$/.test(sizeText)) continue; // skip header rows + const hrefMatch = tds[2].match(/]*href="([^"]+)"/i); + if (!hrefMatch) continue; + const path = hrefMatch[1]; + if (!path.toLowerCase().endsWith(".zip")) continue; + files.push({ + path, + size: Number(sizeText), + mtime: parseStModifiedDate(dateText), + }); + } + return files; +} + +// Parse bbc/sthcollection.html — a hand-rolled list of . +// No size/mtime in the index, so size is filled in later via HEAD requests. +function parseSthCollection(html) { + const hrefs = [...html.matchAll(/href="(sthcollection\/[^"]+\.zip)"/gi)].map((m) => m[1]); + // Strip the leading "sthcollection/" so paths are relative to the category root, + // matching how parseReclist returns them. + return [...new Set(hrefs)].sort().map((href) => ({ + path: href.replace(/^sthcollection\//, ""), + size: null, + mtime: null, + })); +} + +// URL-encode the path components in a STH-relative path, preserving slashes. +function encodePath(path) { + return path.split("/").map(encodeURIComponent).join("/"); +} + +async function ensureDir(dir) { + await mkdir(dir, { recursive: true }); +} + +async function fileExistsWithSize(path, expected) { + try { + const s = await stat(path); + return s.isFile() && (expected == null || s.size === expected); + } catch { + return false; + } +} + +async function downloadFile(url, dest) { + await ensureDir(dirname(dest)); + const res = await fetchWithRetry(url); + const buf = Buffer.from(await res.arrayBuffer()); + await writeFile(dest, buf); + const lastModified = res.headers.get("last-modified"); + const mtime = lastModified ? new Date(lastModified).toISOString() : null; + return { size: buf.length, mtime }; +} + +// A simple bounded-concurrency pool. Returns when every task has settled; +// throws on the first error. +async function runPool(items, concurrency, worker) { + let next = 0; + let firstError = null; + let completed = 0; + const total = items.length; + let lastReport = 0; + + const runners = Array.from({ length: Math.min(concurrency, items.length) }, async () => { + while (true) { + if (firstError) return; + const i = next++; + if (i >= items.length) return; + try { + await worker(items[i], i); + } catch (err) { + if (!firstError) firstError = err; + return; + } + completed++; + const now = Date.now(); + if (now - lastReport > 2000 || completed === total) { + lastReport = now; + stderr.write(` ${completed}/${total}\n`); + } + } + }); + await Promise.all(runners); + if (firstError) throw firstError; +} + +async function indexCategory(cat) { + stderr.write(`\n[${cat.id}] indexing ${cat.indexUrl}\n`); + const res = await fetchWithRetry(cat.indexUrl); + const html = await res.text(); + const entries = cat.indexer(html); + entries.sort((a, b) => a.path.localeCompare(b.path)); + stderr.write(`[${cat.id}] ${entries.length} files\n`); + return entries; +} + +async function fillMissingMetadata(cat, entries, concurrency) { + // sthcollection's index lacks size/mtime — HEAD each file to get them. + const needs = entries.filter((e) => e.size == null || e.mtime == null); + if (needs.length === 0) return; + stderr.write(`[${cat.id}] HEADing ${needs.length} files for size/mtime\n`); + await runPool(needs, concurrency, async (entry) => { + const url = cat.fileBase + encodePath(entry.path); + const res = await fetchWithRetry(url, { method: "HEAD" }); + const len = Number(res.headers.get("content-length")); + if (Number.isFinite(len)) entry.size = len; + const lm = res.headers.get("last-modified"); + if (lm) entry.mtime = new Date(lm).toISOString(); + }); +} + +async function downloadCategory(cat, entries, outRoot, concurrency) { + const categoryRoot = join(outRoot, cat.id); + let skipped = 0; + let downloaded = 0; + await runPool(entries, concurrency, async (entry) => { + const dest = join(categoryRoot, entry.path); + if (await fileExistsWithSize(dest, entry.size)) { + skipped++; + return; + } + const url = cat.fileBase + encodePath(entry.path); + const result = await downloadFile(url, dest); + // Trust the HTTP response if it disagreed with the index — keeps the + // manifest honest even if STH's reclist gets out of sync. + if (entry.size == null) entry.size = result.size; + if (entry.mtime == null) entry.mtime = result.mtime; + downloaded++; + }); + stderr.write(`[${cat.id}] downloaded ${downloaded}, skipped ${skipped} (already present)\n`); +} + +async function writeCategoryManifest(cat, entries, outRoot) { + const dest = join(outRoot, cat.id, "manifest.json"); + await ensureDir(dirname(dest)); + const manifest = { + schemaVersion: SCHEMA_VERSION, + files: entries.map(({ path, size, mtime }) => ({ path, size, mtime })), + }; + await writeFile(dest, JSON.stringify(manifest, null, 2) + "\n"); + return manifest; +} + +async function downloadMetaFiles(outRoot, concurrency) { + stderr.write(`\n[meta] fetching ${META_FILES.length} auxiliary files\n`); + await runPool(META_FILES, concurrency, async (m) => { + const dest = join(outRoot, m.dest); + const res = await fetchWithRetry(m.url); + const buf = Buffer.from(await res.arrayBuffer()); + await ensureDir(dirname(dest)); + await writeFile(dest, buf); + }); +} + +async function writeTopLevelManifest(outRoot, perCategory) { + const manifest = { + schemaVersion: SCHEMA_VERSION, + name: "Stairway to Hell BBC Micro Software Archive", + source: `${STH_ROOT}/bbc/`, + scrapedAt: new Date().toISOString(), + categories: perCategory.map(({ cat, entries }) => ({ + id: cat.id, + title: cat.title, + manifest: posix.join(cat.id, "manifest.json"), + source: cat.source, + fileCount: entries.length, + totalBytes: entries.reduce((sum, e) => sum + (e.size ?? 0), 0), + })), + }; + const dest = join(outRoot, "manifest.json"); + await writeFile(dest, JSON.stringify(manifest, null, 2) + "\n"); + return manifest; +} + +async function main() { + const opts = parseArgs(argv.slice(2)); + const outRoot = opts.out; + await ensureDir(outRoot); + + const wantedCategories = opts.source ? CATEGORIES.filter((c) => c.id === opts.source) : CATEGORIES; + if (opts.source && wantedCategories.length === 0) { + stderr.write(`Unknown --source: ${opts.source}\n`); + stderr.write(`Known sources: ${CATEGORIES.map((c) => c.id).join(", ")}\n`); + exit(2); + } + + const perCategory = []; + for (const cat of wantedCategories) { + const entries = await indexCategory(cat); + if (!opts.quick) { + await fillMissingMetadata(cat, entries, opts.concurrency); + await downloadCategory(cat, entries, outRoot, opts.concurrency); + } + await writeCategoryManifest(cat, entries, outRoot); + perCategory.push({ cat, entries }); + } + + if (!opts.quick) await downloadMetaFiles(outRoot, opts.concurrency); + + // Only rewrite the top-level manifest when scraping the full set, so a + // single-source partial run (e.g. --source diskimages) doesn't drop the + // other categories out of the index. + if (!opts.source) { + const top = await writeTopLevelManifest(outRoot, perCategory); + stderr.write( + `\nWrote top-level manifest: ${top.categories.length} categories, ` + + `${top.categories.reduce((n, c) => n + c.fileCount, 0)} files, ` + + `${(top.categories.reduce((n, c) => n + c.totalBytes, 0) / 1024 / 1024).toFixed(1)} MB\n`, + ); + } +} + +main().catch((err) => { + stderr.write(`\nERROR: ${err.stack ?? err.message ?? err}\n`); + exit(1); +}); From a2acbf07c723650a4cc1671c44f43fef253f3224 Mon Sep 17 00:00:00 2001 From: mattgodbolt-molty Date: Sun, 3 May 2026 12:54:40 -0500 Subject: [PATCH 2/2] feat: extend STH mirror to ROMs and Electron archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds six more categories so the snapshot covers the full set of downloadable archive areas STH hosts, not just the BBC Micro disc and tape sections: * roms/ — 50 BBC + Electron sideways ROMs from /roms/homepage.html * electron/uefarchive/ — ~890 Electron tape image zips, ~15 MB, via reclist.php with the same parser as the BBC tape archive * electron/dfs/ — ~230 Electron DFS disk images * electron/adfs/ — ~23 Electron ADFS disk images * electron/multiplexing/ — single curiosity file * electron/t2p3/ — 4 curiosity files The total mirror grows from ~3,330 to ~4,500 objects and from ~50 MB to ~80 MB; still trivial for S3. S3 paths follow the upstream layout exactly (e.g. archive/sth/electron/uefarchive/Hewson/Uridium-FIXED_E.zip), so provenance is preserved. Generalises parseSthCollection into a reusable parseHomepage that handles any hand-rolled HTML index page, filtering out external, absolute, and upward (../) hrefs and optionally stripping a directory prefix. While doing this, fixes a latent bug where sthcollection's fileBase was set to /bbc/ instead of /bbc/sthcollection/, which would have produced 404s on the first non-quick scrape — not caught by the earlier smoke test because that test only ran the educational category. Adds three new meta files (roms-homepage.html, electron-homepage.html, electron-other.html) and renames the existing bbc/* meta files with a bbc- prefix to disambiguate from the new electron- and roms- ones. Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/README-sth-mirror.md | 48 ++++++++++------ tools/mirror-sth.js | 112 +++++++++++++++++++++++++++++-------- 2 files changed, 119 insertions(+), 41 deletions(-) diff --git a/tools/README-sth-mirror.md b/tools/README-sth-mirror.md index 571026b2..1b0bd214 100644 --- a/tools/README-sth-mirror.md +++ b/tools/README-sth-mirror.md @@ -1,29 +1,43 @@ # Mirroring the Stairway to Hell archive -`tools/mirror-sth.js` scrapes the public BBC Micro software archive at -`https://www.stairwaytohell.com/bbc/` into a local directory tree that is -sync'd into `s3://bbc.xania.org/archive/sth/` by +`tools/mirror-sth.js` scrapes the public BBC Micro, Acorn Electron, and +sideways-ROM areas of `https://www.stairwaytohell.com/` into a local +directory tree that is sync'd into `s3://bbc.xania.org/archive/sth/` by `.github/workflows/mirror-sth.yml`. The mirror exists so jsbeeb's `sth:` URLs and the in-app archive browser keep working if the upstream site disappears. STH has been effectively -frozen since around 2008 (see `meta/disklog.txt` and `meta/tapelog.txt` in -the mirror), so this is a one-shot snapshot rather than a continuous sync. +frozen since around 2008 (see `meta/bbc-disklog.txt` and +`meta/bbc-tapelog.txt` in the mirror), so this is a one-shot snapshot +rather than a continuous sync. ## What is mirrored -| Category | Source on STH | Notes | -| ----------------------- | ---------------------------------------------------------- | ------------------------------ | -| `diskimages/` | `bbc/archive/diskimages/reclist.php?sort=name&filter=.zip` | ~1,600 zips, ~26 MB | -| `tapeimages/` | `bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip` | ~1,500 zips, ~24 MB | -| `sthcollection/` | `bbc/sthcollection.html` | ~140 zips, magazine disk packs | -| `other/educational/` | `bbc/other/educational/reclist.php?...` | ~75 zips | -| `meta/disklog.txt` etc. | `bbc/disklog.txt`, `bbc/tapelog.txt` | upstream changelogs | -| `meta/*.html` | `bbc/homepage.html`, `bbc/diskimages.html`, etc. | site index pages, provenance | - -BBC sideways ROMs are not mirrored — STH does not host them (the linked -`bbc/../roms/*.zip` URLs already 404). jsbeeb ships its own ROMs in -`public/roms/`. +S3 paths mirror STH's upstream layout — `archive/sth//...` matches +the natural URL structure on `stairwaytohell.com`. + +| Category | Source on STH | Notes | +| ------------------------ | ---------------------------------------------------------- | ------------------------------------- | +| `diskimages/` | `bbc/archive/diskimages/reclist.php?sort=name&filter=.zip` | ~1,600 zips, ~26 MB | +| `tapeimages/` | `bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip` | ~1,500 zips, ~24 MB | +| `sthcollection/` | `bbc/sthcollection.html` | ~140 zips, magazine disk packs | +| `other/educational/` | `bbc/other/educational/reclist.php?...` | ~75 zips | +| `roms/` | `roms/homepage.html` | ~50 BBC + Electron sideways ROMs | +| `electron/uefarchive/` | `electron/uefarchive/reclist.php?...` | ~890 Electron tape image zips, ~15 MB | +| `electron/dfs/` | `electron/dfs/homepage.html` | ~230 Electron DFS disk images | +| `electron/adfs/` | `electron/adfs/homepage.html` | ~23 Electron ADFS disk images | +| `electron/multiplexing/` | `electron/multiplexing/homepage.html` | curiosity, 1 file | +| `electron/t2p3/` | `electron/t2p3/homepage.html` | curiosity, 4 files | +| `meta/bbc-disklog.txt` | `bbc/disklog.txt`, `bbc/tapelog.txt` | upstream changelogs | +| `meta/*.html` | `bbc/homepage.html`, `roms/homepage.html`, etc. | site index pages, provenance | + +Total ~4,500 zips, ~80 MB across 10 categories. + +The Electron categories are mirrored even though jsbeeb does not currently +emulate the Electron — the goal is a complete archival snapshot, not just +what the running app uses today. The `roms/` category similarly overlaps +with what jsbeeb already ships in `public/roms/`; mirroring it is purely +archival. ## Manifest format (schemaVersion 1) diff --git a/tools/mirror-sth.js b/tools/mirror-sth.js index 26d24a39..f1981a05 100644 --- a/tools/mirror-sth.js +++ b/tools/mirror-sth.js @@ -21,7 +21,7 @@ const USER_AGENT = "jsbeeb-mirror (+https://github.com/mattgodbolt/jsbeeb)"; const SCHEMA_VERSION = 1; // Each "category" is one downloadable section of the STH archive. -// id — directory name we use under archive/sth/ +// id — directory name we use under archive/sth/, mirrors upstream layout // title — human-friendly label for the top-level manifest // indexUrl — page to fetch to enumerate files // indexer — function that turns the index page into [{ path, size, mtime }] @@ -30,7 +30,7 @@ const SCHEMA_VERSION = 1; const CATEGORIES = [ { id: "diskimages", - title: "Disk Images", + title: "BBC Disk Images", indexUrl: `${STH_ROOT}/bbc/archive/diskimages/reclist.php?sort=name&filter=.zip`, indexer: parseReclist, fileBase: `${STH_ROOT}/bbc/archive/diskimages/`, @@ -38,7 +38,7 @@ const CATEGORIES = [ }, { id: "tapeimages", - title: "Tape Images", + title: "BBC Tape Images", indexUrl: `${STH_ROOT}/bbc/archive/tapeimages/reclist.php?sort=name&filter=.zip`, indexer: parseReclist, fileBase: `${STH_ROOT}/bbc/archive/tapeimages/`, @@ -48,29 +48,80 @@ const CATEGORIES = [ id: "sthcollection", title: "STH Collection", indexUrl: `${STH_ROOT}/bbc/sthcollection.html`, - indexer: parseSthCollection, - fileBase: `${STH_ROOT}/bbc/`, + indexer: (html) => parseHomepage(html, { stripPrefix: "sthcollection/" }), + fileBase: `${STH_ROOT}/bbc/sthcollection/`, source: `${STH_ROOT}/bbc/sthcollection.html`, }, { id: "other/educational", - title: "Educational", + title: "BBC Educational", indexUrl: `${STH_ROOT}/bbc/other/educational/reclist.php?sort=name&filter=.zip`, indexer: parseReclist, fileBase: `${STH_ROOT}/bbc/other/educational/`, source: `${STH_ROOT}/bbc/other/educational/`, }, + { + id: "roms", + title: "BBC and Electron ROMs", + indexUrl: `${STH_ROOT}/roms/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/roms/`, + source: `${STH_ROOT}/roms/homepage.html`, + }, + { + id: "electron/uefarchive", + title: "Electron Tape Images", + indexUrl: `${STH_ROOT}/electron/uefarchive/reclist.php?sort=name&filter=.zip`, + indexer: parseReclist, + fileBase: `${STH_ROOT}/electron/uefarchive/`, + source: `${STH_ROOT}/electron/uefarchive/`, + }, + { + id: "electron/dfs", + title: "Electron DFS Disk Images", + indexUrl: `${STH_ROOT}/electron/dfs/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/dfs/`, + source: `${STH_ROOT}/electron/dfs/homepage.html`, + }, + { + id: "electron/adfs", + title: "Electron ADFS Disk Images", + indexUrl: `${STH_ROOT}/electron/adfs/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/adfs/`, + source: `${STH_ROOT}/electron/adfs/homepage.html`, + }, + { + id: "electron/multiplexing", + title: "Electron Multiplexing", + indexUrl: `${STH_ROOT}/electron/multiplexing/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/multiplexing/`, + source: `${STH_ROOT}/electron/multiplexing/homepage.html`, + }, + { + id: "electron/t2p3", + title: "Electron T2P3", + indexUrl: `${STH_ROOT}/electron/t2p3/homepage.html`, + indexer: parseHomepage, + fileBase: `${STH_ROOT}/electron/t2p3/`, + source: `${STH_ROOT}/electron/t2p3/homepage.html`, + }, ]; // Small auxiliary files saved verbatim under meta/ for provenance. const META_FILES = [ - { url: `${STH_ROOT}/bbc/disklog.txt`, dest: "meta/disklog.txt" }, - { url: `${STH_ROOT}/bbc/tapelog.txt`, dest: "meta/tapelog.txt" }, - { url: `${STH_ROOT}/bbc/homepage.html`, dest: "meta/homepage.html" }, - { url: `${STH_ROOT}/bbc/diskimages.html`, dest: "meta/diskimages.html" }, - { url: `${STH_ROOT}/bbc/tapeimages.html`, dest: "meta/tapeimages.html" }, - { url: `${STH_ROOT}/bbc/sthcollection.html`, dest: "meta/sthcollection.html" }, - { url: `${STH_ROOT}/bbc/other.html`, dest: "meta/other.html" }, + { url: `${STH_ROOT}/bbc/disklog.txt`, dest: "meta/bbc-disklog.txt" }, + { url: `${STH_ROOT}/bbc/tapelog.txt`, dest: "meta/bbc-tapelog.txt" }, + { url: `${STH_ROOT}/bbc/homepage.html`, dest: "meta/bbc-homepage.html" }, + { url: `${STH_ROOT}/bbc/diskimages.html`, dest: "meta/bbc-diskimages.html" }, + { url: `${STH_ROOT}/bbc/tapeimages.html`, dest: "meta/bbc-tapeimages.html" }, + { url: `${STH_ROOT}/bbc/sthcollection.html`, dest: "meta/bbc-sthcollection.html" }, + { url: `${STH_ROOT}/bbc/other.html`, dest: "meta/bbc-other.html" }, + { url: `${STH_ROOT}/roms/homepage.html`, dest: "meta/roms-homepage.html" }, + { url: `${STH_ROOT}/electron/homepage.html`, dest: "meta/electron-homepage.html" }, + { url: `${STH_ROOT}/electron/other.html`, dest: "meta/electron-other.html" }, ]; function parseArgs(args) { @@ -167,17 +218,30 @@ function parseReclist(html) { return files; } -// Parse bbc/sthcollection.html — a hand-rolled list of . -// No size/mtime in the index, so size is filled in later via HEAD requests. -function parseSthCollection(html) { - const hrefs = [...html.matchAll(/href="(sthcollection\/[^"]+\.zip)"/gi)].map((m) => m[1]); - // Strip the leading "sthcollection/" so paths are relative to the category root, - // matching how parseReclist returns them. - return [...new Set(hrefs)].sort().map((href) => ({ - path: href.replace(/^sthcollection\//, ""), - size: null, - mtime: null, - })); +// Parse a hand-rolled HTML index page (e.g. roms/homepage.html, electron/dfs/homepage.html, +// bbc/sthcollection.html). Picks up in the same directory tree, ignoring +// external (http://...), absolute (/...), and upward (../...) references — those belong to +// other categories or unrelated content. +// +// stripPrefix: optional path prefix to remove from each href (e.g. "sthcollection/" when the +// index page sits one level above the files). Hrefs that don't start with the prefix are +// rejected — this is what differentiates real download links from incidental anchors. +// +// No size/mtime in the index, so they're filled in later via HEAD requests. +function parseHomepage(html, { stripPrefix = "" } = {}) { + const hrefs = [...html.matchAll(/href="([^"]+\.zip)"/gi)].map((m) => m[1]); + const filtered = []; + for (const href of new Set(hrefs)) { + if (/^(?:https?:|\/)/i.test(href)) continue; + if (href.startsWith("../")) continue; + if (stripPrefix) { + if (!href.startsWith(stripPrefix)) continue; + filtered.push(href.slice(stripPrefix.length)); + } else { + filtered.push(href); + } + } + return filtered.sort().map((path) => ({ path, size: null, mtime: null })); } // URL-encode the path components in a STH-relative path, preserving slashes.