From 0a527e6a37b9fe7b75a8c37eed7e790efd3c8819 Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Wed, 8 Jul 2026 11:39:46 +0200 Subject: [PATCH 1/3] feat: serve firmware update data from static assets instead of D1 Precompute the lookup data at build time (yarn build:data) into per-manufacturer JSON shards plus a manifest, deployed atomically with the worker via Workers Static Assets. The API routes now look up devices in memory from memoized shards instead of querying D1, and the separate post-deploy upload step (and its flaky admin upload protocol) is gone from CI. The manifest version uses the same content hash as the previous upload flow, so existing edge cache keys remain valid across the cutover. Verified against production: 300+ request pairs across API v1-v4, including region filtering, $if conditions and multi-vendor batches, returned byte-identical responses. The D1 code, admin routes and database are left in place for rollback and will be removed in a follow-up. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-and-release.yml | 9 +- .gitignore | 3 + package.json | 5 +- src/lib/dataFormat.ts | 32 ++++ src/lib/dataOperations.test.ts | 208 +++++++++++++++++++++++++ src/lib/dataOperations.ts | 172 ++++++++++++++++++++ src/maintenance/build-data.ts | 110 +++++++++++++ src/routes/api.ts | 120 ++------------ src/worker.ts | 3 + wrangler.toml | 7 + 10 files changed, 551 insertions(+), 118 deletions(-) create mode 100644 src/lib/dataFormat.ts create mode 100644 src/lib/dataOperations.test.ts create mode 100644 src/lib/dataOperations.ts create mode 100644 src/maintenance/build-data.ts diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.yml index 4998ffb4..df4d9fec 100644 --- a/.github/workflows/test-and-release.yml +++ b/.github/workflows/test-and-release.yml @@ -83,15 +83,12 @@ jobs: - name: Install dependencies run: yarn install --immutable + - name: Build firmware update data + run: yarn build:data + - name: Publish to Cloudflare uses: cloudflare/wrangler-action@v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} packageManager: yarn - - - name: Upload config files to Cloudflare - env: - BASE_URL: ${{ secrets.BASE_URL }} - ADMIN_SECRET: ${{ secrets.ADMIN_SECRET }} - run: yarn upload diff --git a/.gitignore b/.gitignore index fa4729a3..fa14ef09 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ test_requests # Auto-generated firmware index firmwares/index.json + +# Prebuilt firmware update data +dist/ diff --git a/package.json b/package.json index a0b9f6f9..ebc77be9 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "check": "tsc -p tsconfig.build.json --noEmit && tsc -p src/maintenance/tsconfig.json --noEmit && tsc -p .github/scripts/tsconfig.json --noEmit", "check:configs": "yarn tsx src/maintenance/check_configs.ts", "lint": "eslint .", - "test:ci": "NODE_OPTIONS='--loader=tsx' ava", + "build:data": "yarn tsx src/maintenance/build-data.ts", + "test:ci": "yarn build:data && NODE_OPTIONS='--loader=tsx' ava", "test": "yarn test:ci", - "dev": "wrangler dev", + "dev": "yarn build:data && wrangler dev", "upload": "yarn tsx src/maintenance/upload.ts" }, "dependencies": { diff --git a/src/lib/dataFormat.ts b/src/lib/dataFormat.ts new file mode 100644 index 00000000..f36dbf35 --- /dev/null +++ b/src/lib/dataFormat.ts @@ -0,0 +1,32 @@ +import type { ConditionalUpgradeInfo } from "./configSchema.js"; + +export const MANIFEST_PATH = "/manifest.json"; + +export interface DataManifest { + /** Content hash of all config files, used as the cache key version */ + version: string; + /** Manufacturer IDs (formatted as 0x1234) that have a shard file */ + shards: string[]; +} + +export interface ShardDeviceEntry { + productType: string; + productId: string; + /** Inclusive firmware version range, normalized with versionToNumber */ + min: number; + max: number; +} + +/** Devices and upgrades from a single config file, limited to one manufacturer ID */ +export interface ShardConfigEntry { + devices: ShardDeviceEntry[]; + upgrades: ConditionalUpgradeInfo[]; +} + +export interface DataShard { + configs: ShardConfigEntry[]; +} + +export function getShardPath(manufacturerId: string): string { + return `/shards/${manufacturerId}.json`; +} diff --git a/src/lib/dataOperations.test.ts b/src/lib/dataOperations.test.ts new file mode 100644 index 00000000..48ad2e30 --- /dev/null +++ b/src/lib/dataOperations.test.ts @@ -0,0 +1,208 @@ +import test from "ava"; +import type { DataManifest, DataShard } from "./dataFormat.js"; +import { + getDataVersion, + lookupConfig, + lookupConfigsBatch, +} from "./dataOperations.js"; +import { versionToNumber } from "./shared.js"; + +function mockAssets(manifest: DataManifest, shards: Record) { + return { + fetch: (input: any) => { + const path = new URL( + typeof input === "string" ? input : input.url, + ).pathname; + let body: unknown; + if (path === "/manifest.json") { + body = manifest; + } else { + const match = /^\/shards\/(.+)\.json$/.exec(path); + if (match) body = shards[match[1]]; + } + if (body === undefined) { + return Promise.resolve(new Response(null, { status: 404 })); + } + return Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ); + }, + } as any; +} + +const upgrade1 = { + version: "1.5", + changelog: "Fixed stuff", + channel: "stable" as const, + files: [ + { + target: 0, + url: "https://example.com/1.5.otz", + integrity: "sha256:" + "0".repeat(64), + }, + ], +}; + +const upgrade2 = { + $if: "firmwareVersion < 1.0", + version: "1.0", + changelog: "Initial", + channel: "stable" as const, + files: [ + { + target: 0, + url: "https://example.com/1.0.otz", + integrity: "sha256:" + "1".repeat(64), + }, + ], +}; + +const defaultManifest: DataManifest = { + version: "abcd1234", + shards: ["0x0086"], +}; + +const defaultShards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: versionToNumber("0.0.0"), + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade1, upgrade2], + }, + ], + }, +}; + +test("getDataVersion returns the manifest version", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + t.is(await getDataVersion(assets), "abcd1234"); +}); + +test("lookupConfig returns updates for a known device", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"); + + t.truthy(result); + t.is(result!.manufacturerId, "0x0086"); + t.is(result!.firmwareVersion, "1.5.0"); + // The $if condition of upgrade2 does not apply for version 1.5 + t.is(result!.updates.length, 1); + t.is(result!.updates[0].version, "1.5"); +}); + +test("lookupConfig evaluates $if conditions against the device", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"); + + t.truthy(result); + t.deepEqual( + result!.updates.map((u) => u.version), + ["1.5", "1.0"], + ); +}); + +test("lookupConfig respects the device firmware version range", async (t) => { + const shards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: versionToNumber("1.0.0"), + max: versionToNumber("1.255.255"), + }, + ], + upgrades: [upgrade1], + }, + ], + }, + }; + const assets = mockAssets(defaultManifest, shards); + + t.truthy(await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5")); + t.is( + await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "2.0"), + undefined, + ); + t.is( + await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.9"), + undefined, + ); +}); + +test("lookupConfig merges updates from multiple matching configs", async (t) => { + const shards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: 0, + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade1], + }, + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: 0, + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade2], + }, + ], + }, + }; + const assets = mockAssets(defaultManifest, shards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"); + + t.deepEqual( + result!.updates.map((u) => u.version), + ["1.5", "1.0"], + ); +}); + +test("lookupConfigsBatch skips unknown devices", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const results = await lookupConfigsBatch(assets, [ + // Unknown manufacturer (no shard) + { + manufacturerId: "0x9999", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.0", + }, + // Known manufacturer, unknown product + { + manufacturerId: "0x0086", + productType: "0xffff", + productId: "0x0064", + firmwareVersion: "1.0", + }, + // Known device + { + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.0", + }, + ]); + + t.is(results.length, 1); + t.is(results[0].productType, "0x0002"); +}); diff --git a/src/lib/dataOperations.ts b/src/lib/dataOperations.ts new file mode 100644 index 00000000..a9288702 --- /dev/null +++ b/src/lib/dataOperations.ts @@ -0,0 +1,172 @@ +import type { Fetcher } from "@cloudflare/workers-types"; +import { APIv3_UpgradeInfo, APIv4_DeviceInfo } from "../apiDefinitions.js"; +import { + DataManifest, + DataShard, + getShardPath, + MANIFEST_PATH, +} from "./dataFormat.js"; +import { conditionApplies } from "./Logic.js"; +import { formatId, padVersion, versionToNumber } from "./shared.js"; + +export interface DeviceLookupRequest { + manufacturerId: number | string; + productType: number | string; + productId: number | string; + firmwareVersion: string; +} + +// Assets are immutable per deployment and isolates die on redeploy, +// so parsed data can be memoized for the isolate's lifetime +interface DataCache { + manifest?: Promise; + shards: Map>; +} +const dataCaches = new WeakMap(); + +function getDataCache(assets: Fetcher): DataCache { + let cache = dataCaches.get(assets); + if (!cache) { + cache = { shards: new Map() }; + dataCaches.set(assets, cache); + } + return cache; +} + +async function fetchJSON( + assets: Fetcher, + path: string, +): Promise { + // The assets binding only routes on the pathname, the origin is arbitrary + const resp = await assets.fetch(`https://assets.local${path}`); + if (!resp.ok) return undefined; + return (await resp.json()) as T; +} + +function getManifest(assets: Fetcher): Promise { + const cache = getDataCache(assets); + cache.manifest ??= fetchJSON(assets, MANIFEST_PATH).then( + (manifest) => { + // Do not memoize failures, so a transient error heals on the next request + if (!manifest) cache.manifest = undefined; + return manifest; + }, + (e) => { + cache.manifest = undefined; + throw e; + }, + ); + return cache.manifest; +} + +async function getShard( + assets: Fetcher, + manufacturerId: string, +): Promise { + const manifest = await getManifest(assets); + if (!manifest?.shards.includes(manufacturerId)) return undefined; + + const cache = getDataCache(assets); + let shard = cache.shards.get(manufacturerId); + if (!shard) { + shard = fetchJSON(assets, getShardPath(manufacturerId)).then( + (result) => { + if (!result) cache.shards.delete(manufacturerId); + return result; + }, + (e) => { + cache.shards.delete(manufacturerId); + throw e; + }, + ); + cache.shards.set(manufacturerId, shard); + } + return shard; +} + +export async function getDataVersion( + assets: Fetcher, +): Promise { + const manifest = await getManifest(assets); + return manifest?.version; +} + +export async function lookupConfigsBatch( + assets: Fetcher, + devices: DeviceLookupRequest[], +): Promise { + const results: APIv4_DeviceInfo[] = []; + + for (const device of devices) { + const manufacturerId = formatId(device.manufacturerId); + const shard = await getShard(assets, manufacturerId); + if (!shard) continue; + + const productType = formatId(device.productType); + const productId = formatId(device.productId); + const firmwareVersion = padVersion(device.firmwareVersion, "0"); + const versionNumber = versionToNumber(firmwareVersion); + + const matchingConfigs = shard.configs.filter((config) => + config.devices.some( + (d) => + d.productType === productType && + d.productId === productId && + versionNumber >= d.min && + versionNumber <= d.max, + ), + ); + if (matchingConfigs.length === 0) continue; + + const deviceId = { + manufacturerId: parseInt(manufacturerId, 16), + productType: parseInt(productType, 16), + productId: parseInt(productId, 16), + firmwareVersion, + }; + + const updates = matchingConfigs + .flatMap((config) => config.upgrades) + .filter((upgrade) => conditionApplies(upgrade, deviceId)) + .map(({ $if, ...upgrade }): APIv3_UpgradeInfo => { + return { + version: upgrade.version, + changelog: upgrade.changelog, + channel: upgrade.channel, + ...(upgrade.region ? { region: upgrade.region } : {}), + files: upgrade.files, + // These two will be filled in or filtered by the downstream handler + downgrade: undefined as any, + normalizedVersion: undefined as any, + }; + }); + + results.push({ + manufacturerId, + productType, + productId, + firmwareVersion, + updates, + }); + } + + return results; +} + +export async function lookupConfig( + assets: Fetcher, + manufacturerId: number | string, + productType: number | string, + productId: number | string, + firmwareVersion: string, +): Promise { + const results = await lookupConfigsBatch(assets, [ + { + manufacturerId, + productType, + productId, + firmwareVersion, + }, + ]); + return results[0]; +} diff --git a/src/maintenance/build-data.ts b/src/maintenance/build-data.ts new file mode 100644 index 00000000..7331f539 --- /dev/null +++ b/src/maintenance/build-data.ts @@ -0,0 +1,110 @@ +import JSON5 from "json5"; +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "path-browserify"; +import { ConditionalUpdateConfig } from "../lib/config.js"; +import { + DataManifest, + DataShard, + getShardPath, + MANIFEST_PATH, +} from "../lib/dataFormat.js"; +import { getErrorMessage, padVersion, versionToNumber } from "../lib/shared.js"; +import { NodeFS } from "./nodeFS.js"; + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const configDir = path.resolve(__dirname, "../../firmwares"); +const outDir = join(__dirname, "../../dist/data"); + +void (async () => { + const configFiles = (await NodeFS.readDir(configDir, true)) + .filter( + (file) => + file.endsWith(".json") && + !file.endsWith("index.json") && + !path.basename(file).startsWith("_") && + !file.includes("/templates/") && + !file.includes("\\templates\\"), + ) + // Sort for a deterministic version hash and shard layout + .sort(); + + const files: { filename: string; data: string }[] = []; + for (const filePath of configFiles) { + const relativePath = path + .relative(configDir, filePath) + .replace(/\\/g, "/"); + const fileContent = await NodeFS.readFile(filePath); + files.push({ filename: relativePath, data: fileContent }); + } + + const hasher = crypto.createHash("sha256"); + const version = hasher + .update(JSON.stringify(files), "utf8") + .digest("hex") + .slice(0, 8); + + const shards = new Map(); + for (const file of files) { + let config: ConditionalUpdateConfig; + try { + config = new ConditionalUpdateConfig(JSON5.parse(file.data)); + } catch (e) { + console.error( + `Error parsing config file ${file.filename}: ${getErrorMessage(e)}`, + ); + process.exit(1); + } + + const byManufacturer = Map.groupBy( + config.devices, + (d) => d.manufacturerId, + ); + for (const [manufacturerId, devices] of byManufacturer) { + let shard = shards.get(manufacturerId); + if (!shard) { + shard = { configs: [] }; + shards.set(manufacturerId, shard); + } + shard.configs.push({ + devices: devices.map((d) => ({ + productType: d.productType, + productId: d.productId, + min: versionToNumber(padVersion(d.firmwareVersion.min, "0")), + max: versionToNumber( + padVersion(d.firmwareVersion.max, "255"), + ), + })), + upgrades: config.upgrades, + }); + } + } + + await fs.rm(outDir, { recursive: true, force: true }); + await fs.mkdir(join(outDir, "shards"), { recursive: true }); + + for (const [manufacturerId, shard] of shards) { + await fs.writeFile( + join(outDir, getShardPath(manufacturerId)), + JSON.stringify(shard), + "utf8", + ); + } + + const manifest: DataManifest = { + version, + shards: [...shards.keys()].sort(), + }; + await fs.writeFile( + join(outDir, MANIFEST_PATH), + JSON.stringify(manifest), + "utf8", + ); + + console.log( + `Built data version ${version}: ${files.length} config files, ${shards.size} shards`, + ); +})(); diff --git a/src/routes/api.ts b/src/routes/api.ts index 8ff0ecd4..05eec259 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -11,17 +11,12 @@ import { APIv4_Response, } from "../apiDefinitions.js"; import { withCache } from "../lib/cache.js"; -import { - cacheD1Config, - getCurrentVersionCached, - getD1CachedConfig, -} from "../lib/cachedD1Operations.js"; import type { UpgradeInfo } from "../lib/configSchema.js"; import { - DeviceLookupRequest, + getDataVersion, lookupConfig, lookupConfigsBatch, -} from "../lib/d1Operations.js"; +} from "../lib/dataOperations.js"; import { compareVersions, padVersion } from "../lib/shared.js"; import { clientError, @@ -79,11 +74,7 @@ async function handleUpdateRequest( const { manufacturerId, productType, productId, firmwareVersion } = result.data; - const filesVersion = await getCurrentVersionCached( - req.url, - context, - env.CONFIG_FILES, - ); + const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { return serverError("Database empty"); } @@ -113,8 +104,7 @@ async function handleUpdateRequest( }, async () => { const deviceInfo = await lookupConfig( - env.CONFIG_FILES, - filesVersion, + env.DATA, manufacturerId, productType, productId, @@ -324,7 +314,7 @@ export default function register(router: any): void { async ( req: RequestWithProps<[ContentProps]>, env: CloudflareEnvironment, - context: ExecutionContext, + _context: ExecutionContext, ) => { // Parse and validate the v4 request const result = await APIv4_RequestSchema.safeParseAsync( @@ -335,11 +325,7 @@ export default function register(router: any): void { } const { region, devices } = result.data; - const filesVersion = await getCurrentVersionCached( - req.url, - context, - env.CONFIG_FILES, - ); + const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { return serverError("Database empty"); } @@ -374,59 +360,10 @@ export default function register(router: any): void { return a.firmwareVersion.localeCompare(b.firmwareVersion); }); - // Step 1: Try to find cached responses for each unique device - const cacheMisses: DeviceLookupRequest[] = []; - const results: APIv4_DeviceInfo[] = []; - - for (const device of uniqueDevices) { - // Try to get cached config for this device using D1 cache utilities - const cachedConfig = await getD1CachedConfig( - req.url, - filesVersion, - device.manufacturerId, - device.productType, - device.productId, - device.firmwareVersion, - ); - - if (cachedConfig === undefined) { - // Cache miss - add to batch lookup - cacheMisses.push(device); - continue; - } - - if (cachedConfig === null) { - // Cached as "not found" - continue; - } - - // Cache hit - add to results - results.push(cachedConfig); - } - - // Step 2: Perform single batch request to database for all cache misses - if (cacheMisses.length > 0) { - const batchResults = await lookupConfigsBatch( - env.CONFIG_FILES, - filesVersion, - cacheMisses, - ); - results.push(...batchResults); - - // Cache the results of the batch lookup - for (const deviceInfo of batchResults) { - cacheD1Config( - req.url, - context, - filesVersion, - deviceInfo.manufacturerId, - deviceInfo.productType, - deviceInfo.productId, - deviceInfo.firmwareVersion, - deviceInfo, - ); - } - } + const results: APIv4_DeviceInfo[] = await lookupConfigsBatch( + env.DATA, + uniqueDevices, + ); // Post-process the results to apply region filtering etc. const response: APIv4_Response = results.map((device) => { @@ -478,43 +415,6 @@ export default function register(router: any): void { return device; }); - // Track which devices were NOT found in the DB and cache them as `null` - const requestedFingerprints = new Set( - uniqueDevices.map( - (d) => - `${d.manufacturerId}:${d.productType}:${d.productId}:${d.firmwareVersion}`, - ), - ); - const foundFingerprints = new Set( - results.map( - (d) => - `${d.manufacturerId}:${d.productType}:${d.productId}:${d.firmwareVersion}`, - ), - ); - const missingFromDb = - requestedFingerprints.difference(foundFingerprints); - - for (const fingerprint of missingFromDb) { - const [ - manufacturerId, - productType, - productId, - firmwareVersion, - ] = fingerprint.split(":"); - - // Cache as "not found" - cacheD1Config( - req.url, - context, - filesVersion, - manufacturerId, - productType, - productId, - firmwareVersion, - null, - ); - } - return json(response); }, ); diff --git a/src/worker.ts b/src/worker.ts index f5b42f76..566479b1 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -5,6 +5,9 @@ export interface CloudflareEnvironment { CONFIG_FILES: D1Database; + /** Static assets containing the prebuilt firmware update data */ + DATA: Fetcher; + RL_GLOBAL: RateLimit; RL_BURST: RateLimit; diff --git a/wrangler.toml b/wrangler.toml index f3ca18b1..e43d625a 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,6 +15,13 @@ binding = "CONFIG_FILES" database_name = "zwave-js-firmware-updates--prod" database_id = "6d9a21f3-6886-4d53-b7dc-3b03dad70698" +# Prebuilt firmware update data, generated by `yarn build:data`. +# run_worker_first keeps the shards from being publicly routable. +[assets] +directory = "dist/data" +binding = "DATA" +run_worker_first = true + # Global sustained rate limit shared by all requests. # Per-edge-location: ~33 req/s sustained across the 60s window, up to 3M req/day. [[ratelimits]] From c934769d716c18e15b187b12a3e826e2d91a6575 Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Wed, 8 Jul 2026 15:41:18 +0200 Subject: [PATCH 2/3] fix: address review feedback on the assets migration (PR #347 review) - Fail build:data on implausibly small datasets instead of shipping an empty catalog - Extract shared config discovery/hash/shard logic to dataBuild.ts, reuse it in check_configs.ts, and cover it with unit tests - Throw on non-404 asset fetch errors and log all failure paths, including manifest-listed shards that are missing - Guard against malformed manifests without a shards array - Prefetch distinct manufacturer shards in parallel for v4 batches - Restore edge caching for v4, keyed on data version + hash of the normalized (deduplicated, sorted) device list - Keep the file key order of the D1 path for stable response bodies - Reword "Database empty" to "Firmware update data not available" - Test failure recovery (no memoized failures), malformed JSON, and concurrent shard fetch dedup Co-Authored-By: Claude Fable 5 --- src/lib/dataOperations.test.ts | 84 ++++++++++++++++++ src/lib/dataOperations.ts | 37 +++++++- src/maintenance/build-data.ts | 97 ++++++--------------- src/maintenance/check_configs.ts | 11 +-- src/maintenance/dataBuild.test.ts | 118 ++++++++++++++++++++++++++ src/maintenance/dataBuild.ts | 108 ++++++++++++++++++++++++ src/routes/api.ts | 136 ++++++++++++++++++------------ 7 files changed, 453 insertions(+), 138 deletions(-) create mode 100644 src/maintenance/dataBuild.test.ts create mode 100644 src/maintenance/dataBuild.ts diff --git a/src/lib/dataOperations.test.ts b/src/lib/dataOperations.test.ts index 48ad2e30..ae679543 100644 --- a/src/lib/dataOperations.test.ts +++ b/src/lib/dataOperations.test.ts @@ -177,6 +177,90 @@ test("lookupConfig merges updates from multiple matching configs", async (t) => ); }); +test("transient fetch errors are not memoized", async (t) => { + let calls = 0; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + calls++; + if (calls === 1) throw new Error("transient"); + return inner.fetch(input); + }, + } as any; + + await t.throwsAsync(() => getDataVersion(assets)); + // The failed manifest fetch was not cached, so the retry succeeds + t.is(await getDataVersion(assets), "abcd1234"); +}); + +test("non-404 error responses throw instead of reading as missing data", async (t) => { + const assets = { + fetch: () => Promise.resolve(new Response(null, { status: 500 })), + } as any; + await t.throwsAsync(() => getDataVersion(assets), { + message: /500/, + }); +}); + +test("malformed shard JSON is not memoized", async (t) => { + let broken = true; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + const path = new URL(input).pathname; + if (broken && path.startsWith("/shards/")) { + return Promise.resolve( + new Response("{ not json", { status: 200 }), + ); + } + return inner.fetch(input); + }, + } as any; + + await t.throwsAsync(() => + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"), + ); + broken = false; + t.truthy(await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5")); +}); + +test("a shard listed in the manifest but missing degrades to no results", async (t) => { + const assets = mockAssets(defaultManifest, {}); + const results = await lookupConfigsBatch(assets, [ + { + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.5", + }, + ]); + t.deepEqual(results, []); +}); + +test("a manifest without a shards array degrades to no data", async (t) => { + const assets = mockAssets({ version: "abcd1234" } as any, defaultShards); + t.is(await getDataVersion(assets), undefined); +}); + +test("concurrent lookups share one shard fetch", async (t) => { + let shardFetches = 0; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + if (new URL(input).pathname.startsWith("/shards/")) shardFetches++; + return inner.fetch(input); + }, + } as any; + + const [a, b] = await Promise.all([ + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"), + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"), + ]); + t.truthy(a); + t.truthy(b); + t.is(shardFetches, 1); +}); + test("lookupConfigsBatch skips unknown devices", async (t) => { const assets = mockAssets(defaultManifest, defaultShards); const results = await lookupConfigsBatch(assets, [ diff --git a/src/lib/dataOperations.ts b/src/lib/dataOperations.ts index a9288702..f3b0f89b 100644 --- a/src/lib/dataOperations.ts +++ b/src/lib/dataOperations.ts @@ -39,7 +39,11 @@ async function fetchJSON( ): Promise { // The assets binding only routes on the pathname, the origin is arbitrary const resp = await assets.fetch(`https://assets.local${path}`); - if (!resp.ok) return undefined; + // A missing asset is expected (unknown manufacturer), anything else is not + if (resp.status === 404) return undefined; + if (!resp.ok) { + throw new Error(`Fetching asset ${path} failed with ${resp.status}`); + } return (await resp.json()) as T; } @@ -48,11 +52,16 @@ function getManifest(assets: Fetcher): Promise { cache.manifest ??= fetchJSON(assets, MANIFEST_PATH).then( (manifest) => { // Do not memoize failures, so a transient error heals on the next request - if (!manifest) cache.manifest = undefined; + if (!manifest || !Array.isArray(manifest.shards)) { + cache.manifest = undefined; + console.error("Data manifest is missing or malformed"); + return undefined; + } return manifest; }, (e) => { cache.manifest = undefined; + console.error(`Failed to load data manifest: ${e}`); throw e; }, ); @@ -71,11 +80,18 @@ async function getShard( if (!shard) { shard = fetchJSON(assets, getShardPath(manufacturerId)).then( (result) => { - if (!result) cache.shards.delete(manufacturerId); + if (!result) { + cache.shards.delete(manufacturerId); + // The manifest promised this shard, so this is a broken deployment + console.error( + `Shard ${manufacturerId} is listed in the manifest but missing`, + ); + } return result; }, (e) => { cache.shards.delete(manufacturerId); + console.error(`Failed to load shard ${manufacturerId}: ${e}`); throw e; }, ); @@ -97,6 +113,14 @@ export async function lookupConfigsBatch( ): Promise { const results: APIv4_DeviceInfo[] = []; + // Load all needed shards in parallel before the sequential matching below + const manufacturerIds = new Set( + devices.map((d) => formatId(d.manufacturerId)), + ); + await Promise.all( + [...manufacturerIds].map((id) => getShard(assets, id)), + ); + for (const device of devices) { const manufacturerId = formatId(device.manufacturerId); const shard = await getShard(assets, manufacturerId); @@ -134,7 +158,12 @@ export async function lookupConfigsBatch( changelog: upgrade.changelog, channel: upgrade.channel, ...(upgrade.region ? { region: upgrade.region } : {}), - files: upgrade.files, + // Keep the key order the D1 path produced, for stable response bodies + files: upgrade.files.map((f) => ({ + target: f.target, + url: f.url, + integrity: f.integrity, + })), // These two will be filled in or filtered by the downstream handler downgrade: undefined as any, normalizedVersion: undefined as any, diff --git a/src/maintenance/build-data.ts b/src/maintenance/build-data.ts index 7331f539..2fba6a27 100644 --- a/src/maintenance/build-data.ts +++ b/src/maintenance/build-data.ts @@ -1,15 +1,12 @@ -import JSON5 from "json5"; -import crypto from "node:crypto"; import fs from "node:fs/promises"; import path from "path-browserify"; -import { ConditionalUpdateConfig } from "../lib/config.js"; +import { getShardPath, MANIFEST_PATH } from "../lib/dataFormat.js"; import { - DataManifest, - DataShard, - getShardPath, - MANIFEST_PATH, -} from "../lib/dataFormat.js"; -import { getErrorMessage, padVersion, versionToNumber } from "../lib/shared.js"; + buildDataShards, + buildManifest, + hashConfigFiles, + readConfigFiles, +} from "./dataBuild.js"; import { NodeFS } from "./nodeFS.js"; import { dirname, join } from "node:path"; @@ -19,68 +16,28 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const configDir = path.resolve(__dirname, "../../firmwares"); const outDir = join(__dirname, "../../dist/data"); -void (async () => { - const configFiles = (await NodeFS.readDir(configDir, true)) - .filter( - (file) => - file.endsWith(".json") && - !file.endsWith("index.json") && - !path.basename(file).startsWith("_") && - !file.includes("/templates/") && - !file.includes("\\templates\\"), - ) - // Sort for a deterministic version hash and shard layout - .sort(); +// Fail the build (and thereby the deploy) if the dataset shrinks implausibly, +// e.g. because a failed directory read silently yielded an empty file list +const MIN_CONFIG_FILES = 100; +const MIN_SHARDS = 10; - const files: { filename: string; data: string }[] = []; - for (const filePath of configFiles) { - const relativePath = path - .relative(configDir, filePath) - .replace(/\\/g, "/"); - const fileContent = await NodeFS.readFile(filePath); - files.push({ filename: relativePath, data: fileContent }); +void (async () => { + const files = await readConfigFiles(NodeFS, configDir); + const version = hashConfigFiles(files); + + let shards; + try { + shards = buildDataShards(files); + } catch (e) { + console.error(e instanceof Error ? e.message : e); + process.exit(1); } - const hasher = crypto.createHash("sha256"); - const version = hasher - .update(JSON.stringify(files), "utf8") - .digest("hex") - .slice(0, 8); - - const shards = new Map(); - for (const file of files) { - let config: ConditionalUpdateConfig; - try { - config = new ConditionalUpdateConfig(JSON5.parse(file.data)); - } catch (e) { - console.error( - `Error parsing config file ${file.filename}: ${getErrorMessage(e)}`, - ); - process.exit(1); - } - - const byManufacturer = Map.groupBy( - config.devices, - (d) => d.manufacturerId, + if (files.length < MIN_CONFIG_FILES || shards.size < MIN_SHARDS) { + console.error( + `ERROR: Implausibly small dataset (${files.length} config files, ${shards.size} shards), refusing to build`, ); - for (const [manufacturerId, devices] of byManufacturer) { - let shard = shards.get(manufacturerId); - if (!shard) { - shard = { configs: [] }; - shards.set(manufacturerId, shard); - } - shard.configs.push({ - devices: devices.map((d) => ({ - productType: d.productType, - productId: d.productId, - min: versionToNumber(padVersion(d.firmwareVersion.min, "0")), - max: versionToNumber( - padVersion(d.firmwareVersion.max, "255"), - ), - })), - upgrades: config.upgrades, - }); - } + process.exit(1); } await fs.rm(outDir, { recursive: true, force: true }); @@ -94,13 +51,9 @@ void (async () => { ); } - const manifest: DataManifest = { - version, - shards: [...shards.keys()].sort(), - }; await fs.writeFile( join(outDir, MANIFEST_PATH), - JSON.stringify(manifest), + JSON.stringify(buildManifest(version, shards)), "utf8", ); diff --git a/src/maintenance/check_configs.ts b/src/maintenance/check_configs.ts index 88833202..9dff6303 100644 --- a/src/maintenance/check_configs.ts +++ b/src/maintenance/check_configs.ts @@ -4,6 +4,7 @@ import path from "path-browserify"; import { ConditionalUpdateConfig } from "../lib/config.js"; import { parseLogic } from "../lib/Logic.js"; import { getErrorMessage } from "../lib/shared.js"; +import { discoverConfigFiles } from "./dataBuild.js"; import { NodeFS } from "./nodeFS.js"; import { dirname } from "path"; @@ -128,15 +129,7 @@ void (async () => { try { console.log("🔍 Validating config files...\n"); - // Find all config files (same logic as upload.ts) - const configFiles = (await NodeFS.readDir(configDir, true)).filter( - (file) => - file.endsWith(".json") && - !file.endsWith("index.json") && - !path.basename(file).startsWith("_") && - !file.includes("/templates/") && - !file.includes("\\templates\\"), - ); + const configFiles = await discoverConfigFiles(NodeFS, configDir); console.log(`Found ${configFiles.length} config files to validate.`); diff --git a/src/maintenance/dataBuild.test.ts b/src/maintenance/dataBuild.test.ts new file mode 100644 index 00000000..968998e5 --- /dev/null +++ b/src/maintenance/dataBuild.test.ts @@ -0,0 +1,118 @@ +import test from "ava"; +import { versionToNumber } from "../lib/shared.js"; +import { + buildDataShards, + buildManifest, + hashConfigFiles, + isConfigFile, +} from "./dataBuild.js"; + +const upgrade = { + version: "1.5", + changelog: "Fixed stuff", + url: "https://example.com/1.5.otz", + integrity: "sha256:" + "0".repeat(64), +}; + +function configFile(devices: any[], upgrades: any[] = [upgrade]) { + return JSON.stringify({ devices, upgrades }); +} + +const device1 = { + brand: "Test", + model: "Device 1", + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", +}; + +test("isConfigFile applies the discovery filter", (t) => { + t.true(isConfigFile("vendor/device.json")); + t.false(isConfigFile("vendor/device.md")); + t.false(isConfigFile("index.json")); + t.false(isConfigFile("vendor/_draft.json")); + t.false(isConfigFile("vendor/templates/base.json")); +}); + +test("buildDataShards groups devices by manufacturer ID", (t) => { + const shards = buildDataShards([ + { filename: "a.json", data: configFile([device1]) }, + { + filename: "b.json", + data: configFile([ + { ...device1, model: "Device 2", productId: "0x0065" }, + { ...device1, model: "Other vendor", manufacturerId: "0x027a" }, + ]), + }, + ]); + + t.deepEqual([...shards.keys()].sort(), ["0x0086", "0x027a"].sort()); + // a.json and b.json each contribute one config entry to 0x0086 + t.is(shards.get("0x0086")!.configs.length, 2); + // The multi-manufacturer file lands in both shards with the same upgrades + const otherVendor = shards.get("0x027a")!.configs[0]; + t.is(otherVendor.devices.length, 1); + t.is(otherVendor.upgrades[0].version, "1.5"); +}); + +test("buildDataShards normalizes firmware version ranges", (t) => { + const shards = buildDataShards([ + { + filename: "a.json", + data: configFile([ + device1, + { + ...device1, + productId: "0x0065", + firmwareVersion: { min: "1.5", max: "2.0" }, + }, + ]), + }, + ]); + + const [defaultRange, explicitRange] = shards.get("0x0086")!.configs[0] + .devices; + // No range in the config means "all versions" + t.is(defaultRange.min, 0); + t.is(defaultRange.max, versionToNumber("255.255.255")); + // min is padded with .0, max with .255 + t.is(explicitRange.min, versionToNumber("1.5.0")); + t.is(explicitRange.max, versionToNumber("2.0.255")); +}); + +test("buildDataShards names the offending file for invalid configs", (t) => { + const error = t.throws(() => + buildDataShards([ + { filename: "vendor/broken.json", data: "{ not valid" }, + ]), + ); + t.true(error!.message.includes("vendor/broken.json")); +}); + +test("hashConfigFiles is deterministic and content-sensitive", (t) => { + const files = [{ filename: "a.json", data: configFile([device1]) }]; + const hash = hashConfigFiles(files); + t.is(hash, hashConfigFiles([...files.map((f) => ({ ...f }))])); + t.regex(hash, /^[0-9a-f]{8}$/); + t.not( + hash, + hashConfigFiles([{ filename: "a.json", data: configFile([device1], [ + { ...upgrade, version: "1.6" }, + ]) }]), + ); +}); + +test("buildManifest sorts shard IDs and carries the version", (t) => { + const shards = buildDataShards([ + { + filename: "a.json", + data: configFile([ + { ...device1, manufacturerId: "0x027a" }, + { ...device1, manufacturerId: "0x0086" }, + ]), + }, + ]); + const manifest = buildManifest("abcd1234", shards); + t.is(manifest.version, "abcd1234"); + t.deepEqual(manifest.shards, ["0x0086", "0x027a"]); +}); diff --git a/src/maintenance/dataBuild.ts b/src/maintenance/dataBuild.ts new file mode 100644 index 00000000..01e804a4 --- /dev/null +++ b/src/maintenance/dataBuild.ts @@ -0,0 +1,108 @@ +import JSON5 from "json5"; +import crypto from "node:crypto"; +import path from "path-browserify"; +import { ConditionalUpdateConfig } from "../lib/config.js"; +import { DataManifest, DataShard } from "../lib/dataFormat.js"; +import type { FileSystem } from "../lib/fs/filesystem.js"; +import { getErrorMessage, padVersion, versionToNumber } from "../lib/shared.js"; + +export interface ConfigFileEntry { + filename: string; + data: string; +} + +export function isConfigFile(file: string): boolean { + return ( + file.endsWith(".json") && + !file.endsWith("index.json") && + !path.basename(file).startsWith("_") && + !file.includes("/templates/") && + !file.includes("\\templates\\") + ); +} + +/** Finds all config files below the given directory, sorted for deterministic output */ +export async function discoverConfigFiles( + fs: FileSystem, + configDir: string, +): Promise { + return (await fs.readDir(configDir, true)).filter(isConfigFile).sort(); +} + +export async function readConfigFiles( + fs: FileSystem, + configDir: string, +): Promise { + const configFiles = await discoverConfigFiles(fs, configDir); + + const files: ConfigFileEntry[] = []; + for (const filePath of configFiles) { + const relativePath = path + .relative(configDir, filePath) + .replace(/\\/g, "/"); + const fileContent = await fs.readFile(filePath); + files.push({ filename: relativePath, data: fileContent }); + } + return files; +} + +export function hashConfigFiles(files: ConfigFileEntry[]): string { + return crypto + .createHash("sha256") + .update(JSON.stringify(files), "utf8") + .digest("hex") + .slice(0, 8); +} + +/** Groups all config files into one shard per manufacturer ID. Throws on invalid config files. */ +export function buildDataShards( + files: ConfigFileEntry[], +): Map { + const shards = new Map(); + + for (const file of files) { + let config: ConditionalUpdateConfig; + try { + config = new ConditionalUpdateConfig(JSON5.parse(file.data)); + } catch (e) { + throw new Error( + `Error parsing config file ${file.filename}: ${getErrorMessage(e)}`, + ); + } + + const byManufacturer = Map.groupBy( + config.devices, + (d) => d.manufacturerId, + ); + for (const [manufacturerId, devices] of byManufacturer) { + let shard = shards.get(manufacturerId); + if (!shard) { + shard = { configs: [] }; + shards.set(manufacturerId, shard); + } + shard.configs.push({ + devices: devices.map((d) => ({ + productType: d.productType, + productId: d.productId, + min: versionToNumber(padVersion(d.firmwareVersion.min, "0")), + max: versionToNumber( + padVersion(d.firmwareVersion.max, "255"), + ), + })), + upgrades: config.upgrades, + }); + } + } + + return shards; +} + +export function buildManifest( + version: string, + shards: Map, +): DataManifest { + return { + version, + shards: [...shards.keys()].sort(), + }; +} diff --git a/src/routes/api.ts b/src/routes/api.ts index 05eec259..45bbff1d 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -17,7 +17,7 @@ import { lookupConfig, lookupConfigsBatch, } from "../lib/dataOperations.js"; -import { compareVersions, padVersion } from "../lib/shared.js"; +import { array2hex, compareVersions, padVersion } from "../lib/shared.js"; import { clientError, ContentProps, @@ -76,7 +76,7 @@ async function handleUpdateRequest( const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { - return serverError("Database empty"); + return serverError("Firmware update data not available"); } // Figure out if this info is already cached @@ -314,7 +314,7 @@ export default function register(router: any): void { async ( req: RequestWithProps<[ContentProps]>, env: CloudflareEnvironment, - _context: ExecutionContext, + context: ExecutionContext, ) => { // Parse and validate the v4 request const result = await APIv4_RequestSchema.safeParseAsync( @@ -327,7 +327,7 @@ export default function register(router: any): void { const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { - return serverError("Database empty"); + return serverError("Firmware update data not available"); } // Remove duplicates, normalize and sort devices for consistent processing @@ -360,62 +360,92 @@ export default function register(router: any): void { return a.firmwareVersion.localeCompare(b.firmwareVersion); }); - const results: APIv4_DeviceInfo[] = await lookupConfigsBatch( - env.DATA, - uniqueDevices, + // The devices are deduplicated and sorted, so identical queries against + // the same data version share one cache entry regardless of input order + const fingerprint = JSON.stringify({ + region: region ?? null, + devices: uniqueDevices, + }); + const fingerprintHash = array2hex( + new Uint8Array( + await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(fingerprint), + ), + ), ); + const cacheKey = new URL( + `./${filesVersion}/${fingerprintHash}`, + req.url.endsWith("/") ? req.url : req.url + "/", + ).toString(); - // Post-process the results to apply region filtering etc. - const response: APIv4_Response = results.map((device) => { - if (!device.updates) return device; - let filteredUpgrades = device.updates - // Filter out upgrades for a different region - .filter((u) => !u.region || u.region === region) - // Filter out the current version - .filter( - (u) => - padVersion(u.version) !== - padVersion(device.firmwareVersion), - ) - // Add missing fields to the returned objects + return withCache( + { + req, + context, + cacheKey, + // Same policy as v1-v3: 1 hour on the client, 1 day at the edge + // (the cache key includes the data version) + maxAge: 60 * 60, + sMaxAge: 60 * 60 * 24, + }, + async () => { + const results: APIv4_DeviceInfo[] = + await lookupConfigsBatch(env.DATA, uniqueDevices); + + // Post-process the results to apply region filtering etc. + const response: APIv4_Response = results.map((device) => { + if (!device.updates) return device; + let filteredUpgrades = device.updates + // Filter out upgrades for a different region + .filter((u) => !u.region || u.region === region) + // Filter out the current version + .filter( + (u) => + padVersion(u.version) !== + padVersion(device.firmwareVersion), + ) + // Add missing fields to the returned objects - .map((u: UpgradeInfo) => { - const downgrade = - compareVersions(u.version, device.firmwareVersion) < - 0; - let normalizedVersion = padVersion(u.version, "0"); - if (u.channel === "beta") normalizedVersion += "-beta"; + .map((u: UpgradeInfo) => { + const downgrade = + compareVersions(u.version, device.firmwareVersion) < + 0; + let normalizedVersion = padVersion(u.version, "0"); + if (u.channel === "beta") normalizedVersion += "-beta"; - return { - ...u, - downgrade, - normalizedVersion, - }; - }) - // Sort by version ascending... - .sort((a: any, b: any) => { - const ret = compare( - a.normalizedVersion, - b.normalizedVersion, + return { + ...u, + downgrade, + normalizedVersion, + }; + }) + // Sort by version ascending... + .sort((a: any, b: any) => { + const ret = compare( + a.normalizedVersion, + b.normalizedVersion, + ); + if (ret !== 0) return ret; + // ... and put updates for a specific region first + return -(a.region ?? "").localeCompare(b.region ?? ""); + }); + // If there are multiple updates for the same version, only return the first one + // This happens when there are updates for a specific region and a general update for the same version + filteredUpgrades = filteredUpgrades.filter( + (u: any, i: number, arr: any[]) => { + if (i > 0 && u.version === arr[i - 1].version) + return false; + return true; + }, ); - if (ret !== 0) return ret; - // ... and put updates for a specific region first - return -(a.region ?? "").localeCompare(b.region ?? ""); + device.updates = filteredUpgrades; + return device; }); - // If there are multiple updates for the same version, only return the first one - // This happens when there are updates for a specific region and a general update for the same version - filteredUpgrades = filteredUpgrades.filter( - (u: any, i: number, arr: any[]) => { - if (i > 0 && u.version === arr[i - 1].version) - return false; - return true; - }, - ); - device.updates = filteredUpgrades; - return device; - }); - return json(response); + return json(response); + }, + ); }, ); } From 68ece45f20e046eed5ec5ab84a45cffc9ee48e1b Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Thu, 9 Jul 2026 14:44:47 +0200 Subject: [PATCH 3/3] refactor: index prefetched shards in a map for v4 lookups (PR #347 review) Build a manufacturerId->shard map from the parallel prefetch and index into it in the matching loop, instead of calling getShard again and relying on its internal memoization. Co-Authored-By: Claude Opus 4.8 --- src/lib/dataOperations.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lib/dataOperations.ts b/src/lib/dataOperations.ts index f3b0f89b..dbce3986 100644 --- a/src/lib/dataOperations.ts +++ b/src/lib/dataOperations.ts @@ -114,16 +114,20 @@ export async function lookupConfigsBatch( const results: APIv4_DeviceInfo[] = []; // Load all needed shards in parallel before the sequential matching below - const manufacturerIds = new Set( - devices.map((d) => formatId(d.manufacturerId)), - ); - await Promise.all( - [...manufacturerIds].map((id) => getShard(assets, id)), + const manufacturerIds = [ + ...new Set(devices.map((d) => formatId(d.manufacturerId))), + ]; + const shardsById = new Map( + await Promise.all( + manufacturerIds.map( + async (id) => [id, await getShard(assets, id)] as const, + ), + ), ); for (const device of devices) { const manufacturerId = formatId(device.manufacturerId); - const shard = await getShard(assets, manufacturerId); + const shard = shardsById.get(manufacturerId); if (!shard) continue; const productType = formatId(device.productType);