diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.yml index 4998ffb..df4d9fe 100644 --- a/.github/workflows/test-and-release.yml +++ b/.github/workflows/test-and-release.yml @@ -83,15 +83,12 @@ jobs: - name: Install dependencies run: yarn install --immutable + - name: Build firmware update data + run: yarn build:data + - name: Publish to Cloudflare uses: cloudflare/wrangler-action@v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} packageManager: yarn - - - name: Upload config files to Cloudflare - env: - BASE_URL: ${{ secrets.BASE_URL }} - ADMIN_SECRET: ${{ secrets.ADMIN_SECRET }} - run: yarn upload diff --git a/.gitignore b/.gitignore index fa4729a..fa14ef0 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ test_requests # Auto-generated firmware index firmwares/index.json + +# Prebuilt firmware update data +dist/ diff --git a/package.json b/package.json index a0b9f6f..ebc77be 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "check": "tsc -p tsconfig.build.json --noEmit && tsc -p src/maintenance/tsconfig.json --noEmit && tsc -p .github/scripts/tsconfig.json --noEmit", "check:configs": "yarn tsx src/maintenance/check_configs.ts", "lint": "eslint .", - "test:ci": "NODE_OPTIONS='--loader=tsx' ava", + "build:data": "yarn tsx src/maintenance/build-data.ts", + "test:ci": "yarn build:data && NODE_OPTIONS='--loader=tsx' ava", "test": "yarn test:ci", - "dev": "wrangler dev", + "dev": "yarn build:data && wrangler dev", "upload": "yarn tsx src/maintenance/upload.ts" }, "dependencies": { diff --git a/src/lib/dataFormat.ts b/src/lib/dataFormat.ts new file mode 100644 index 0000000..f36dbf3 --- /dev/null +++ b/src/lib/dataFormat.ts @@ -0,0 +1,32 @@ +import type { ConditionalUpgradeInfo } from "./configSchema.js"; + +export const MANIFEST_PATH = "/manifest.json"; + +export interface DataManifest { + /** Content hash of all config files, used as the cache key version */ + version: string; + /** Manufacturer IDs (formatted as 0x1234) that have a shard file */ + shards: string[]; +} + +export interface ShardDeviceEntry { + productType: string; + productId: string; + /** Inclusive firmware version range, normalized with versionToNumber */ + min: number; + max: number; +} + +/** Devices and upgrades from a single config file, limited to one manufacturer ID */ +export interface ShardConfigEntry { + devices: ShardDeviceEntry[]; + upgrades: ConditionalUpgradeInfo[]; +} + +export interface DataShard { + configs: ShardConfigEntry[]; +} + +export function getShardPath(manufacturerId: string): string { + return `/shards/${manufacturerId}.json`; +} diff --git a/src/lib/dataOperations.test.ts b/src/lib/dataOperations.test.ts new file mode 100644 index 0000000..ae67954 --- /dev/null +++ b/src/lib/dataOperations.test.ts @@ -0,0 +1,292 @@ +import test from "ava"; +import type { DataManifest, DataShard } from "./dataFormat.js"; +import { + getDataVersion, + lookupConfig, + lookupConfigsBatch, +} from "./dataOperations.js"; +import { versionToNumber } from "./shared.js"; + +function mockAssets(manifest: DataManifest, shards: Record) { + return { + fetch: (input: any) => { + const path = new URL( + typeof input === "string" ? input : input.url, + ).pathname; + let body: unknown; + if (path === "/manifest.json") { + body = manifest; + } else { + const match = /^\/shards\/(.+)\.json$/.exec(path); + if (match) body = shards[match[1]]; + } + if (body === undefined) { + return Promise.resolve(new Response(null, { status: 404 })); + } + return Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ); + }, + } as any; +} + +const upgrade1 = { + version: "1.5", + changelog: "Fixed stuff", + channel: "stable" as const, + files: [ + { + target: 0, + url: "https://example.com/1.5.otz", + integrity: "sha256:" + "0".repeat(64), + }, + ], +}; + +const upgrade2 = { + $if: "firmwareVersion < 1.0", + version: "1.0", + changelog: "Initial", + channel: "stable" as const, + files: [ + { + target: 0, + url: "https://example.com/1.0.otz", + integrity: "sha256:" + "1".repeat(64), + }, + ], +}; + +const defaultManifest: DataManifest = { + version: "abcd1234", + shards: ["0x0086"], +}; + +const defaultShards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: versionToNumber("0.0.0"), + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade1, upgrade2], + }, + ], + }, +}; + +test("getDataVersion returns the manifest version", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + t.is(await getDataVersion(assets), "abcd1234"); +}); + +test("lookupConfig returns updates for a known device", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"); + + t.truthy(result); + t.is(result!.manufacturerId, "0x0086"); + t.is(result!.firmwareVersion, "1.5.0"); + // The $if condition of upgrade2 does not apply for version 1.5 + t.is(result!.updates.length, 1); + t.is(result!.updates[0].version, "1.5"); +}); + +test("lookupConfig evaluates $if conditions against the device", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"); + + t.truthy(result); + t.deepEqual( + result!.updates.map((u) => u.version), + ["1.5", "1.0"], + ); +}); + +test("lookupConfig respects the device firmware version range", async (t) => { + const shards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: versionToNumber("1.0.0"), + max: versionToNumber("1.255.255"), + }, + ], + upgrades: [upgrade1], + }, + ], + }, + }; + const assets = mockAssets(defaultManifest, shards); + + t.truthy(await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5")); + t.is( + await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "2.0"), + undefined, + ); + t.is( + await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.9"), + undefined, + ); +}); + +test("lookupConfig merges updates from multiple matching configs", async (t) => { + const shards: Record = { + "0x0086": { + configs: [ + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: 0, + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade1], + }, + { + devices: [ + { + productType: "0x0002", + productId: "0x0064", + min: 0, + max: versionToNumber("255.255.255"), + }, + ], + upgrades: [upgrade2], + }, + ], + }, + }; + const assets = mockAssets(defaultManifest, shards); + const result = await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"); + + t.deepEqual( + result!.updates.map((u) => u.version), + ["1.5", "1.0"], + ); +}); + +test("transient fetch errors are not memoized", async (t) => { + let calls = 0; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + calls++; + if (calls === 1) throw new Error("transient"); + return inner.fetch(input); + }, + } as any; + + await t.throwsAsync(() => getDataVersion(assets)); + // The failed manifest fetch was not cached, so the retry succeeds + t.is(await getDataVersion(assets), "abcd1234"); +}); + +test("non-404 error responses throw instead of reading as missing data", async (t) => { + const assets = { + fetch: () => Promise.resolve(new Response(null, { status: 500 })), + } as any; + await t.throwsAsync(() => getDataVersion(assets), { + message: /500/, + }); +}); + +test("malformed shard JSON is not memoized", async (t) => { + let broken = true; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + const path = new URL(input).pathname; + if (broken && path.startsWith("/shards/")) { + return Promise.resolve( + new Response("{ not json", { status: 200 }), + ); + } + return inner.fetch(input); + }, + } as any; + + await t.throwsAsync(() => + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"), + ); + broken = false; + t.truthy(await lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5")); +}); + +test("a shard listed in the manifest but missing degrades to no results", async (t) => { + const assets = mockAssets(defaultManifest, {}); + const results = await lookupConfigsBatch(assets, [ + { + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.5", + }, + ]); + t.deepEqual(results, []); +}); + +test("a manifest without a shards array degrades to no data", async (t) => { + const assets = mockAssets({ version: "abcd1234" } as any, defaultShards); + t.is(await getDataVersion(assets), undefined); +}); + +test("concurrent lookups share one shard fetch", async (t) => { + let shardFetches = 0; + const inner = mockAssets(defaultManifest, defaultShards); + const assets = { + fetch: (input: any) => { + if (new URL(input).pathname.startsWith("/shards/")) shardFetches++; + return inner.fetch(input); + }, + } as any; + + const [a, b] = await Promise.all([ + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "1.5"), + lookupConfig(assets, "0x0086", "0x0002", "0x0064", "0.5"), + ]); + t.truthy(a); + t.truthy(b); + t.is(shardFetches, 1); +}); + +test("lookupConfigsBatch skips unknown devices", async (t) => { + const assets = mockAssets(defaultManifest, defaultShards); + const results = await lookupConfigsBatch(assets, [ + // Unknown manufacturer (no shard) + { + manufacturerId: "0x9999", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.0", + }, + // Known manufacturer, unknown product + { + manufacturerId: "0x0086", + productType: "0xffff", + productId: "0x0064", + firmwareVersion: "1.0", + }, + // Known device + { + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", + firmwareVersion: "1.0", + }, + ]); + + t.is(results.length, 1); + t.is(results[0].productType, "0x0002"); +}); diff --git a/src/lib/dataOperations.ts b/src/lib/dataOperations.ts new file mode 100644 index 0000000..dbce398 --- /dev/null +++ b/src/lib/dataOperations.ts @@ -0,0 +1,205 @@ +import type { Fetcher } from "@cloudflare/workers-types"; +import { APIv3_UpgradeInfo, APIv4_DeviceInfo } from "../apiDefinitions.js"; +import { + DataManifest, + DataShard, + getShardPath, + MANIFEST_PATH, +} from "./dataFormat.js"; +import { conditionApplies } from "./Logic.js"; +import { formatId, padVersion, versionToNumber } from "./shared.js"; + +export interface DeviceLookupRequest { + manufacturerId: number | string; + productType: number | string; + productId: number | string; + firmwareVersion: string; +} + +// Assets are immutable per deployment and isolates die on redeploy, +// so parsed data can be memoized for the isolate's lifetime +interface DataCache { + manifest?: Promise; + shards: Map>; +} +const dataCaches = new WeakMap(); + +function getDataCache(assets: Fetcher): DataCache { + let cache = dataCaches.get(assets); + if (!cache) { + cache = { shards: new Map() }; + dataCaches.set(assets, cache); + } + return cache; +} + +async function fetchJSON( + assets: Fetcher, + path: string, +): Promise { + // The assets binding only routes on the pathname, the origin is arbitrary + const resp = await assets.fetch(`https://assets.local${path}`); + // A missing asset is expected (unknown manufacturer), anything else is not + if (resp.status === 404) return undefined; + if (!resp.ok) { + throw new Error(`Fetching asset ${path} failed with ${resp.status}`); + } + return (await resp.json()) as T; +} + +function getManifest(assets: Fetcher): Promise { + const cache = getDataCache(assets); + cache.manifest ??= fetchJSON(assets, MANIFEST_PATH).then( + (manifest) => { + // Do not memoize failures, so a transient error heals on the next request + if (!manifest || !Array.isArray(manifest.shards)) { + cache.manifest = undefined; + console.error("Data manifest is missing or malformed"); + return undefined; + } + return manifest; + }, + (e) => { + cache.manifest = undefined; + console.error(`Failed to load data manifest: ${e}`); + throw e; + }, + ); + return cache.manifest; +} + +async function getShard( + assets: Fetcher, + manufacturerId: string, +): Promise { + const manifest = await getManifest(assets); + if (!manifest?.shards.includes(manufacturerId)) return undefined; + + const cache = getDataCache(assets); + let shard = cache.shards.get(manufacturerId); + if (!shard) { + shard = fetchJSON(assets, getShardPath(manufacturerId)).then( + (result) => { + if (!result) { + cache.shards.delete(manufacturerId); + // The manifest promised this shard, so this is a broken deployment + console.error( + `Shard ${manufacturerId} is listed in the manifest but missing`, + ); + } + return result; + }, + (e) => { + cache.shards.delete(manufacturerId); + console.error(`Failed to load shard ${manufacturerId}: ${e}`); + throw e; + }, + ); + cache.shards.set(manufacturerId, shard); + } + return shard; +} + +export async function getDataVersion( + assets: Fetcher, +): Promise { + const manifest = await getManifest(assets); + return manifest?.version; +} + +export async function lookupConfigsBatch( + assets: Fetcher, + devices: DeviceLookupRequest[], +): Promise { + const results: APIv4_DeviceInfo[] = []; + + // Load all needed shards in parallel before the sequential matching below + const manufacturerIds = [ + ...new Set(devices.map((d) => formatId(d.manufacturerId))), + ]; + const shardsById = new Map( + await Promise.all( + manufacturerIds.map( + async (id) => [id, await getShard(assets, id)] as const, + ), + ), + ); + + for (const device of devices) { + const manufacturerId = formatId(device.manufacturerId); + const shard = shardsById.get(manufacturerId); + if (!shard) continue; + + const productType = formatId(device.productType); + const productId = formatId(device.productId); + const firmwareVersion = padVersion(device.firmwareVersion, "0"); + const versionNumber = versionToNumber(firmwareVersion); + + const matchingConfigs = shard.configs.filter((config) => + config.devices.some( + (d) => + d.productType === productType && + d.productId === productId && + versionNumber >= d.min && + versionNumber <= d.max, + ), + ); + if (matchingConfigs.length === 0) continue; + + const deviceId = { + manufacturerId: parseInt(manufacturerId, 16), + productType: parseInt(productType, 16), + productId: parseInt(productId, 16), + firmwareVersion, + }; + + const updates = matchingConfigs + .flatMap((config) => config.upgrades) + .filter((upgrade) => conditionApplies(upgrade, deviceId)) + .map(({ $if, ...upgrade }): APIv3_UpgradeInfo => { + return { + version: upgrade.version, + changelog: upgrade.changelog, + channel: upgrade.channel, + ...(upgrade.region ? { region: upgrade.region } : {}), + // Keep the key order the D1 path produced, for stable response bodies + files: upgrade.files.map((f) => ({ + target: f.target, + url: f.url, + integrity: f.integrity, + })), + // These two will be filled in or filtered by the downstream handler + downgrade: undefined as any, + normalizedVersion: undefined as any, + }; + }); + + results.push({ + manufacturerId, + productType, + productId, + firmwareVersion, + updates, + }); + } + + return results; +} + +export async function lookupConfig( + assets: Fetcher, + manufacturerId: number | string, + productType: number | string, + productId: number | string, + firmwareVersion: string, +): Promise { + const results = await lookupConfigsBatch(assets, [ + { + manufacturerId, + productType, + productId, + firmwareVersion, + }, + ]); + return results[0]; +} diff --git a/src/maintenance/build-data.ts b/src/maintenance/build-data.ts new file mode 100644 index 0000000..2fba6a2 --- /dev/null +++ b/src/maintenance/build-data.ts @@ -0,0 +1,63 @@ +import fs from "node:fs/promises"; +import path from "path-browserify"; +import { getShardPath, MANIFEST_PATH } from "../lib/dataFormat.js"; +import { + buildDataShards, + buildManifest, + hashConfigFiles, + readConfigFiles, +} from "./dataBuild.js"; +import { NodeFS } from "./nodeFS.js"; + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const configDir = path.resolve(__dirname, "../../firmwares"); +const outDir = join(__dirname, "../../dist/data"); + +// Fail the build (and thereby the deploy) if the dataset shrinks implausibly, +// e.g. because a failed directory read silently yielded an empty file list +const MIN_CONFIG_FILES = 100; +const MIN_SHARDS = 10; + +void (async () => { + const files = await readConfigFiles(NodeFS, configDir); + const version = hashConfigFiles(files); + + let shards; + try { + shards = buildDataShards(files); + } catch (e) { + console.error(e instanceof Error ? e.message : e); + process.exit(1); + } + + if (files.length < MIN_CONFIG_FILES || shards.size < MIN_SHARDS) { + console.error( + `ERROR: Implausibly small dataset (${files.length} config files, ${shards.size} shards), refusing to build`, + ); + process.exit(1); + } + + await fs.rm(outDir, { recursive: true, force: true }); + await fs.mkdir(join(outDir, "shards"), { recursive: true }); + + for (const [manufacturerId, shard] of shards) { + await fs.writeFile( + join(outDir, getShardPath(manufacturerId)), + JSON.stringify(shard), + "utf8", + ); + } + + await fs.writeFile( + join(outDir, MANIFEST_PATH), + JSON.stringify(buildManifest(version, shards)), + "utf8", + ); + + console.log( + `Built data version ${version}: ${files.length} config files, ${shards.size} shards`, + ); +})(); diff --git a/src/maintenance/check_configs.ts b/src/maintenance/check_configs.ts index 8883320..9dff630 100644 --- a/src/maintenance/check_configs.ts +++ b/src/maintenance/check_configs.ts @@ -4,6 +4,7 @@ import path from "path-browserify"; import { ConditionalUpdateConfig } from "../lib/config.js"; import { parseLogic } from "../lib/Logic.js"; import { getErrorMessage } from "../lib/shared.js"; +import { discoverConfigFiles } from "./dataBuild.js"; import { NodeFS } from "./nodeFS.js"; import { dirname } from "path"; @@ -128,15 +129,7 @@ void (async () => { try { console.log("🔍 Validating config files...\n"); - // Find all config files (same logic as upload.ts) - const configFiles = (await NodeFS.readDir(configDir, true)).filter( - (file) => - file.endsWith(".json") && - !file.endsWith("index.json") && - !path.basename(file).startsWith("_") && - !file.includes("/templates/") && - !file.includes("\\templates\\"), - ); + const configFiles = await discoverConfigFiles(NodeFS, configDir); console.log(`Found ${configFiles.length} config files to validate.`); diff --git a/src/maintenance/dataBuild.test.ts b/src/maintenance/dataBuild.test.ts new file mode 100644 index 0000000..968998e --- /dev/null +++ b/src/maintenance/dataBuild.test.ts @@ -0,0 +1,118 @@ +import test from "ava"; +import { versionToNumber } from "../lib/shared.js"; +import { + buildDataShards, + buildManifest, + hashConfigFiles, + isConfigFile, +} from "./dataBuild.js"; + +const upgrade = { + version: "1.5", + changelog: "Fixed stuff", + url: "https://example.com/1.5.otz", + integrity: "sha256:" + "0".repeat(64), +}; + +function configFile(devices: any[], upgrades: any[] = [upgrade]) { + return JSON.stringify({ devices, upgrades }); +} + +const device1 = { + brand: "Test", + model: "Device 1", + manufacturerId: "0x0086", + productType: "0x0002", + productId: "0x0064", +}; + +test("isConfigFile applies the discovery filter", (t) => { + t.true(isConfigFile("vendor/device.json")); + t.false(isConfigFile("vendor/device.md")); + t.false(isConfigFile("index.json")); + t.false(isConfigFile("vendor/_draft.json")); + t.false(isConfigFile("vendor/templates/base.json")); +}); + +test("buildDataShards groups devices by manufacturer ID", (t) => { + const shards = buildDataShards([ + { filename: "a.json", data: configFile([device1]) }, + { + filename: "b.json", + data: configFile([ + { ...device1, model: "Device 2", productId: "0x0065" }, + { ...device1, model: "Other vendor", manufacturerId: "0x027a" }, + ]), + }, + ]); + + t.deepEqual([...shards.keys()].sort(), ["0x0086", "0x027a"].sort()); + // a.json and b.json each contribute one config entry to 0x0086 + t.is(shards.get("0x0086")!.configs.length, 2); + // The multi-manufacturer file lands in both shards with the same upgrades + const otherVendor = shards.get("0x027a")!.configs[0]; + t.is(otherVendor.devices.length, 1); + t.is(otherVendor.upgrades[0].version, "1.5"); +}); + +test("buildDataShards normalizes firmware version ranges", (t) => { + const shards = buildDataShards([ + { + filename: "a.json", + data: configFile([ + device1, + { + ...device1, + productId: "0x0065", + firmwareVersion: { min: "1.5", max: "2.0" }, + }, + ]), + }, + ]); + + const [defaultRange, explicitRange] = shards.get("0x0086")!.configs[0] + .devices; + // No range in the config means "all versions" + t.is(defaultRange.min, 0); + t.is(defaultRange.max, versionToNumber("255.255.255")); + // min is padded with .0, max with .255 + t.is(explicitRange.min, versionToNumber("1.5.0")); + t.is(explicitRange.max, versionToNumber("2.0.255")); +}); + +test("buildDataShards names the offending file for invalid configs", (t) => { + const error = t.throws(() => + buildDataShards([ + { filename: "vendor/broken.json", data: "{ not valid" }, + ]), + ); + t.true(error!.message.includes("vendor/broken.json")); +}); + +test("hashConfigFiles is deterministic and content-sensitive", (t) => { + const files = [{ filename: "a.json", data: configFile([device1]) }]; + const hash = hashConfigFiles(files); + t.is(hash, hashConfigFiles([...files.map((f) => ({ ...f }))])); + t.regex(hash, /^[0-9a-f]{8}$/); + t.not( + hash, + hashConfigFiles([{ filename: "a.json", data: configFile([device1], [ + { ...upgrade, version: "1.6" }, + ]) }]), + ); +}); + +test("buildManifest sorts shard IDs and carries the version", (t) => { + const shards = buildDataShards([ + { + filename: "a.json", + data: configFile([ + { ...device1, manufacturerId: "0x027a" }, + { ...device1, manufacturerId: "0x0086" }, + ]), + }, + ]); + const manifest = buildManifest("abcd1234", shards); + t.is(manifest.version, "abcd1234"); + t.deepEqual(manifest.shards, ["0x0086", "0x027a"]); +}); diff --git a/src/maintenance/dataBuild.ts b/src/maintenance/dataBuild.ts new file mode 100644 index 0000000..01e804a --- /dev/null +++ b/src/maintenance/dataBuild.ts @@ -0,0 +1,108 @@ +import JSON5 from "json5"; +import crypto from "node:crypto"; +import path from "path-browserify"; +import { ConditionalUpdateConfig } from "../lib/config.js"; +import { DataManifest, DataShard } from "../lib/dataFormat.js"; +import type { FileSystem } from "../lib/fs/filesystem.js"; +import { getErrorMessage, padVersion, versionToNumber } from "../lib/shared.js"; + +export interface ConfigFileEntry { + filename: string; + data: string; +} + +export function isConfigFile(file: string): boolean { + return ( + file.endsWith(".json") && + !file.endsWith("index.json") && + !path.basename(file).startsWith("_") && + !file.includes("/templates/") && + !file.includes("\\templates\\") + ); +} + +/** Finds all config files below the given directory, sorted for deterministic output */ +export async function discoverConfigFiles( + fs: FileSystem, + configDir: string, +): Promise { + return (await fs.readDir(configDir, true)).filter(isConfigFile).sort(); +} + +export async function readConfigFiles( + fs: FileSystem, + configDir: string, +): Promise { + const configFiles = await discoverConfigFiles(fs, configDir); + + const files: ConfigFileEntry[] = []; + for (const filePath of configFiles) { + const relativePath = path + .relative(configDir, filePath) + .replace(/\\/g, "/"); + const fileContent = await fs.readFile(filePath); + files.push({ filename: relativePath, data: fileContent }); + } + return files; +} + +export function hashConfigFiles(files: ConfigFileEntry[]): string { + return crypto + .createHash("sha256") + .update(JSON.stringify(files), "utf8") + .digest("hex") + .slice(0, 8); +} + +/** Groups all config files into one shard per manufacturer ID. Throws on invalid config files. */ +export function buildDataShards( + files: ConfigFileEntry[], +): Map { + const shards = new Map(); + + for (const file of files) { + let config: ConditionalUpdateConfig; + try { + config = new ConditionalUpdateConfig(JSON5.parse(file.data)); + } catch (e) { + throw new Error( + `Error parsing config file ${file.filename}: ${getErrorMessage(e)}`, + ); + } + + const byManufacturer = Map.groupBy( + config.devices, + (d) => d.manufacturerId, + ); + for (const [manufacturerId, devices] of byManufacturer) { + let shard = shards.get(manufacturerId); + if (!shard) { + shard = { configs: [] }; + shards.set(manufacturerId, shard); + } + shard.configs.push({ + devices: devices.map((d) => ({ + productType: d.productType, + productId: d.productId, + min: versionToNumber(padVersion(d.firmwareVersion.min, "0")), + max: versionToNumber( + padVersion(d.firmwareVersion.max, "255"), + ), + })), + upgrades: config.upgrades, + }); + } + } + + return shards; +} + +export function buildManifest( + version: string, + shards: Map, +): DataManifest { + return { + version, + shards: [...shards.keys()].sort(), + }; +} diff --git a/src/routes/api.ts b/src/routes/api.ts index 8ff0ecd..45bbff1 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -11,18 +11,13 @@ import { APIv4_Response, } from "../apiDefinitions.js"; import { withCache } from "../lib/cache.js"; -import { - cacheD1Config, - getCurrentVersionCached, - getD1CachedConfig, -} from "../lib/cachedD1Operations.js"; import type { UpgradeInfo } from "../lib/configSchema.js"; import { - DeviceLookupRequest, + getDataVersion, lookupConfig, lookupConfigsBatch, -} from "../lib/d1Operations.js"; -import { compareVersions, padVersion } from "../lib/shared.js"; +} from "../lib/dataOperations.js"; +import { array2hex, compareVersions, padVersion } from "../lib/shared.js"; import { clientError, ContentProps, @@ -79,13 +74,9 @@ async function handleUpdateRequest( const { manufacturerId, productType, productId, firmwareVersion } = result.data; - const filesVersion = await getCurrentVersionCached( - req.url, - context, - env.CONFIG_FILES, - ); + const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { - return serverError("Database empty"); + return serverError("Firmware update data not available"); } // Figure out if this info is already cached @@ -113,8 +104,7 @@ async function handleUpdateRequest( }, async () => { const deviceInfo = await lookupConfig( - env.CONFIG_FILES, - filesVersion, + env.DATA, manufacturerId, productType, productId, @@ -335,13 +325,9 @@ export default function register(router: any): void { } const { region, devices } = result.data; - const filesVersion = await getCurrentVersionCached( - req.url, - context, - env.CONFIG_FILES, - ); + const filesVersion = await getDataVersion(env.DATA); if (!filesVersion) { - return serverError("Database empty"); + return serverError("Firmware update data not available"); } // Remove duplicates, normalize and sort devices for consistent processing @@ -374,148 +360,92 @@ export default function register(router: any): void { return a.firmwareVersion.localeCompare(b.firmwareVersion); }); - // Step 1: Try to find cached responses for each unique device - const cacheMisses: DeviceLookupRequest[] = []; - const results: APIv4_DeviceInfo[] = []; - - for (const device of uniqueDevices) { - // Try to get cached config for this device using D1 cache utilities - const cachedConfig = await getD1CachedConfig( - req.url, - filesVersion, - device.manufacturerId, - device.productType, - device.productId, - device.firmwareVersion, - ); - - if (cachedConfig === undefined) { - // Cache miss - add to batch lookup - cacheMisses.push(device); - continue; - } - - if (cachedConfig === null) { - // Cached as "not found" - continue; - } - - // Cache hit - add to results - results.push(cachedConfig); - } - - // Step 2: Perform single batch request to database for all cache misses - if (cacheMisses.length > 0) { - const batchResults = await lookupConfigsBatch( - env.CONFIG_FILES, - filesVersion, - cacheMisses, - ); - results.push(...batchResults); - - // Cache the results of the batch lookup - for (const deviceInfo of batchResults) { - cacheD1Config( - req.url, - context, - filesVersion, - deviceInfo.manufacturerId, - deviceInfo.productType, - deviceInfo.productId, - deviceInfo.firmwareVersion, - deviceInfo, - ); - } - } - - // Post-process the results to apply region filtering etc. - const response: APIv4_Response = results.map((device) => { - if (!device.updates) return device; - let filteredUpgrades = device.updates - // Filter out upgrades for a different region - .filter((u) => !u.region || u.region === region) - // Filter out the current version - .filter( - (u) => - padVersion(u.version) !== - padVersion(device.firmwareVersion), - ) - // Add missing fields to the returned objects - - .map((u: UpgradeInfo) => { - const downgrade = - compareVersions(u.version, device.firmwareVersion) < - 0; - let normalizedVersion = padVersion(u.version, "0"); - if (u.channel === "beta") normalizedVersion += "-beta"; - - return { - ...u, - downgrade, - normalizedVersion, - }; - }) - // Sort by version ascending... - .sort((a: any, b: any) => { - const ret = compare( - a.normalizedVersion, - b.normalizedVersion, - ); - if (ret !== 0) return ret; - // ... and put updates for a specific region first - return -(a.region ?? "").localeCompare(b.region ?? ""); - }); - // If there are multiple updates for the same version, only return the first one - // This happens when there are updates for a specific region and a general update for the same version - filteredUpgrades = filteredUpgrades.filter( - (u: any, i: number, arr: any[]) => { - if (i > 0 && u.version === arr[i - 1].version) - return false; - return true; - }, - ); - device.updates = filteredUpgrades; - return device; + // The devices are deduplicated and sorted, so identical queries against + // the same data version share one cache entry regardless of input order + const fingerprint = JSON.stringify({ + region: region ?? null, + devices: uniqueDevices, }); - - // Track which devices were NOT found in the DB and cache them as `null` - const requestedFingerprints = new Set( - uniqueDevices.map( - (d) => - `${d.manufacturerId}:${d.productType}:${d.productId}:${d.firmwareVersion}`, + const fingerprintHash = array2hex( + new Uint8Array( + await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(fingerprint), + ), ), ); - const foundFingerprints = new Set( - results.map( - (d) => - `${d.manufacturerId}:${d.productType}:${d.productId}:${d.firmwareVersion}`, - ), - ); - const missingFromDb = - requestedFingerprints.difference(foundFingerprints); + const cacheKey = new URL( + `./${filesVersion}/${fingerprintHash}`, + req.url.endsWith("/") ? req.url : req.url + "/", + ).toString(); + + return withCache( + { + req, + context, + cacheKey, + // Same policy as v1-v3: 1 hour on the client, 1 day at the edge + // (the cache key includes the data version) + maxAge: 60 * 60, + sMaxAge: 60 * 60 * 24, + }, + async () => { + const results: APIv4_DeviceInfo[] = + await lookupConfigsBatch(env.DATA, uniqueDevices); + + // Post-process the results to apply region filtering etc. + const response: APIv4_Response = results.map((device) => { + if (!device.updates) return device; + let filteredUpgrades = device.updates + // Filter out upgrades for a different region + .filter((u) => !u.region || u.region === region) + // Filter out the current version + .filter( + (u) => + padVersion(u.version) !== + padVersion(device.firmwareVersion), + ) + // Add missing fields to the returned objects - for (const fingerprint of missingFromDb) { - const [ - manufacturerId, - productType, - productId, - firmwareVersion, - ] = fingerprint.split(":"); + .map((u: UpgradeInfo) => { + const downgrade = + compareVersions(u.version, device.firmwareVersion) < + 0; + let normalizedVersion = padVersion(u.version, "0"); + if (u.channel === "beta") normalizedVersion += "-beta"; - // Cache as "not found" - cacheD1Config( - req.url, - context, - filesVersion, - manufacturerId, - productType, - productId, - firmwareVersion, - null, - ); - } + return { + ...u, + downgrade, + normalizedVersion, + }; + }) + // Sort by version ascending... + .sort((a: any, b: any) => { + const ret = compare( + a.normalizedVersion, + b.normalizedVersion, + ); + if (ret !== 0) return ret; + // ... and put updates for a specific region first + return -(a.region ?? "").localeCompare(b.region ?? ""); + }); + // If there are multiple updates for the same version, only return the first one + // This happens when there are updates for a specific region and a general update for the same version + filteredUpgrades = filteredUpgrades.filter( + (u: any, i: number, arr: any[]) => { + if (i > 0 && u.version === arr[i - 1].version) + return false; + return true; + }, + ); + device.updates = filteredUpgrades; + return device; + }); - return json(response); + return json(response); + }, + ); }, ); } diff --git a/src/worker.ts b/src/worker.ts index f5b42f7..566479b 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -5,6 +5,9 @@ export interface CloudflareEnvironment { CONFIG_FILES: D1Database; + /** Static assets containing the prebuilt firmware update data */ + DATA: Fetcher; + RL_GLOBAL: RateLimit; RL_BURST: RateLimit; diff --git a/wrangler.toml b/wrangler.toml index f3ca18b..e43d625 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -15,6 +15,13 @@ binding = "CONFIG_FILES" database_name = "zwave-js-firmware-updates--prod" database_id = "6d9a21f3-6886-4d53-b7dc-3b03dad70698" +# Prebuilt firmware update data, generated by `yarn build:data`. +# run_worker_first keeps the shards from being publicly routable. +[assets] +directory = "dist/data" +binding = "DATA" +run_worker_first = true + # Global sustained rate limit shared by all requests. # Per-edge-location: ~33 req/s sustained across the 60s window, up to 3M req/day. [[ratelimits]]