From f580c60d6f18e9cac78c0237e2367899ebb86680 Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Thu, 9 Jul 2026 20:50:02 +0200 Subject: [PATCH] chore: remove the retired D1 code path (Phase 4 cleanup) Now that firmware update data is served from static assets (#347) and has been verified healthy in production, remove the unused D1 backend: - Delete d1Operations, cachedD1Operations, uploadSchema, the /admin/* upload routes and upload.ts, plus the D1 SQL migrations - Drop the CONFIG_FILES D1 binding from wrangler.toml and the CONFIG_FILES/ADMIN_SECRET fields from CloudflareEnvironment - Remove the now-orphaned safeCompare helper and the `upload` script - Rewrite docs/testing-locally.md for the assets flow (yarn dev builds and serves the data; no ADMIN_SECRET, no manual upload) The D1 database and the ADMIN_SECRET/BASE_URL repo secrets are left in place as a rollback anchor; deleting them is a manual follow-up. The Durable Object migrations in wrangler.toml are unrelated and untouched. Co-Authored-By: Claude Opus 4.8 --- docs/testing-locally.md | 29 +- migrations/0001_create_initial_tables.sql | 58 --- .../0002_add_normalized_firmware_versions.sql | 67 --- .../0003_add_upgrade_files_position.sql | 12 - package.json | 3 +- src/app.ts | 2 - src/lib/cachedD1Operations.ts | 196 --------- src/lib/d1Operations.ts | 392 ------------------ src/lib/shared_cloudflare.ts | 17 - src/lib/uploadSchema.ts | 27 -- src/maintenance/upload.ts | 115 ----- src/routes/admin.ts | 110 ----- src/worker.ts | 4 - wrangler.toml | 5 - 14 files changed, 3 insertions(+), 1034 deletions(-) delete mode 100644 migrations/0001_create_initial_tables.sql delete mode 100644 migrations/0002_add_normalized_firmware_versions.sql delete mode 100644 migrations/0003_add_upgrade_files_position.sql delete mode 100644 src/lib/cachedD1Operations.ts delete mode 100644 src/lib/d1Operations.ts delete mode 100644 src/lib/uploadSchema.ts delete mode 100644 src/maintenance/upload.ts delete mode 100644 src/routes/admin.ts diff --git a/docs/testing-locally.md b/docs/testing-locally.md index 53b1cc32..c3655689 100644 --- a/docs/testing-locally.md +++ b/docs/testing-locally.md @@ -2,17 +2,10 @@ ## Getting started -The firmware update service can be tested locally. To get started, install and build everything: +The firmware update service can be tested locally. To get started, install everything: ```bash yarn -yarn build -``` - -and create a file `.env` with the following content - -```ini -ADMIN_SECRET=your-random-admin-secret ``` To start the service, run @@ -21,25 +14,7 @@ To start the service, run yarn dev ``` -This will also rebuild on changes. - -## Uploading firmware files - -The database behind the local service is initially empty and decoupled from the repository contents. Firmware definition files need to be uploaded manually. - -To do so, build the index, which also checks the files for errors: - -```bash -yarn build:index -``` - -Then upload the files to the running service: - -```bash -ADMIN_SECRET=your-random-admin-secret BASE_URL=http://localhost:8787 yarn upload -``` - -These steps need to be repeated whenever the firmware files change. +This compiles the firmware definition files into the data served by the worker (`yarn build:data`) and starts a local server. Unlike production, it does not rebuild automatically when the firmware files change — rerun `yarn dev` (or `yarn build:data`) after editing them. ## Using the local service diff --git a/migrations/0001_create_initial_tables.sql b/migrations/0001_create_initial_tables.sql deleted file mode 100644 index 7bd806ec..00000000 --- a/migrations/0001_create_initial_tables.sql +++ /dev/null @@ -1,58 +0,0 @@ --- Create the main tables for firmware update definitions - --- Track configuration versions -CREATE TABLE IF NOT EXISTS config_versions ( - version TEXT PRIMARY KEY, - active BOOLEAN NOT NULL DEFAULT FALSE -); - --- Devices table - stores device information from the JSON configs -CREATE TABLE IF NOT EXISTS devices ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - version TEXT NOT NULL, - brand TEXT NOT NULL, - model TEXT NOT NULL, - manufacturer_id TEXT NOT NULL, - product_type TEXT NOT NULL, - product_id TEXT NOT NULL, - firmware_version_min TEXT NOT NULL DEFAULT '0.0', - firmware_version_max TEXT NOT NULL DEFAULT '255.255', - FOREIGN KEY (version) REFERENCES config_versions(version) ON DELETE CASCADE -); - --- Upgrades table - stores unique upgrade information -CREATE TABLE IF NOT EXISTS upgrades ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - firmware_version TEXT NOT NULL, -- target firmware version - changelog TEXT NOT NULL, - channel TEXT NOT NULL DEFAULT 'stable', - region TEXT, - condition TEXT -- the $if condition -); - --- Junction table for device-upgrade relationships (N:M) -CREATE TABLE IF NOT EXISTS device_upgrades ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_id INTEGER NOT NULL, - upgrade_id INTEGER NOT NULL, - FOREIGN KEY (device_id) REFERENCES devices(id) ON DELETE CASCADE, - FOREIGN KEY (upgrade_id) REFERENCES upgrades(id) ON DELETE CASCADE, - UNIQUE(device_id, upgrade_id) -); - --- Upgrade files table - stores file information for each upgrade -CREATE TABLE IF NOT EXISTS upgrade_files ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - upgrade_id INTEGER NOT NULL, - target INTEGER NOT NULL DEFAULT 0, - url TEXT NOT NULL, - integrity TEXT NOT NULL, - FOREIGN KEY (upgrade_id) REFERENCES upgrades(id) ON DELETE CASCADE -); - --- Create indexes for efficient queries -CREATE INDEX IF NOT EXISTS idx_devices_lookup ON devices(version, manufacturer_id, product_type, product_id); -CREATE INDEX IF NOT EXISTS idx_device_upgrades_device ON device_upgrades(device_id); -CREATE INDEX IF NOT EXISTS idx_device_upgrades_upgrade ON device_upgrades(upgrade_id); -CREATE INDEX IF NOT EXISTS idx_upgrade_files_upgrade ON upgrade_files(upgrade_id); -CREATE INDEX IF NOT EXISTS idx_config_versions_active ON config_versions(active) WHERE active = TRUE; \ No newline at end of file diff --git a/migrations/0002_add_normalized_firmware_versions.sql b/migrations/0002_add_normalized_firmware_versions.sql deleted file mode 100644 index 94c9506b..00000000 --- a/migrations/0002_add_normalized_firmware_versions.sql +++ /dev/null @@ -1,67 +0,0 @@ --- Add normalized firmware version columns for efficient comparison - --- Step 1: Add new columns as optional -ALTER TABLE devices ADD COLUMN firmware_version_min_normalized INTEGER; -ALTER TABLE devices ADD COLUMN firmware_version_max_normalized INTEGER; - --- Step 2: Populate the normalized columns from existing version strings --- Handle both x.y and x.y.z formats by padding with .0 as needed -UPDATE devices SET - firmware_version_min_normalized = ( - CAST(SUBSTR(firmware_version_min || '.0.0', 1, INSTR(firmware_version_min || '.0.0', '.') - 1) AS INTEGER) * 65536 + - CAST(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1, - CASE WHEN INSTR(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1), '.') > 0 - THEN INSTR(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1), '.') - 1 - ELSE LENGTH(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1)) END) AS INTEGER) * 256 + - CAST(CASE WHEN INSTR(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1), '.') > 0 - THEN SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + INSTR(SUBSTR(firmware_version_min || '.0.0', INSTR(firmware_version_min || '.0.0', '.') + 1), '.') + 1) - ELSE '0' END AS INTEGER) - ), - firmware_version_max_normalized = ( - CAST(SUBSTR(firmware_version_max || '.0.0', 1, INSTR(firmware_version_max || '.0.0', '.') - 1) AS INTEGER) * 65536 + - CAST(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1, - CASE WHEN INSTR(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1), '.') > 0 - THEN INSTR(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1), '.') - 1 - ELSE LENGTH(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1)) END) AS INTEGER) * 256 + - CAST(CASE WHEN INSTR(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1), '.') > 0 - THEN SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + INSTR(SUBSTR(firmware_version_max || '.0.0', INSTR(firmware_version_max || '.0.0', '.') + 1), '.') + 1) - ELSE '0' END AS INTEGER) - ); - --- Step 3: Create a new table with the required constraints and copy data -CREATE TABLE devices_new ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - version TEXT NOT NULL, - brand TEXT NOT NULL, - model TEXT NOT NULL, - manufacturer_id TEXT NOT NULL, - product_type TEXT NOT NULL, - product_id TEXT NOT NULL, - firmware_version_min TEXT NOT NULL DEFAULT '0.0', - firmware_version_max TEXT NOT NULL DEFAULT '255.255', - firmware_version_min_normalized INTEGER NOT NULL, - firmware_version_max_normalized INTEGER NOT NULL, - FOREIGN KEY (version) REFERENCES config_versions(version) ON DELETE CASCADE -); - --- Copy data from old table -INSERT INTO devices_new ( - id, version, brand, model, manufacturer_id, product_type, product_id, - firmware_version_min, firmware_version_max, - firmware_version_min_normalized, firmware_version_max_normalized -) -SELECT - id, version, brand, model, manufacturer_id, product_type, product_id, - firmware_version_min, firmware_version_max, - firmware_version_min_normalized, firmware_version_max_normalized -FROM devices; - --- Drop the old table and rename the new one -DROP TABLE devices; -ALTER TABLE devices_new RENAME TO devices; - --- Recreate the index for the devices table -CREATE INDEX IF NOT EXISTS idx_devices_lookup ON devices(version, manufacturer_id, product_type, product_id); - --- Create an index on the normalized version columns for efficient range queries -CREATE INDEX IF NOT EXISTS idx_devices_firmware_versions ON devices(firmware_version_min_normalized, firmware_version_max_normalized); diff --git a/migrations/0003_add_upgrade_files_position.sql b/migrations/0003_add_upgrade_files_position.sql deleted file mode 100644 index f7e46763..00000000 --- a/migrations/0003_add_upgrade_files_position.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Preserve the original order of files within an upgrade. --- Multi-chip updates depend on execution order (e.g. target 1 before target 0). - -ALTER TABLE upgrade_files ADD COLUMN position INTEGER NOT NULL DEFAULT 0; - --- Backfill existing rows: assign sequential positions per upgrade based on id order -UPDATE upgrade_files SET position = ( - SELECT COUNT(*) - FROM upgrade_files AS uf2 - WHERE uf2.upgrade_id = upgrade_files.upgrade_id - AND uf2.id <= upgrade_files.id -) - 1; diff --git a/package.json b/package.json index ebc77be9..60406288 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ "build:data": "yarn tsx src/maintenance/build-data.ts", "test:ci": "yarn build:data && NODE_OPTIONS='--loader=tsx' ava", "test": "yarn test:ci", - "dev": "yarn build:data && wrangler dev", - "upload": "yarn tsx src/maintenance/upload.ts" + "dev": "yarn build:data && wrangler dev" }, "dependencies": { "itty-router": "^5.0.22", diff --git a/src/app.ts b/src/app.ts index 5c380ff4..daa5f945 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,5 +1,4 @@ import { error, Router, withContent } from "itty-router"; -import registerAdmin from "./routes/admin.js"; import registerAPI from "./routes/api.js"; // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types @@ -26,7 +25,6 @@ export function build() { router.post("*", withContent); registerAPI(router); - registerAdmin(router); router.all("*", () => error(404)); diff --git a/src/lib/cachedD1Operations.ts b/src/lib/cachedD1Operations.ts deleted file mode 100644 index 16a24d46..00000000 --- a/src/lib/cachedD1Operations.ts +++ /dev/null @@ -1,196 +0,0 @@ -import type { D1Database } from "@cloudflare/workers-types"; -import { APIv4_DeviceInfo } from "../apiDefinitions.js"; -import { getCurrentVersion } from "./d1Operations.js"; -import { array2hex } from "./shared.js"; - -const CACHE_KEY_PREFIX = "/__d1-cache/"; - -// Cache for 24 hours to avoid hitting D1 too often. -// Since we use the database version as part of the cache key, -// we don't need to purge them when serving a new version. -const oneDayInSeconds = 24 * 60 * 60; - -/** - * Create cache key for D1 device lookups, similar to getUpdatesCacheUrl - */ -export function getD1CacheKey( - baseURL: string, - filesVersion: string, - manufacturerId: number | string, - productType: number | string, - productId: number | string, - firmwareVersion: string, -): string { - const cacheKeySuffix = [ - filesVersion, - manufacturerId, - productType, - productId, - firmwareVersion, - ].join("/"); - - return new URL( - CACHE_KEY_PREFIX + encodeURIComponent(cacheKeySuffix), - baseURL, - ).toString(); -} - -/** - * D1-specific wrapper around getCachedResponse for device configurations. - * Returns `null` if there is no update (cache hit) and `undefined` if cache miss. - */ -export async function getD1CachedConfig( - baseURL: string, - filesVersion: string, - manufacturerId: number | string, - productType: number | string, - productId: number | string, - firmwareVersion: string, -): Promise { - // FIXME: Use a tagged union type to make it clearer what null (cache hit, device not in DB) and undefined (cache miss) mean - const cacheKey = getD1CacheKey( - baseURL, - filesVersion, - manufacturerId, - productType, - productId, - firmwareVersion, - ); - - const cachedResponse = await getCachedResponse(cacheKey); - if (cachedResponse) { - try { - return await cachedResponse.json(); - } catch { - // Cache corruption, return undefined to indicate cache miss - return undefined; - } - } - - // Cache miss - return undefined; -} - -/** - * D1-specific wrapper around cacheResponse for device configurations - */ -export function cacheD1Config( - baseURL: string, - context: ExecutionContext, - filesVersion: string, - manufacturerId: number | string, - productType: number | string, - productId: number | string, - firmwareVersion: string, - config: APIv4_DeviceInfo | null, -): void { - const cacheKey = getD1CacheKey( - baseURL, - filesVersion, - manufacturerId, - productType, - productId, - firmwareVersion, - ); - - const response = new Response(JSON.stringify(config), { - status: 200, - headers: { - "Content-Type": "application/json", - }, - }); - - context.waitUntil(cacheResponse(cacheKey, context, response)); -} - -/** - * Helper function similar to withCache but only returns cached response if available - */ -async function getCachedResponse( - cacheKey: string, -): Promise { - const cache = caches.default; - const cachedResponse = await cache.match(cacheKey, { - ignoreMethod: true, - }); - - if (cachedResponse && cachedResponse.status === 200) { - return cachedResponse; - } - - return undefined; -} - -/** - * Cache a response with proper headers and ETag - */ -async function cacheResponse( - cacheKey: string, - context: ExecutionContext, - response: Response, - sMaxAge: number = oneDayInSeconds, - maxAge: number = 60 * 60, -): Promise { - if (response.status === 200) { - const responseBody = await response.clone().text(); - const hash = array2hex( - new Uint8Array( - await crypto.subtle.digest( - "SHA-256", - new TextEncoder().encode(responseBody), - ), - ), - ); - - // Cache the response - response.headers.set( - "Cache-Control", - `public, s-maxage=${sMaxAge}, max-age=${maxAge}, stale-while-revalidate`, - ); - response.headers.set("ETag", hash); - - const cache = caches.default; - await cache.put(cacheKey, response); - } -} - -/** - * Cached wrapper for getCurrentVersion - */ -export async function getCurrentVersionCached( - baseURL: string, - context: ExecutionContext, - db: D1Database, -): Promise { - const cacheKey = new URL( - CACHE_KEY_PREFIX + "current-version", - baseURL, - ).toString(); - - // Try to get from cache first - const cachedResponse = await getCachedResponse(cacheKey); - if (cachedResponse) { - try { - return await cachedResponse.text(); - } catch { - // Cache corruption, continue to database lookup - } - } - - // Cache miss - query database - const version = await getCurrentVersion(db); - if (version) { - // Cache the result - const response = new Response(version, { - status: 200, - headers: { - "Content-Type": "text/plain", - }, - }); - - // Cache current version at the edge for 1 minute - context.waitUntil(cacheResponse(cacheKey, context, response, 60, 0)); - } - - return version; -} diff --git a/src/lib/d1Operations.ts b/src/lib/d1Operations.ts deleted file mode 100644 index 56a73ba7..00000000 --- a/src/lib/d1Operations.ts +++ /dev/null @@ -1,392 +0,0 @@ -import type { - D1Database, - D1PreparedStatement, -} from "@cloudflare/workers-types"; -import { APIv3_UpgradeInfo, APIv4_DeviceInfo } from "../apiDefinitions.js"; -import type { - ConditionalUpgradeInfo, - DeviceIdentifier, -} from "./configSchema.js"; -import { conditionApplies } from "./Logic.js"; -import { formatId, padVersion, versionToNumber } from "./shared.js"; - -// Schema for the joined device + upgrade + file query result -export interface UpgradesQueryRow { - // devices table - brand: string; - model: string; - manufacturer_id: string; - product_type: string; - product_id: string; - firmware_version: string; - firmware_version_min: string; - firmware_version_max: string; - firmware_version_min_normalized: number; - firmware_version_max_normalized: number; - - // upgrades table - upgrade_id: number; - upgrade_firmware_version: string; - changelog: string; - channel: string; - region: string | null; - condition: string | null; - - // upgrade_files table - target: number; - url: string; - integrity: string; - position: number; -} - -export async function createConfigVersion( - db: D1Database, - version: string, -): Promise { - await db - .prepare( - "INSERT OR REPLACE INTO config_versions (version, active) VALUES (?, FALSE)", - ) - .bind(version) - .run(); -} - -export async function getCurrentVersion( - db: D1Database, -): Promise { - const result = await db - .prepare( - "SELECT version FROM config_versions WHERE active = TRUE LIMIT 1", - ) - .first<{ version: string }>(); - - return result?.version; -} - -export async function enableConfigVersion( - db: D1Database, - version: string, -): Promise { - // Disable all versions, enable the new one, then delete old inactive versions - all in one batch - const statements: D1PreparedStatement[] = [ - // Disable all currently active versions - db.prepare( - "UPDATE config_versions SET active = FALSE WHERE active = TRUE", - ), - // Enable the new version - db - .prepare( - "UPDATE config_versions SET active = TRUE WHERE version = ?", - ) - .bind(version), - // Delete all inactive versions (cleanup old data) - db - .prepare( - "DELETE FROM config_versions WHERE active = FALSE AND version != ?", - ) - .bind(version), - ]; - - await db.batch(statements); -} - -export interface DeviceLookupRequest { - manufacturerId: number | string; - productType: number | string; - productId: number | string; - firmwareVersion: string; -} - -// D1 has a limit of 100 variables per query -// We use 5 variables per device + 1 for version = max 19 devices per chunk -const D1_MAX_VARIABLES = 100; -const VARIABLES_PER_DEVICE = 5; -const CHUNK_SIZE = Math.floor((D1_MAX_VARIABLES - 1) / VARIABLES_PER_DEVICE); // -1 for version variable - -async function lookupConfigsChunk( - db: D1Database, - filesVersion: string, - devices: DeviceLookupRequest[], -): Promise { - // Build device conditions and bind parameters for the query - const bindParams: any[] = []; - - for (const device of devices) { - bindParams.push( - formatId(device.manufacturerId), - formatId(device.productType), - formatId(device.productId), - padVersion(device.firmwareVersion, "0"), - versionToNumber(device.firmwareVersion), - ); - } - - // Single query to get all devices and their upgrades - const query = ` - WITH fingerprints(manufacturer_id, product_type, product_id, firmware_version, firmware_version_normalized) AS ( - VALUES - ${devices.map(() => `(?, ?, ?, ?, ?)`).join(",")} - ) - SELECT - d.brand, - d.model, - d.manufacturer_id, - d.product_type, - d.product_id, - f.firmware_version, - d.firmware_version_min, - d.firmware_version_max, - d.firmware_version_min_normalized, - d.firmware_version_max_normalized, - u.id as upgrade_id, - u.firmware_version as upgrade_firmware_version, - u.changelog, - u.channel, - u.region, - u.condition, - uf.target, - uf.url, - uf.integrity - FROM fingerprints f - JOIN devices d - ON d.manufacturer_id = f.manufacturer_id - AND d.product_type = f.product_type - AND d.product_id = f.product_id - AND f.firmware_version_normalized BETWEEN d.firmware_version_min_normalized AND d.firmware_version_max_normalized - LEFT JOIN device_upgrades du ON d.id = du.device_id - LEFT JOIN upgrades u ON du.upgrade_id = u.id - LEFT JOIN upgrade_files uf ON u.id = uf.upgrade_id - WHERE d.version = ? - ORDER BY d.id, u.id, uf.position - `; - - bindParams.push(filesVersion); - - const queryResults = await db - .prepare(query) - .bind(...bindParams) - .all(); - - return queryResults.results; -} - -export async function lookupConfigsBatch( - db: D1Database, - filesVersion: string, - devices: DeviceLookupRequest[], -): Promise { - if (devices.length === 0) return []; - - // Process devices in chunks to avoid D1's variable limit - const allResults: UpgradesQueryRow[] = []; - - for (let i = 0; i < devices.length; i += CHUNK_SIZE) { - const chunk = devices.slice(i, i + CHUNK_SIZE); - const chunkResults = await lookupConfigsChunk(db, filesVersion, chunk); - allResults.push(...chunkResults); - } - - // Group rows by device ID - return Map.groupBy( - allResults, - (row) => - `${row.manufacturer_id}:${row.product_type}:${row.product_id}:${row.firmware_version}`, - ) - .values() - .map((deviceRows) => { - // All rows in each deviceRows array are for the same device. They are essentially an expansion device x upgrades x files - const deviceRow = deviceRows[0]; - const deviceId = { - manufacturerId: parseInt(deviceRow.manufacturer_id, 16), - productType: parseInt(deviceRow.product_type, 16), - productId: parseInt(deviceRow.product_id, 16), - firmwareVersion: deviceRow.firmware_version, - }; - - const updates = Map.groupBy(deviceRows, (row) => row.upgrade_id) - .values() - // All rows in each upgradeRows array are for the same upgrade. They are essentially an expansion upgrade x files - .filter((upgradeRows) => { - const upgrade = upgradeRows[0]; - // Apply conditional logic if condition exists - if (upgrade.condition) { - return conditionApplies( - { $if: upgrade.condition, ...upgrade }, - deviceId, - ); - } - return true; - }) - .map((upgradeRows) => { - const upgradeRow = upgradeRows[0]; - const upgrade: APIv3_UpgradeInfo = { - version: upgradeRow.upgrade_firmware_version, - changelog: upgradeRow.changelog, - channel: upgradeRow.channel as "stable" | "beta", - ...(upgradeRow.region - ? { region: upgradeRow.region as any } - : {}), - files: upgradeRows.map((file) => ({ - target: file.target, - url: file.url, - integrity: file.integrity, - })), - // These two will be filled in or filtered by the downstream handler - downgrade: undefined as any, - normalizedVersion: undefined as any, - }; - return upgrade; - }) - .toArray(); - - const device: APIv4_DeviceInfo = { - manufacturerId: deviceRow.manufacturer_id, - productType: deviceRow.product_type, - productId: deviceRow.product_id, - firmwareVersion: deviceRow.firmware_version, - updates, - }; - - return device; - }) - .toArray(); -} - -export async function lookupConfig( - db: D1Database, - filesVersion: string, - manufacturerId: number | string, - productType: number | string, - productId: number | string, - firmwareVersion: string, -): Promise { - const results = await lookupConfigsBatch(db, filesVersion, [ - { - manufacturerId, - productType, - productId, - firmwareVersion, - }, - ]); - return results[0]; -} - -export async function insertSingleConfigData( - db: D1Database, - version: string, - config: { devices: DeviceIdentifier[]; upgrades: ConditionalUpgradeInfo[] }, -): Promise { - const { devices, upgrades } = config; - - // Insert devices and get their IDs - const deviceIds: number[] = []; - for (const device of devices) { - const firmwareVersionMin = padVersion(device.firmwareVersion.min, "0"); - const firmwareVersionMax = padVersion( - device.firmwareVersion.max, - "255", - ); - const minVersionNormalized = versionToNumber(firmwareVersionMin); - const maxVersionNormalized = versionToNumber(firmwareVersionMax); - - const result = await db - .prepare( - ` - INSERT INTO devices ( - version, brand, model, manufacturer_id, product_type, product_id, - firmware_version_min, firmware_version_max, - firmware_version_min_normalized, firmware_version_max_normalized - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - RETURNING id - `, - ) - .bind( - version, - device.brand, - device.model, - device.manufacturerId, - device.productType, - device.productId, - firmwareVersionMin, - firmwareVersionMax, - minVersionNormalized, - maxVersionNormalized, - ) - .first<{ id: number }>(); - - if (result) { - deviceIds.push(result.id); - } - } - - // Insert upgrades and get their IDs - const upgradeIds: number[] = []; - for (const upgrade of upgrades) { - const result = await db - .prepare( - ` - INSERT INTO upgrades ( - firmware_version, changelog, channel, region, condition - ) VALUES (?, ?, ?, ?, ?) - RETURNING id - `, - ) - .bind( - upgrade.version, - upgrade.changelog, - upgrade.channel, - upgrade.region || null, - upgrade.$if || null, - ) - .first<{ id: number }>(); - - if (result) { - upgradeIds.push(result.id); - } - } - - // Create device-upgrade relationships in the junction table - const junctionStatements: D1PreparedStatement[] = []; - for (const deviceId of deviceIds) { - for (const upgradeId of upgradeIds) { - junctionStatements.push( - db - .prepare( - ` - INSERT INTO device_upgrades (device_id, upgrade_id) VALUES (?, ?) - `, - ) - .bind(deviceId, upgradeId), - ); - } - } - - if (junctionStatements.length > 0) { - await db.batch(junctionStatements); - } - - // Insert upgrade files - const fileStatements: D1PreparedStatement[] = []; - for (let i = 0; i < upgrades.length; i++) { - const upgrade = upgrades[i]; - const upgradeId = upgradeIds[i]; - - for (let j = 0; j < upgrade.files.length; j++) { - const file = upgrade.files[j]; - fileStatements.push( - db - .prepare( - ` - INSERT INTO upgrade_files (upgrade_id, target, url, integrity, position) - VALUES (?, ?, ?, ?, ?) - `, - ) - .bind(upgradeId, file.target, file.url, file.integrity, j), - ); - } - } - - if (fileStatements.length > 0) { - await db.batch(fileStatements); - } -} diff --git a/src/lib/shared_cloudflare.ts b/src/lib/shared_cloudflare.ts index db17c491..08edf254 100644 --- a/src/lib/shared_cloudflare.ts +++ b/src/lib/shared_cloudflare.ts @@ -1,22 +1,5 @@ import { error } from "itty-router"; -/** Constant-time string comparison */ -export function safeCompare(expected: string, actual: string): boolean { - const lenExpected = expected.length; - let result = 0; - - if (lenExpected !== actual.length) { - actual = expected; - result = 1; - } - - for (let i = 0; i < lenExpected; i++) { - result |= expected.charCodeAt(i) ^ actual.charCodeAt(i); - } - - return result === 0; -} - export function clientError( message: BodyInit | Record | undefined, code: number = 400, diff --git a/src/lib/uploadSchema.ts b/src/lib/uploadSchema.ts deleted file mode 100644 index c5ff3ae4..00000000 --- a/src/lib/uploadSchema.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { z } from "zod"; - -const versionSchema = z - .string() - .length(8) - .regex(/[0-9a-f]{8}/); - -export const createSchema = z.object({ - task: z.literal("create"), -}); - -export const putSchema = z.object({ - task: z.literal("put"), - filename: z.string().min(1), - data: z.any(), -}); - -export const enableSchema = z.object({ - task: z.literal("enable"), -}); - -export const uploadSchema = z.object({ - version: versionSchema, - actions: z.union([createSchema, putSchema, enableSchema]).array(), -}); - -export type UploadPayload = z.infer; diff --git a/src/maintenance/upload.ts b/src/maintenance/upload.ts deleted file mode 100644 index 95a5681a..00000000 --- a/src/maintenance/upload.ts +++ /dev/null @@ -1,115 +0,0 @@ -import ky from "ky"; -import crypto from "node:crypto"; -import path from "path-browserify"; -import type { UploadPayload } from "../lib/uploadSchema.js"; -import { NodeFS } from "./nodeFS.js"; - -import { argv } from "node:process"; -import { dirname } from "path"; -import { fileURLToPath } from "url"; -const __dirname = dirname(fileURLToPath(import.meta.url)); - -const configDir = path.resolve(__dirname, "../../firmwares"); -const MAX_FILES_PER_REQUEST = 50; // limited by no. of subrequests in Cloudflare Workers -const baseURL = process.env.BASE_URL; -const adminSecret = process.env.ADMIN_SECRET; - -if (!baseURL) { - console.error(); - console.error("ERROR: Missing BASE_URL environment variable"); - process.exit(1); -} else if (!adminSecret) { - console.error(); - console.error("ERROR: Missing ADMIN_SECRET environment variable"); - process.exit(1); -} - -void (async () => { - // Find all config files directly instead of using index.json - const configFiles = (await NodeFS.readDir(configDir, true)).filter( - (file) => - file.endsWith(".json") && - !file.endsWith("index.json") && - !path.basename(file).startsWith("_") && - !file.includes("/templates/") && - !file.includes("\\templates\\"), - ); - - const files: { filename: string; data: string }[] = []; - - for (const filePath of configFiles) { - const relativePath = path - .relative(configDir, filePath) - .replace(/\\/g, "/"); - const fileContent = await NodeFS.readFile(filePath); - files.push({ filename: relativePath, data: fileContent }); - } - - const hasher = crypto.createHash("sha256"); - const data = JSON.stringify(files); - const version = hasher.update(data, "utf8").digest("hex").slice(0, 8); - - const onlineVersion = await ky - .get(new URL("/admin/config/version", baseURL).toString(), { - headers: { "x-admin-secret": adminSecret }, - }) - .text(); - - if (onlineVersion === version && !argv.includes("--force")) { - console.log("No change in config files, skipping upload..."); - return; - } - - // First, create the new config version - console.log("Creating config version..."); - const createPayload: UploadPayload = { - version, - actions: [{ task: "create" }], - }; - await ky.post(new URL("/admin/config/upload", baseURL).toString(), { - headers: { "x-admin-secret": adminSecret }, - json: createPayload, - timeout: false, - }); - - let cursor = 0; - - while (cursor < files.length) { - const currentBatch = files.slice( - cursor, - cursor + MAX_FILES_PER_REQUEST, - ); - - const payload: UploadPayload = { - version, - actions: currentBatch.map((f) => ({ - task: "put", - ...f, - })), - }; - console.log( - `Uploading files ${cursor + 1}...${ - cursor + currentBatch.length - } of ${files.length}...`, - ); - await ky.post(new URL("/admin/config/upload", baseURL).toString(), { - headers: { "x-admin-secret": adminSecret }, - json: payload, - timeout: false, - }); - - cursor += MAX_FILES_PER_REQUEST; - } - - const finalizePayload: UploadPayload = { - version, - actions: [{ task: "enable" }], - }; - console.log("finalizing..."); - await ky.post(new URL("/admin/config/upload", baseURL).toString(), { - headers: { "x-admin-secret": adminSecret }, - json: finalizePayload, - timeout: false, - }); - console.log("done!"); -})(); diff --git a/src/routes/admin.ts b/src/routes/admin.ts deleted file mode 100644 index 7a8a7940..00000000 --- a/src/routes/admin.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { json, text } from "itty-router"; -import JSON5 from "json5"; -import { ConditionalUpdateConfig } from "../lib/config.js"; -import { - createConfigVersion, - enableConfigVersion, - getCurrentVersion, - insertSingleConfigData, -} from "../lib/d1Operations.js"; -import { - clientError, - ContentProps, - safeCompare, - type RequestWithProps, -} from "../lib/shared_cloudflare.js"; -import { uploadSchema } from "../lib/uploadSchema.js"; -import type { CloudflareEnvironment } from "../worker.js"; - -export default function register(router: any): void { - // Verify the admin token - router.post("/admin/*", (req: Request, env: CloudflareEnvironment) => { - // Avoid timing attacks, but still do not do unnecessary work - const secret = req.headers.get("x-admin-secret"); - if ( - !secret || - !env.ADMIN_SECRET || - !safeCompare(env.ADMIN_SECRET, secret) - ) { - return new Response(undefined, { status: 404 }); - } - }); - - router.post( - "/admin/config/upload", - async ( - req: RequestWithProps<[ContentProps]>, - env: CloudflareEnvironment, - _context: ExecutionContext, - ) => { - try { - const result = await uploadSchema.safeParseAsync(req.content); - if (!result.success) { - return clientError(result.error.format() as any); - } - - const newVersion = result.data.version; - - for (const action of result.data.actions) { - if (action.task === "create") { - // Create a new config version in the database - await createConfigVersion(env.CONFIG_FILES, newVersion); - } else if (action.task === "put") { - // Process config file data for D1 insertion - if (action.filename === "index.json") { - // Skip index.json as we don't need it anymore - continue; - } - - try { - const definition = JSON5.parse(action.data); - const config = new ConditionalUpdateConfig( - definition, - ); - - // Insert this single config immediately - await insertSingleConfigData( - env.CONFIG_FILES, - newVersion, - { - devices: config.devices, - upgrades: config.upgrades, - }, - ); - } catch (e) { - console.error( - `Error parsing config file ${action.filename}:`, - e, - ); - // Skip invalid files but don't fail the whole upload - continue; - } - } else if (action.task === "enable") { - // Enable the new version and clean up old data - await enableConfigVersion(env.CONFIG_FILES, newVersion); - - // Make sure not to process any more files after this - break; - } - } - } catch (e: any) { - console.error(e.stack); - throw e; - } - - return json({ ok: true }); - }, - ); - - router.get( - "/admin/config/version", - async ( - req: Request, - env: CloudflareEnvironment, - _context: ExecutionContext, - ) => { - const ret = await getCurrentVersion(env.CONFIG_FILES); - return text(ret || ""); - }, - ); -} diff --git a/src/worker.ts b/src/worker.ts index 566479b1..7bf2d528 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,10 +1,6 @@ import { build } from "./app.js"; export interface CloudflareEnvironment { - ADMIN_SECRET?: string; - - CONFIG_FILES: D1Database; - /** Static assets containing the prebuilt firmware update data */ DATA: Fetcher; diff --git a/wrangler.toml b/wrangler.toml index e43d625a..f4fcc355 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -10,11 +10,6 @@ new_classes = ["RateLimiterDurableObject"] tag = "v2" deleted_classes = ["RateLimiterDurableObject"] -[[d1_databases]] -binding = "CONFIG_FILES" -database_name = "zwave-js-firmware-updates--prod" -database_id = "6d9a21f3-6886-4d53-b7dc-3b03dad70698" - # Prebuilt firmware update data, generated by `yarn build:data`. # run_worker_first keeps the shards from being publicly routable. [assets]