From 60d2c123438d05e939b46bc71f7f4b78dde311ec Mon Sep 17 00:00:00 2001 From: Saagar Date: Sat, 4 Jul 2026 01:58:54 -0700 Subject: [PATCH 1/3] fix: remove lhci js-yaml path --- .prettierignore | 1 + package.json | 7 +- pnpm-lock.yaml | 107 --------- pnpm-workspace.yaml | 3 - scripts/perf/run-lighthouse-budget.mjs | 288 +++++++++++++++++++++++++ 5 files changed, 293 insertions(+), 113 deletions(-) create mode 100644 .prettierignore create mode 100644 scripts/perf/run-lighthouse-budget.mjs diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..bd5535a --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +pnpm-lock.yaml diff --git a/package.json b/package.json index 526efcc..e24fe33 100644 --- a/package.json +++ b/package.json @@ -47,8 +47,10 @@ "build:ui": "vite build", "perf:bundle": "node scripts/perf/vite-bundle-report.mjs", "perf:build": "node scripts/perf/measure-build-time.mjs", - "perf:lhci": "lhci autorun", - "perf:lhci:prod": "lhci autorun --config=.lighthouserc.production.json", + "perf:lighthouse": "node scripts/perf/run-lighthouse-budget.mjs", + "perf:lighthouse:prod": "node scripts/perf/run-lighthouse-budget.mjs --config=.lighthouserc.production.json", + "perf:lhci": "pnpm perf:lighthouse", + "perf:lhci:prod": "pnpm perf:lighthouse:prod", "perf:api": "bash scripts/perf/run-k6.sh tests/perf/api.k6.js --summary-export=.perf-results/api-summary.json", "perf:db": "bash scripts/perf/db/check-pg-stat.sh", "perf:db:enforce": "bash scripts/perf/db/enforce-pg-stat.sh", @@ -108,7 +110,6 @@ "@commitlint/config-conventional": "^19.8.1", "@commitlint/cz-commitlint": "^21.2.0", "@eslint/js": "^9.39.4", - "@lhci/cli": "^0.15.1", "@playwright/test": "^1.59.1", "@tauri-apps/cli": "^2.11.2", "@testing-library/react": "^16.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cbde5cc..2e333c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,8 +5,6 @@ settings: excludeLinksFromLockfile: false overrides: - '@lhci/cli>tmp': ^0.2.7 - '@lhci/cli>uuid': ^11.1.1 '@babel/core': ^7.29.6 basic-ftp: ^5.3.0 commitizen>lodash: ^4.17.23 @@ -14,7 +12,6 @@ overrides: eslint>ajv: ^6.14.0 external-editor>tmp: ^0.2.7 flatted: ^3.4.2 - js-yaml@<3.15.0: ^3.15.0 js-yaml@>=4.0.0 <=4.1.1: ^4.2.0 minimatch: ^10.2.3 qs@>=6.11.1 <=6.15.1: ^6.15.2 @@ -64,9 +61,6 @@ importers: '@eslint/js': specifier: ^9.39.4 version: 9.39.4 - '@lhci/cli': - specifier: ^0.15.1 - version: 0.15.1 '@playwright/test': specifier: ^1.59.1 version: 1.59.1 @@ -721,13 +715,6 @@ packages: '@keyv/serialize@1.1.1': resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} - '@lhci/cli@0.15.1': - resolution: {integrity: sha512-yhC0oXnXqGHYy1xl4D8YqaydMZ/khFAnXGY/o2m/J3PqPa/D0nj3V6TLoH02oVMFeEF2AQim7UbmdXMiXx2tOw==} - hasBin: true - - '@lhci/utils@0.15.1': - resolution: {integrity: sha512-WclJnUQJeOMY271JSuaOjCv/aA0pgvuHZS29NFNdIeI14id8eiFsjith85EGKYhljgoQhJ2SiW4PsVfFiakNNw==} - '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2660,10 +2647,6 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@3.15.0: - resolution: {integrity: sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==} - hasBin: true - js-yaml@4.3.0: resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} hasBin: true @@ -2744,11 +2727,6 @@ packages: lighthouse-stack-packs@1.12.3: resolution: {integrity: sha512-d8IsOpE83kbANgnM+Tp8+x6HcMpX9o2ITBiUERssgzAIFdZCQzs/f4k6D0DLQTE59enml9mbAOU52Wu35exWtg==} - lighthouse@12.6.1: - resolution: {integrity: sha512-85WDkjcXAVdlFem9Y6SSxqoKiz/89UsDZhLpeLJIsJ4LlHxw047XTZhlFJmjYCB7K5S1erSBAf5cYLcfyNbH3A==} - engines: {node: '>=18.20'} - hasBin: true - lighthouse@13.3.0: resolution: {integrity: sha512-jlsJi7+2i2UQWJY8M+gNDEGoDL1sfZ5J2hArCvmQgMTwKq1KQs5ou2pmasyhrRafdU6jh/Prjuz8rZLbqGuydQ==} engines: {node: '>=22.19'} @@ -4709,48 +4687,6 @@ snapshots: '@keyv/serialize@1.1.1': {} - '@lhci/cli@0.15.1': - dependencies: - '@lhci/utils': 0.15.1 - chrome-launcher: 0.13.4 - compression: 1.8.1 - debug: 4.4.3 - express: 4.22.1 - inquirer: 6.5.2 - isomorphic-fetch: 3.0.0 - lighthouse: 12.6.1 - lighthouse-logger: 1.2.0 - open: 7.4.2 - proxy-agent: 6.5.0 - tmp: 0.2.7 - uuid: 11.1.1 - yargs: 15.4.1 - yargs-parser: 13.1.2 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - bufferutil - - encoding - - react-native-b4a - - supports-color - - utf-8-validate - - '@lhci/utils@0.15.1': - dependencies: - debug: 4.4.3 - isomorphic-fetch: 3.0.0 - js-yaml: 3.15.0 - lighthouse: 12.6.1 - tree-kill: 1.2.2 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - bufferutil - - encoding - - react-native-b4a - - supports-color - - utf-8-validate - '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -6830,11 +6766,6 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@3.15.0: - dependencies: - argparse: 1.0.10 - esprima: 4.0.1 - js-yaml@4.3.0: dependencies: argparse: 2.0.1 @@ -6930,44 +6861,6 @@ snapshots: lighthouse-stack-packs@1.12.3: {} - lighthouse@12.6.1: - dependencies: - '@paulirish/trace_engine': 0.0.53 - '@sentry/node': 7.120.4 - axe-core: 4.11.4 - chrome-launcher: 1.2.1 - configstore: 5.0.1 - csp_evaluator: 1.1.5 - devtools-protocol: 0.0.1467305 - enquirer: 2.4.1 - http-link-header: 1.1.3 - intl-messageformat: 10.7.18 - jpeg-js: 0.4.4 - js-library-detector: 6.7.0 - lighthouse-logger: 2.0.2 - lighthouse-stack-packs: 1.12.2 - lodash-es: 4.18.1 - lookup-closest-locale: 6.2.0 - metaviewport-parser: 0.3.0 - open: 8.4.2 - parse-cache-control: 1.0.1 - puppeteer-core: 24.43.1 - robots-parser: 3.0.1 - semver: 5.7.2 - speedline-core: 1.4.3 - third-party-web: 0.26.7 - tldts-icann: 6.1.86 - ws: 7.5.11 - yargs: 17.7.2 - yargs-parser: 21.1.1 - transitivePeerDependencies: - - bare-abort-controller - - bare-buffer - - bufferutil - - react-native-b4a - - supports-color - - utf-8-validate - lighthouse@13.3.0: dependencies: '@paulirish/trace_engine': 0.0.64 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 37dfb4f..fbeb9fa 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,8 +6,6 @@ minimumReleaseAgeExclude: - uuid@11.1.1 - ws@8.20.1 overrides: - "@lhci/cli>tmp": ^0.2.7 - "@lhci/cli>uuid": ^11.1.1 "@babel/core": ^7.29.6 basic-ftp: ^5.3.0 commitizen>lodash: ^4.17.23 @@ -15,7 +13,6 @@ overrides: eslint>ajv: ^6.14.0 external-editor>tmp: ^0.2.7 flatted: ^3.4.2 - js-yaml@<3.15.0: ^3.15.0 js-yaml@>=4.0.0 <=4.1.1: ^4.2.0 minimatch: ^10.2.3 qs@>=6.11.1 <=6.15.1: ^6.15.2 diff --git a/scripts/perf/run-lighthouse-budget.mjs b/scripts/perf/run-lighthouse-budget.mjs new file mode 100644 index 0000000..c4b3527 --- /dev/null +++ b/scripts/perf/run-lighthouse-budget.mjs @@ -0,0 +1,288 @@ +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const configArg = process.argv.find((arg) => arg.startsWith("--config=")); +const configPath = path.resolve( + root, + configArg ? configArg.slice("--config=".length) : ".lighthouserc.json", +); +const config = JSON.parse(readFileSync(configPath, "utf8")); +const collect = config.ci?.collect ?? {}; +const assertions = config.ci?.assert?.assertions ?? {}; +const urls = Array.isArray(collect.url) + ? collect.url + : [collect.url].filter(Boolean); +const numberOfRuns = Number(collect.numberOfRuns ?? 1); +const outputDir = path.join(root, ".perf-results", "lighthouse"); +const chromeFlags = + process.env.LIGHTHOUSE_CHROME_FLAGS ?? "--headless=new --no-sandbox"; + +if (!urls.length) { + console.error(`No Lighthouse URLs configured in ${configPath}.`); + process.exit(1); +} + +if (collect.startServerCommand?.includes("preview") && !existsSync("dist")) { + console.error( + "dist is missing; run pnpm perf:build or pnpm build:ui before Lighthouse.", + ); + process.exit(1); +} + +mkdirSync(outputDir, { recursive: true }); + +function packageManagerArgs(commandArgs) { + if (process.env.npm_execpath) { + return { + command: process.execPath, + args: [process.env.npm_execpath, ...commandArgs], + }; + } + + return { command: "pnpm", args: commandArgs }; +} + +function waitForServer(command, readyPattern) { + const timeoutMs = Number(process.env.LIGHTHOUSE_SERVER_TIMEOUT_MS ?? 60_000); + + return new Promise((resolve, reject) => { + const child = spawn(command, { + cwd: root, + detached: process.platform !== "win32", + env: process.env, + shell: true, + stdio: ["ignore", "pipe", "pipe"], + }); + let ready = !readyPattern; + let output = ""; + + const cleanupTimer = setTimeout(() => { + reject( + new Error( + `Timed out waiting for Lighthouse server readiness pattern: ${readyPattern}`, + ), + ); + }, timeoutMs); + + const observe = (chunk) => { + const text = chunk.toString(); + output += text; + process.stdout.write(text); + if (!ready && text.includes(readyPattern)) { + ready = true; + clearTimeout(cleanupTimer); + resolve(child); + } + }; + + child.stdout.on("data", observe); + child.stderr.on("data", observe); + child.on("exit", (code) => { + clearTimeout(cleanupTimer); + if (!ready) { + reject( + new Error( + `Lighthouse server exited before it was ready (${code ?? "signal"}):\n${output}`, + ), + ); + } + }); + + if (ready) { + clearTimeout(cleanupTimer); + resolve(child); + } + }); +} + +function stopServer(child) { + if (!child || child.killed) { + return; + } + + child.stdout?.removeAllListeners("data"); + child.stderr?.removeAllListeners("data"); + + try { + if (process.platform !== "win32" && child.pid) { + process.kill(-child.pid, "SIGTERM"); + return; + } + } catch { + // Fall through to killing the direct child process. + } + + child.kill("SIGTERM"); +} + +function safeName(url) { + return url + .replace(/[^a-z0-9]+/gi, "-") + .replace(/^-|-$/g, "") + .toLowerCase(); +} + +function runLighthouse(url, runNumber) { + const outputPath = path.join( + outputDir, + `${safeName(url)}-run-${runNumber}.json`, + ); + const pm = packageManagerArgs([ + "exec", + "lighthouse", + url, + "--quiet", + "--output=json", + `--output-path=${outputPath}`, + `--chrome-flags=${chromeFlags}`, + ]); + + const result = spawnSync(pm.command, pm.args, { + cwd: root, + env: process.env, + stdio: "inherit", + }); + + if (result.status !== 0) { + process.exit(result.status ?? 1); + } + + return JSON.parse(readFileSync(outputPath, "utf8")); +} + +function normalizeAssertion(assertion) { + if (Array.isArray(assertion)) { + return { level: assertion[0], options: assertion[1] ?? {} }; + } + + return { level: "error", options: assertion ?? {} }; +} + +function getMetric(lhr, key) { + if (key.startsWith("categories:")) { + const category = key.slice("categories:".length); + return { + label: key, + value: lhr.categories?.[category]?.score, + }; + } + + const audit = lhr.audits?.[key]; + return { + label: key, + value: + typeof audit?.numericValue === "number" + ? audit.numericValue + : audit?.score, + }; +} + +function evaluate(lhr, runLabel) { + const results = []; + + for (const [key, rawAssertion] of Object.entries(assertions)) { + const { level, options } = normalizeAssertion(rawAssertion); + if (level === "off") { + continue; + } + + const metric = getMetric(lhr, key); + if (typeof metric.value !== "number") { + results.push({ + key, + level, + run: runLabel, + status: level === "warn" ? "warn" : "fail", + message: `${key} did not produce a numeric Lighthouse metric.`, + }); + continue; + } + + if (typeof options.minScore === "number") { + const pass = metric.value >= options.minScore; + results.push({ + key, + level, + run: runLabel, + status: pass ? "pass" : level === "warn" ? "warn" : "fail", + value: metric.value, + threshold: options.minScore, + comparator: ">=", + }); + continue; + } + + if (typeof options.maxNumericValue === "number") { + const pass = metric.value <= options.maxNumericValue; + results.push({ + key, + level, + run: runLabel, + status: pass ? "pass" : level === "warn" ? "warn" : "fail", + value: metric.value, + threshold: options.maxNumericValue, + comparator: "<=", + }); + } + } + + return results; +} + +let server; +const allResults = []; + +try { + if (collect.startServerCommand) { + server = await waitForServer( + collect.startServerCommand, + collect.startServerReadyPattern, + ); + } + + for (const url of urls) { + for (let run = 1; run <= numberOfRuns; run += 1) { + console.log(`\n== Lighthouse ${url} run ${run}/${numberOfRuns} ==`); + const lhr = runLighthouse(url, run); + allResults.push(...evaluate(lhr, `${safeName(url)}-${run}`)); + } + } +} finally { + stopServer(server); +} + +const failures = allResults.filter((result) => result.status === "fail"); +const warnings = allResults.filter((result) => result.status === "warn"); +const summary = { + config: path.relative(root, configPath), + capturedAt: new Date().toISOString(), + runs: urls.length * numberOfRuns, + failures: failures.length, + warnings: warnings.length, + results: allResults, +}; + +writeFileSync( + path.join(outputDir, "summary.json"), + JSON.stringify(summary, null, 2), +); + +for (const result of allResults) { + const prefix = + result.status === "pass" + ? "PASS" + : result.status === "warn" + ? "WARN" + : "FAIL"; + const detail = + typeof result.value === "number" + ? `${result.value.toFixed(3)} ${result.comparator} ${result.threshold}` + : result.message; + console.log(`${prefix} ${result.run} ${result.key}: ${detail}`); +} + +if (failures.length > 0) { + process.exit(1); +} From f15afbb80962b6a3197de4e49439f5df9d3c32de Mon Sep 17 00:00:00 2001 From: Saagar Date: Sat, 4 Jul 2026 02:08:04 -0700 Subject: [PATCH 2/3] fix: preserve lighthouse aggregation semantics --- scripts/perf/run-lighthouse-budget.mjs | 79 ++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/scripts/perf/run-lighthouse-budget.mjs b/scripts/perf/run-lighthouse-budget.mjs index c4b3527..5d74827 100644 --- a/scripts/perf/run-lighthouse-budget.mjs +++ b/scripts/perf/run-lighthouse-budget.mjs @@ -11,6 +11,7 @@ const configPath = path.resolve( const config = JSON.parse(readFileSync(configPath, "utf8")); const collect = config.ci?.collect ?? {}; const assertions = config.ci?.assert?.assertions ?? {}; +const aggregationMethod = config.ci?.assert?.aggregationMethod ?? "optimistic"; const urls = Array.isArray(collect.url) ? collect.url : [collect.url].filter(Boolean); @@ -146,7 +147,13 @@ function runLighthouse(url, runNumber) { }); if (result.status !== 0) { - process.exit(result.status ?? 1); + const error = new Error( + `Lighthouse failed for ${url} run ${runNumber} with status ${ + result.status ?? 1 + }.`, + ); + error.exitCode = result.status ?? 1; + throw error; } return JSON.parse(readFileSync(outputPath, "utf8")); @@ -179,8 +186,29 @@ function getMetric(lhr, key) { }; } -function evaluate(lhr, runLabel) { +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} + +function aggregate(values, favorsHigher) { + if (aggregationMethod === "median") { + return median(values); + } + + if (aggregationMethod === "pessimistic") { + return favorsHigher ? Math.min(...values) : Math.max(...values); + } + + return favorsHigher ? Math.max(...values) : Math.min(...values); +} + +function evaluate(url, lhrRuns) { const results = []; + const runLabel = safeName(url); for (const [key, rawAssertion] of Object.entries(assertions)) { const { level, options } = normalizeAssertion(rawAssertion); @@ -188,8 +216,11 @@ function evaluate(lhr, runLabel) { continue; } - const metric = getMetric(lhr, key); - if (typeof metric.value !== "number") { + const samples = lhrRuns.map(({ lhr, run }) => ({ + run, + value: getMetric(lhr, key).value, + })); + if (samples.some((sample) => typeof sample.value !== "number")) { results.push({ key, level, @@ -201,29 +232,41 @@ function evaluate(lhr, runLabel) { } if (typeof options.minScore === "number") { - const pass = metric.value >= options.minScore; + const value = aggregate( + samples.map((sample) => sample.value), + true, + ); + const pass = value >= options.minScore; results.push({ key, level, run: runLabel, status: pass ? "pass" : level === "warn" ? "warn" : "fail", - value: metric.value, + value, threshold: options.minScore, comparator: ">=", + aggregationMethod, + samples, }); continue; } if (typeof options.maxNumericValue === "number") { - const pass = metric.value <= options.maxNumericValue; + const value = aggregate( + samples.map((sample) => sample.value), + false, + ); + const pass = value <= options.maxNumericValue; results.push({ key, level, run: runLabel, status: pass ? "pass" : level === "warn" ? "warn" : "fail", - value: metric.value, + value, threshold: options.maxNumericValue, comparator: "<=", + aggregationMethod, + samples, }); } } @@ -233,6 +276,7 @@ function evaluate(lhr, runLabel) { let server; const allResults = []; +let runError; try { if (collect.startServerCommand) { @@ -243,20 +287,30 @@ try { } for (const url of urls) { + const lhrRuns = []; for (let run = 1; run <= numberOfRuns; run += 1) { console.log(`\n== Lighthouse ${url} run ${run}/${numberOfRuns} ==`); const lhr = runLighthouse(url, run); - allResults.push(...evaluate(lhr, `${safeName(url)}-${run}`)); + lhrRuns.push({ lhr, run }); } + allResults.push(...evaluate(url, lhrRuns)); } +} catch (error) { + runError = error; } finally { stopServer(server); } +if (runError) { + console.error(runError.message); + process.exit(runError.exitCode ?? 1); +} + const failures = allResults.filter((result) => result.status === "fail"); const warnings = allResults.filter((result) => result.status === "warn"); const summary = { config: path.relative(root, configPath), + aggregationMethod, capturedAt: new Date().toISOString(), runs: urls.length * numberOfRuns, failures: failures.length, @@ -280,7 +334,12 @@ for (const result of allResults) { typeof result.value === "number" ? `${result.value.toFixed(3)} ${result.comparator} ${result.threshold}` : result.message; - console.log(`${prefix} ${result.run} ${result.key}: ${detail}`); + const samples = result.samples + ? ` (${result.aggregationMethod}; samples ${result.samples + .map((sample) => `${sample.run}:${sample.value.toFixed(3)}`) + .join(", ")})` + : ""; + console.log(`${prefix} ${result.run} ${result.key}: ${detail}${samples}`); } if (failures.length > 0) { From c459ef470b02d1e5a35e70cc54de716f378c029d Mon Sep 17 00:00:00 2001 From: Saagar Date: Sat, 4 Jul 2026 02:13:55 -0700 Subject: [PATCH 3/3] fix: clean up lighthouse readiness timeouts --- scripts/perf/run-lighthouse-budget.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/perf/run-lighthouse-budget.mjs b/scripts/perf/run-lighthouse-budget.mjs index 5d74827..21583d2 100644 --- a/scripts/perf/run-lighthouse-budget.mjs +++ b/scripts/perf/run-lighthouse-budget.mjs @@ -60,6 +60,7 @@ function waitForServer(command, readyPattern) { let output = ""; const cleanupTimer = setTimeout(() => { + stopServer(child); reject( new Error( `Timed out waiting for Lighthouse server readiness pattern: ${readyPattern}`,