From adaa3c767a449cc7ab604eab50ee6cb3745c29ae Mon Sep 17 00:00:00 2001 From: Matt MacDougall Date: Wed, 1 Jul 2026 19:04:52 -0400 Subject: [PATCH 1/2] feat: free Lighthouse audits via PageSpeed Insights when PAGESPEED_API_KEY is set Site-audit Lighthouse currently runs on the metered DataForSEO OnPage endpoint ($0.00425/page) even though Google's PageSpeed Insights API returns the same Lighthouse report for free (~25k requests/day with a free API key). - Add src/server/lib/pagespeedLighthousePayload.ts: fetches PSI and normalizes the report into the exact same StoredLighthousePayload the DataForSEO path produces, so R2 payloads, scores, metrics, and the issue-details view are provider-agnostic. Handles Lighthouse 13.x returning details.items as an object for some audits, and a 90s timeout for slow runs. - Route the audit Lighthouse phase to PSI only when PAGESPEED_API_KEY is set, falling back to the existing DataForSEO path otherwise, so current deployments keep their behavior with zero action. (Keyless PSI is not viable: the anonymous shared quota returns HTTP 429 almost always.) - Widen the stored-payload source enum to accept both providers; existing stored DataForSEO payloads still parse. - Default "Include Lighthouse" to on. Without Lighthouse the results view has no Performance tab and the only recovery is re-crawling the whole site. - Plumb PAGESPEED_API_KEY through .env.example, compose.yaml, env.d.ts, and document it in docs/SELF_HOSTING_DOCKER.md. Verified end to end on a live 10-page audit: 8/8 Lighthouse runs via PSI (mobile + desktop), scores and per-page issue details render unchanged, and no DataForSEO credits are spent. Co-Authored-By: Claude Opus 4.8 --- .env.example | 6 + compose.yaml | 3 + docs/SELF_HOSTING_DOCKER.md | 11 ++ .../features/audit/launch/LaunchFormCard.tsx | 4 + src/client/features/audit/launch/types.ts | 4 +- src/env.d.ts | 4 + src/server/lib/audit/lighthouse.ts | 13 +- src/server/lib/lighthouseStoredPayload.ts | 2 +- src/server/lib/pagespeedLighthousePayload.ts | 175 ++++++++++++++++++ 9 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 src/server/lib/pagespeedLighthousePayload.ts diff --git a/.env.example b/.env.example index 671bff39..5e69cdfb 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,12 @@ # Required for all modes. See README.md for setup instructions. # DATAFORSEO_API_KEY= +# Optional: free Google PageSpeed Insights API key. When set, site-audit +# Lighthouse runs on Google's free API (~25k requests/day) instead of the +# metered DataForSEO OnPage endpoint. Create one (2 minutes, no billing): +# https://developers.google.com/speed/docs/insights/v5/get-started +# PAGESPEED_API_KEY= + # Optional app port # PORT=3001 diff --git a/compose.yaml b/compose.yaml index 87b1dad9..efea2e76 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,9 @@ services: - ALLOWED_HOST=${ALLOWED_HOST:-} - AUTH_MODE=local_noauth - DATAFORSEO_API_KEY=${DATAFORSEO_API_KEY} + # Optional: free Google PageSpeed Insights key. When set, site-audit + # Lighthouse runs on Google's free API instead of DataForSEO OnPage. + - PAGESPEED_API_KEY=${PAGESPEED_API_KEY:-} # Optional: Google Search Console. See # docs/SELF_HOSTING_GOOGLE_SEARCH_CONSOLE.md - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-} diff --git a/docs/SELF_HOSTING_DOCKER.md b/docs/SELF_HOSTING_DOCKER.md index 66ef274f..a8c171ea 100644 --- a/docs/SELF_HOSTING_DOCKER.md +++ b/docs/SELF_HOSTING_DOCKER.md @@ -29,6 +29,17 @@ Optional env values: - `ALLOWED_HOST` (single reverse-proxy hostname to allow in Vite preview) - `AUTH_MODE=local_noauth` (already set in compose) - `OPEN_SEO_IMAGE` (defaults to `ghcr.io/every-app/open-seo:latest`) +- `PAGESPEED_API_KEY` (free Lighthouse for site audits, see below) + +## Free Lighthouse audits (PAGESPEED_API_KEY) + +Site audits run Lighthouse through the metered DataForSEO OnPage endpoint by +default. If you set a free Google PageSpeed Insights API key, Lighthouse runs +on Google's API instead at no cost (about 25k requests/day per key): + +1. Create an API key (2 minutes, no billing): https://developers.google.com/speed/docs/insights/v5/get-started +2. Set `PAGESPEED_API_KEY` in `.env` +3. Recreate the container: `docker compose up -d --force-recreate open-seo` If you are putting Docker behind a reverse proxy or a temporary tunnel, remember that Docker self-hosting runs with app auth disabled. Only expose it behind your own auth-protected reverse proxy, tunnel, or private network, and add the public hostname before restarting: diff --git a/src/client/features/audit/launch/LaunchFormCard.tsx b/src/client/features/audit/launch/LaunchFormCard.tsx index 72997e4d..b50ea1c8 100644 --- a/src/client/features/audit/launch/LaunchFormCard.tsx +++ b/src/client/features/audit/launch/LaunchFormCard.tsx @@ -148,6 +148,10 @@ function LighthouseOptions({ launchForm }: Pick) { We choose a sample of 20 pages to audit, removing pages from duplicate templates.

+

+ Runs on the free Google PageSpeed Insights API when + PAGESPEED_API_KEY is set. +

) : null } diff --git a/src/client/features/audit/launch/types.ts b/src/client/features/audit/launch/types.ts index 8fec84c6..ac81bd59 100644 --- a/src/client/features/audit/launch/types.ts +++ b/src/client/features/audit/launch/types.ts @@ -10,5 +10,7 @@ export type LaunchFormValues = { export const DEFAULT_LAUNCH_FORM_VALUES: LaunchFormValues = { url: "", maxPagesInput: "50", - runLighthouse: false, + // On by default: without Lighthouse the results view has no Performance tab + // and the only way to add one is re-crawling the whole site. + runLighthouse: true, }; diff --git a/src/env.d.ts b/src/env.d.ts index 1b2205dd..7b9d507f 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -28,6 +28,10 @@ declare namespace Cloudflare { // DataForSEO API Basic auth value (base64 of login:password) DATAFORSEO_API_KEY: string; + // Optional (free) Google PageSpeed Insights API key. When set, site-audit + // Lighthouse runs on Google's free API instead of DataForSEO OnPage. + PAGESPEED_API_KEY?: string; + // OpenRouter API key for the onboarding chat. OPENROUTER_API_KEY?: string; // Optional OpenRouter model slug override (defaults in openrouter.ts). diff --git a/src/server/lib/audit/lighthouse.ts b/src/server/lib/audit/lighthouse.ts index 85a1b19a..6f6119dc 100644 --- a/src/server/lib/audit/lighthouse.ts +++ b/src/server/lib/audit/lighthouse.ts @@ -1,6 +1,8 @@ import { detectUrlTemplate } from "./url-utils"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { createDataforseoClient } from "@/server/lib/dataforseo"; +import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload"; +import { getOptionalEnvValue } from "@/server/lib/runtime-env"; import type { LighthouseResult, LighthouseStrategy } from "./types"; import { putTextToR2 } from "@/server/lib/r2"; @@ -22,6 +24,13 @@ async function fetchLighthouseResult( ): Promise { let lastError: Error | null = null; const dataforseo = createDataforseoClient(billingCustomer); + // When a (free) Google PageSpeed Insights API key is configured, run + // Lighthouse there instead of the metered DataForSEO OnPage endpoint. Both + // paths normalize to the same StoredLighthousePayload, so everything + // downstream (R2 payloads, scores, issue details) is provider-agnostic. + const usePagespeed = Boolean( + (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(), + ); for (let attempt = 0; attempt < 3; attempt++) { try { @@ -32,7 +41,9 @@ async function fetchLighthouseResult( ); } - const data = await dataforseo.lighthouse.live({ url, strategy }); + const data = usePagespeed + ? await fetchPagespeedLighthouse({ url, strategy }) + : await dataforseo.lighthouse.live({ url, strategy }); return { result: { diff --git a/src/server/lib/lighthouseStoredPayload.ts b/src/server/lib/lighthouseStoredPayload.ts index 03376d56..82304d15 100644 --- a/src/server/lib/lighthouseStoredPayload.ts +++ b/src/server/lib/lighthouseStoredPayload.ts @@ -55,7 +55,7 @@ const storedLighthouseIssueSchema = z.object({ export const storedLighthousePayloadSchema = z.object({ version: z.literal(2), - source: z.literal("dataforseo-lighthouse"), + source: z.enum(["dataforseo-lighthouse", "pagespeed-insights"]), hasIssueDetails: z.boolean(), metadata: z.object({ requestedUrl: z.string(), diff --git a/src/server/lib/pagespeedLighthousePayload.ts b/src/server/lib/pagespeedLighthousePayload.ts new file mode 100644 index 00000000..2a10a91a --- /dev/null +++ b/src/server/lib/pagespeedLighthousePayload.ts @@ -0,0 +1,175 @@ +import { z } from "zod"; +import { + buildStoredLighthouseIssues, + buildStoredLighthouseMetrics, + type RawLighthouseAudit, + type RawLighthouseCategory, + scoreToPercent, + type StoredLighthousePayload, +} from "@/server/lib/lighthouseStoredPayload"; +import { getOptionalEnvValue } from "@/server/lib/runtime-env"; + +type LighthouseStrategy = "mobile" | "desktop"; + +// Google PageSpeed Insights runs Lighthouse on Google's own infrastructure and +// returns the standard Lighthouse report under `lighthouseResult`. It is free, +// but a (free) PAGESPEED_API_KEY is effectively required: keyless requests share +// a tiny global anonymous quota that is almost always exhausted (HTTP 429). With +// a key each project gets ~25k requests/day, which is why the audit only routes +// here when PAGESPEED_API_KEY is set (see src/server/lib/audit/lighthouse.ts) +// and otherwise falls back to DataForSEO OnPage. The report is normalised into +// the exact same StoredLighthousePayload the DataForSEO path produces, so the +// audit UI (scores, metrics, and per-audit issues) is provider-agnostic. +const PSI_ENDPOINT = + "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"; +// Lighthouse runs on Google's side are slow; some pages take well over a minute. +const PSI_REQUEST_TIMEOUT_MS = 90_000; +const PSI_REQUEST_CATEGORIES = [ + "PERFORMANCE", + "ACCESSIBILITY", + "BEST_PRACTICES", + "SEO", +] as const; + +// Newer Lighthouse (13.x) returns `details.items` as an object for some audits +// (e.g. document-latency-insight) instead of an array. Accept either and +// normalise to an array so one stray audit shape can't fail the whole parse. +const psiAuditItemsSchema = z + .union([ + z.array(z.record(z.string(), z.unknown())), + z.record(z.string(), z.unknown()), + ]) + .transform((items) => (Array.isArray(items) ? items : [items])); + +const psiAuditSchema = z + .object({ + score: z.number().nullable().optional(), + displayValue: z.string().optional(), + numericValue: z.number().optional(), + title: z.string().optional(), + description: z.string().optional(), + scoreDisplayMode: z.string().optional(), + details: z + .object({ + overallSavingsMs: z.number().optional(), + overallSavingsBytes: z.number().optional(), + items: psiAuditItemsSchema.optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +const psiCategorySchema = z + .object({ + score: z.number().nullable().optional(), + auditRefs: z + .array(z.object({ id: z.string().optional() }).passthrough()) + .optional(), + }) + .passthrough(); + +const psiResponseSchema = z + .object({ + lighthouseResult: z + .object({ + requestedUrl: z.string().optional(), + finalUrl: z.string().optional(), + finalDisplayedUrl: z.string().optional(), + lighthouseVersion: z.string().optional(), + categories: z + .record(z.string(), psiCategorySchema) + .optional() + .default({}), + audits: z.record(z.string(), psiAuditSchema).optional().default({}), + }) + .passthrough(), + }) + .passthrough(); + +function summarizeZodIssues(error: z.ZodError, maxIssues = 3): string { + return error.issues + .slice(0, maxIssues) + .map((issue) => { + const path = issue.path.length > 0 ? issue.path.join(".") : ""; + return `${path}: ${issue.message}`; + }) + .join("; "); +} + +export async function fetchPagespeedLighthouse(input: { + url: string; + strategy: LighthouseStrategy; +}): Promise { + const params = new URLSearchParams(); + params.set("url", input.url); + params.set("strategy", input.strategy); + for (const category of PSI_REQUEST_CATEGORIES) { + params.append("category", category); + } + const apiKey = (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(); + if (apiKey) params.set("key", apiKey); + + const response = await fetch(`${PSI_ENDPOINT}?${params.toString()}`, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(PSI_REQUEST_TIMEOUT_MS), + }); + + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `PageSpeed Insights HTTP ${response.status}${ + body ? `: ${body.slice(0, 200)}` : "" + }`, + ); + } + + const parsed = psiResponseSchema.safeParse(await response.json()); + if (!parsed.success) { + throw new Error( + `PageSpeed Insights returned an invalid response: ${summarizeZodIssues(parsed.error)}`, + ); + } + + const result = parsed.data.lighthouseResult; + const categories: Record = + result.categories ?? {}; + const audits: Record = result.audits ?? {}; + const issueReport = buildStoredLighthouseIssues({ audits, categories }); + const metrics = buildStoredLighthouseMetrics({ audits }); + + const storedPayload: StoredLighthousePayload = { + version: 2, + source: "pagespeed-insights", + hasIssueDetails: issueReport.hasIssueDetails, + metadata: { + requestedUrl: result.requestedUrl ?? input.url, + finalUrl: result.finalUrl ?? result.finalDisplayedUrl ?? input.url, + strategy: input.strategy, + fetchedAt: new Date().toISOString(), + lighthouseVersion: result.lighthouseVersion ?? null, + taskId: null, + // PageSpeed Insights is free; no per-request cost to record. + cost: 0, + }, + scores: { + performance: scoreToPercent(categories.performance?.score), + accessibility: scoreToPercent(categories.accessibility?.score), + "best-practices": scoreToPercent(categories["best-practices"]?.score), + seo: scoreToPercent(categories.seo?.score), + }, + metrics, + issues: issueReport.issues, + }; + + const allScoresMissing = Object.values(storedPayload.scores).every( + (score) => score == null, + ); + if (allScoresMissing) { + throw new Error( + `PageSpeed Insights returned no category scores for ${storedPayload.metadata.finalUrl}`, + ); + } + + return storedPayload; +} From 1d7cd3a2a5f8b761d14073afd98bda301b43555d Mon Sep 17 00:00:00 2001 From: Matt MacDougall Date: Sat, 4 Jul 2026 15:27:31 -0400 Subject: [PATCH 2/2] review: cap PSI concurrency, keep Lighthouse opt-in, add pagespeed:check script Addresses the review on #52: - Root cause of the timeouts/500s: the audit's Lighthouse batch fires up to 10 URLs x 2 strategies at PSI simultaneously, and PSI's Lighthouse backend rejects parallel bursts (HTTP 500 'Lighthouse returned error') and queues the rest past the request timeout. Measured live: 20 concurrent runs fail ~10% instantly; 4 concurrent completes 20/20. All PSI calls now go through a small in-module queue (4 at a time, queue wait does not consume the request timeout) and the per-request timeout is raised to 120s (successful runs under load measured at 60-90s). - runLighthouse defaults back to false (no surprise spend for anyone without the free key), and the extra checkbox copy is removed. - New scripts/pagespeed-lighthouse-check.ts (pnpm pagespeed:check): exercises the exact audit code path (same module, same queue, same retry shape) against any URLs. Verified 6/6 against the URLs from the review. Co-Authored-By: Claude Opus 4.8 --- package.json | 1 + scripts/pagespeed-lighthouse-check.ts | 133 ++++++++++++++++++ .../features/audit/launch/LaunchFormCard.tsx | 4 - src/client/features/audit/launch/types.ts | 4 +- src/server/lib/pagespeedLighthousePayload.ts | 65 +++++++-- 5 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 scripts/pagespeed-lighthouse-check.ts diff --git a/package.json b/package.json index 36478d2c..109e5dba 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "test:e2e:domain": "playwright test e2e/domain-overview-filters.spec.ts", "test:e2e:domain:perf": "playwright test e2e/domain-overview-filters.perf.spec.ts", "test:e2e:keywords": "playwright test e2e/keyword-research-navigation.spec.ts", + "pagespeed:check": "tsx scripts/pagespeed-lighthouse-check.ts", "billing:backlinks": "tsx scripts/backlinks-cost-profile.ts", "billing:brand-lookup": "tsx scripts/brand-lookup-cost-profile.ts", "cleanup:default-projects:d1": "tsx scripts/d1-default-project-cleanup.ts", diff --git a/scripts/pagespeed-lighthouse-check.ts b/scripts/pagespeed-lighthouse-check.ts new file mode 100644 index 00000000..8c7ade3c --- /dev/null +++ b/scripts/pagespeed-lighthouse-check.ts @@ -0,0 +1,133 @@ +import process from "node:process"; +import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload"; +import { loadLocalEnv, parseArgs } from "./cli-utils"; + +/** + * Exercises the PageSpeed Insights Lighthouse path exactly as the site audit + * does (same module, same concurrency queue, same parsing), without running a + * crawl or touching DataForSEO. + * + * Usage: + * pnpm pagespeed:check --url=https://openseo.so + * pnpm pagespeed:check --url=https://openseo.so/,https://openseo.so/docs/skills + * pnpm pagespeed:check --url=https://openseo.so --strategy=mobile + * pnpm pagespeed:check --url=... --sequential=true # one run at a time + * + * By default every URL runs on both strategies, all fired at once, which is + * the same burst shape as an audit's Lighthouse batch, and each run retries + * up to 3 attempts with backoff exactly like the audit does (PSI is slow and + * occasionally errors under load; pass --attempts=1 to see raw single-shot + * behavior). Requires PAGESPEED_API_KEY in the environment or .env.local / + * .env (free key: + * https://developers.google.com/speed/docs/insights/v5/get-started). + */ + +loadLocalEnv(); + +const args = parseArgs(process.argv.slice(2)); + +await main(); + +async function main() { + if (!process.env.PAGESPEED_API_KEY?.trim()) { + printUsageAndExit( + "PAGESPEED_API_KEY is not set. Add it to .env.local (free key: https://developers.google.com/speed/docs/insights/v5/get-started).", + ); + } + + const urls = (args.url ?? "") + .split(",") + .map((url) => url.trim()) + .filter(Boolean); + if (urls.length === 0) { + printUsageAndExit("Pass at least one URL via --url=https://example.com"); + } + + const strategies: Array<"mobile" | "desktop"> = + args.strategy === "mobile" + ? ["mobile"] + : args.strategy === "desktop" + ? ["desktop"] + : ["mobile", "desktop"]; + + const runs = urls.flatMap((url) => + strategies.map((strategy) => ({ url, strategy })), + ); + console.log( + `Running ${runs.length} Lighthouse check(s) via PageSpeed Insights` + + `${args.sequential === "true" ? " (sequential)" : ""}...\n`, + ); + + const startedAt = Date.now(); + let results: Array<{ ok: boolean; line: string }>; + if (args.sequential === "true") { + results = []; + for (const run of runs) { + results.push(await checkOne(run)); + } + } else { + // Fire everything at once: the module's internal queue caps how many + // requests actually reach PSI in parallel, mirroring an audit batch. + results = await Promise.all(runs.map((run) => checkOne(run))); + } + + console.log(`\n${"-".repeat(72)}`); + const failed = results.filter((result) => !result.ok).length; + console.log( + `${results.length - failed}/${results.length} succeeded in ${elapsedSince(startedAt)}.`, + ); + if (failed > 0) { + console.log( + "PSI occasionally fails individual runs; the audit retries each URL up to 3 times with backoff.", + ); + process.exitCode = 1; + } +} + +async function checkOne(run: { + url: string; + strategy: "mobile" | "desktop"; +}): Promise<{ ok: boolean; line: string }> { + // Same retry shape as the audit's Lighthouse phase: up to 3 attempts with + // 2s/4s backoff, so the result reflects what an audit would actually get. + const maxAttempts = Math.max(1, Number(args.attempts ?? "3") || 3); + const startedAt = Date.now(); + let lastError: unknown = null; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (attempt > 1) { + await new Promise((resolve) => + setTimeout(resolve, 2000 * Math.pow(2, attempt - 2)), + ); + } + try { + const payload = await fetchPagespeedLighthouse(run); + const { performance, accessibility, seo } = payload.scores; + const bestPractices = payload.scores["best-practices"]; + const line = + `OK ${run.strategy.padEnd(7)} ${run.url} ` + + `perf=${performance} a11y=${accessibility} bp=${bestPractices} seo=${seo} ` + + `(${elapsedSince(startedAt)}, attempt ${attempt}/${maxAttempts}, lighthouse ${payload.metadata.lighthouseVersion ?? "?"})`; + console.log(line); + return { ok: true, line }; + } catch (error) { + lastError = error; + } + } + const message = + lastError instanceof Error ? lastError.message : String(lastError); + const line = `FAIL ${run.strategy.padEnd(7)} ${run.url} (${elapsedSince(startedAt)}, ${maxAttempts} attempts) ${message}`; + console.log(line); + return { ok: false, line }; +} + +function elapsedSince(startedAt: number): string { + return `${((Date.now() - startedAt) / 1000).toFixed(1)}s`; +} + +function printUsageAndExit(message: string): never { + console.error(`\n${message}\n`); + console.error( + "Usage: pnpm pagespeed:check --url=https://example.com[,https://example.com/page] [--strategy=mobile|desktop] [--sequential=true] [--attempts=1]", + ); + process.exit(1); +} diff --git a/src/client/features/audit/launch/LaunchFormCard.tsx b/src/client/features/audit/launch/LaunchFormCard.tsx index b50ea1c8..72997e4d 100644 --- a/src/client/features/audit/launch/LaunchFormCard.tsx +++ b/src/client/features/audit/launch/LaunchFormCard.tsx @@ -148,10 +148,6 @@ function LighthouseOptions({ launchForm }: Pick) { We choose a sample of 20 pages to audit, removing pages from duplicate templates.

-

- Runs on the free Google PageSpeed Insights API when - PAGESPEED_API_KEY is set. -

) : null } diff --git a/src/client/features/audit/launch/types.ts b/src/client/features/audit/launch/types.ts index ac81bd59..8fec84c6 100644 --- a/src/client/features/audit/launch/types.ts +++ b/src/client/features/audit/launch/types.ts @@ -10,7 +10,5 @@ export type LaunchFormValues = { export const DEFAULT_LAUNCH_FORM_VALUES: LaunchFormValues = { url: "", maxPagesInput: "50", - // On by default: without Lighthouse the results view has no Performance tab - // and the only way to add one is re-crawling the whole site. - runLighthouse: true, + runLighthouse: false, }; diff --git a/src/server/lib/pagespeedLighthousePayload.ts b/src/server/lib/pagespeedLighthousePayload.ts index 2a10a91a..3dc80791 100644 --- a/src/server/lib/pagespeedLighthousePayload.ts +++ b/src/server/lib/pagespeedLighthousePayload.ts @@ -22,8 +22,9 @@ type LighthouseStrategy = "mobile" | "desktop"; // audit UI (scores, metrics, and per-audit issues) is provider-agnostic. const PSI_ENDPOINT = "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"; -// Lighthouse runs on Google's side are slow; some pages take well over a minute. -const PSI_REQUEST_TIMEOUT_MS = 90_000; +// Lighthouse runs on Google's side are slow; under load some runs take 60-90s+. +// (Queue wait for a concurrency slot below does not count against this budget.) +const PSI_REQUEST_TIMEOUT_MS = 120_000; const PSI_REQUEST_CATEGORIES = [ "PERFORMANCE", "ACCESSIBILITY", @@ -31,6 +32,36 @@ const PSI_REQUEST_CATEGORIES = [ "SEO", ] as const; +// PSI's Lighthouse backend rejects bursts of parallel runs (HTTP 500 +// "Lighthouse returned error: Something went wrong.") and queues the rest past +// our timeout. The audit's Lighthouse phase fires batches of up to 10 URLs x 2 +// strategies at once, so every PSI call is funneled through this small +// in-isolate queue. Measured against a live site: 20 concurrent runs fail ~10% +// instantly, while 4 concurrent completes 20/20. +const PSI_MAX_CONCURRENT_REQUESTS = 4; +let psiSlotsAvailable = PSI_MAX_CONCURRENT_REQUESTS; +const psiSlotWaiters: Array<() => void> = []; + +async function withPsiSlot(run: () => Promise): Promise { + if (psiSlotsAvailable > 0) { + psiSlotsAvailable -= 1; + } else { + await new Promise((resolve) => psiSlotWaiters.push(resolve)); + } + try { + return await run(); + } finally { + const next = psiSlotWaiters.shift(); + // Hand the slot directly to the next waiter so a newly arriving caller + // can't jump the queue between release and resume. + if (next) { + next(); + } else { + psiSlotsAvailable += 1; + } + } +} + // Newer Lighthouse (13.x) returns `details.items` as an object for some audits // (e.g. document-latency-insight) instead of an array. Accept either and // normalise to an array so one stray audit shape can't fail the whole parse. @@ -110,21 +141,25 @@ export async function fetchPagespeedLighthouse(input: { const apiKey = (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(); if (apiKey) params.set("key", apiKey); - const response = await fetch(`${PSI_ENDPOINT}?${params.toString()}`, { - headers: { accept: "application/json" }, - signal: AbortSignal.timeout(PSI_REQUEST_TIMEOUT_MS), - }); + const raw = await withPsiSlot(async () => { + const response = await fetch(`${PSI_ENDPOINT}?${params.toString()}`, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(PSI_REQUEST_TIMEOUT_MS), + }); - if (!response.ok) { - const body = await response.text().catch(() => ""); - throw new Error( - `PageSpeed Insights HTTP ${response.status}${ - body ? `: ${body.slice(0, 200)}` : "" - }`, - ); - } + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `PageSpeed Insights HTTP ${response.status}${ + body ? `: ${body.slice(0, 200)}` : "" + }`, + ); + } + + return response.json(); + }); - const parsed = psiResponseSchema.safeParse(await response.json()); + const parsed = psiResponseSchema.safeParse(raw); if (!parsed.success) { throw new Error( `PageSpeed Insights returned an invalid response: ${summarizeZodIssues(parsed.error)}`,