diff --git a/.env.example b/.env.example index ab00267e..5f559ff2 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,12 @@ # Required for all modes. See README.md for setup instructions. # DATAFORSEO_API_KEY= +# Optional: free Google PageSpeed Insights API key. When set, site-audit +# Lighthouse runs on Google's free API (~25k requests/day) instead of the +# metered DataForSEO OnPage endpoint. Create one (2 minutes, no billing): +# https://developers.google.com/speed/docs/insights/v5/get-started +# PAGESPEED_API_KEY= + # Optional app port # PORT=3001 diff --git a/compose.yaml b/compose.yaml index 87b1dad9..efea2e76 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,9 @@ services: - ALLOWED_HOST=${ALLOWED_HOST:-} - AUTH_MODE=local_noauth - DATAFORSEO_API_KEY=${DATAFORSEO_API_KEY} + # Optional: free Google PageSpeed Insights key. When set, site-audit + # Lighthouse runs on Google's free API instead of DataForSEO OnPage. + - PAGESPEED_API_KEY=${PAGESPEED_API_KEY:-} # Optional: Google Search Console. See # docs/SELF_HOSTING_GOOGLE_SEARCH_CONSOLE.md - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-} diff --git a/docs/SELF_HOSTING_DOCKER.md b/docs/SELF_HOSTING_DOCKER.md index 66ef274f..a8c171ea 100644 --- a/docs/SELF_HOSTING_DOCKER.md +++ b/docs/SELF_HOSTING_DOCKER.md @@ -29,6 +29,17 @@ Optional env values: - `ALLOWED_HOST` (single reverse-proxy hostname to allow in Vite preview) - `AUTH_MODE=local_noauth` (already set in compose) - `OPEN_SEO_IMAGE` (defaults to `ghcr.io/every-app/open-seo:latest`) +- `PAGESPEED_API_KEY` (free Lighthouse for site audits, see below) + +## Free Lighthouse audits (PAGESPEED_API_KEY) + +Site audits run Lighthouse through the metered DataForSEO OnPage endpoint by +default. If you set a free Google PageSpeed Insights API key, Lighthouse runs +on Google's API instead at no cost (about 25k requests/day per key): + +1. Create an API key (2 minutes, no billing): https://developers.google.com/speed/docs/insights/v5/get-started +2. Set `PAGESPEED_API_KEY` in `.env` +3. Recreate the container: `docker compose up -d --force-recreate open-seo` If you are putting Docker behind a reverse proxy or a temporary tunnel, remember that Docker self-hosting runs with app auth disabled. Only expose it behind your own auth-protected reverse proxy, tunnel, or private network, and add the public hostname before restarting: diff --git a/package.json b/package.json index 45ee417d..4018e311 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "test:e2e:domain": "playwright test e2e/domain-overview-filters.spec.ts", "test:e2e:domain:perf": "playwright test e2e/domain-overview-filters.perf.spec.ts", "test:e2e:keywords": "playwright test e2e/keyword-research-navigation.spec.ts", + "pagespeed:check": "tsx scripts/pagespeed-lighthouse-check.ts", "billing:backlinks": "tsx scripts/backlinks-cost-profile.ts", "billing:brand-lookup": "tsx scripts/brand-lookup-cost-profile.ts", "cleanup:default-projects:d1": "tsx scripts/d1-default-project-cleanup.ts", diff --git a/scripts/pagespeed-lighthouse-check.ts b/scripts/pagespeed-lighthouse-check.ts new file mode 100644 index 00000000..8c7ade3c --- /dev/null +++ b/scripts/pagespeed-lighthouse-check.ts @@ -0,0 +1,133 @@ +import process from "node:process"; +import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload"; +import { loadLocalEnv, parseArgs } from "./cli-utils"; + +/** + * Exercises the PageSpeed Insights Lighthouse path exactly as the site audit + * does (same module, same concurrency queue, same parsing), without running a + * crawl or touching DataForSEO. + * + * Usage: + * pnpm pagespeed:check --url=https://openseo.so + * pnpm pagespeed:check --url=https://openseo.so/,https://openseo.so/docs/skills + * pnpm pagespeed:check --url=https://openseo.so --strategy=mobile + * pnpm pagespeed:check --url=... --sequential=true # one run at a time + * + * By default every URL runs on both strategies, all fired at once, which is + * the same burst shape as an audit's Lighthouse batch, and each run retries + * up to 3 attempts with backoff exactly like the audit does (PSI is slow and + * occasionally errors under load; pass --attempts=1 to see raw single-shot + * behavior). Requires PAGESPEED_API_KEY in the environment or .env.local / + * .env (free key: + * https://developers.google.com/speed/docs/insights/v5/get-started). + */ + +loadLocalEnv(); + +const args = parseArgs(process.argv.slice(2)); + +await main(); + +async function main() { + if (!process.env.PAGESPEED_API_KEY?.trim()) { + printUsageAndExit( + "PAGESPEED_API_KEY is not set. Add it to .env.local (free key: https://developers.google.com/speed/docs/insights/v5/get-started).", + ); + } + + const urls = (args.url ?? "") + .split(",") + .map((url) => url.trim()) + .filter(Boolean); + if (urls.length === 0) { + printUsageAndExit("Pass at least one URL via --url=https://example.com"); + } + + const strategies: Array<"mobile" | "desktop"> = + args.strategy === "mobile" + ? ["mobile"] + : args.strategy === "desktop" + ? ["desktop"] + : ["mobile", "desktop"]; + + const runs = urls.flatMap((url) => + strategies.map((strategy) => ({ url, strategy })), + ); + console.log( + `Running ${runs.length} Lighthouse check(s) via PageSpeed Insights` + + `${args.sequential === "true" ? " (sequential)" : ""}...\n`, + ); + + const startedAt = Date.now(); + let results: Array<{ ok: boolean; line: string }>; + if (args.sequential === "true") { + results = []; + for (const run of runs) { + results.push(await checkOne(run)); + } + } else { + // Fire everything at once: the module's internal queue caps how many + // requests actually reach PSI in parallel, mirroring an audit batch. + results = await Promise.all(runs.map((run) => checkOne(run))); + } + + console.log(`\n${"-".repeat(72)}`); + const failed = results.filter((result) => !result.ok).length; + console.log( + `${results.length - failed}/${results.length} succeeded in ${elapsedSince(startedAt)}.`, + ); + if (failed > 0) { + console.log( + "PSI occasionally fails individual runs; the audit retries each URL up to 3 times with backoff.", + ); + process.exitCode = 1; + } +} + +async function checkOne(run: { + url: string; + strategy: "mobile" | "desktop"; +}): Promise<{ ok: boolean; line: string }> { + // Same retry shape as the audit's Lighthouse phase: up to 3 attempts with + // 2s/4s backoff, so the result reflects what an audit would actually get. + const maxAttempts = Math.max(1, Number(args.attempts ?? "3") || 3); + const startedAt = Date.now(); + let lastError: unknown = null; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (attempt > 1) { + await new Promise((resolve) => + setTimeout(resolve, 2000 * Math.pow(2, attempt - 2)), + ); + } + try { + const payload = await fetchPagespeedLighthouse(run); + const { performance, accessibility, seo } = payload.scores; + const bestPractices = payload.scores["best-practices"]; + const line = + `OK ${run.strategy.padEnd(7)} ${run.url} ` + + `perf=${performance} a11y=${accessibility} bp=${bestPractices} seo=${seo} ` + + `(${elapsedSince(startedAt)}, attempt ${attempt}/${maxAttempts}, lighthouse ${payload.metadata.lighthouseVersion ?? "?"})`; + console.log(line); + return { ok: true, line }; + } catch (error) { + lastError = error; + } + } + const message = + lastError instanceof Error ? lastError.message : String(lastError); + const line = `FAIL ${run.strategy.padEnd(7)} ${run.url} (${elapsedSince(startedAt)}, ${maxAttempts} attempts) ${message}`; + console.log(line); + return { ok: false, line }; +} + +function elapsedSince(startedAt: number): string { + return `${((Date.now() - startedAt) / 1000).toFixed(1)}s`; +} + +function printUsageAndExit(message: string): never { + console.error(`\n${message}\n`); + console.error( + "Usage: pnpm pagespeed:check --url=https://example.com[,https://example.com/page] [--strategy=mobile|desktop] [--sequential=true] [--attempts=1]", + ); + process.exit(1); +} diff --git a/src/env.d.ts b/src/env.d.ts index 6f420ddb..d377e0c0 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -40,6 +40,10 @@ declare namespace Cloudflare { // DataForSEO API Basic auth value (base64 of login:password) DATAFORSEO_API_KEY: string; + // Optional (free) Google PageSpeed Insights API key. When set, site-audit + // Lighthouse runs on Google's free API instead of DataForSEO OnPage. + PAGESPEED_API_KEY?: string; + // OpenRouter API key for the in-app chat agents (onboarding + SAM). OPENROUTER_API_KEY?: string; // Optional OpenRouter model slug override (defaults in openrouter.ts). diff --git a/src/server/lib/audit/lighthouse.ts b/src/server/lib/audit/lighthouse.ts index 05907750..2021cff3 100644 --- a/src/server/lib/audit/lighthouse.ts +++ b/src/server/lib/audit/lighthouse.ts @@ -1,6 +1,8 @@ import { detectUrlTemplate, canonicalUrlKey } from "./url-utils"; import type { BillingCustomerContext } from "@/server/billing/subscription"; import { createDataforseoClient } from "@/server/lib/dataforseo"; +import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload"; +import { getOptionalEnvValue } from "@/server/lib/runtime-env"; import type { LighthouseResult, LighthouseStrategy } from "./types"; import { putTextToR2 } from "@/server/lib/r2"; @@ -30,6 +32,13 @@ async function fetchLighthouseResult( ): Promise { let lastError: Error | null = null; const dataforseo = createDataforseoClient(billingCustomer); + // When a (free) Google PageSpeed Insights API key is configured, run + // Lighthouse there instead of the metered DataForSEO OnPage endpoint. Both + // paths normalize to the same StoredLighthousePayload, so everything + // downstream (R2 payloads, scores, issue details) is provider-agnostic. + const usePagespeed = Boolean( + (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(), + ); for (let attempt = 0; attempt < 3; attempt++) { try { @@ -40,7 +49,9 @@ async function fetchLighthouseResult( ); } - const data = await dataforseo.lighthouse.live({ url, strategy }); + const data = usePagespeed + ? await fetchPagespeedLighthouse({ url, strategy }) + : await dataforseo.lighthouse.live({ url, strategy }); return { result: { diff --git a/src/server/lib/lighthouseStoredPayload.ts b/src/server/lib/lighthouseStoredPayload.ts index 03376d56..82304d15 100644 --- a/src/server/lib/lighthouseStoredPayload.ts +++ b/src/server/lib/lighthouseStoredPayload.ts @@ -55,7 +55,7 @@ const storedLighthouseIssueSchema = z.object({ export const storedLighthousePayloadSchema = z.object({ version: z.literal(2), - source: z.literal("dataforseo-lighthouse"), + source: z.enum(["dataforseo-lighthouse", "pagespeed-insights"]), hasIssueDetails: z.boolean(), metadata: z.object({ requestedUrl: z.string(), diff --git a/src/server/lib/pagespeedLighthousePayload.ts b/src/server/lib/pagespeedLighthousePayload.ts new file mode 100644 index 00000000..3dc80791 --- /dev/null +++ b/src/server/lib/pagespeedLighthousePayload.ts @@ -0,0 +1,210 @@ +import { z } from "zod"; +import { + buildStoredLighthouseIssues, + buildStoredLighthouseMetrics, + type RawLighthouseAudit, + type RawLighthouseCategory, + scoreToPercent, + type StoredLighthousePayload, +} from "@/server/lib/lighthouseStoredPayload"; +import { getOptionalEnvValue } from "@/server/lib/runtime-env"; + +type LighthouseStrategy = "mobile" | "desktop"; + +// Google PageSpeed Insights runs Lighthouse on Google's own infrastructure and +// returns the standard Lighthouse report under `lighthouseResult`. It is free, +// but a (free) PAGESPEED_API_KEY is effectively required: keyless requests share +// a tiny global anonymous quota that is almost always exhausted (HTTP 429). With +// a key each project gets ~25k requests/day, which is why the audit only routes +// here when PAGESPEED_API_KEY is set (see src/server/lib/audit/lighthouse.ts) +// and otherwise falls back to DataForSEO OnPage. The report is normalised into +// the exact same StoredLighthousePayload the DataForSEO path produces, so the +// audit UI (scores, metrics, and per-audit issues) is provider-agnostic. +const PSI_ENDPOINT = + "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"; +// Lighthouse runs on Google's side are slow; under load some runs take 60-90s+. +// (Queue wait for a concurrency slot below does not count against this budget.) +const PSI_REQUEST_TIMEOUT_MS = 120_000; +const PSI_REQUEST_CATEGORIES = [ + "PERFORMANCE", + "ACCESSIBILITY", + "BEST_PRACTICES", + "SEO", +] as const; + +// PSI's Lighthouse backend rejects bursts of parallel runs (HTTP 500 +// "Lighthouse returned error: Something went wrong.") and queues the rest past +// our timeout. The audit's Lighthouse phase fires batches of up to 10 URLs x 2 +// strategies at once, so every PSI call is funneled through this small +// in-isolate queue. Measured against a live site: 20 concurrent runs fail ~10% +// instantly, while 4 concurrent completes 20/20. +const PSI_MAX_CONCURRENT_REQUESTS = 4; +let psiSlotsAvailable = PSI_MAX_CONCURRENT_REQUESTS; +const psiSlotWaiters: Array<() => void> = []; + +async function withPsiSlot(run: () => Promise): Promise { + if (psiSlotsAvailable > 0) { + psiSlotsAvailable -= 1; + } else { + await new Promise((resolve) => psiSlotWaiters.push(resolve)); + } + try { + return await run(); + } finally { + const next = psiSlotWaiters.shift(); + // Hand the slot directly to the next waiter so a newly arriving caller + // can't jump the queue between release and resume. + if (next) { + next(); + } else { + psiSlotsAvailable += 1; + } + } +} + +// Newer Lighthouse (13.x) returns `details.items` as an object for some audits +// (e.g. document-latency-insight) instead of an array. Accept either and +// normalise to an array so one stray audit shape can't fail the whole parse. +const psiAuditItemsSchema = z + .union([ + z.array(z.record(z.string(), z.unknown())), + z.record(z.string(), z.unknown()), + ]) + .transform((items) => (Array.isArray(items) ? items : [items])); + +const psiAuditSchema = z + .object({ + score: z.number().nullable().optional(), + displayValue: z.string().optional(), + numericValue: z.number().optional(), + title: z.string().optional(), + description: z.string().optional(), + scoreDisplayMode: z.string().optional(), + details: z + .object({ + overallSavingsMs: z.number().optional(), + overallSavingsBytes: z.number().optional(), + items: psiAuditItemsSchema.optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +const psiCategorySchema = z + .object({ + score: z.number().nullable().optional(), + auditRefs: z + .array(z.object({ id: z.string().optional() }).passthrough()) + .optional(), + }) + .passthrough(); + +const psiResponseSchema = z + .object({ + lighthouseResult: z + .object({ + requestedUrl: z.string().optional(), + finalUrl: z.string().optional(), + finalDisplayedUrl: z.string().optional(), + lighthouseVersion: z.string().optional(), + categories: z + .record(z.string(), psiCategorySchema) + .optional() + .default({}), + audits: z.record(z.string(), psiAuditSchema).optional().default({}), + }) + .passthrough(), + }) + .passthrough(); + +function summarizeZodIssues(error: z.ZodError, maxIssues = 3): string { + return error.issues + .slice(0, maxIssues) + .map((issue) => { + const path = issue.path.length > 0 ? issue.path.join(".") : ""; + return `${path}: ${issue.message}`; + }) + .join("; "); +} + +export async function fetchPagespeedLighthouse(input: { + url: string; + strategy: LighthouseStrategy; +}): Promise { + const params = new URLSearchParams(); + params.set("url", input.url); + params.set("strategy", input.strategy); + for (const category of PSI_REQUEST_CATEGORIES) { + params.append("category", category); + } + const apiKey = (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(); + if (apiKey) params.set("key", apiKey); + + const raw = await withPsiSlot(async () => { + const response = await fetch(`${PSI_ENDPOINT}?${params.toString()}`, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(PSI_REQUEST_TIMEOUT_MS), + }); + + if (!response.ok) { + const body = await response.text().catch(() => ""); + throw new Error( + `PageSpeed Insights HTTP ${response.status}${ + body ? `: ${body.slice(0, 200)}` : "" + }`, + ); + } + + return response.json(); + }); + + const parsed = psiResponseSchema.safeParse(raw); + if (!parsed.success) { + throw new Error( + `PageSpeed Insights returned an invalid response: ${summarizeZodIssues(parsed.error)}`, + ); + } + + const result = parsed.data.lighthouseResult; + const categories: Record = + result.categories ?? {}; + const audits: Record = result.audits ?? {}; + const issueReport = buildStoredLighthouseIssues({ audits, categories }); + const metrics = buildStoredLighthouseMetrics({ audits }); + + const storedPayload: StoredLighthousePayload = { + version: 2, + source: "pagespeed-insights", + hasIssueDetails: issueReport.hasIssueDetails, + metadata: { + requestedUrl: result.requestedUrl ?? input.url, + finalUrl: result.finalUrl ?? result.finalDisplayedUrl ?? input.url, + strategy: input.strategy, + fetchedAt: new Date().toISOString(), + lighthouseVersion: result.lighthouseVersion ?? null, + taskId: null, + // PageSpeed Insights is free; no per-request cost to record. + cost: 0, + }, + scores: { + performance: scoreToPercent(categories.performance?.score), + accessibility: scoreToPercent(categories.accessibility?.score), + "best-practices": scoreToPercent(categories["best-practices"]?.score), + seo: scoreToPercent(categories.seo?.score), + }, + metrics, + issues: issueReport.issues, + }; + + const allScoresMissing = Object.values(storedPayload.scores).every( + (score) => score == null, + ); + if (allScoresMissing) { + throw new Error( + `PageSpeed Insights returned no category scores for ${storedPayload.metadata.finalUrl}`, + ); + } + + return storedPayload; +}