feat: free Lighthouse audits via PageSpeed Insights when PAGESPEED_API_KEY is set

.env.example

@@ -4,6 +4,12 @@
 # Required for all modes. See README.md for setup instructions.
 # DATAFORSEO_API_KEY=
 
+# Optional: free Google PageSpeed Insights API key. When set, site-audit
+# Lighthouse runs on Google's free API (~25k requests/day) instead of the
+# metered DataForSEO OnPage endpoint. Create one (2 minutes, no billing):
+# https://developers.google.com/speed/docs/insights/v5/get-started
+# PAGESPEED_API_KEY=
+
 # Optional app port
 # PORT=3001
 

compose.yaml

@@ -9,6 +9,9 @@ services:
       - ALLOWED_HOST=${ALLOWED_HOST:-}
       - AUTH_MODE=local_noauth
       - DATAFORSEO_API_KEY=${DATAFORSEO_API_KEY}
+      # Optional: free Google PageSpeed Insights key. When set, site-audit
+      # Lighthouse runs on Google's free API instead of DataForSEO OnPage.
+      - PAGESPEED_API_KEY=${PAGESPEED_API_KEY:-}
       # Optional: Google Search Console. See
       # docs/SELF_HOSTING_GOOGLE_SEARCH_CONSOLE.md
       - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID:-}

docs/SELF_HOSTING_DOCKER.md

@@ -29,6 +29,17 @@ Optional env values:
 - `ALLOWED_HOST` (single reverse-proxy hostname to allow in Vite preview)
 - `AUTH_MODE=local_noauth` (already set in compose)
 - `OPEN_SEO_IMAGE` (defaults to `ghcr.io/every-app/open-seo:latest`)
+- `PAGESPEED_API_KEY` (free Lighthouse for site audits, see below)
+
+## Free Lighthouse audits (PAGESPEED_API_KEY)
+
+Site audits run Lighthouse through the metered DataForSEO OnPage endpoint by
+default. If you set a free Google PageSpeed Insights API key, Lighthouse runs
+on Google's API instead at no cost (about 25k requests/day per key):
+
+1. Create an API key (2 minutes, no billing): https://developers.google.com/speed/docs/insights/v5/get-started
+2. Set `PAGESPEED_API_KEY` in `.env`
+3. Recreate the container: `docker compose up -d --force-recreate open-seo`
 
 If you are putting Docker behind a reverse proxy or a temporary tunnel, remember that Docker self-hosting runs with app auth disabled. Only expose it behind your own auth-protected reverse proxy, tunnel, or private network, and add the public hostname before restarting:
 

package.json

@@ -40,6 +40,7 @@
     "test:e2e:domain": "playwright test e2e/domain-overview-filters.spec.ts",
     "test:e2e:domain:perf": "playwright test e2e/domain-overview-filters.perf.spec.ts",
     "test:e2e:keywords": "playwright test e2e/keyword-research-navigation.spec.ts",
+    "pagespeed:check": "tsx scripts/pagespeed-lighthouse-check.ts",
     "billing:backlinks": "tsx scripts/backlinks-cost-profile.ts",
     "billing:brand-lookup": "tsx scripts/brand-lookup-cost-profile.ts",
     "cleanup:default-projects:d1": "tsx scripts/d1-default-project-cleanup.ts",

scripts/pagespeed-lighthouse-check.ts

@@ -0,0 +1,133 @@
+import process from "node:process";
+import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload";
+import { loadLocalEnv, parseArgs } from "./cli-utils";
+
+/**
+ * Exercises the PageSpeed Insights Lighthouse path exactly as the site audit
+ * does (same module, same concurrency queue, same parsing), without running a
+ * crawl or touching DataForSEO.
+ *
+ * Usage:
+ *   pnpm pagespeed:check --url=https://openseo.so
+ *   pnpm pagespeed:check --url=https://openseo.so/,https://openseo.so/docs/skills
+ *   pnpm pagespeed:check --url=https://openseo.so --strategy=mobile
+ *   pnpm pagespeed:check --url=... --sequential=true   # one run at a time
+ *
+ * By default every URL runs on both strategies, all fired at once, which is
+ * the same burst shape as an audit's Lighthouse batch, and each run retries
+ * up to 3 attempts with backoff exactly like the audit does (PSI is slow and
+ * occasionally errors under load; pass --attempts=1 to see raw single-shot
+ * behavior). Requires PAGESPEED_API_KEY in the environment or .env.local /
+ * .env (free key:
+ * https://developers.google.com/speed/docs/insights/v5/get-started).
+ */
+
+loadLocalEnv();
+
+const args = parseArgs(process.argv.slice(2));
+
+await main();
+
+async function main() {
+  if (!process.env.PAGESPEED_API_KEY?.trim()) {
+    printUsageAndExit(
+      "PAGESPEED_API_KEY is not set. Add it to .env.local (free key: https://developers.google.com/speed/docs/insights/v5/get-started).",
+    );
+  }
+
+  const urls = (args.url ?? "")
+    .split(",")
+    .map((url) => url.trim())
+    .filter(Boolean);
+  if (urls.length === 0) {
+    printUsageAndExit("Pass at least one URL via --url=https://example.com");
+  }
+
+  const strategies: Array<"mobile" | "desktop"> =
+    args.strategy === "mobile"
+      ? ["mobile"]
+      : args.strategy === "desktop"
+        ? ["desktop"]
+        : ["mobile", "desktop"];
+
+  const runs = urls.flatMap((url) =>
+    strategies.map((strategy) => ({ url, strategy })),
+  );
+  console.log(
+    `Running ${runs.length} Lighthouse check(s) via PageSpeed Insights` +
+      `${args.sequential === "true" ? " (sequential)" : ""}...\n`,
+  );
+
+  const startedAt = Date.now();
+  let results: Array<{ ok: boolean; line: string }>;
+  if (args.sequential === "true") {
+    results = [];
+    for (const run of runs) {
+      results.push(await checkOne(run));
+    }
+  } else {
+    // Fire everything at once: the module's internal queue caps how many
+    // requests actually reach PSI in parallel, mirroring an audit batch.
+    results = await Promise.all(runs.map((run) => checkOne(run)));
+  }
+
+  console.log(`\n${"-".repeat(72)}`);
+  const failed = results.filter((result) => !result.ok).length;
+  console.log(
+    `${results.length - failed}/${results.length} succeeded in ${elapsedSince(startedAt)}.`,
+  );
+  if (failed > 0) {
+    console.log(
+      "PSI occasionally fails individual runs; the audit retries each URL up to 3 times with backoff.",
+    );
+    process.exitCode = 1;
+  }
+}
+
+async function checkOne(run: {
+  url: string;
+  strategy: "mobile" | "desktop";
+}): Promise<{ ok: boolean; line: string }> {
+  // Same retry shape as the audit's Lighthouse phase: up to 3 attempts with
+  // 2s/4s backoff, so the result reflects what an audit would actually get.
+  const maxAttempts = Math.max(1, Number(args.attempts ?? "3") || 3);
+  const startedAt = Date.now();
+  let lastError: unknown = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    if (attempt > 1) {
+      await new Promise((resolve) =>
+        setTimeout(resolve, 2000 * Math.pow(2, attempt - 2)),
+      );
+    }
+    try {
+      const payload = await fetchPagespeedLighthouse(run);
+      const { performance, accessibility, seo } = payload.scores;
+      const bestPractices = payload.scores["best-practices"];
+      const line =
+        `OK   ${run.strategy.padEnd(7)} ${run.url} ` +
+        `perf=${performance} a11y=${accessibility} bp=${bestPractices} seo=${seo} ` +
+        `(${elapsedSince(startedAt)}, attempt ${attempt}/${maxAttempts}, lighthouse ${payload.metadata.lighthouseVersion ?? "?"})`;
+      console.log(line);
+      return { ok: true, line };
+    } catch (error) {
+      lastError = error;
+    }
+  }
+  const message =
+    lastError instanceof Error ? lastError.message : String(lastError);
+  const line = `FAIL ${run.strategy.padEnd(7)} ${run.url} (${elapsedSince(startedAt)}, ${maxAttempts} attempts) ${message}`;
+  console.log(line);
+  return { ok: false, line };
+}
+
+function elapsedSince(startedAt: number): string {
+  return `${((Date.now() - startedAt) / 1000).toFixed(1)}s`;
+}
+
+function printUsageAndExit(message: string): never {
+  console.error(`\n${message}\n`);
+  console.error(
+    "Usage: pnpm pagespeed:check --url=https://example.com[,https://example.com/page] [--strategy=mobile|desktop] [--sequential=true] [--attempts=1]",
+  );
+  process.exit(1);
+}

src/env.d.ts

@@ -40,6 +40,10 @@ declare namespace Cloudflare {
     // DataForSEO API Basic auth value (base64 of login:password)
     DATAFORSEO_API_KEY: string;
 
+    // Optional (free) Google PageSpeed Insights API key. When set, site-audit
+    // Lighthouse runs on Google's free API instead of DataForSEO OnPage.
+    PAGESPEED_API_KEY?: string;
+
     // OpenRouter API key for the in-app chat agents (onboarding + SAM).
     OPENROUTER_API_KEY?: string;
     // Optional OpenRouter model slug override (defaults in openrouter.ts).

src/server/lib/audit/lighthouse.ts

@@ -1,6 +1,8 @@
 import { detectUrlTemplate, canonicalUrlKey } from "./url-utils";
 import type { BillingCustomerContext } from "@/server/billing/subscription";
 import { createDataforseoClient } from "@/server/lib/dataforseo";
+import { fetchPagespeedLighthouse } from "@/server/lib/pagespeedLighthousePayload";
+import { getOptionalEnvValue } from "@/server/lib/runtime-env";
 import type { LighthouseResult, LighthouseStrategy } from "./types";
 import { putTextToR2 } from "@/server/lib/r2";
 
@@ -30,6 +32,13 @@ async function fetchLighthouseResult(
 ): Promise<LighthouseFetchResult> {
   let lastError: Error | null = null;
   const dataforseo = createDataforseoClient(billingCustomer);
+  // When a (free) Google PageSpeed Insights API key is configured, run
+  // Lighthouse there instead of the metered DataForSEO OnPage endpoint. Both
+  // paths normalize to the same StoredLighthousePayload, so everything
+  // downstream (R2 payloads, scores, issue details) is provider-agnostic.
+  const usePagespeed = Boolean(
+    (await getOptionalEnvValue("PAGESPEED_API_KEY"))?.trim(),
+  );
 
   for (let attempt = 0; attempt < 3; attempt++) {
     try {
@@ -40,7 +49,9 @@ async function fetchLighthouseResult(
         );
       }
 
-      const data = await dataforseo.lighthouse.live({ url, strategy });
+      const data = usePagespeed
+        ? await fetchPagespeedLighthouse({ url, strategy })
+        : await dataforseo.lighthouse.live({ url, strategy });
 
       return {
         result: {

src/server/lib/lighthouseStoredPayload.ts

@@ -55,7 +55,7 @@ const storedLighthouseIssueSchema = z.object({
 
 export const storedLighthousePayloadSchema = z.object({
   version: z.literal(2),
-  source: z.literal("dataforseo-lighthouse"),
+  source: z.enum(["dataforseo-lighthouse", "pagespeed-insights"]),
   hasIssueDetails: z.boolean(),
   metadata: z.object({
     requestedUrl: z.string(),