fix(trigger): stop prod runs failing on backfill + transient upstreams

.gitignore

@@ -77,3 +77,6 @@ worktrees/
 # Claude Code local project memory
 .claude/projects/
 .fallow/
+
+# Trigger.dev local build/temp state
+.trigger/

src/trigger/enrich-ai.ts

@@ -229,6 +229,16 @@ export const enrichAiTask = task({
   id: "enrich-ai",
   queue: scrapeQueue,
   maxDuration: 120,
+  // Reads/writes `enrichments` via neon-http. Transient Neon HTTP errors
+  // (rate-limit/timeout during a fan-out burst) need room to clear — the
+  // global 3×/1s default exhausts in ~6s, too tight to outlast a blip.
+  retry: {
+    maxAttempts: 5,
+    minTimeoutInMs: 2000,
+    maxTimeoutInMs: 30_000,
+    factor: 2,
+    randomize: true,
+  },
 
   onFailure: async ({
     error,
@@ -239,6 +249,16 @@ export const enrichAiTask = task({
   }) => {
     const db = getDb();
     const message = error instanceof Error ? error.message : String(error);
+    // neon-http masks DB failures as a generic `Failed query: <sql>` and
+    // strips the cause — surface it so prod failures aren't a black box.
+    logger.error("enrich-ai: run failed", {
+      runId: ctx.run.id,
+      cause:
+        (error as { cause?: unknown }).cause ??
+        (error as { sourceError?: unknown }).sourceError ??
+        null,
+      message,
+    });
     await db
       .update(schema.aiRuns)
       .set({

src/trigger/enrich-broadband.ts

@@ -35,6 +35,16 @@ export const enrichBroadbandTask = task({
   id: "enrich-broadband",
   queue: scrapeQueue,
   maxDuration: 60,
+  // Writes to `enrichments` via neon-http. Transient Neon HTTP errors
+  // (rate-limit/timeout during a fan-out burst) need room to clear — the
+  // global 3×/1s default exhausts in ~6s, too tight to outlast a blip.
+  retry: {
+    maxAttempts: 5,
+    minTimeoutInMs: 2000,
+    maxTimeoutInMs: 30_000,
+    factor: 2,
+    randomize: true,
+  },
 
   run: async (
     payload: EnrichBroadbandPayload

src/trigger/enrich-epc.ts

@@ -304,12 +304,25 @@ async function searchEpcCerts(
     query: { postcode, size: 100 },
   });
   if (search.error) {
+    const status = search.response?.status;
     const message =
       typeof search.error === "object" &&
       search.error !== null &&
       "message" in search.error
         ? String((search.error as { message: unknown }).message)
         : JSON.stringify(search.error);
+    // Upstream outage — HTTP 5xx, or a network/parse failure that left no
+    // response status (the register sometimes serves an HTML "Error 500"
+    // page). Skip this round instead of failing the run: EPC enrichment
+    // re-runs on cadence and will pick the cluster up next time. Genuine
+    // 4xx (bad postcode, auth) still throws so real misconfig stays loud.
+    if (status === undefined || status >= 500) {
+      logger.warn("enrich-epc: EPC register unavailable, skipping", {
+        postcode,
+        status: status ?? null,
+      });
+      return [];
+    }
     throw new Error(`enrich-epc: EPC search failed: ${message}`);
   }
   return extractCertRows(search.data);

src/trigger/enrich-helpers.ts

@@ -8,6 +8,7 @@
  * lives here. The `numeric`-column coercion is shared for the same
  * reason.
  */
+import { logger } from "@trigger.dev/sdk";
 import { and, eq } from "drizzle-orm";
 import { nanoid } from "nanoid";
 import type { getDb } from "../../db";
@@ -47,33 +48,50 @@ export async function upsertEnrichmentForListings(
 ): Promise<number> {
   let touched = 0;
   for (const listingId of listingIds) {
-    const inserted = await db
-      .insert(schema.enrichments)
-      .values({
-        id: nanoid(),
-        listingId,
-        promptVersion: PROMPT_VERSION,
-        features: {},
-        ...patch,
-      })
-      .onConflictDoNothing({
-        target: [
-          schema.enrichments.listingId,
-          schema.enrichments.promptVersion,
-        ],
-      })
-      .returning({ id: schema.enrichments.id });
+    try {
+      const inserted = await db
+        .insert(schema.enrichments)
+        .values({
+          id: nanoid(),
+          listingId,
+          promptVersion: PROMPT_VERSION,
+          features: {},
+          ...patch,
+        })
+        .onConflictDoNothing({
+          target: [
+            schema.enrichments.listingId,
+            schema.enrichments.promptVersion,
+          ],
+        })
+        .returning({ id: schema.enrichments.id });
 
-    if (inserted.length === 0) {
-      await db
-        .update(schema.enrichments)
-        .set(patch)
-        .where(
-          and(
-            eq(schema.enrichments.listingId, listingId),
-            eq(schema.enrichments.promptVersion, PROMPT_VERSION)
-          )
-        );
+      if (inserted.length === 0) {
+        await db
+          .update(schema.enrichments)
+          .set(patch)
+          .where(
+            and(
+              eq(schema.enrichments.listingId, listingId),
+              eq(schema.enrichments.promptVersion, PROMPT_VERSION)
+            )
+          );
+      }
+    } catch (err) {
+      // The neon-http driver rethrows as a generic `Failed query: <sql>`
+      // and strips the underlying error, so prod failures are a black box.
+      // Surface the real cause (rate-limit / timeout / Postgres error)
+      // before letting the task retry on it.
+      logger.error("enrich-helpers: enrichment write failed", {
+        listingId,
+        columns: Object.keys(patch),
+        cause:
+          (err as { cause?: unknown }).cause ??
+          (err as { sourceError?: unknown }).sourceError ??
+          null,
+        message: err instanceof Error ? err.message : String(err),
+      });
+      throw err;
     }
     touched += 1;
   }

src/trigger/scrape-portal.ts

@@ -971,7 +971,30 @@ export const scrapePortalTask = task({
             url,
             page === 0 ? scopeFor(target.label) : null
           );
-          const parsed = parseSearchPage(portal, html);
+          let parsed: ListingSummary[];
+          try {
+            parsed = parseSearchPage(portal, html);
+          } catch (err) {
+            // A page without the portal's JSON island past page 0 means we
+            // walked off the end of the result set (or hit a soft block
+            // mid-backfill). Keep the summaries already collected and stop
+            // paginating instead of failing the whole run — otherwise a
+            // backfill that reached page 2 throws away pages 0/1 too. Page 0
+            // failing is a genuine block: rethrow so the task retries.
+            if (page === 0) {
+              throw err;
+            }
+            logger.warn(
+              "scrape-portal: unparseable page, stopping pagination",
+              {
+                portal,
+                outcode: target.label,
+                page,
+                error: err instanceof Error ? err.message : String(err),
+              }
+            );
+            break;
+          }
           let added = 0;
           for (const s of parsed) {
             if (!seen.has(s.portalListingId)) {