fix(cli): stop dev runs crashing when a rebuild removes an in-use build dir (#4276)

Author: ericallam

.changeset/fix-dev-build-dir-race.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes intermittent `trigger dev` run crashes where a run could fail at boot with a cryptic `Cannot find module .../dev-run-worker.mjs` after a rebuild had cleaned up the build directory the run was launched against. Dev runs now retry cleanly instead of hard-crashing when their build directory is missing, the dev watchdog no longer removes the build tree of a still-running session, and a run assigned to a worker version that was superseded by a rebuild now fails fast with a clear message instead of silently hanging until it times out.

packages/cli-v3/src/dev/devSupervisor.ts

@@ -1,7 +1,9 @@
 import { tryCatch } from "@trigger.dev/core/utils";
+import { TaskRunErrorCodes } from "@trigger.dev/core/v3";
 import type {
   BuildManifest,
   CreateBackgroundWorkerRequestBody,
+  DequeuedMessage,
   DevConfigResponseBody,
   WorkerManifest,
 } from "@trigger.dev/core/v3";
@@ -224,6 +226,7 @@ class DevSupervisor implements WorkerRuntime {
 
     this.activeRunsPath = join(triggerDir, `active-runs${suffix}.json`);
     this.watchdogPidPath = join(triggerDir, `watchdog${suffix}.pid`);
+    const lockFilePath = join(triggerDir, safeBranch ? `dev.${safeBranch}.lock` : "dev.lock");
 
     // Write empty active-runs file
     this.#updateActiveRunsFile();
@@ -249,6 +252,7 @@ class DevSupervisor implements WorkerRuntime {
           WATCHDOG_ACTIVE_RUNS: this.activeRunsPath,
           WATCHDOG_PID_FILE: this.watchdogPidPath,
           WATCHDOG_TMP_DIR: getTmpRoot(this.options.config.workingDir, this.options.branch),
+          WATCHDOG_LOCK_FILE: lockFilePath,
         },
       });
 
@@ -478,7 +482,9 @@ class DevSupervisor implements WorkerRuntime {
             }
           );
 
-          //todo call the API to crash the run with a good message
+          this.#failRunWithMissingWorker(message).catch((error) => {
+            logger.debug("[DevSupervisor] Failed to fail run with missing worker", { error });
+          });
           continue;
         }
 
@@ -588,6 +594,33 @@ class DevSupervisor implements WorkerRuntime {
     }
   }
 
+  async #failRunWithMissingWorker(message: DequeuedMessage) {
+    const start = await this.options.client.dev.startRunAttempt(
+      message.run.friendlyId,
+      message.snapshot.friendlyId
+    );
+
+    if (!start.success) {
+      return;
+    }
+
+    const { run, snapshot, execution } = start.data;
+
+    await this.options.client.dev.completeRunAttempt(run.friendlyId, snapshot.friendlyId, {
+      completion: {
+        id: execution.run.id,
+        ok: false,
+        retry: undefined,
+        error: {
+          type: "INTERNAL_ERROR",
+          code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR,
+          message:
+            "This run was assigned to a background worker version that is no longer available in the dev session because it was superseded by a rebuild. Trigger the run again to use the current version.",
+        },
+      },
+    });
+  }
+
   async #startPresenceConnection() {
     try {
       const eventSource = this.options.client.dev.presenceConnection();

packages/cli-v3/src/dev/devWatchdog.ts

@@ -34,6 +34,7 @@ const apiKey = process.env.WATCHDOG_API_KEY!;
 const activeRunsPath = process.env.WATCHDOG_ACTIVE_RUNS!;
 const pidFilePath = process.env.WATCHDOG_PID_FILE!;
 const tmpDir = process.env.WATCHDOG_TMP_DIR;
+const lockFilePath = process.env.WATCHDOG_LOCK_FILE;
 
 if (!parentPid || !apiUrl || !apiKey || !activeRunsPath || !pidFilePath) {
   process.exit(1);
@@ -77,8 +78,25 @@ function cleanup() {
   } catch {}
 }
 
+function tmpDirOwnedByLiveSession(): boolean {
+  if (!lockFilePath) return false;
+  try {
+    const pid = Number(readFileSync(lockFilePath, "utf8").trim());
+    if (!pid || pid === parentPid) return false;
+    try {
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
 function cleanupTmpDir() {
   if (!tmpDir) return;
+  if (tmpDirOwnedByLiveSession()) return;
   try {
     rmSync(tmpDir, { recursive: true, force: true });
   } catch {

packages/cli-v3/src/entryPoints/dev-run-controller.ts

@@ -14,6 +14,7 @@ import {
   isOOMRunError,
   SuspendedProcessError,
 } from "@trigger.dev/core/v3";
+import { UnexpectedExitError } from "@trigger.dev/core/v3/errors";
 import { type WorkloadRunAttemptStartResponseBody } from "@trigger.dev/core/v3/workers";
 import { setTimeout as sleep } from "timers/promises";
 import type { CliApiClient } from "../apiClient.js";
@@ -22,6 +23,7 @@ import { assertExhaustive } from "../utilities/assertExhaustive.js";
 import { logger } from "../utilities/logger.js";
 import { sanitizeEnvVars } from "../utilities/sanitizeEnvVars.js";
 import { join } from "node:path";
+import { existsSync } from "node:fs";
 import type { BackgroundWorker } from "../dev/backgroundWorker.js";
 import { eventBus } from "../utilities/eventBus.js";
 import type { TaskRunProcessPool } from "../dev/taskRunProcessPool.js";
@@ -600,6 +602,15 @@ export class DevRunController {
       throw new Error(`No worker manifest for Dev ${run.friendlyId}`);
     }
 
+    const workerEntryPoint = this.opts.worker.manifest.workerEntryPoint;
+    if (!existsSync(workerEntryPoint)) {
+      throw new UnexpectedExitError(
+        1,
+        null,
+        `Dev worker build directory was removed before the run could start, likely cleaned up by a concurrent rebuild. Missing worker entry: ${workerEntryPoint}`
+      );
+    }
+
     this.snapshotPoller.start();
 
     logger.debug("getProcess", {