fix: preserve runtime wake across pod deletion

Author: bashybaranaba

apps/api/src/services/kubernetes-pods.test.ts

@@ -51,4 +51,62 @@ describe("idempotent Kubernetes pod creation", () => {
     ).resolves.toBe("created");
     expect(api.createNamespacedPod).toHaveBeenCalledTimes(2);
   });
+
+  it("retries when creation conflicts again after the old pod disappears", async () => {
+    const api = {
+      createNamespacedPod: jest
+        .fn()
+        .mockRejectedValueOnce(statusError(409))
+        .mockRejectedValueOnce(statusError(409))
+        .mockResolvedValueOnce({}),
+      readNamespacedPod: jest
+        .fn()
+        .mockResolvedValueOnce({
+          metadata: {
+            name: "computer-one",
+            deletionTimestamp: "2026-07-11T00:00:00Z",
+          },
+        })
+        .mockRejectedValueOnce(statusError(404))
+        .mockRejectedValueOnce(statusError(404)),
+    };
+
+    await expect(
+      createKubernetesPodIdempotently(
+        api,
+        "tenant-one",
+        { metadata: { name: "computer-one" } },
+        { deletionTimeoutMs: 100, pollIntervalMs: 0 },
+      ),
+    ).resolves.toBe("created");
+    expect(api.createNamespacedPod).toHaveBeenCalledTimes(3);
+  });
+
+  it("accepts a concurrently-created replacement after deletion", async () => {
+    const api = {
+      createNamespacedPod: jest
+        .fn()
+        .mockRejectedValueOnce(statusError(409))
+        .mockRejectedValueOnce(statusError(409)),
+      readNamespacedPod: jest
+        .fn()
+        .mockResolvedValueOnce({
+          metadata: {
+            name: "computer-one",
+            deletionTimestamp: "2026-07-11T00:00:00Z",
+          },
+        })
+        .mockRejectedValueOnce(statusError(404))
+        .mockResolvedValueOnce({ metadata: { name: "computer-one" } }),
+    };
+
+    await expect(
+      createKubernetesPodIdempotently(
+        api,
+        "tenant-one",
+        { metadata: { name: "computer-one" } },
+        { deletionTimeoutMs: 100, pollIntervalMs: 0 },
+      ),
+    ).resolves.toBe("existing");
+  });
 });

apps/api/src/services/kubernetes-pods.ts

@@ -48,40 +48,41 @@ export async function createKubernetesPodIdempotently<TPod extends PodLike>(
   const name = body.metadata?.name;
   if (!name) throw new Error("Kubernetes pod metadata.name is required");
 
-  try {
-    await coreApi.createNamespacedPod({ namespace, body });
-    return "created";
-  } catch (error) {
-    if (kubernetesStatusCode(error) !== 409) throw error;
-  }
+  const timeoutMs = options.deletionTimeoutMs ?? 180_000;
+  const intervalMs = options.pollIntervalMs ?? 250;
+  const deadline = Date.now() + timeoutMs;
 
-  let current: PodLike;
-  try {
-    current = responsePod(
-      await coreApi.readNamespacedPod({ namespace, name }),
-    );
-  } catch (error) {
-    if (kubernetesStatusCode(error) === 404) {
+  while (Date.now() < deadline) {
+    try {
       await coreApi.createNamespacedPod({ namespace, body });
       return "created";
+    } catch (error) {
+      if (kubernetesStatusCode(error) !== 409) throw error;
     }
-    throw error;
-  }
-  if (!current.metadata?.deletionTimestamp) return "existing";
 
-  const timeoutMs = options.deletionTimeoutMs ?? 45_000;
-  const intervalMs = options.pollIntervalMs ?? 250;
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    await wait(intervalMs);
+    let current: PodLike;
     try {
-      await coreApi.readNamespacedPod({ namespace, name });
+      current = responsePod(
+        await coreApi.readNamespacedPod({ namespace, name }),
+      );
     } catch (error) {
-      if (kubernetesStatusCode(error) === 404) {
-        await coreApi.createNamespacedPod({ namespace, body });
-        return "created";
+      if (kubernetesStatusCode(error) !== 404) throw error;
+      await wait(intervalMs);
+      continue;
+    }
+    if (!current.metadata?.deletionTimestamp) return "existing";
+
+    while (Date.now() < deadline) {
+      await wait(intervalMs);
+      try {
+        current = responsePod(
+          await coreApi.readNamespacedPod({ namespace, name }),
+        );
+        if (!current.metadata?.deletionTimestamp) return "existing";
+      } catch (error) {
+        if (kubernetesStatusCode(error) === 404) break;
+        throw error;
       }
-      throw error;
     }
   }