open-element · SisyphusZheng · Sep 3, 2026 · Sep 3, 2026
diff --git a/tools/check-coverage.test.ts b/tools/check-coverage.test.ts
@@ -0,0 +1,119 @@
+import { assertEquals, assertRejects, assertStringIncludes } from '@std/assert';
+import {
+  classifyTestExit,
+  describeNativeCrash,
+  runTestSuiteWithCrashRetry,
+} from './check-coverage.ts';
+
+function scriptedRunner(
+  codes: number[],
+): { runner: () => Promise<{ code: number }>; calls: () => number } {
+  let calls = 0;
+  return {
+    runner: () => {
+      const code = codes[Math.min(calls, codes.length - 1)];
+      calls++;
+      return Promise.resolve({ code });
+    },
+    calls: () => calls,
+  };
+}
+
+Deno.test('classifyTestExit separates real failures from native crashes (#1278)', () => {
+  assertEquals(classifyTestExit(0), 'ok');
+  // deno test reports assertion failures as exit code 1; usage and spawn
+  // errors stay below the 128 + signal floor. None of these may be retried.
+  assertEquals(classifyTestExit(1), 'test-failure');
+  assertEquals(classifyTestExit(2), 'test-failure');
+  assertEquals(classifyTestExit(127), 'test-failure');
+  // Signal-terminated processes surface as 128 + signal number.
+  assertEquals(classifyTestExit(128), 'native-crash');
+  assertEquals(classifyTestExit(132), 'native-crash'); // SIGILL
+  assertEquals(classifyTestExit(134), 'native-crash'); // SIGABRT
+  assertEquals(classifyTestExit(139), 'native-crash'); // SIGSEGV
+});
+
+Deno.test('describeNativeCrash names known signals and stays explicit for unknown ones', () => {
+  assertStringIncludes(describeNativeCrash(139), 'SIGSEGV');
+  assertStringIncludes(describeNativeCrash(139), '139');
+  assertStringIncludes(describeNativeCrash(134), 'SIGABRT');
+  assertStringIncludes(describeNativeCrash(200), 'signal 72');
+  assertStringIncludes(describeNativeCrash(200), '200');
+});
+
+Deno.test('a crash without any assertion failure is retried and can recover', async () => {
+  const { runner, calls } = scriptedRunner([139, 139, 0]);
+  const crashes: number[] = [];
+
+  const result = await runTestSuiteWithCrashRetry(runner, {
+    maxAttempts: 3,
+    onCrash: ({ attempt }) => crashes.push(attempt),
+  });
+
+  assertEquals(result, { crashes: 2 });
+  assertEquals(calls(), 3);
+  // Every crash is reported loudly, in order, so flakes stay countable.
+  assertEquals(crashes, [1, 2]);
+});
+
+Deno.test('a real assertion failure fails immediately without any retry', async () => {
+  const { runner, calls } = scriptedRunner([1, 0]);
+  let crashes = 0;
+
+  const error = await assertRejects(
+    () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 3, onCrash: () => crashes++ }),
+    Error,
+  );
+
+  assertStringIncludes(error.message, 'tests failed with code 1');
+  assertEquals(calls(), 1);
+  assertEquals(crashes, 0);
+});
+
+Deno.test('crash exhaustion fails loudly after the bounded attempt count', async () => {
+  const { runner, calls } = scriptedRunner([139]);
+  const crashes: Array<{ attempt: number; maxAttempts: number; code: number }> = [];
+
+  const error = await assertRejects(
+    () =>
+      runTestSuiteWithCrashRetry(runner, {
+        maxAttempts: 3,
+        onCrash: (event) => crashes.push(event),
+      }),
+    Error,
+  );
+
+  assertEquals(calls(), 3);
+  assertStringIncludes(error.message, 'crashed natively');
+  assertStringIncludes(error.message, 'SIGSEGV');
+  assertStringIncludes(error.message, '3');
+  assertEquals(crashes, [
+    { attempt: 1, maxAttempts: 3, code: 139 },
+    { attempt: 2, maxAttempts: 3, code: 139 },
+    { attempt: 3, maxAttempts: 3, code: 139 },
+  ]);
+});
+
+Deno.test('different crash signals across attempts still count toward the same bound', async () => {
+  const { runner, calls } = scriptedRunner([134, 139, 0]);
+
+  const result = await runTestSuiteWithCrashRetry(runner, { maxAttempts: 3 });
+
+  assertEquals(result, { crashes: 2 });
+  assertEquals(calls(), 3);
+});
+
+Deno.test('maxAttempts must be a positive integer', async () => {
+  const { runner, calls } = scriptedRunner([0]);
+  await assertRejects(
+    () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 0 }),
+    Error,
+    'positive integer',
+  );
+  await assertRejects(
+    () => runTestSuiteWithCrashRetry(runner, { maxAttempts: 1.5 }),
+    Error,
+    'positive integer',
+  );
+  assertEquals(calls(), 0);
+});
diff --git a/tools/check-coverage.ts b/tools/check-coverage.ts
@@ -20,26 +20,117 @@ function getNumberArg(flag: string, fallback: number): number {
   return value;
 }
 
-async function runCoverage(): Promise<string> {
+// Issue #1278: the `deno test --coverage` subprocess has repeatedly died by
+// native crash (observed exit 139 = SIGSEGV, rolldown/workerd class) with no
+// test assertion failure. Signal-terminated processes surface as exit code
+// 128 + signal number; only those are retryable. Any exit code below the
+// floor — including 1, the deno test assertion-failure code — is a real
+// failure and must fail the gate immediately, never retried.
+const NATIVE_CRASH_FLOOR = 128;
+
+const SIGNAL_NAMES: Record<number, string> = {
+  4: 'SIGILL',
+  5: 'SIGTRAP',
+  6: 'SIGABRT',
+  7: 'SIGBUS',
+  8: 'SIGFPE',
+  11: 'SIGSEGV',
+};
+
+export type TestExitKind = 'ok' | 'test-failure' | 'native-crash';
+
+export function classifyTestExit(code: number): TestExitKind {
+  if (code === 0) return 'ok';
+  return code >= NATIVE_CRASH_FLOOR ? 'native-crash' : 'test-failure';
+}
+
+export function describeNativeCrash(code: number): string {
+  const signal = code - NATIVE_CRASH_FLOOR;
+  const name = SIGNAL_NAMES[signal];
+  return name ? `signal ${name} (exit code ${code})` : `signal ${signal} (exit code ${code})`;
+}
+
+export interface CrashRetryEvent {
+  attempt: number;
+  maxAttempts: number;
+  code: number;
+}
+
+// Runs the coverage test suite with a bounded, fail-loud native-crash retry:
+// every crash is reported through onCrash, real test failures abort without
+// retry, and exhausting maxAttempts on crashes alone throws. Returns the
+// number of crashes observed so the caller can keep recovered flakes visible.
+export async function runTestSuiteWithCrashRetry(
+  runner: () => Promise<{ code: number }>,
+  options: { maxAttempts: number; onCrash?: (event: CrashRetryEvent) => void },
+): Promise<{ crashes: number }> {
+  const { maxAttempts, onCrash } = options;
+  if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
+    throw new Error('maxAttempts must be a positive integer');
+  }
+  let crashes = 0;
+  for (let attempt = 1;; attempt++) {
+    const { code } = await runner();
+    const kind = classifyTestExit(code);
+    if (kind === 'ok') return { crashes };
+    if (kind === 'test-failure') throw new Error(`tests failed with code ${code}`);
+    crashes++;
+    onCrash?.({ attempt, maxAttempts, code });
+    if (attempt >= maxAttempts) {
+      throw new Error(
+        `coverage test run crashed natively (${describeNativeCrash(code)}) on all ` +
+          `${maxAttempts} attempts; refusing to pass the gate on repeated native ` +
+          'crashes (#1278)',
+      );
+    }
+  }
+}
+
+async function runCoverage(crashRetries: number): Promise<string> {
   const coverageDir = '.coverage-check';
   try {
-    const test = await new Deno.Command(Deno.execPath(), {
-      args: [
-        'test',
-        '--no-lock',
-        `--coverage=${coverageDir}`,
-        '--allow-read',
-        '--allow-write',
-        '--allow-env',
-        '--allow-net',
-        '--allow-run',
-        '--allow-ffi',
-        '--allow-sys',
-      ],
-      stdout: 'inherit',
-      stderr: 'inherit',
-    }).spawn().status;
-    if (!test.success) throw new Error(`tests failed with code ${test.code}`);
+    const { crashes } = await runTestSuiteWithCrashRetry(
+      async () => {
+        // A crashed attempt can leave partial coverage profiles behind that
+        // `deno coverage` would choke on; each attempt starts from a clean dir.
+        await Deno.remove(coverageDir, { recursive: true }).catch(() => undefined);
+        return await new Deno.Command(Deno.execPath(), {
+          args: [
+            'test',
+            '--no-lock',
+            `--coverage=${coverageDir}`,
+            '--allow-read',
+            '--allow-write',
+            '--allow-env',
+            '--allow-net',
+            '--allow-run',
+            '--allow-ffi',
+            '--allow-sys',
+          ],
+          stdout: 'inherit',
+          stderr: 'inherit',
+        }).spawn().status;
+      },
+      {
+        maxAttempts: crashRetries + 1,
+        onCrash: ({ attempt, maxAttempts, code }) => {
+          console.error(
+            `\n[check-coverage] NATIVE CRASH: deno test terminated by ${
+              describeNativeCrash(code)
+            } ` +
+              `on attempt ${attempt}/${maxAttempts} with no test assertion failure (#1278). ` +
+              (attempt < maxAttempts ? 'Retrying.' : 'No attempts left.'),
+          );
+        },
+      },
+    );
+    if (crashes > 0) {
+      console.error(
+        `\n[check-coverage] WARNING: coverage run recovered after ${crashes} native ` +
+          `crash(es) (#1278). The gate passed, but the flake stays visible — count ` +
+          'these lines in CI logs when trending the crash rate.',
+      );
+    }
 
     const report = await new Deno.Command(Deno.execPath(), {
       args: ['coverage', coverageDir, '--lcov'],
@@ -80,7 +171,11 @@ function formatMetric(name: string, metric: CoverageMetric, threshold: number):
 
 async function main(): Promise<void> {
   await ensureWwwBuildOutput();
-  const lcov = await runCoverage();
+  // Bounded crash retry for #1278: one clean run plus `--crash-retries`
+  // retries that only fire on native-crash exits (>= 128 + signal), never on
+  // assertion failures. Loud by design: every crash prints to stderr.
+  const crashRetries = getNumberArg('--crash-retries', 2);
+  const lcov = await runCoverage(crashRetries);
   const profiledFiles = lcovFilePaths(lcov);
 
   // Threshold baseline: 2026-08-04 (v0.42.0-alpha.14 cycle), measured with the