Generally reject exit codes other than 0 and 1

tests/test_typecheck_benchmark_isolation.py

@@ -17,6 +17,7 @@
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from typecheck_benchmark.daily_runner import (
+    DEFAULT_TYPE_CHECKERS,
     _benchmark_package,
     create_package_venv,
     install_deps,
@@ -38,6 +39,27 @@ def _ok_result(**_: object) -> dict[str, object]:
     }
 
 
+@pytest.mark.parametrize("checker", DEFAULT_TYPE_CHECKERS)
+@pytest.mark.parametrize("capture_output", [True, False])
+@pytest.mark.parametrize(
+    ("returncode", "expected_ok"),
+    [(0, True), (1, True), (2, False), (101, False), (-9, False)],
+)
+def test_checker_exit_code(
+    checker: str, capture_output: bool, returncode: int, expected_ok: bool, tmp_path: Path
+) -> None:
+    process_result = _ok_result()
+    process_result["returncode"] = returncode
+    with patch(
+        "typecheck_benchmark.daily_runner.run_process_with_timeout", return_value=process_result
+    ):
+        result = run_checker(checker, tmp_path, None, timeout=60, capture_output=capture_output)
+
+    assert result["ok"] is expected_ok
+    if not expected_ok:
+        assert f"exit code {returncode}" in (result.get("error_message") or "")
+
+
 # ---------------------------------------------------------------------------
 # venv_python
 # ---------------------------------------------------------------------------

typecheck_benchmark/daily_runner.py

@@ -670,7 +670,7 @@ def run_checker(
     """Run a single type checker and return timing metrics only.
 
     When *capture_output* is False, output is discarded (no pipe overhead)
-    and error detection is skipped — caller must have done a capture pass first.
+    and output-based error detection is skipped. Exit codes are always checked.
 
     When *venv* is given, every checker is pointed at that environment
     explicitly (and ``VIRTUAL_ENV`` is set), so third-party imports resolve
@@ -756,15 +756,7 @@ def run_checker(
             "error_message": msg,
         }
 
-    if not capture_output:
-        return {
-            "ok": True,
-            "execution_time_s": result["execution_time_s"],
-            "peak_memory_mb": result.get("peak_memory_mb", 0.0),
-        }
-
-    # Detect fatal errors: mypy exits with code 2 for fatal errors (code 1 = type errors found).
-    # Also check for "errors prevented further checking" which means mypy bailed out early.
+    # Also detect mypy errors that prevent it from completing the check.
     stderr = result.get("stderr", "")
     stdout = result.get("stdout", "")
     combined_output = stderr + stdout
@@ -777,8 +769,10 @@ def run_checker(
             "error_message": "Fatal: errors prevented further checking",
         }
 
-    # mypy return code 2 = fatal error (not type errors)
-    if checker == "mypy" and result.get("returncode", 0) == 2:
+    # All supported checkers use 0 for success and 1 for diagnostics.
+    # Other codes indicate an incomplete check, a crash, or a killed process.
+    returncode = result["returncode"]
+    if returncode not in (0, 1):
         # Extract first error line from output for context
         first_error = ""
         for line in combined_output.splitlines():
@@ -789,7 +783,8 @@ def run_checker(
             "ok": False,
             "execution_time_s": result["execution_time_s"],
             "peak_memory_mb": result.get("peak_memory_mb", 0.0),
-            "error_message": first_error or "Fatal error (exit code 2)",
+            "error_message": f"Fatal error (exit code {returncode})"
+            + (f": {first_error}" if first_error else ""),
         }
 
     return {