fix: runner pre-flight + telemetry — close the gap for existing users

src/cli/agent-runner.ts

@@ -31,6 +31,7 @@ import {
   currentAgentEnvPathForHome,
   managedAgentEnvPathForHome,
   primaryRepoFromConfig,
+  providerEnvPathForHome,
   readSetupConfigFromHome,
   resolveAgentRailHome,
   type ConnectedRepo,
@@ -41,8 +42,10 @@ import type { SetupConfig } from "./setup-config.ts";
 import type { TaskBlocker, TaskPriority, TaskStatus } from "../task-store.ts";
 import type {
   TelemetryClient,
+  TelemetryDurationBucket,
   TelemetryEventName,
   TelemetryFailureCategory,
+  TelemetryOutcome,
   TelemetryProperties,
   TelemetryTaskStatus,
 } from "../telemetry.ts";
@@ -709,6 +712,7 @@ async function executeAgentRun({
           fetchImpl,
           runnerTimeoutMs,
           runnerPolicy: setupConfig?.runnerPolicy ?? null,
+          setupConfig,
           telemetry,
         });
       } catch (error) {
@@ -844,6 +848,7 @@ async function executeSingleTaskRun({
   fetchImpl,
   runnerTimeoutMs,
   runnerPolicy,
+  setupConfig,
   telemetry,
 }: {
   agentId: string;
@@ -865,6 +870,7 @@ async function executeSingleTaskRun({
   fetchImpl: typeof globalThis.fetch;
   runnerTimeoutMs: number;
   runnerPolicy?: RunnerExecutionPolicyLike | null;
+  setupConfig?: SetupConfigLike | null;
   telemetry?: TelemetryClient;
 }): Promise<AgentRunRecord> {
   const timestamp = now().toISOString();
@@ -980,8 +986,48 @@ async function executeSingleTaskRun({
 
   let filesystemSnapshot: RunnerPolicyFilesystemSnapshot | null = null;
   let runnerFailureCaptured = false;
+  let runnerStartedAt: number | null = null;
+  let runnerCompletionCaptured = false;
+
+  const captureRunnerCompletionOnce = (outcome: TelemetryOutcome): void => {
+    if (runnerStartedAt === null || runnerCompletionCaptured) {
+      return;
+    }
+    runnerCompletionCaptured = true;
+    captureRunnerTelemetry(telemetry, "runner_completed", {
+      runner: safeTelemetryRunner(runner),
+      outcome,
+      durationBucket: bucketDuration(Date.now() - runnerStartedAt),
+    });
+  };
 
   try {
+    const submitProviderReady = await detectSubmitProviderReady(setupConfig, homePath);
+    if (!submitProviderReady.ok) {
+      captureRunnerTelemetry(telemetry, "runner_failed", {
+        runner: safeTelemetryRunner(runner),
+        failureCategory: "provider_not_configured",
+      });
+      runnerFailureCaptured = true;
+      return await blockManagedRunAwaitingUser({
+        baseUrl,
+        apiKey,
+        fetchImpl,
+        task,
+        runId,
+        agentId,
+        runStore,
+        homePath,
+        markdownEnabled,
+        now,
+        summary: submitProviderReady.message,
+        blocker: {
+          reason: "provider_not_configured",
+          actionRequired: submitProviderReady.message,
+          resumeInstructions: "Run `agentrail provider connect github`, then `agentrail task resume <task-id>` to retry.",
+        },
+      });
+    }
     await prepareWorktree({
       repoPath: repo.path,
       worktreePath,
@@ -1014,6 +1060,11 @@ async function executeSingleTaskRun({
         actionRequired: `Choose a runner/policy AgentRail can enforce, or configure an external sandbox/advisory mode. ${summary}`,
         resumeInstructions: "Update the agent runner policy or selected runner, then run `agentrail task resume <task-id>` so AgentRail can start a fresh managed run.",
       };
+      captureRunnerTelemetry(telemetry, "runner_failed", {
+        runner: safeTelemetryRunner(runner),
+        failureCategory: "runner_policy_denied",
+      });
+      runnerFailureCaptured = true;
       return await blockManagedRunAwaitingUser({
         baseUrl,
         apiKey,
@@ -1034,6 +1085,11 @@ async function executeSingleTaskRun({
     filesystemSnapshot = filesystemPreflight.snapshot;
     if (!filesystemPreflight.ok) {
       const summary = filesystemPreflight.summary;
+      captureRunnerTelemetry(telemetry, "runner_failed", {
+        runner: safeTelemetryRunner(runner),
+        failureCategory: "runner_policy_denied",
+      });
+      runnerFailureCaptured = true;
       return await blockManagedRunAwaitingUser({
         baseUrl,
         apiKey,
@@ -1057,6 +1113,7 @@ async function executeSingleTaskRun({
     await writeRunnerPolicyGeneratedFiles(runnerPlan);
 
     captureRunnerTelemetry(telemetry, "runner_started", buildRunnerStartedTelemetryProperties(runner, task.status));
+    runnerStartedAt = Date.now();
     const result = await launchRunner({
       runner,
       executable: runnerPlan.executable,
@@ -1090,6 +1147,14 @@ async function executeSingleTaskRun({
     const filesystemPostRun = await validateRunnerPolicyFilesystemPostRun(runnerPlan, filesystemSnapshot);
     if (!filesystemPostRun.ok) {
       const summary = filesystemPostRun.summary;
+      if (!runnerFailureCaptured) {
+        captureRunnerTelemetry(telemetry, "runner_failed", {
+          runner: safeTelemetryRunner(runner),
+          failureCategory: "runner_policy_violation",
+        });
+        runnerFailureCaptured = true;
+      }
+      captureRunnerCompletionOnce("blocked");
       return await blockManagedRunAwaitingUser({
         baseUrl,
         apiKey,
@@ -1147,14 +1212,17 @@ async function executeSingleTaskRun({
     if (markdownEnabled) {
       await writeRunMarkdown({ homePath, run: completed });
     }
+    captureRunnerCompletionOnce(runnerStatusToOutcome(finalResult.status));
     return completed;
   } catch (error) {
     if (!runnerFailureCaptured) {
       captureRunnerTelemetry(telemetry, "runner_failed", {
         runner: safeTelemetryRunner(runner),
         failureCategory: classifyRunnerFailure(error),
       });
+      runnerFailureCaptured = true;
     }
+    captureRunnerCompletionOnce("failed");
     if (runnerPlan && filesystemSnapshot) {
       try {
         await restoreProtectedInstructionFileModes(runnerPlan, filesystemSnapshot);
@@ -1252,20 +1320,109 @@ function safeTelemetryTaskStatus(value: unknown): TelemetryTaskStatus | undefine
   }
 }
 
-function classifyRunnerFailure(error: unknown): TelemetryFailureCategory {
+async function detectSubmitProviderReady(
+  setupConfig: SetupConfigLike | null | undefined,
+  homePath: string,
+): Promise<{ ok: boolean; message: string }> {
+  const github = setupConfig?.providers?.github;
+  const mode = github?.mode === "real" || github?.mode === "disabled" ? github.mode : "real";
+  if (mode === "disabled") {
+    return { ok: true, message: "" };
+  }
+  const tokenEnvName = github?.tokenEnv?.trim() || "GITHUB_TOKEN";
+  if (process.env[tokenEnvName]?.trim()) {
+    return { ok: true, message: "" };
+  }
+  try {
+    const content = await readFile(providerEnvPathForHome(homePath), "utf8");
+    if (parseSimpleEnv(content)[tokenEnvName]?.trim()) {
+      return { ok: true, message: "" };
+    }
+  } catch {
+    // Missing provider.env is treated as "not configured".
+  }
+  return {
+    ok: false,
+    message:
+      "AgentRail cannot publish a branch or open a pull request without a GitHub provider. "
+      + "Run `agentrail provider connect github` to enable submissions, then retry. "
+      + `(Looking for ${tokenEnvName} in shell env or ~/.agentrail/provider.env.)`,
+  };
+}
+
+export function classifyRunnerFailure(error: unknown): TelemetryFailureCategory {
   const text = runnerFailureText(error);
   if (/\b(etimedout|timeout|timed out)\b/u.test(text)) {
     return "runner_timeout";
   }
+  if (/\b(econnrefused|enetunreach|enotfound|getaddrinfo)\b/u.test(text)
+    || /\bfetch failed\b/u.test(text)
+    || /network unreachable/u.test(text)) {
+    return "network_error";
+  }
   if (/\b(eacces|eperm)\b/u.test(text) || /sandbox|permission|denied/u.test(text)) {
     return "sandbox_blocked";
   }
+  if (/\bprompt\b[^.]{0,80}(too (large|long)|exceeded|max tokens|token limit)/u.test(text)
+    || /context (window|length).{0,40}exceed/u.test(text)) {
+    return "prompt_too_large";
+  }
+  if (/(refused|declined|cannot comply|will not).{0,40}(request|task|instruction|prompt)/u.test(text)) {
+    return "model_refused";
+  }
+  if (/template (not modified|unedited|unchanged)/u.test(text)
+    || /result template.{0,40}(unedited|not modified)/u.test(text)) {
+    return "result_template_unedited";
+  }
+  if (/\bhandoff\b/u.test(text)
+    || /result\.json.{0,40}(malformed|invalid|missing|not (found|written))/u.test(text)) {
+    return "handoff_invalid";
+  }
   if (/\b(enoent)\b/u.test(text) || /not found|command not found|executable|missing/u.test(text)) {
     return "runner_missing";
   }
+  if (/exit code|exited with|non-?zero exit/u.test(text)) {
+    return "nonzero_exit";
+  }
   return "unknown";
 }
 
+export function bucketDuration(elapsedMs: number): TelemetryDurationBucket {
+  if (!Number.isFinite(elapsedMs) || elapsedMs < 0) {
+    return "sub_1s";
+  }
+  if (elapsedMs < 1000) {
+    return "sub_1s";
+  }
+  if (elapsedMs < 5000) {
+    return "1_5s";
+  }
+  if (elapsedMs < 30000) {
+    return "5_30s";
+  }
+  if (elapsedMs < 120000) {
+    return "30_120s";
+  }
+  return "over_120s";
+}
+
+function runnerStatusToOutcome(status: unknown): TelemetryOutcome {
+  switch (status) {
+    case "succeeded":
+      return "succeeded";
+    case "failed":
+      return "failed";
+    case "cancelled":
+      return "cancelled";
+    case "awaiting_user":
+      return "awaiting_user";
+    case "blocked":
+      return "blocked";
+    default:
+      return "unknown";
+  }
+}
+
 function runnerFailureText(error: unknown): string {
   const parts: string[] = [];
   if (isRecord(error) && typeof error.code === "string") {

src/telemetry.ts

@@ -17,6 +17,7 @@ export const TELEMETRY_EVENT_NAMES = [
   "runner_wake_attempted",
   "runner_started",
   "runner_failed",
+  "runner_completed",
   "pr_opened",
   "ci_status_observed",
   "review_status_observed",
@@ -47,6 +48,15 @@ const TELEMETRY_FAILURE_CATEGORY_VALUES = [
   "runner_timeout",
   "sandbox_blocked",
   "routing_no_agent",
+  "nonzero_exit",
+  "handoff_invalid",
+  "result_template_unedited",
+  "network_error",
+  "runner_policy_denied",
+  "runner_policy_violation",
+  "prompt_too_large",
+  "model_refused",
+  "provider_not_configured",
   "unknown",
 ] as const;
 const TELEMETRY_CI_STATUS_VALUES = [
@@ -84,6 +94,8 @@ const TELEMETRY_OUTCOME_VALUES = [
   "no_route",
   "conflict",
   "awaiting_user",
+  "succeeded",
+  "cancelled",
   "unknown",
 ] as const;
 const TELEMETRY_DURATION_BUCKET_VALUES = ["sub_1s", "1_5s", "5_30s", "30_120s", "over_120s"] as const;