From aa7e880173b377c66552c9ea7e1a32be414ec7d6 Mon Sep 17 00:00:00 2001 From: Hessabi Date: Fri, 23 Oct 2026 11:27:45 -0500 Subject: [PATCH] Treat the CLI's first-run welcome screen as a failed run Right after the first devin auth login, devin -p printed its welcome screen and exited 0 without running the prompt, and the job was recorded as completed with that text as the result. The new isOnboardingOutput helper (ANSI-stripped) detects it and the job is marked failed with a hint to run again. The check runs before normal rendering and is skipped when a sandbox hint applies, so the sandbox failure path is unchanged. --- plugins/devin/scripts/devin-companion.mjs | 18 +++++++++++++ plugins/devin/scripts/lib/devin.mjs | 11 ++++++++ tests/companion.test.mjs | 24 +++++++++++++++++ tests/devin.test.mjs | 33 +++++++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 tests/devin.test.mjs diff --git a/plugins/devin/scripts/devin-companion.mjs b/plugins/devin/scripts/devin-companion.mjs index 5fa0c4d..1944915 100755 --- a/plugins/devin/scripts/devin-companion.mjs +++ b/plugins/devin/scripts/devin-companion.mjs @@ -27,6 +27,7 @@ import { getDevinBinary, getDevinAvailability, getResolvedDevinBinary, + isOnboardingOutput, normalizePermissionMode, runDevinPrint, sandboxFailureHint @@ -249,6 +250,23 @@ async function runDevinJob({ cwd, kind, prompt, promptIsFile = false, model = nu const rawOutput = result.stdout.trim(); const devinSession = findLatestDevinSession(cwd, { createdAfterMs: startedMs - 60_000 }); + + // The first run after `devin auth login` shows a welcome screen instead of running the + // prompt. Record it as failed so it is not stored as Devin's answer. + if (!sandboxHint && isOnboardingOutput(rawOutput)) { + return { + exitStatus: 1, + payload: { + rawOutput, + stderr: result.stderr.trim() || null, + exitStatus: result.exitStatus + }, + rendered: "Devin CLI showed its first-run welcome screen instead of running the task. Run the command again.\n", + summary: "Devin CLI first-run onboarding", + devinSessionId: devinSession?.id ?? devinSession?.short_id ?? null + }; + } + return { exitStatus: result.exitStatus, payload: { diff --git a/plugins/devin/scripts/lib/devin.mjs b/plugins/devin/scripts/lib/devin.mjs index a65c5bf..8d5bd79 100644 --- a/plugins/devin/scripts/lib/devin.mjs +++ b/plugins/devin/scripts/lib/devin.mjs @@ -217,6 +217,17 @@ export function sandboxFailureHint({ platform = process.platform, stderr = "" } ].join("\n"); } +// The CLI prints a welcome screen and exits 0 without running the prompt right after +// the first `devin auth login`. ANSI bold codes may wrap some of the words. +export function isOnboardingOutput(text) { + const stripped = String(text ?? "").replace(/\x1b\[[0-9;]*m/g, ""); + return ( + /Welcome to Devin CLI!/i.test(stripped) && + /Logged in as/i.test(stripped) && + /Run .*devin.* to get started/i.test(stripped) + ); +} + export function listDevinSessions(cwd) { const result = runCommand(getDevinBinary(), ["list", "--format", "json"], { cwd }); if (result.error || result.status !== 0) { diff --git a/tests/companion.test.mjs b/tests/companion.test.mjs index 99ed541..f418d60 100644 --- a/tests/companion.test.mjs +++ b/tests/companion.test.mjs @@ -222,3 +222,27 @@ test("cancel reports finished for completed jobs", () => { fake.cleanup(); } }); + +test("task with onboarding output is marked as failed with hint", () => { + const repo = createTempRepo(); + const onboardingText = "Welcome to Devin CLI!\n ✓ Logged in as someone@example.com.\nYou're all set. Run devin to get started."; + const fake = createFakeDevin({ responseText: onboardingText }); + initGitRepo(repo.dir); + const dataDir = fs.mkdtempSync("/tmp/devin-plugin-data-"); + try { + const env = { ...fake.env, CLAUDE_PLUGIN_DATA: dataDir }; + const result = runCompanion(["task", "do something"], { cwd: repo.dir, env }); + assert.match(result.stdout, /Devin CLI showed its first-run welcome screen/); + + const status = runCompanion(["status", "--json"], { cwd: repo.dir, env }); + const payload = JSON.parse(status.stdout); + assert.equal(payload.jobs.length, 1); + assert.equal(payload.jobs[0].status, "failed"); + + const rendered = runCompanion(["result", payload.jobs[0].id], { cwd: repo.dir, env }); + assert.match(rendered.stdout, /Devin CLI showed its first-run welcome screen instead of running the task/); + } finally { + repo.cleanup(); + fake.cleanup(); + } +}); diff --git a/tests/devin.test.mjs b/tests/devin.test.mjs new file mode 100644 index 0000000..9e6f01b --- /dev/null +++ b/tests/devin.test.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { isOnboardingOutput } from "../plugins/devin/scripts/lib/devin.mjs"; + +test("isOnboardingOutput matches welcome screen with ANSI codes", () => { + const withAnsi = "\x1b[1mWelcome to Devin CLI!\x1b[0m\n \x1b[32m✓\x1b[0m Logged in as someone@example.com.\nYou're all set. Run devin to get started."; + assert.equal(isOnboardingOutput(withAnsi), true); +}); + +test("isOnboardingOutput matches welcome screen without ANSI codes", () => { + const withoutAnsi = "Welcome to Devin CLI!\n ✓ Logged in as someone@example.com.\nYou're all set. Run devin to get started."; + assert.equal(isOnboardingOutput(withoutAnsi), true); +}); + +test("isOnboardingOutput does not match normal review output", () => { + const reviewOutput = "## Verdict\nAPPROVE\n\nThe changes look good."; + assert.equal(isOnboardingOutput(reviewOutput), false); +}); + +test("isOnboardingOutput does not match partial welcome patterns", () => { + const partial = "Welcome to Devin CLI!\nSome other text here."; + assert.equal(isOnboardingOutput(partial), false); +}); + +test("isOnboardingOutput does not match empty output", () => { + assert.equal(isOnboardingOutput(""), false); +}); + +test("isOnboardingOutput is case-insensitive for key phrases", () => { + const mixedCase = "WELCOME TO DEVIN CLI!\n ✓ LOGGED IN AS someone@example.com.\nYou're all set. Run devin to get started."; + assert.equal(isOnboardingOutput(mixedCase), true); +});