From ebb706fc2947a3391ffc62294c83663cdfb2ead4 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Sat, 5 Sep 2026 16:59:39 -0700 Subject: [PATCH] fix(computer-use): win32 backend reports PowerShell failures truthfully Every win32 action now runs self-contained in its own PowerShell process: the User32 Add-Type definition travels with each invocation instead of a bootstrap process whose type registration could never reach the fresh powershell.exe spawned per action (issue #5896 defect 1). withUser32()/psJson() now go through psOk(), which rejects nonzero exits, timeouts, and spawn failures, so input methods can no longer return action_sent:true after a failed subprocess (defect 2). left_mouse_down's ternary no longer swallows the LEFTDOWN press when a target is supplied - a targeted press both moves and presses (defect 3). list_windows now parses its JSON via psJson instead of mapping the raw run result, and write_clipboard fails truthfully on error. Tests: fake powershell.exe on PATH captures decoded -EncodedCommand payloads; covers exit!=0 -> error, spawn failure -> error, success -> success, and targeted mouse-down emitting SetCursorPos + LEFTDOWN in one self-contained command. Signed-off-by: CodeWhale Bot --- .../computer-use/src/backends/win32.mjs | 49 +++--- .../computer-use/tests/win32-input.test.mjs | 164 ++++++++++++++++++ 2 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 crates/tui/plugins/computer-use/tests/win32-input.test.mjs diff --git a/crates/tui/plugins/computer-use/src/backends/win32.mjs b/crates/tui/plugins/computer-use/src/backends/win32.mjs index 40c8c07202..d73b8bb8e8 100644 --- a/crates/tui/plugins/computer-use/src/backends/win32.mjs +++ b/crates/tui/plugins/computer-use/src/backends/win32.mjs @@ -18,8 +18,16 @@ function ps(script, opts = {}) { }); } -async function psJson(script, opts = {}) { +/** ps() but truthful: timeout, nonzero exit, and spawn failure all throw. */ +async function psOk(script, opts = {}) { const r = await ps(script, opts); + if (r.timedOut) throw new ExecError(`powershell timed out after ${opts.timeoutMs ?? 25_000}ms`, r); + if (r.code !== 0) throw new ExecError(`powershell.exe exited ${r.code}: ${(r.stderr || r.stdout).trim().slice(0, 300)}`, r); + return r; +} + +async function psJson(script, opts = {}) { + const r = await psOk(script, opts); const out = r.stdout.trim(); const j = tryJson(out, null); if (!j) throw new ExecError(`powershell did not return JSON: ${(r.stderr || out).trim().slice(0, 300)}`, r); @@ -65,22 +73,24 @@ const VK = { }; const MODVK = { ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b }; +// Every action runs in a fresh powershell.exe process, so a bootstrap process +// can never register the User32 type for later spawns. Each User32-backed +// invocation therefore carries its own type definition via this prelude +// (Add-Type re-definition is tolerated through -ErrorAction SilentlyContinue). +const USER32_PRELUDE = `Add-Type -TypeDefinition @'\n${USER32}\n'@ -ErrorAction SilentlyContinue;`; + export function create() { - const bootstrapped = (async () => { - await ps(`Add-Type -TypeDefinition @'\n${USER32}\n'@ -ErrorAction SilentlyContinue; [User32] | Out-Null`, { timeoutMs: 30_000 }); - })(); let lastRaster = null; let recording = null; // {id, pid, file, startedAt, mode} + /** Self-contained User32 invocation: prelude + script, fails truthfully. */ async function withUser32(script, opts) { - await bootstrapped; - return ps(script, opts); + return psOk(`${USER32_PRELUDE}\n${script}`, opts); } return { platform: "win32", probe: async () => { - await bootstrapped.catch(() => {}); let ffmpeg = true; try { await runOk("ffmpeg", ["-version"], { timeoutMs: 10_000 }); } catch { ffmpeg = false; } const psOk = await ps("Write-Output 'ok'").then((r) => r.code === 0).catch(() => false); @@ -92,7 +102,6 @@ export function create() { }; }, list_displays: async () => { - await bootstrapped; const d = await psJson(`Add-Type -AssemblyName System.Windows.Forms; $out = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object { [pscustomobject]@{ name = $_.DeviceName; primary = $_.Primary; x = $_.Bounds.X; y = $_.Bounds.Y; w = $_.Bounds.Width; h = $_.Bounds.Height; } } | ConvertTo-Json -Compress; Write-Output ('{"displays": ' + ($out -replace '^\\[','[' -replace '\\]$/',']') + '}');`, { timeoutMs: 15_000 }).catch(async () => { @@ -105,7 +114,6 @@ Write-Output ('{"displays": ' + (ConvertTo-Json $arr -Compress) + '}');`, { time }, async switch_display({ index }) { return { activeDisplay: index ?? 1, note: "windows screenshots grab the virtual screen; per-display crop applies where supported" }; }, list_apps: async () => { - await bootstrapped; const j = await psJson(`Add-Type -AssemblyName System.Windows.Forms; $out = Get-Process | Where-Object { $_.MainWindowTitle } | ForEach-Object { [pscustomobject]@{ name = $_.ProcessName; pid2 = $_.Id; title = $_.MainWindowTitle } } | ConvertTo-Json -Compress; if (-not $out) { $out = '[]' } @@ -113,7 +121,7 @@ Write-Output ('{"apps": ' + $out + '}');`); return { apps: (Array.isArray(j.apps) ? j.apps : [j.apps]).map((a) => ({ name: a.name, pid: a.pid2, title: a.title })) }; }, list_windows: async () => { - await withUser32(`Add-Type -AssemblyName System.Windows.Forms; + const j = await psJson(`Add-Type -AssemblyName System.Windows.Forms; Add-Type -TypeDefinition @' using System; using System.Text; @@ -147,12 +155,13 @@ public static class WinEnum { '@; $json = (WinEnum::List() | ForEach-Object { $p = $_.Split('|', 2); $parts = $p[1].Split('|', 2); [pscustomobject]@{ pid2 = [int]$p[0]; geom = $parts[0]; title = $parts[1] } } | ConvertTo-Json -Compress; if (-not $json) { $json = '[]' } -Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 }).then((j) => ({ +Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 }); + return { windows: (Array.isArray(j.windows) ? j.windows : [j.windows]).map((w) => { const g = String(w.geom).split(",").map(Number); return { pid: w.pid2, title: w.title, position: { x: g[0], y: g[1] }, size: { w: g[2], h: g[3] } }; }), - })).catch((e) => { throw e; }); + }; }, open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => { const target = name ?? bid; @@ -162,7 +171,6 @@ Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 }).then((j) = return { launched: true, name: target, url: urlArg ?? null, activate }; }, get_app_state: async ({ app_ref, detail } = {}) => { - await bootstrapped; const filter = app_ref?.name ? app_ref.name.replace(/'/g, "''") : ""; const maxEls = detail === "full" ? 800 : 400; const j = await psJson(`Add-Type -AssemblyName UIAutomationClient; @@ -197,7 +205,6 @@ Write-Output ($result | ConvertTo-Json -Depth 6 -Compress);`, { timeoutMs: 60_00 return j; }, screenshot: async ({ display, region, path: outPath } = {}) => { - await bootstrapped; const dir = recordingsDir(); fs.mkdirSync(dir, { recursive: true }); const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`); @@ -256,7 +263,10 @@ Write-Output '{"ok": true}';`, { timeoutMs: 20_000 }); return { action_sent: true, from, to }; }, left_mouse_down: async ({ target }) => { - await withUser32(target ? `[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;` : "" + `[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`); + // Ternary must select ONLY the optional move prefix; the LEFTDOWN press + // always runs, so a targeted press both moves and presses. + const move = target ? `[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;\n` : ""; + await withUser32(`${move}[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`); return { action_sent: true }; }, left_mouse_up: async () => { @@ -276,7 +286,6 @@ Write-Output '{"ok": true}';`); }, type: async ({ text }) => { if (!text) return { action_sent: false, note: "empty text" }; - await bootstrapped; const b64 = Buffer.from(String(text), "utf16le").toString("base64"); const script = `Add-Type -TypeDefinition @' using System; @@ -337,7 +346,6 @@ Write-Output '{"ok": true}';`, { timeoutMs: Math.max(10_000, d * 1000 + 8000) }) }, set_value: async ({ target, value }) => { // UIA ValuePattern via a re-walk to target.path from the desktop root. - await bootstrapped; const b64path = Buffer.from(JSON.stringify(target.path ?? []), "utf8").toString("base64"); const b64val = Buffer.from(String(value ?? ""), "utf16le").toString("base64"); const j = await psJson(`Add-Type -AssemblyName UIAutomationClient; Add-Type -AssemblyName UIAutomationTypes; @@ -365,7 +373,6 @@ try { }, select_text: async () => { throw new ExecError("select_text is not implemented on the win32 backend yet — fail-closed"); }, perform_action: async ({ target, action }) => { - await bootstrapped; const b64path = Buffer.from(JSON.stringify(target.path ?? []), "utf8").toString("base64"); const act = String(action ?? "Invoke").replace(/'/g, ""); const j = await psJson(`Add-Type -AssemblyName UIAutomationClient; Add-Type -AssemblyName UIAutomationTypes; @@ -402,13 +409,13 @@ Write-Output ('{"text": ' + ($t | ConvertTo-Json -Compress) + '}');`, { timeoutM }, write_clipboard: async ({ text }) => { const b64 = Buffer.from(String(text ?? ""), "utf16le").toString("base64"); - await ps(`Set-Clipboard -Value ([System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'))); + await psOk(`Set-Clipboard -Value ([System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'))); Write-Output '{"ok": true}';`, { timeoutMs: 10_000 }); return { written: String(text ?? "").length }; }, cursor_position: async () => { - await bootstrapped; - const j = await psJson(`$p = New-Object User32+POINT; + const j = await psJson(`${USER32_PRELUDE} +$p = New-Object User32+POINT; [void][User32]::GetCursorPos([ref]$p); Write-Output ('{"x": ' + $p.X + ', "y": ' + $p.Y + '}');`); return { x: j.x, y: j.y }; diff --git a/crates/tui/plugins/computer-use/tests/win32-input.test.mjs b/crates/tui/plugins/computer-use/tests/win32-input.test.mjs new file mode 100644 index 0000000000..5bb45974a6 --- /dev/null +++ b/crates/tui/plugins/computer-use/tests/win32-input.test.mjs @@ -0,0 +1,164 @@ +// Windows backend input truthfulness tests. No Windows box (or GUI) is +// involved: a fake powershell.exe is placed on PATH, captures the decoded +// -EncodedCommand payload of every spawn, and exits with a code the test +// controls — mirroring how the issue was verified upstream (issue #5896). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import win32 from "../src/backends/win32.mjs"; +import { ExecError } from "../src/exec.mjs"; + +// Fake powershell.exe logic. It lives in a .mjs file because Node's loader +// refuses to execute a script whose name ends in .exe; powershell.exe itself +// is a #!/bin/sh launcher that execs node on this file (extension is +// irrelevant to the kernel, only to Node's module loader). +const FAKE_PS_MJS = ` +import fs from "node:fs"; +const args = process.argv.slice(2); +const i = args.indexOf("-EncodedCommand"); +const script = i >= 0 ? Buffer.from(args[i + 1], "base64").toString("utf16le") : ""; +if (process.env.CU_FAKE_PS_CAPTURE) { + fs.appendFileSync(process.env.CU_FAKE_PS_CAPTURE, JSON.stringify({ args, script }) + "\\n"); +} +if (process.env.CU_FAKE_PS_STDOUT) process.stdout.write(process.env.CU_FAKE_PS_STDOUT); +if (process.env.CU_FAKE_PS_STDERR) process.stderr.write(process.env.CU_FAKE_PS_STDERR); +process.exit(Number(process.env.CU_FAKE_PS_EXIT || 0)); +`; + +/** + * Install a controllable fake powershell.exe on PATH (or, with onPath:false, + * guarantee no powershell.exe at all) and restore every mutation in t.after. + * Returns calls(): the decoded {args, script} of each spawn so far. + */ +function fakePowershell(t, { exit = 0, stdout = "", stderr = "", onPath = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-win32-test-")); + const capture = path.join(dir, "captured.jsonl"); + if (onPath) { + const mjs = path.join(dir, "ps-fake.mjs"); + fs.writeFileSync(mjs, FAKE_PS_MJS); + const bin = path.join(dir, "powershell.exe"); + fs.writeFileSync(bin, `#!/bin/sh\nexec node ${JSON.stringify(mjs)} "$@"\n`); + fs.chmodSync(bin, 0o755); + } + const saved = { + PATH: process.env.PATH, + CU_FAKE_PS_EXIT: process.env.CU_FAKE_PS_EXIT, + CU_FAKE_PS_STDOUT: process.env.CU_FAKE_PS_STDOUT, + CU_FAKE_PS_STDERR: process.env.CU_FAKE_PS_STDERR, + CU_FAKE_PS_CAPTURE: process.env.CU_FAKE_PS_CAPTURE, + }; + process.env.PATH = onPath ? `${dir}${path.delimiter}${process.env.PATH}` : dir; + process.env.CU_FAKE_PS_EXIT = String(exit); + process.env.CU_FAKE_PS_STDOUT = stdout; + process.env.CU_FAKE_PS_STDERR = stderr; + process.env.CU_FAKE_PS_CAPTURE = capture; + t.after(() => { + process.env.PATH = saved.PATH; + for (const [k, v] of Object.entries({ + CU_FAKE_PS_EXIT: saved.CU_FAKE_PS_EXIT, + CU_FAKE_PS_STDOUT: saved.CU_FAKE_PS_STDOUT, + CU_FAKE_PS_STDERR: saved.CU_FAKE_PS_STDERR, + CU_FAKE_PS_CAPTURE: saved.CU_FAKE_PS_CAPTURE, + })) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + const calls = () => + fs.existsSync(capture) + ? fs.readFileSync(capture, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) + : []; + return { calls }; +} + +test("win32: targeted left_mouse_down both moves and presses in one self-contained command", async (t) => { + const fake = fakePowershell(t); + const backend = win32.create(); + const r = await backend.left_mouse_down({ target: { x: 12, y: 34 } }); + assert.deepEqual(r, { action_sent: true }); + const calls = fake.calls(); + assert.equal(calls.length, 1, "exactly one PowerShell process — no bootstrap process is relied on"); + assert.ok(calls[0].args.includes("-EncodedCommand"), "script travels as an encoded command"); + const script = calls[0].script; + assert.match(script, /Add-Type -TypeDefinition/, "the action command carries the type definition itself"); + assert.match(script, /public static class User32/); + assert.match(script, /\[User32\]::SetCursorPos\(12, 34\)/, "targeted press moves the cursor"); + assert.match(script, /\[User32\]::mouse_event\(\[User32\]::LEFTDOWN/, "targeted press also presses"); +}); + +test("win32: PowerShell exit != 0 becomes an error, never action_sent:true", async (t) => { + fakePowershell(t, { exit: 1, stderr: "unable to find type [User32]" }); + const backend = win32.create(); + const cases = [ + ["left_mouse_down", () => backend.left_mouse_down({ target: { x: 1, y: 2 } })], + ["mouse_move", () => backend.mouse_move({ target: { x: 1, y: 2 } })], + ["key", () => backend.key({ text: "enter" })], + ]; + for (const [name, fn] of cases) { + await assert.rejects(fn(), (e) => { + assert.ok(e instanceof ExecError, `${name} rejects with ExecError`); + assert.match(e.message, /exited 1/, `${name} reports the nonzero exit code`); + assert.match(e.message, /unable to find type/, `${name} surfaces PowerShell stderr`); + return true; + }, `${name} must not report success when PowerShell exits nonzero`); + } +}); + +test("win32: spawn failure (powershell.exe missing) becomes an error, never action_sent:true", async (t) => { + fakePowershell(t, { onPath: false }); + const backend = win32.create(); + await assert.rejects(backend.left_mouse_down({ target: { x: 1, y: 2 } }), (e) => { + assert.ok(e instanceof ExecError); + assert.match(e.message, /exited -1|ENOENT/); + return true; + }); +}); + +test("win32: successful input still reports success", async (t) => { + const fake = fakePowershell(t); + const backend = win32.create(); + assert.deepEqual(await backend.mouse_move({ target: { x: 5, y: 6 } }), { action_sent: true, at: { x: 5, y: 6 } }); + const click = await backend.left_click({ target: { x: 9, y: 8 } }); + assert.equal(click.action_sent, true); + const clickScript = fake.calls()[1].script; + assert.match(clickScript, /\[User32\]::SetCursorPos\(9, 8\)/); + assert.match(clickScript, /\[User32\]::LEFTDOWN/); + assert.match(clickScript, /\[User32\]::LEFTUP/); +}); + +test("win32: every User32-backed action carries the Add-Type definition in its own process", async (t) => { + const fake = fakePowershell(t); + const backend = win32.create(); + const at = { x: 3, y: 4 }; + const actions = [ + ["mouse_move", () => backend.mouse_move({ target: at })], + ["left_mouse_up", () => backend.left_mouse_up()], + ["left_click", () => backend.left_click({ target: at })], + ["double_click", () => backend.double_click({ target: at })], + ["right_click", () => backend.right_click({ target: at })], + ["middle_click", () => backend.middle_click({ target: at })], + ["left_click_drag", () => backend.left_click_drag({ from_target: at, to: { x: 10, y: 11 } })], + ["scroll", () => backend.scroll({ target: at, direction: "down", amount: 2 })], + ["key", () => backend.key({ text: "ctrl+a" })], + ["hold_key", () => backend.hold_key({ text: "a", duration: 0.05 })], + ]; + for (const [, fn] of actions) await fn(); + const calls = fake.calls(); + assert.equal(calls.length, actions.length, "exactly one self-contained PowerShell process per action"); + for (const [i, [name]] of actions.entries()) { + assert.match(calls[i].script, /Add-Type -TypeDefinition/, `${name} is self-contained`); + assert.match(calls[i].script, /public static class User32/, `${name} defines User32 inline`); + } +}); + +test("win32: cursor_position is self-contained and parses JSON output", async (t) => { + const fake = fakePowershell(t, { stdout: '{"x": 11, "y": 22}' }); + const backend = win32.create(); + assert.deepEqual(await backend.cursor_position(), { x: 11, y: 22 }); + const script = fake.calls()[0].script; + assert.match(script, /Add-Type -TypeDefinition/, "cursor_position carries the type definition"); + assert.match(script, /GetCursorPos/); +});