From e44d86357966f79f603bea521c749cd9d22ad61c Mon Sep 17 00:00:00 2001 From: delta456 Date: Mon, 24 Aug 2026 13:22:54 +0530 Subject: [PATCH 1/4] fix(agent): repair broken Anthropic goal execution and missing result output --- .../src/ui/ink/components/FinalSummary.tsx | 33 +++++--- packages/core/src/agent/loop.ts | 68 +++++++++++++-- packages/core/src/device/android-power.ts | 45 ++++++++++ packages/core/src/llm/provider.ts | 46 ++++++++++ packages/core/src/mcp/keyboard.ts | 2 +- tests/sdk/llm-thinking.test.ts | 83 +++++++++++++++++++ tests/ui/ink-ui.test.tsx | 59 ++++++++++++- 7 files changed, 312 insertions(+), 24 deletions(-) create mode 100644 packages/core/src/device/android-power.ts create mode 100644 tests/sdk/llm-thinking.test.ts diff --git a/packages/cli/src/ui/ink/components/FinalSummary.tsx b/packages/cli/src/ui/ink/components/FinalSummary.tsx index f6a3c85..73dc1e5 100644 --- a/packages/cli/src/ui/ink/components/FinalSummary.tsx +++ b/packages/cli/src/ui/ink/components/FinalSummary.tsx @@ -53,20 +53,27 @@ export function FinalSummary({ data }: { data: JourneySummaryData }) { {data.subGoals.map((sg, i) => { const sgOk = sg.status === 'completed'; return ( - - - - {sgOk ? symbols.check : symbols.cross} - - - - {sg.goal} - - - - {sgOk ? 'pass' : 'FAIL'} - + + + + + {sgOk ? symbols.check : symbols.cross} + + + + {sg.goal} + + + + {sgOk ? 'pass' : 'FAIL'} + + + {sg.result ? ( + + {sg.result} + + ) : null} ); })} diff --git a/packages/core/src/agent/loop.ts b/packages/core/src/agent/loop.ts index ef701bf..f09359a 100644 --- a/packages/core/src/agent/loop.ts +++ b/packages/core/src/agent/loop.ts @@ -15,6 +15,7 @@ import { typeViaKeyboard, detectDeviceUdid, pressEnterKey } from '../mcp/keyboar import type { LLMProvider, AgentContext, ToolCallDecision } from '../llm/provider.js'; import type { ActionResult } from '../llm/schemas.js'; import { getScreenState } from '../perception/screen.js'; +import { detectScreenOff } from '../device/android-power.js'; import { diffScreen, computePerceptionHash } from '../perception/screen-diff.js'; import { createStuckDetector } from './stuck.js'; import { shouldPreferVisionLocateTap } from './vision-tap-policy.js'; @@ -135,6 +136,22 @@ export interface StepRecord { const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +/** + * Describe a screen-capture failure, appending a device-state hint when a + * sleeping display is the cause. Only probes on timeouts — every other failure + * mode already reports itself, and the probe costs an adb round-trip. + */ +async function describeScreenFailure( + err: unknown, + platform: 'android' | 'ios', + deviceUdid?: string | null +): Promise { + const msg = err instanceof Error ? err.message : String(err); + if (platform !== 'android' || !/timed out|timeout/i.test(msg)) return msg; + const hint = await detectScreenOff(deviceUdid ?? undefined); + return hint ? `${msg} — ${hint}` : msg; +} + export async function runAgent(options: AgentOptions): Promise { let { goal } = options; const { @@ -298,8 +315,9 @@ export async function runAgent(options: AgentOptions): Promise { ); } catch (err) { ui.stopSpinner(); - ui.printError(`Step ${step + 1}: Failed to get screen state`, String(err)); - lastResult = `Screen capture failed: ${err}`; + const detail = await describeScreenFailure(err, detectedPlatform, deviceUdid); + ui.printError(`Step ${step + 1}: Failed to get screen state`, detail); + lastResult = `Screen capture failed: ${detail}`; cachedPostScreen = undefined; // Invalidate cache on failure continue; } @@ -343,6 +361,19 @@ export async function runAgent(options: AgentOptions): Promise { modelName, totalCachedTokens ); + + // Second early-return path with the same gap as the LLM "done" + // branch above: screenEvaluator can conclude the goal is already + // satisfied from a screen diff alone, without an LLM tool call — + // still needs an onStep so the answer reaches the logger/SDK/JSON. + onStep?.({ + step, + decision: { toolName: 'done', args: { reason: evaluation.reason } }, + result: { success: true, message: evaluation.reason }, + elementsCount: screen.elementCount, + screenshot: postActionScreenshot, + }); + return { success: true, reason: evaluation.reason, @@ -574,6 +605,8 @@ export async function runAgent(options: AgentOptions): Promise { } catch (err: any) { const errName = err?.name ?? ''; const errMsg = err?.message ?? ''; + const status = err?.statusCode ?? err?.status; + const apiErrorType = err?.data?.error?.type ?? err?.responseBody?.error?.type; if ( errName.includes('UnsupportedModel') || errName.includes('AuthenticationError') || @@ -581,23 +614,27 @@ export async function runAgent(options: AgentOptions): Promise { errMsg.includes('API key') || errMsg.includes('is not found') || errMsg.includes('NOT_FOUND') || - err?.statusCode === 401 || - err?.statusCode === 404 + // A malformed request never succeeds on retry — a bad thinking/tool + // config would otherwise burn the entire step budget on identical 400s. + status === 400 || + status === 401 || + status === 404 || + apiErrorType === 'invalid_request_error' ) { ui.stopStreaming(); ui.stopSpinner(); - ui.printError('Fatal LLM error', err.message ?? String(err)); + ui.printError('Fatal LLM error', errMsg || String(err)); return { success: false, - reason: `Fatal LLM error: ${err.message ?? err}`, + reason: `Fatal LLM error: ${errMsg || err}`, stepsUsed: step + 1, history, }; } ui.stopStreaming(); ui.stopSpinner(); - ui.printError(`Step ${step + 1}: LLM error`, String(err)); - lastResult = `LLM call failed: ${err}`; + ui.printError(`Step ${step + 1}: LLM error`, errMsg || String(err)); + lastResult = `LLM call failed: ${errMsg || err}`; continue; } @@ -708,7 +745,7 @@ export async function runAgent(options: AgentOptions): Promise { } catch (err) { // Verification failed to execute — log but accept the done to avoid blocking indefinitely ui.printWarning( - `Done verification failed: ${err instanceof Error ? err.message : String(err)}` + `Done verification failed: ${await describeScreenFailure(err, detectedPlatform, deviceUdid)}` ); } @@ -722,6 +759,19 @@ export async function runAgent(options: AgentOptions): Promise { const cost = (totalInputTokens / 1_000_000) * pricing[0] + (totalOutputTokens / 1_000_000) * pricing[1]; ui.printTokenSummary(totalInputTokens, totalOutputTokens, cost, modelName, totalCachedTokens); + + // The "done" branch returns here rather than falling through to the + // bottom-of-loop onStep() call below — without this, the step carrying + // the LLM's actual answer (reason) never reaches the session logger, + // the SDK's onStep consumers, or the JSON event stream. + onStep?.({ + step, + decision, + result: { success: true, message: reason }, + elementsCount: screen.elementCount, + screenshot: postActionScreenshot, + }); + return { success: true, reason, diff --git a/packages/core/src/device/android-power.ts b/packages/core/src/device/android-power.ts new file mode 100644 index 0000000..b55f44c --- /dev/null +++ b/packages/core/src/device/android-power.ts @@ -0,0 +1,45 @@ +/** + * Android display-power probe. + * + * UiAutomator2 builds a page source by walking the focused window. While the + * display is asleep there is no focused window, so the dump never returns and + * the call burns the entire MCP request budget (MCP_TIMEOUT_MS, default 120 s) + * before failing with a bare "Request timed out" — which says nothing about the + * actual cause. Probing the power state turns that into an actionable message. + */ + +import { exec } from 'child_process'; +import { promisify } from 'util'; +import { getADBPath } from '../mcp/keyboard.js'; + +const execAsync = promisify(exec); + +/** Appended to screen-capture timeouts that a sleeping display explains. */ +export const SCREEN_OFF_HINT = + 'device screen is off — UiAutomator cannot read the screen while the display sleeps. ' + + 'Wake it with `adb shell input keyevent KEYCODE_WAKEUP`; ' + + 'for longer runs `adb shell svc power stayon true` keeps it awake while charging.'; + +/** + * Resolve SCREEN_OFF_HINT when the Android display is asleep, else undefined. + * + * Never throws and never blocks for long — a diagnostic must not mask or delay + * the failure it is explaining. Returns undefined when adb is missing, the + * device is gone, or the probe itself times out. + */ +export async function detectScreenOff(deviceUdid?: string): Promise { + const deviceFlag = deviceUdid ? `-s ${deviceUdid}` : ''; + try { + // Filter on-device: a full `dumpsys power` dump runs to megabytes and blows + // past exec's 1 MB maxBuffer, which would fail the probe before it ever + // reads the field. Grepping on the device returns a couple of lines. + const { stdout } = await execAsync( + `${getADBPath()} ${deviceFlag} shell "dumpsys power | grep mWakefulness"`, + { timeout: 5000 } + ); + // Asleep = display off. Dozing = ambient/always-on display, equally unreadable. + return /mWakefulness=(Asleep|Dozing)/.test(stdout) ? SCREEN_OFF_HINT : undefined; + } catch { + return undefined; + } +} diff --git a/packages/core/src/llm/provider.ts b/packages/core/src/llm/provider.ts index 998fd12..845e964 100644 --- a/packages/core/src/llm/provider.ts +++ b/packages/core/src/llm/provider.ts @@ -381,6 +381,20 @@ function isGemini25Family(modelId: string): boolean { return /gemini-2\.5/i.test(modelId); } +/** + * Adaptive thinking landed with Claude 4.6 and is the only thinking mode the + * 5 family accepts. Pre-4.6 models (4.5 and older) support only the budgeted + * `{type:'enabled', budgetTokens}` form. + */ +function supportsAdaptiveThinking(modelId: string): boolean { + const id = modelId.toLowerCase(); + // 5 family: claude-opus-5, claude-sonnet-5, claude-fable-5, claude-mythos-5 + if (/^claude-[a-z]+-5\b/.test(id)) return true; + // 4.x: adaptive from 4-6 onward (4-6, 4-7, 4-8, …) + const minor = id.match(/^claude-[a-z]+-4-(\d+)/); + return minor ? Number(minor[1]) >= 6 : false; +} + export function buildThinkingOptions(config: AppClawConfig): Record | undefined { if (config.LLM_THINKING !== 'on') return undefined; if (!THINKING_PROVIDERS.has(config.LLM_PROVIDER)) return undefined; @@ -394,6 +408,23 @@ export function buildThinkingOptions(config: AppClawConfig): Record switch (config.LLM_PROVIDER) { case 'anthropic': + // Anthropic rejects `{type:'enabled'}` when tool_choice forces tool use + // ("Thinking may not be enabled when tool_choice forces tool use"), and + // getDecision always sends toolChoice:'required'. Adaptive is the only + // thinking mode the API permits alongside forced tools — and the + // documented replacement for budgetTokens on 4.6+ regardless. + // `display:'summarized'` is required for reasoning to reach the UI; the + // API default omits it and streams empty thinking blocks. + if (supportsAdaptiveThinking(modelId)) { + return { + anthropic: { + thinking: { type: 'adaptive', display: 'summarized' }, + }, + }; + } + // Pre-4.6 models have no adaptive mode. The budgeted form still cannot + // coexist with forced tool use; the agent loop reports that 400 as fatal + // rather than retrying it. return { anthropic: { thinking: { type: 'enabled', budgetTokens: budget }, @@ -534,7 +565,15 @@ export function createLLMProvider(config: AppClawConfig, mcpTools: MCPToolInfo[] // textStream omits reasoning — use fullStream so goal-based runs show thinking. let reasoningText = ''; let streamingStarted = false; + let streamError: unknown; for await (const part of stream.fullStream) { + // Provider failures arrive as `error` parts. The promises awaited + // below then throw a generic NoOutputGeneratedError that carries no + // `cause`, so the actionable message is lost unless captured here. + if (part.type === 'error') { + streamError = (part as any).error; + continue; + } const chunk = part.type === 'reasoning-delta' ? part.text @@ -551,6 +590,13 @@ export function createLLMProvider(config: AppClawConfig, mcpTools: MCPToolInfo[] } if (streamingStarted) callbacks.onDone?.(); + // Rethrow the provider's own error rather than letting the awaits below + // surface it as "No output generated. Check the stream for errors." + if (streamError) { + clearTimeout(abortTimer); + throw streamError; + } + // Await final results after stream completes const [streamUsage, streamTotalUsage, toolCalls, text, providerMeta, response] = await Promise.all([ diff --git a/packages/core/src/mcp/keyboard.ts b/packages/core/src/mcp/keyboard.ts index c656e17..f9dc018 100644 --- a/packages/core/src/mcp/keyboard.ts +++ b/packages/core/src/mcp/keyboard.ts @@ -118,7 +118,7 @@ export async function typeViaSetValue(mcp: MCPClient, text: string): Promise { + test.each([ + ['claude-sonnet-4-6', true], + ['claude-opus-5', true], + ['claude-sonnet-5', true], + ['claude-fable-5', true], + ['claude-opus-4-8', true], + ['claude-opus-4-7', true], + ['claude-sonnet-4-5', false], + ['claude-haiku-4-5', false], + ['claude-3-5-sonnet', false], + ])('%s → adaptive: %s', (model, expectAdaptive) => { + const config = loadConfig({ LLM_PROVIDER: 'anthropic', LLM_MODEL: model, LLM_THINKING: 'on' }); + const options = buildThinkingOptions(config); + expect(options?.anthropic?.thinking?.type).toBe(expectAdaptive ? 'adaptive' : 'enabled'); + }); + + test('adaptive thinking requests display:summarized so reasoning reaches the UI', () => { + const config = loadConfig({ + LLM_PROVIDER: 'anthropic', + LLM_MODEL: 'claude-sonnet-4-6', + LLM_THINKING: 'on', + }); + const options = buildThinkingOptions(config); + // The API default for `display` is 'omitted', which streams empty + // thinking blocks — 'summarized' is required for the terminal's + // "Reasoning…" panel to actually show anything. + expect(options?.anthropic?.thinking?.display).toBe('summarized'); + }); + + test('pre-4.6 models still carry a budgetTokens value (no adaptive support)', () => { + const config = loadConfig({ + LLM_PROVIDER: 'anthropic', + LLM_MODEL: 'claude-sonnet-4-5', + LLM_THINKING: 'on', + }); + const options = buildThinkingOptions(config); + expect(options?.anthropic?.thinking).toMatchObject({ type: 'enabled' }); + expect(typeof options?.anthropic?.thinking?.budgetTokens).toBe('number'); + }); + + test('LLM_THINKING=off disables thinking regardless of model', () => { + const config = loadConfig({ + LLM_PROVIDER: 'anthropic', + LLM_MODEL: 'claude-sonnet-4-6', + LLM_THINKING: 'off', + }); + expect(buildThinkingOptions(config)).toBeUndefined(); + }); + + test('Gemini routing is unaffected by the Anthropic adaptive-thinking change', () => { + const gemini25 = loadConfig({ + LLM_PROVIDER: 'gemini', + LLM_MODEL: 'gemini-2.5-flash', + LLM_THINKING: 'on', + }); + expect(buildThinkingOptions(gemini25)).toMatchObject({ + google: { thinkingConfig: { thinkingBudget: expect.any(Number) } }, + }); + + const gemini3 = loadConfig({ + LLM_PROVIDER: 'gemini', + LLM_MODEL: 'gemini-3.1-flash-lite', + LLM_THINKING: 'on', + }); + expect(buildThinkingOptions(gemini3)).toMatchObject({ + google: { thinkingConfig: { thinkingLevel: expect.any(String) } }, + }); + }); +}); diff --git a/tests/ui/ink-ui.test.tsx b/tests/ui/ink-ui.test.tsx index 780f31c..d26f80b 100644 --- a/tests/ui/ink-ui.test.tsx +++ b/tests/ui/ink-ui.test.tsx @@ -7,11 +7,12 @@ import React from 'react'; import { describe, test, expect } from 'vitest'; import { render } from 'ink-testing-library'; import { RunScreen, tailSlice } from '@appclaw/cli/ui/ink/RunScreen'; -import type { TimelineEntry } from '@appclaw/cli/ui/ink/store'; +import type { TimelineEntry, JourneySummaryData } from '@appclaw/cli/ui/ink/store'; import { store } from '@appclaw/cli/ui/ink/store'; import { askUserViaInk } from '@appclaw/cli/ui/ink/InkRenderer'; import { PlaygroundApp } from '@appclaw/cli/ui/ink/PlaygroundApp'; import { pgStore } from '@appclaw/cli/ui/ink/playground-store'; +import { FinalSummary } from '@appclaw/cli/ui/ink/components/FinalSummary'; const tick = (ms = 30) => new Promise((r) => setTimeout(r, ms)); @@ -278,3 +279,59 @@ describe('Ink PlaygroundApp', () => { unmount(); }); }); + +describe('Ink FinalSummary', () => { + const journeyData = ( + overrides: Partial = {}, + subGoals: JourneySummaryData['subGoals'] = [] + ): JourneySummaryData => ({ + success: true, + overallGoal: 'open settings and get the wifi name', + subGoals, + totalSteps: 3, + durationMs: 58400, + tokens: { input: 17425, output: 86, cost: 0.0536, model: 'claude-sonnet-4-6' }, + ...overrides, + }); + + // Regression test for the bug where a sub-goal's `done` reason — e.g. the + // Wi-Fi name the agent was asked to find — reached this component but was + // never rendered, only its goal text and pass/fail were. + test('renders each sub-goal result text, not just goal + pass/fail', () => { + const data = journeyData({}, [ + { goal: 'Open the Settings app', status: 'completed', result: 'Opened Settings' }, + { + goal: 'Navigate to Network & internet → Internet and identify the Wi-Fi name', + status: 'completed', + result: "The connected Wi-Fi network is 'AndroidWifi'", + }, + ]); + const { lastFrame, unmount } = render(); + const out = lastFrame() ?? ''; + expect(out).toContain('AndroidWifi'); + expect(out).toContain('Opened Settings'); + unmount(); + }); + + test('renders a failed sub-goal result alongside FAIL', () => { + const data = journeyData({ success: false }, [ + { + goal: "Tap on 'Network & internet'", + status: 'failed', + result: 'UiAutomator2 driver crashed', + }, + ]); + const { lastFrame, unmount } = render(); + const out = lastFrame() ?? ''; + expect(out).toContain('FAIL'); + expect(out).toContain('UiAutomator2 driver crashed'); + unmount(); + }); + + test('omits the result line when a sub-goal has no result text', () => { + const data = journeyData({}, [{ goal: 'Open the Settings app', status: 'completed' }]); + const { lastFrame, unmount } = render(); + expect(lastFrame()).toContain('Open the Settings app'); + unmount(); + }); +}); From 17368d6bf979f6ab1b0e338220c81a356ae75a1d Mon Sep 17 00:00:00 2001 From: delta456 Date: Wed, 26 Aug 2026 13:59:09 +0530 Subject: [PATCH 2/4] feat: introduce Terminal Studio (TUI) --- .agents/skills/use-appclaw-cli/SKILL.md | 22 +- .gitignore | 3 + .kiro/steering/product.md | 2 +- .kiro/steering/structure.md | 3 +- CLAUDE.md | 21 +- README.md | 17 +- landing/public/usage.html | 521 ++++- package-lock.json | 4 +- packages/cli/package.json | 3 +- packages/cli/src/cli/doctor.ts | 3 + packages/cli/src/index.ts | 72 +- packages/cli/src/playground/index.ts | 1678 ----------------- .../cli/src/step-recorder/flow-builder.ts | 158 ++ packages/cli/src/step-recorder/json-bridge.ts | 567 ++++++ .../cli/src/step-recorder/memory-inspect.ts | 129 ++ packages/cli/src/step-recorder/screen-info.ts | 79 + packages/cli/src/tui/TuiApp.tsx | 99 + packages/cli/src/tui/capture-console.ts | 65 + packages/cli/src/tui/commands.ts | 551 ++++++ .../cli/src/tui/components/CommandPalette.tsx | 175 ++ .../cli/src/tui/components/ConfirmDialog.tsx | 73 + packages/cli/src/tui/components/Header.tsx | 19 + .../cli/src/tui/components/OutputDialog.tsx | 211 +++ .../cli/src/tui/components/ProgressDialog.tsx | 57 + packages/cli/src/tui/components/StatusBar.tsx | 42 + .../cli/src/tui/components/StreamPanel.tsx | 201 ++ packages/cli/src/tui/highlight.ts | 259 +++ packages/cli/src/tui/index.ts | 755 ++++++++ packages/cli/src/tui/input-history.ts | 25 + .../src/tui/screens/DevicePickerScreen.tsx | 165 ++ .../cli/src/tui/screens/HistoryScreen.tsx | 161 ++ packages/cli/src/tui/screens/MainScreen.tsx | 364 ++++ .../cli/src/tui/screens/SettingsScreen.tsx | 123 ++ .../cli/src/tui/screens/WelcomeScreen.tsx | 116 ++ packages/cli/src/tui/session-log.ts | 195 ++ packages/cli/src/tui/store.ts | 430 +++++ packages/cli/src/tui/stream/capture.ts | 107 ++ packages/cli/src/tui/stream/frame-loop.ts | 184 ++ packages/cli/src/tui/stream/halfblock.ts | 68 + packages/cli/src/tui/stream/kitty.ts | 63 + packages/cli/src/tui/stream/layout.ts | 222 +++ packages/cli/src/tui/stream/placeholder.ts | 96 + packages/cli/src/tui/stream/terminal-caps.ts | 33 + packages/cli/src/tui/useLayout.ts | 26 + packages/cli/src/ui/ink/PlaygroundApp.tsx | 94 - .../ui/ink/components/PlaygroundBottomBar.tsx | 79 - packages/cli/src/ui/ink/playground-runner.tsx | 53 - packages/cli/src/ui/ink/playground-store.ts | 64 - packages/core/src/agent/app-resolver.ts | 2 +- packages/core/src/agent/loop.ts | 4 +- packages/core/src/config.ts | 15 +- packages/core/src/device/emulator-list.ts | 107 ++ packages/core/src/device/index.ts | 2 +- packages/core/src/flow/run-instruction.ts | 8 +- packages/core/src/flow/run-yaml-flow.ts | 6 +- packages/core/src/flow/vision-execute.ts | 4 +- packages/core/src/sdk/goal-export.ts | 75 +- packages/core/src/sdk/index.ts | 4 +- packages/core/src/sdk/step-runner.ts | 6 +- packages/core/src/sdk/types.ts | 2 +- packages/core/src/ui/step-printer.ts | 6 +- packages/core/src/ui/terminal.ts | 3 +- tests/sdk/goal-export.test.ts | 76 + tests/ui/command-palette.test.tsx | 82 + tests/ui/complete-command.test.ts | 42 + tests/ui/ink-ui.test.tsx | 146 +- tests/ui/main-screen-layout.test.tsx | 76 + tests/ui/output-dialog.test.tsx | 158 ++ tests/ui/tui-stream.test.ts | 302 +++ vscode-extension/README.md | 18 +- vscode-extension/package.json | 2 +- vscode-extension/src/extension.ts | 9 +- 72 files changed, 7426 insertions(+), 2186 deletions(-) delete mode 100644 packages/cli/src/playground/index.ts create mode 100644 packages/cli/src/step-recorder/flow-builder.ts create mode 100644 packages/cli/src/step-recorder/json-bridge.ts create mode 100644 packages/cli/src/step-recorder/memory-inspect.ts create mode 100644 packages/cli/src/step-recorder/screen-info.ts create mode 100644 packages/cli/src/tui/TuiApp.tsx create mode 100644 packages/cli/src/tui/capture-console.ts create mode 100644 packages/cli/src/tui/commands.ts create mode 100644 packages/cli/src/tui/components/CommandPalette.tsx create mode 100644 packages/cli/src/tui/components/ConfirmDialog.tsx create mode 100644 packages/cli/src/tui/components/Header.tsx create mode 100644 packages/cli/src/tui/components/OutputDialog.tsx create mode 100644 packages/cli/src/tui/components/ProgressDialog.tsx create mode 100644 packages/cli/src/tui/components/StatusBar.tsx create mode 100644 packages/cli/src/tui/components/StreamPanel.tsx create mode 100644 packages/cli/src/tui/highlight.ts create mode 100644 packages/cli/src/tui/index.ts create mode 100644 packages/cli/src/tui/input-history.ts create mode 100644 packages/cli/src/tui/screens/DevicePickerScreen.tsx create mode 100644 packages/cli/src/tui/screens/HistoryScreen.tsx create mode 100644 packages/cli/src/tui/screens/MainScreen.tsx create mode 100644 packages/cli/src/tui/screens/SettingsScreen.tsx create mode 100644 packages/cli/src/tui/screens/WelcomeScreen.tsx create mode 100644 packages/cli/src/tui/session-log.ts create mode 100644 packages/cli/src/tui/store.ts create mode 100644 packages/cli/src/tui/stream/capture.ts create mode 100644 packages/cli/src/tui/stream/frame-loop.ts create mode 100644 packages/cli/src/tui/stream/halfblock.ts create mode 100644 packages/cli/src/tui/stream/kitty.ts create mode 100644 packages/cli/src/tui/stream/layout.ts create mode 100644 packages/cli/src/tui/stream/placeholder.ts create mode 100644 packages/cli/src/tui/stream/terminal-caps.ts create mode 100644 packages/cli/src/tui/useLayout.ts delete mode 100644 packages/cli/src/ui/ink/PlaygroundApp.tsx delete mode 100644 packages/cli/src/ui/ink/components/PlaygroundBottomBar.tsx delete mode 100644 packages/cli/src/ui/ink/playground-runner.tsx delete mode 100644 packages/cli/src/ui/ink/playground-store.ts create mode 100644 packages/core/src/device/emulator-list.ts create mode 100644 tests/sdk/goal-export.test.ts create mode 100644 tests/ui/command-palette.test.tsx create mode 100644 tests/ui/complete-command.test.ts create mode 100644 tests/ui/main-screen-layout.test.tsx create mode 100644 tests/ui/output-dialog.test.tsx create mode 100644 tests/ui/tui-stream.test.ts diff --git a/.agents/skills/use-appclaw-cli/SKILL.md b/.agents/skills/use-appclaw-cli/SKILL.md index 50f2c07..2da3056 100644 --- a/.agents/skills/use-appclaw-cli/SKILL.md +++ b/.agents/skills/use-appclaw-cli/SKILL.md @@ -1,7 +1,7 @@ --- name: use-appclaw-cli description: > - Use the AppClaw CLI to run YAML flows, start interactive playground, explore apps, + Use the AppClaw CLI to run YAML flows, start the interactive TUI shell, explore apps, record/replay sessions, configure devices, and troubleshoot. Trigger for any request involving appclaw commands, device setup, .env configuration, running flows, vision setup, or debugging execution failures. @@ -91,17 +91,19 @@ appclaw --flow tests/flows/youtube-phased.yaml --env dev **No LLM key needed** unless the flow has steps that fall back to LLM parsing (unrecognized natural language). -### 3. Playground — interactive REPL +### 3. Terminal Studio (`--tui`, alias `--playground`) ```sh -appclaw --playground -appclaw --playground --platform ios --device-type simulator -appclaw --playground --device "iPhone 17 Pro" +appclaw --tui +appclaw --tui --platform ios --device-type simulator +appclaw --tui --device "iPhone 17 Pro" ``` -Type natural language commands that execute live on the device. Steps accumulate and can be exported to a YAML flow. +The interactive mode. Type natural language commands that execute live on the device; steps accumulate and can be exported as a YAML flow or an `@appclaw/runner` spec via `/export`. -**REPL commands:** `/help`, `/steps`, `/export`, `/clear`, `/device`, `/disconnect` +`--playground` is an alias for `--tui` — the old playground REPL was removed. (`--json --playground` is different: a headless NDJSON bridge used by the VS Code / Cursor extension, not something to run by hand.) + +Full-screen Ink shell: platform/device picker, slash-command palette, goal REPL, settings, run history. `/stream` (Android only) mirrors the device screen **inside the terminal** — Kitty graphics on Ghostty/kitty/WezTerm, 24-bit ANSI half-blocks everywhere else — at ~5fps via `adb screencap`; `/stream-close` stops it. Requires an interactive terminal; incompatible with `--json`. ### 4. Explorer — PRD to test flows @@ -317,7 +319,7 @@ Records successful trajectories to `~/.appclaw/trajectories.json` and reuses the - `appclaw "goal"` (agent mode — uses LLM credits, takes actions on device) - `appclaw --explore` (LLM credits + device crawling) - `appclaw --record` (agent mode + saves recording) -- `appclaw --playground` (interactive device session) +- `appclaw --tui` (interactive device session; goals typed inside it use LLM credits — `--playground` is an alias for it) Why: agent and explorer modes consume LLM API credits and take real actions on the connected device. @@ -390,8 +392,8 @@ appclaw --flow tests/flows/youtube-phased.yaml --env dev ### Quick test on iOS simulator ```sh -appclaw --platform ios --device-type simulator --playground -# In REPL: type commands, test them, /export to YAML +appclaw --platform ios --device-type simulator --tui +# In the TUI: type commands, test them, /export to YAML ``` ### Generate test flows from a PRD diff --git a/.gitignore b/.gitignore index 60b3a2d..61dac7d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ dist/ .env *.log .DS_Store +# JetBrains workspace state — per-developer, never shared +.idea/ +*.iml .claude/ recordings/ logs/ diff --git a/.kiro/steering/product.md b/.kiro/steering/product.md index 0bd7d6f..26531a4 100644 --- a/.kiro/steering/product.md +++ b/.kiro/steering/product.md @@ -6,7 +6,7 @@ AppClaw is an agentic AI layer for mobile automation on Android and iOS. Users d - **Agent mode** — LLM-driven goal execution (e.g. `appclaw "Send a WhatsApp message to Mom"`) - **YAML flows** — declarative, zero-LLM automation steps defined in YAML files -- **Playground** — interactive REPL for building flows live on a device +- **Terminal Studio** (`--tui`, alias `--playground`) — the interactive mode: step recorder, device picker, command palette, run history, and in-terminal device stream - **Explorer** — generates YAML test flows from a PRD or app description - **Record/Replay** — capture and adaptively replay goal executions - **Report** — Express server serving HTML run reports diff --git a/.kiro/steering/structure.md b/.kiro/steering/structure.md index 1b6bcc3..196f9c8 100644 --- a/.kiro/steering/structure.md +++ b/.kiro/steering/structure.md @@ -36,7 +36,8 @@ appclaw/ | `device/` | Device setup pipeline — platform/device picker, iOS setup, Appium session | | `memory/` | Episodic memory — trajectory recording, fingerprinting, retrieval | | `explorer/` | PRD → YAML flow generation, screen crawler | -| `playground/` | Interactive REPL for building flows | +| `step-recorder/` | Shared step-recording helpers + the headless `--json --playground` NDJSON bridge | +| `tui/` | Terminal Studio — multi-screen Ink app (`--tui`, alias `--playground`) | | `recording/` | Session recorder and adaptive replayer | | `report/` | Run artifact collection, HTML report rendering, Express server | | `sdk/` | Public SDK — `GoalRunner`, `FlowRunner`, `StepRunner`, config builder | diff --git a/CLAUDE.md b/CLAUDE.md index 58dc1ca..4cfd0ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,12 +27,12 @@ The CLI routes to 6 modes based on flags: - **Interactive** (default) — prompts for platform/device/goal, runs agent loop - **YAML Flow** (`--flow file.yaml`) — declarative automation, zero LLM cost -- **Playground** (`--playground`) — interactive REPL for building flows +- **Terminal Studio** (`--tui`, or its alias `--playground`) — multi-screen Ink app: platform/device picker, slash-command palette, step recording, `/goal` agent runs, settings, run history, and device mirroring — `/stream` renders the screen inside the terminal (see Terminal Studio below). This is the interactive mode; the old `--playground` REPL was removed and the flag now routes here. - **Explorer** (`--explore`) — PRD → YAML test flow generation - **Record/Replay** (`--record`, `--replay`) — capture and replay sessions - **Report** (`--report`) — Express server serving HTML reports from `.appclaw/runs/` -The interactive and goal-direct paths also accept `--export [path]` (optionally `--export-dir `) to write a replayable vitest spec when the goal completes. Path resolution: empty → `EXPORT_DIR/.test.ts`; bare filename → `EXPORT_DIR/`; anything with a directory hint → used verbatim. Implementation in `src/sdk/goal-export.ts` (translator + renderer) called from `src/index.ts` after the agent loop. +The interactive and goal-direct paths also accept `--export [path]` (optionally `--export-dir `) to write a replayable `@appclaw/runner` spec when the goal completes. Path resolution: empty → `EXPORT_DIR/.test.ts` (EXPORT_DIR defaults to `tests`, the runner's own testDir, so an export is runnable where it lands); bare filename → `EXPORT_DIR/`; anything with a directory hint → used verbatim. Implementation in `src/sdk/goal-export.ts` (translator + renderer) called from `src/index.ts` after the agent loop. ### SDK (`src/sdk/`) @@ -41,10 +41,10 @@ Public TypeScript API consumed by external tests (vitest/jest/mocha). Single ent - `app.run(instruction, options?)` — one natural-language step, non-throwing, returns `{ success, action, message }`. `options` (`RunOptions`) applies per-command overrides for this call only: `waitTimeout`/`waitInterval` (implicit-wait poll budget) and `scrollMode`/`scrollTimes` (scroll/swipe distance + count). Instance-wide defaults for all four live on `AppClawOptions`; per-call values win. Implicit wait: every element-bearing action polls its target until present (DOM re-reads page source, vision re-captures the screenshot) or the budget is exhausted — `WAIT_TIMEOUT`/`WAIT_INTERVAL` env, default 10s/300ms - `app.verify(claim)` — assertion. Throws `AppClawAssertionError` on failure (includes `claim`, `result`, and `screenContents` from DOM page-source in DOM mode — in vision mode the LLM's reason is already in `result.message`) - `app.runFlow(path)` — wraps the YAML flow engine -- `app.runGoal(goal, { exportPath?, exportConfig? })` — wraps the agent loop. When `exportPath` is set, the trajectory is filtered with `keepOnlyFinalAttempt()` (drops the branch before any rejected `done`) then rendered as a vitest spec via `generateSdkTest()` +- `app.runGoal(goal, { exportPath?, exportConfig? })` — wraps the agent loop. When `exportPath` is set, the trajectory is filtered with `keepOnlyFinalAttempt()` (drops the branch before any rejected `done`) then rendered as an `@appclaw/runner` spec via `generateSdkTest()` - `app.teardown()` — finalize report, close MCP -Helpers in `src/sdk/goal-export.ts`: `keepOnlyFinalAttempt`, `instructionsFromHistory`, `decisionToInstruction`, `generateSdkTest`, `generateSdkTestFromInstructions` (the last is used by the playground's `/export *.test.ts`). +Helpers in `src/sdk/goal-export.ts`: `keepOnlyFinalAttempt`, `instructionsFromHistory`, `decisionToInstruction`, `generateSdkTest`, `generateSdkTestFromInstructions` (the last is used by the step recorders' `/export *.test.ts` — see `packages/cli/src/step-recorder/`). ### Core Agent Loop (`src/agent/loop.ts`) @@ -60,7 +60,7 @@ Supporting agent modules: planner (goal decomposition), recovery (checkpointing) ### Key Module Responsibilities -- **`src/sdk/`** — Public TypeScript API for external tests. `index.ts` is the `AppClaw` class; `goal-export.ts` translates agent histories back to natural-language `app.run(...)` calls and renders vitest specs; `step-runner.ts` adapts the YAML flow engine to single-instruction calls; `screen-snapshot.ts` captures visible DOM text for assertion-error context. +- **`src/sdk/`** — Public TypeScript API for external tests. `index.ts` is the `AppClaw` class; `goal-export.ts` translates agent histories back to natural-language `app.run(...)` calls and renders `@appclaw/runner` specs; `step-runner.ts` adapts the YAML flow engine to single-instruction calls; `screen-snapshot.ts` captures visible DOM text for assertion-error context. - **`src/llm/`** — Multi-provider LLM integration. `provider.ts` is the factory; `prompts.ts` builds system/user messages; `schemas.ts` defines action schemas. Tools from appium-mcp are dynamically converted to Vercel AI SDK format. - **`src/mcp/`** — Appium MCP client wrapper. Connects via stdio (subprocess) or SSE. Handles tool calling, element finding, screenshots, keyboard input. - **`src/perception/`** — Screen parsing. Android (`android-parser.ts`) and iOS (`ios-parser.ts`) XML parsers. `dom-trimmer.ts` compacts DOM for LLM token efficiency. @@ -70,6 +70,17 @@ Supporting agent modules: planner (goal decomposition), recovery (checkpointing) - **`src/memory/`** — Episodic memory. Records successful trajectories to `~/.appclaw/trajectories.json`, retrieves relevant past experiences via fingerprinting. - **`src/report/`** — Execution reporting. `writer.ts` collects artifacts; `renderer.ts` generates HTML reports; `server.ts` serves them. - **`src/ui/terminal.ts`** — Rich terminal output (spinners, boxes, gradient headers, markdown rendering). JSON output mode for IDE integration (`json-emitter.ts`). +- **`packages/cli/src/step-recorder/`** — Shared by every step-recording surface. `flow-builder.ts` renders a recorded `FlowStep[]` as YAML or an `@appclaw/runner` spec and resolves `/export` paths; `screen-info.ts` answers "what's on screen?" via one vision call; `memory-inspect.ts` backs `/memory`. `json-bridge.ts` is the headless NDJSON-over-stdio recorder behind `appclaw --json --playground`, which the VS Code / Cursor extension spawns — its wire protocol is a shipped contract (see `vscode-extension/src/bridge.ts`). + +### Terminal Studio (`packages/cli/src/tui/`) + +`appclaw --tui` (and its alias `appclaw --playground`) is a separate, multi-screen Ink app (distinct from the single-screen agent-run UI in `packages/cli/src/ui/ink/`) with its own observable store (`store.ts`, a small subscribe/snapshot pub-sub) and a screen router (`TuiApp.tsx`) switching between `screens/{Welcome,DevicePicker,Main,Settings,History}Screen.tsx`. `commands.ts` defines the `/`-prefixed command palette shown on the Main screen. Anything NOT starting with `/` is one deterministic instruction — `runOneInstruction()` — appended to `store.steps`; that recording is what `/list`, `/yaml`, `/edit` and `/export` operate on, and a step that fails is reported but not recorded. `/goal ` is the opt-in to the autonomous loop (a single flat `runAgent()` call, no multi-sub-goal planner — that stays on the plain `appclaw "goal"` path) and records nothing. Device listing goes straight through `adb`/`xcrun simctl` (`@appclaw/core/device/emulator-list.ts`), not through an MCP session, so the picker works before any Appium session exists. + +**Device mirroring** is Android-only, and `adb -s ` is device-agnostic — the same path covers a running emulator, a physical phone and a headless emulator. + +`/stream` renders the device **inside the terminal** (`packages/cli/src/tui/stream/`). Frames go to the side panel on the main screen (`components/StreamPanel.tsx`), which draws only chrome and leaves a blank region so the command palette and prompt stay live alongside it; `frame-loop.ts` polls `adb exec-out screencap` every 200ms and paints that region with direct `process.stdout.write`s — never from React, since Ink rewrites its whole frame on every state change. `terminal-caps.ts` picks the backend from env (no capability query — Ink owns stdin in raw mode): **Kitty graphics** for Ghostty/kitty/WezTerm (`kitty.ts` sends a PNG by file path, `a=T,f=100,t=f,c=…,r=…`, so the terminal does the scaling), otherwise **24-bit ANSI half-blocks** (`halfblock.ts` downsamples the raw RGBA framebuffer from `screencap` with no `-p`, so no PNG decoder and no new dependency). `layout.ts` holds the geometry both sides agree on. Force a backend with `APPCLAW_STREAM_BACKEND=kitty|halfblock`. + +`/stream-close` stops it; device switches and `quit()` tear the frame loop down the same way, via `resetStream()`. ### Configuration (`src/config.ts`) diff --git a/README.md b/README.md index cce7805..6d9eded 100644 --- a/README.md +++ b/README.md @@ -44,23 +44,26 @@ appclaw "open the settings app and turn on airplane mode" You'll need **Node.js 22+**, a connected device / emulator / simulator, and an **LLM API key** (Anthropic, OpenAI, Google, Groq, or local Ollama). `appclaw doctor` checks all of this in seconds and prints fix hints for anything missing (`--full` also spawns appium-mcp for a real handshake). Full setup → **[appclaw.in](https://appclaw.in)**. +Terminal Studio (`appclaw --tui`) can mirror the device inside the terminal with `/stream` (Android only). It looks best on a terminal that speaks the kitty graphics protocol — Ghostty, kitty, WezTerm — and falls back to half-block characters elsewhere. + ## What it can do - **Agent mode** — plain-English goals; the LLM drives the device (tap, type, swipe) - **YAML flows** — deterministic, zero-LLM automation with [structured selectors and state assertions](docs/structured-selectors.md) -- **Test runner** — vitest-style specs across real devices; scaffold with `appclaw init` +- **Terminal Studio** (`appclaw --tui`) — record steps one at a time, watch the device mirrored in the terminal, then `/export` a runnable spec +- **Test runner** — vitest-style specs across real devices, run with `appclaw test`; scaffold with `appclaw init` - **SDK** — drive AppClaw from your own vitest / jest / mocha -- **Playground, cloud devices, record & replay, PRD explorer**, and more +- **Cloud devices, record & replay, PRD explorer**, and more Every mode is documented at **[appclaw.in](https://appclaw.in)**. ## Packages -| Install | Package | For | -| ----------------------- | ----------------- | --------------------------------------------------------- | -| `npm i -g @appclaw/cli` | `@appclaw/cli` | the `appclaw` command — goals, flows, playground, reports | -| via `appclaw init` | `@appclaw/runner` | vitest-style test runner (`appclaw-runner`) | -| `npm i @appclaw/core` | `@appclaw/core` | the SDK / headless engine | +| Install | Package | For | +| ----------------------- | ----------------- | -------------------------------------------------------------- | +| `npm i -g @appclaw/cli` | `@appclaw/cli` | the `appclaw` command — goals, flows, Terminal Studio, reports | +| via `appclaw init` | `@appclaw/runner` | vitest-style test runner (`appclaw test`) | +| `npm i @appclaw/core` | `@appclaw/core` | the SDK / headless engine | ## Local development diff --git a/landing/public/usage.html b/landing/public/usage.html index 57c2643..911883b 100644 --- a/landing/public/usage.html +++ b/landing/public/usage.html @@ -1247,6 +1247,160 @@ animation: none; } } + + /* ── TUI layout diagram ──────────────────────────────────────────── + A picture of the shell rather than ASCII art in a code block: the real + thing is coloured and proportioned, and box-drawing characters + reflow badly on narrow screens. Colours mirror the TUI's own theme. */ + .tui-diagram { + --tui-bg: #14161b; + --tui-brand: #fc8eac; + --tui-dim: #6b7280; + --tui-step: #9cc6f5; + --tui-green: #46c26a; + --tui-text: #e6e6e6; + background: var(--tui-bg); + border: 1px solid var(--rule-strong); + border-radius: 10px; + overflow: hidden; + margin: 1.5rem 0; + font-family: var(--font-mono); + font-size: 0.78rem; + line-height: 1.7; + } + .tui-diagram .tui-bar { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.55rem 0.85rem; + background: rgba(255, 255, 255, 0.04); + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + } + .tui-diagram .tui-bar i { + width: 10px; + height: 10px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.18); + display: inline-block; + } + .tui-diagram .tui-bar span { + color: var(--tui-dim); + margin-left: 0.35rem; + } + .tui-diagram .tui-head { + padding: 0.7rem 1rem 0.2rem; + color: var(--tui-brand); + font-weight: 600; + } + .tui-diagram .tui-head em { + display: block; + color: var(--tui-dim); + font-style: normal; + font-weight: 400; + } + .tui-diagram .tui-cols { + display: flex; + gap: 0.75rem; + padding: 0.6rem 1rem 1rem; + align-items: stretch; + } + .tui-diagram .tui-left { + flex: 0 0 45%; + display: flex; + flex-direction: column; + gap: 0.6rem; + min-width: 0; + } + .tui-diagram .tui-right { + flex: 1; + min-width: 0; + display: flex; + } + .tui-diagram .pane { + border: 1px solid rgba(252, 142, 172, 0.35); + border-radius: 6px; + padding: 0.5rem 0.7rem; + min-width: 0; + } + .tui-diagram .pane.muted { + border-color: rgba(255, 255, 255, 0.14); + } + .tui-diagram .pane .t { + color: var(--tui-dim); + display: block; + margin-bottom: 0.25rem; + } + .tui-diagram .pane.stream { + flex: 1; + display: flex; + flex-direction: column; + } + .tui-diagram .screen { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + min-height: 190px; + } + .tui-diagram .phone { + width: 74px; + height: 152px; + border-radius: 9px; + border: 1px solid rgba(255, 255, 255, 0.22); + background: linear-gradient(170deg, #26303e 0%, #171b22 60%); + position: relative; + } + .tui-diagram .phone::after { + content: ''; + position: absolute; + left: 50%; + bottom: 7px; + width: 26px; + height: 2px; + border-radius: 2px; + transform: translateX(-50%); + background: rgba(255, 255, 255, 0.3); + } + .tui-diagram .line { + color: var(--tui-text); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + .tui-diagram .in { + color: var(--tui-brand); + } + .tui-diagram .ok { + color: var(--tui-green); + } + .tui-diagram .cmd { + color: var(--tui-step); + } + .tui-diagram .dim { + color: var(--tui-dim); + } + .tui-diagram .prompt .in { + margin-right: 0.4rem; + } + .tui-diagram .note { + border-top: 1px solid rgba(255, 255, 255, 0.08); + padding: 0.5rem 1rem 0.65rem; + color: var(--tui-dim); + display: flex; + gap: 1.25rem; + flex-wrap: wrap; + } + @media (max-width: 720px) { + .tui-diagram .tui-cols { + flex-direction: column; + } + .tui-diagram .tui-left { + flex: 1 1 auto; + } + .tui-diagram .screen { + min-height: 150px; + } + } @@ -1352,6 +1506,7 @@

Usage Guide

  • Quick Start
  • CLI Options
  • Execution Modes
    • +
  • Terminal Studio
    • @@ -1588,8 +1743,8 @@

    Quick Start

    # Run a YAML flow appclaw --flow my-test.yaml -# Interactive playground -appclaw --playground +# Interactive shell +appclaw --tui

    For a full test project — config file, spec files, npm scripts — scaffold it with @@ -1786,13 +1941,17 @@

    CLI Options

    Custom Capabilities. + + --tui + Launch Terminal Studio — step recorder, device picker, device mirroring + --playground - Launch the interactive REPL for building flows + Alias for --tui --export [path] - Write a replayable SDK vitest spec after a goal completes + Write a replayable @appclaw/runner spec after a goal completes --export-dir <dir> @@ -1877,44 +2036,359 @@

    YAML Flows

    appclaw --flow tests/youtube-search.yaml --env dev
    -

    Playground

    +

    Terminal Studio

    +

    + A full-screen terminal workspace where you type one instruction at a time and see it + execute immediately, alongside a device picker, a slash-command palette and optional + device mirroring. Great for exploring an app and building flows interactively. + --playground is an alias for it. +

    - An interactive REPL where you type one instruction at a time and see it execute - immediately. Great for exploring an app and building flows interactively. + Requirements. The shell itself needs only an interactive terminal (at + least 27 rows — it will tell you if the window is too small). The optional device + mirroring is Android-only: /stream mirrors the screen inside the + shell. Terminals that support the kitty graphics protocol (Ghostty, kitty, WezTerm) show + real pixels; everywhere else falls back to coloured half-block characters. No install + needed.

    - Playground + Terminal Studio
    - 
    appclaw --playground --platform ios --device-type simulator
    + 
    appclaw --tui --platform ios --device-type simulator

    When you're happy with the recorded steps, type /export to save them. By - default this writes an SDK vitest spec (.test.ts); pass a - .yaml filename to export a YAML flow instead. + default this writes an @appclaw/runner spec (.spec.ts); + pass a .yaml filename to export a YAML flow instead.

    - Exporting from the REPL + Exporting from the shell
    - 
    # In the REPL prompt:
-/export                  # → SDK vitest spec (flow-<ts>.test.ts)
-/export login.test.ts    # → SDK vitest spec
+            
    # At the shell prompt:
+/export                  # → runner spec (flow-<ts>.spec.ts)
+/export login.spec.ts    # → runner spec, run with: appclaw test login
 /export login.yaml       # → YAML flow

    - Bare-filename exports land in EXPORT_DIR (default - .appclaw/exports). Override it per run with --export-dir, and - load a custom .env with --env-path: + Bare-filename exports land in EXPORT_DIR (default tests, the + runner's own testDir, so an export is runnable where it lands). Override it per run with + --export-dir, and load a custom .env with + --env-path:

    - Playground with overrides + Terminal Studio with overrides +
    + 
    appclaw --tui --env-path path/to/.env --export-dir tests/generated
    +
    + + + + + +
    +

    Terminal Studio

    +

    + appclaw --tui is a full-screen terminal workspace for building a flow by + hand: you type one instruction at a time, it runs on the device immediately, and every + step that succeeds is recorded. When the flow looks right, /export turns it + into a runnable spec. --playground is an alias for it. +

    + +

    The layout

    +

    + Two columns. On the left, a scrollable transcript of everything that has run, the + command palette, and the prompt. On the right, the device screen, mirrored live. +

    +
    +
    appclaw --tui
    +
    + ◆ AppClaw + android · Pixel 8 · 4 steps recorded +
    +
    +
    +
    + Transcript +
    open youtube
    +
    + ✓ open youtube (1.4s) +
    +
    Launched youtube
    +
    tap search
    +
    + ✓ tap "search" (159ms) +
    +
    +
    + Command palette +
    + /list List all recorded steps +
    +
    + /export Export steps +
    +
    … +14 more — type / to filter
    +
    +
    + ❯type a step or /command +
    +
    +
    +
    + Device stream · kitty graphics +
    +
    +
    +
    +
    + ↑↓ history + tab complete + ⇧tab focus + ctrl+c quit
    - 
    appclaw --playground --env-path path/to/.env --export-dir tests/generated
    +
    + +

    Recording steps

    +

    + Anything you type that is not a slash command runs as a single deterministic + instruction and is appended to the recording. A step that fails is reported but not + recorded, so the flow you export only contains steps that actually worked. +

    +

    + /goal <text> is the exception: it hands the line to the autonomous + agent, which may take many steps to get there. Agent runs are not + recorded — use it to explore, then record the steps you settle on. +

    + +

    Commands

    +

    + Type / to filter the palette; Tab completes. The full list is + in /help. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    CommandWhat it does
    /listShow the recorded steps
    /yaml · /previewPreview the YAML flow or generated spec without saving
    /export [file] + Write a runner spec (.spec.ts) or YAML flow (.yaml) +
    + /undo · /edit · /insert · + /delete + Fix the recording without starting over
    /metaSet flow name, appId or platform
    /goal <text>Run the autonomous agent (not recorded)
    /stream · /stream-closeMirror the device in the side panel, or stop mirroring
    /device · /platformSwitch device or platform mid-session
    /settings · /history · /doctorEdit .env, browse past runs, run the preflight
    /sessionPath to this session's JSON log
    + +

    Keys

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    KeyAt the promptIn the transcript
    ↑ ↓Recall previous instructionsScroll
    Shift+TabMove focus between the prompt and the transcript
    TabComplete a slash command—
    Ctrl+CQuit (asks first if steps are unexported)
    +

    + History is per-session and never written to disk. Typing any character while the + transcript has focus jumps back to the prompt and keeps the keystroke. +

    + +

    Mirroring the device

    +

    + /stream draws the device screen in the right-hand panel a few times a + second while the prompt stays usable. How it draws depends on your terminal, and it + always works — the only question is fidelity. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    RequirementNeeded forIf missing
    Android device or emulatorMirroring at all/stream refuses on iOS — use the Simulator window
    adb on PATHCapturing frames (adb exec-out screencap)Stream does not start
    Kitty graphics protocolReal pixelsFalls back to half-blocks automatically
    24-bit colourThe half-block fallbackColours are approximated by the terminal
    +

    + appclaw doctor checks the first two rows — adb and a + connected Android device — and prints the exact commands for anything missing. The + last two only change fidelity, never whether the stream runs. +

    + +

    Which terminals give you real pixels

    +

    + AppClaw picks the backend from environment variables rather than asking the terminal. A + capability probe would be answered on stdin, which the shell already holds in raw mode + — the reply would arrive as keystrokes. So detection is: +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TerminalDetected viaResult
    GhosttyTERM_PROGRAM=ghosttykitty graphics
    kitty + KITTY_WINDOW_ID, or TERM containing kitty + kitty graphics
    WezTermWEZTERM_EXECUTABLEkitty graphics
    Everything else—ANSI half-blocks
    +

    + That includes Terminal.app, iTerm2 and the VS Code terminal: they still stream, just as + coloured half-block characters — enough to see which screen you are on, not enough + to read small text. Multiplexers such as tmux and screen sit between AppClaw and the + terminal and generally will not pass graphics through, so expect the fallback there too. +

    +
    +
    +
    + Force a backend +
    + 
    # Exercise the fallback on a terminal that supports graphics
+APPCLAW_STREAM_BACKEND=halfblock appclaw --tui
    +
    +

    + /stream-close stops the mirror and hands the panel back; switching device + or quitting stops it for you. +

    + +

    Session log

    +

    + Every session writes .appclaw/sessions/<id>.json as it goes — + each instruction, what it resolved to, how long it took, and the failures that never + became steps. It is rewritten after every event, so a crash still leaves a readable + file, and past sessions show up in /history alongside YAML flow runs. +

    + +

    Exporting

    +

    + /export writes an @appclaw/runner spec by + default, or a YAML flow if you give it a .yaml name. Bare filenames land in + EXPORT_DIR (default tests), so the file is runnable where it + lands: +

    +
    +
    +
    + Record, export, run +
    + 
    appclaw --tui
+# …record steps at the prompt, then:
+/export login.spec.ts
+
+# then run it like any other spec
+appclaw test login
    @@ -2931,7 +3405,8 @@

    Execution Tuning

    EXPORT_DIR Default directory for bare-filename exports / --export writes - (default: .appclaw/exports). Overridden by --export-dir. + (default: tests, the runner's own testDir). Overridden by + --export-dir. @@ -3006,7 +3481,7 @@

    Where to set it

    • CLI: --caps path/to/caps.json (interactive, YAML flow, - --playground, --record — every mode). + --tui, --record — every mode).
    • Env var: CAPABILITIES_FILE=path/to/caps.json in @@ -3637,7 +4112,7 @@

      runGoal()

      run()

      Execute a single natural-language instruction directly on the device — the programmatic - equivalent of typing a command in the playground REPL. Each call is one atomic action: + equivalent of typing an instruction in Terminal Studio. Each call is one atomic action: parse the instruction, execute it, return the result.

      diff --git a/package-lock.json b/package-lock.json index 057eb06..556b960 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5947,7 +5947,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/appium-uiautomator2-driver/-/appium-uiautomator2-driver-8.0.1.tgz", "integrity": "sha512-NCTEk3Ou13MoGsPE8hE613EjvEuxrqorx19uBDwvBkcS2GkHhvnhLkJmUR/GeSPI3TO0epDdZMpYhroeHS+bKw==", - "hasShrinkwrap": true, "license": "Apache-2.0", "dependencies": { "@appium/css-locator-to-native": "^1.0.1", @@ -24653,7 +24652,8 @@ "ink": "^5.2.1", "ink-spinner": "^5.0.0", "ink-text-input": "^6.0.0", - "react": "^18.3.1" + "react": "^18.3.1", + "wrap-ansi": "^9.0.2" }, "bin": { "appclaw": "bin/appclaw.js" diff --git a/packages/cli/package.json b/packages/cli/package.json index f132fd3..85b5d72 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -30,7 +30,8 @@ "ink": "^5.2.1", "ink-spinner": "^5.0.0", "ink-text-input": "^6.0.0", - "react": "^18.3.1" + "react": "^18.3.1", + "wrap-ansi": "^9.0.2" }, "engines": { "node": ">=22" diff --git a/packages/cli/src/cli/doctor.ts b/packages/cli/src/cli/doctor.ts index aaa571b..c9cf863 100644 --- a/packages/cli/src/cli/doctor.ts +++ b/packages/cli/src/cli/doctor.ts @@ -340,6 +340,9 @@ async function checkHandshake( } export async function runDoctor(args: string[]): Promise { + // Module-global tallies must reset — the TUI's /doctor command makes this + // re-entrant within one process. + counts.ok = counts.warn = counts.fail = 0; const parsed = parseArgs(args); if (parsed.help) { printHelp(); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 2326338..8ce0420 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -50,7 +50,7 @@ import { getStarkVisionModel } from '@appclaw/core/vision/locate-enabled'; import { prepareScreenshotForLlm } from '@appclaw/core/vision/prepare-screenshot-for-llm'; import { runExplorer } from './explorer/index.js'; import type { ExplorerConfig } from './explorer/types.js'; -import { runPlayground } from './playground/index.js'; +import { runTui } from './tui/index.js'; import { setupDevice } from '@appclaw/core/device/index'; import { loadAppGuide } from '@appclaw/core/appguides/index'; import * as ui from '@appclaw/core/ui/terminal'; @@ -67,6 +67,7 @@ interface CLIArgs { replay: string | null; flow: string | null; playground: boolean; + tui: boolean; plan: boolean; explore: string | null; report: boolean; @@ -86,7 +87,7 @@ interface CLIArgs { env: string | null; /** * Path to a dotenv (`KEY=value`) file to load into `process.env` before the - * run starts. Useful for the playground/goal modes when the `.env` lives + * run starts. Useful for the TUI/goal modes when the `.env` lives * outside the current working directory. Values override the process env. * Named `--env-path` (not `--env-file`) to avoid colliding with Node's own * reserved `--env-file` CLI flag. @@ -163,7 +164,10 @@ function printHelp(): void { ` ${c.flag('--caps')} ${c.arg('')} ${c.desc('JSON file of extra Appium capabilities merged into the session')}` ); console.log( - ` ${c.flag('--playground')} ${c.desc('Interactive REPL for building flows')}` + ` ${c.flag('--tui')} ${c.desc('Terminal Studio: step recorder, device picker, device stream')}` + ); + console.log( + ` ${c.flag('--playground')} ${c.desc('Alias for --tui (with --json: headless NDJSON bridge for IDEs)')}` ); console.log( ` ${c.flag('--explore')} ${c.arg('')} ${c.desc('Generate test flows from a PRD')}` @@ -226,11 +230,11 @@ function printHelp(): void { ` ${c.example('appclaw --platform ios --device-type real --udid 00008120-XXXX "Open Settings"')}` ); console.log(); - console.log(` ${c.comment('# Playground on iOS')}`); - console.log(` ${c.example('appclaw --playground --platform ios --device-type simulator')}`); + console.log(` ${c.comment('# Interactive shell on iOS')}`); + console.log(` ${c.example('appclaw --tui --platform ios --device-type simulator')}`); console.log(); - console.log(` ${c.comment('# Playground with a custom .env file')}`); - console.log(` ${c.example('appclaw --playground --env-path path/to/.env')}`); + console.log(` ${c.comment('# Interactive shell with a custom .env file')}`); + console.log(` ${c.example('appclaw --tui --env-path path/to/.env')}`); console.log(); console.log(` ${c.comment('# YAML flow on Android')}`); console.log(` ${c.example('appclaw --flow examples/flows/google-search.yaml')}`); @@ -280,6 +284,7 @@ function parseArgs(): CLIArgs { let replay: string | null = null; let flow: string | null = null; let playground = false; + let tui = false; let plan = false; let explore: string | null = null; let report = false; @@ -332,6 +337,8 @@ function parseArgs(): CLIArgs { flow = args[++i] ?? null; } else if (args[i] === '--playground') { playground = true; + } else if (args[i] === '--tui') { + tui = true; } else if (args[i] === '--plan') { plan = true; } else if (args[i] === '--explore') { @@ -390,6 +397,7 @@ function parseArgs(): CLIArgs { replay, flow, playground, + tui, plan, explore, report, @@ -545,21 +553,45 @@ async function main() { return; } - // ─── Playground mode (interactive REPL → YAML) ─────────── - if (cliArgs.playground) { - // JSON playground for IDE extensions + // ─── Headless step-recorder bridge (`--json --playground`) ─── + // + // The only surviving non-interactive `--playground` behaviour: the VS Code / + // Cursor extension spawns this and speaks NDJSON over stdio, so the protocol + // is a shipped contract. Checked before the TUI block so the interactive form + // of `--playground` falls through to it as an alias. + if (cliArgs.playground && isJsonMode()) { + const { runPlaygroundJson } = await import('./step-recorder/json-bridge.js'); + await runPlaygroundJson({ + platform: cliArgs.platform, + deviceType: cliArgs.deviceType, + udid: cliArgs.deviceUdid, + deviceName: cliArgs.deviceName, + exportDir: cliArgs.exportDir, + }); + return; + } + + // ─── Terminal Studio (multi-screen Ink app + device stream) ─ + // + // `--playground` is an alias: the interactive REPL it used to start has been + // replaced by this shell, which records the same step list. + if (cliArgs.tui || cliArgs.playground) { if (isJsonMode()) { - const { runPlaygroundJson } = await import('./playground/index.js'); - await runPlaygroundJson({ - platform: cliArgs.platform, - deviceType: cliArgs.deviceType, - udid: cliArgs.deviceUdid, - deviceName: cliArgs.deviceName, - exportDir: cliArgs.exportDir, - }); - return; + // silenceTerminalUI() has already filtered stdout to JSON-shaped lines, + // which would swallow every Ink frame — write the refusal to stderr. + process.stderr.write( + '--tui cannot be combined with --json (there is no machine-readable TUI mode).\n' + ); + process.exit(1); + } + if (!process.stdout.isTTY || !process.stdin.isTTY) { + ui.printError( + '--tui requires an interactive terminal', + 'It uses raw-mode keyboard input, so it cannot run piped, in CI, or with stdin redirected.' + ); + process.exit(1); } - await runPlayground({ + await runTui({ platform: cliArgs.platform, deviceType: cliArgs.deviceType, udid: cliArgs.deviceUdid, diff --git a/packages/cli/src/playground/index.ts b/packages/cli/src/playground/index.ts deleted file mode 100644 index d792634..0000000 --- a/packages/cli/src/playground/index.ts +++ /dev/null @@ -1,1678 +0,0 @@ -/** - * Playground — interactive REPL that connects to a real device, - * executes commands live, and records steps for YAML export. - * - * Type natural-language commands (tap, swipe, type, etc.) - * → each one runs immediately on the device via Appium - * → accumulated steps can be exported as a YAML flow file. - */ - -import readline from 'node:readline'; -import { writeFileSync, mkdirSync } from 'node:fs'; -import path from 'node:path'; -import chalk from 'chalk'; -import { stringify } from 'yaml'; - -import { loadConfig, Config } from '@appclaw/core/config'; -import { createMCPClient } from '@appclaw/core/mcp/client'; -import { extractText } from '@appclaw/core/mcp/tools'; -import { setupDevice } from '@appclaw/core/device/index'; -import { AppResolver } from '@appclaw/core/agent/app-resolver'; -import { tryParseNaturalFlowLine } from '@appclaw/core/flow/natural-line'; -import { runOneInstruction, DEFAULT_MIN_MATCH_SCORE } from '@appclaw/core/flow/run-instruction'; -import { resetVisionTokens, getVisionTokens } from '@appclaw/core/vision/vision-token-tracker'; -import { MODEL_PRICING, DEFAULT_MODELS } from '@appclaw/core/constants'; -import { getStarkVisionModel, isVisionLocateEnabled } from '@appclaw/core/vision/locate-enabled'; -import { generateSdkTestFromInstructions } from '@appclaw/core/sdk/goal-export'; -import { stepAction, stepTarget } from '@appclaw/core/ui/step-printer'; -import type { FlowStep, FlowMeta } from '@appclaw/core/flow/types'; -import type { MCPClient } from '@appclaw/core/mcp/types'; -import { - theme, - printBox, - printPanel, - printTable, - hr, - appGradient, - printMarkdown, - progressBar, -} from '@appclaw/core/ui/terminal'; -import * as ui from '@appclaw/core/ui/terminal'; -import Table from 'cli-table3'; - -// executeStep / FlowTapPollOptions are now consumed by src/flow/run-instruction.js -import { loadStore, getTrajectoryStorePath } from '@appclaw/core/memory/store'; -import { loadProcedures, getProceduresStorePath } from '@appclaw/core/memory/procedures'; - -// ─── State ────────────────────────────────────────────── - -interface PlaygroundState { - steps: FlowStep[]; - meta: FlowMeta; - mcp: MCPClient | null; - appResolver: AppResolver | null; -} - -const state: PlaygroundState = { - steps: [], - meta: {}, - mcp: null, - appResolver: null, -}; - -// ─── Cost helpers ─────────────────────────────────────── - -function calcCost(inputTokens: number, outputTokens: number, modelName: string): number { - const pricing = MODEL_PRICING[modelName] ?? [0, 0]; - return (inputTokens / 1_000_000) * pricing[0] + (outputTokens / 1_000_000) * pricing[1]; -} - -function visionCost(inputTokens: number, outputTokens: number): number { - return calcCost(inputTokens, outputTokens, getStarkVisionModel()); -} - -function llmCost(inputTokens: number, outputTokens: number): number { - const modelName = Config.LLM_MODEL || DEFAULT_MODELS[Config.LLM_PROVIDER] || ''; - return calcCost(inputTokens, outputTokens, modelName); -} - -// ─── Formatting helpers ───────────────────────────────── - -// `stepAction` and `stepTarget` live in src/ui/step-printer.ts so the SDK's -// StepRunner can use the same formatting. Imported below from that module. - -function stepToDisplay(step: FlowStep, index: number): string { - const num = theme.brand(`${(index + 1).toString().padStart(2)}.`); - const action = theme.step.bold(stepAction(step).padEnd(7)); - const target = theme.white(stepTarget(step)); - return `${num} ${action} ${target}`; -} - -function spinnerDetail(step: FlowStep): string { - switch (step.kind) { - case 'tap': - return 'tapping the screen…'; - case 'doubleTap': - return 'double-tapping the screen…'; - case 'longPress': - return 'long-pressing the screen…'; - case 'type': - return 'typing into the field…'; - case 'swipe': - return 'swiping the screen…'; - case 'zoom': - return `zooming ${step.scale >= 1 ? 'in' : 'out'}…`; - case 'scrollAssert': - return 'scanning the screen…'; - case 'assert': - return 'verifying the screen…'; - case 'launchApp': - return 'launching the app…'; - case 'openApp': - return 'opening the app…'; - case 'wait': - return 'waiting…'; - case 'waitUntil': - return 'waiting for condition…'; - case 'enter': - return 'pressing enter…'; - case 'back': - return 'navigating back…'; - case 'home': - return 'going home…'; - case 'getInfo': - return 'reading the screen…'; - case 'done': - return 'wrapping up…'; - default: - return 'executing on device…'; - } -} - -// Action glyph per step kind — mirrors the Ink StepLine look. -const KIND_ICON: Record = { - tap: '●', - doubleTap: '●', - longPress: '●', - type: '⊞', - swipe: '↕', - scrollAssert: '↕', - zoom: '⊕', - drag: '↔', - assert: '◈', - launchApp: '↗', - openApp: '↗', - wait: '…', - waitUntil: '…', - enter: '↵', - back: '‹', - home: '⌂', - getInfo: '◎', - done: '✓', -}; - -function fmtMs(ms?: number): string { - if (ms == null) return ''; - return ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(1)}s`; -} - -/** - * Colorful, aligned single-line step row (matches the Ink RunScreen vibe): - * - * ✓ 3 tap "Login" ● 0.8s - * ↳ Tapped "Login" - */ -function printPlaygroundStep( - stepNum: number, - step: FlowStep, - success: boolean, - message: string, - ms?: number -): void { - const TARGET_W = 42; - const icon = success ? theme.success('✓') : theme.error('✗'); - const num = theme.dim(String(stepNum).padStart(2)); - const verbRaw = stepAction(step).padEnd(8); - const verb = success ? theme.step.bold(verbRaw) : theme.error.bold(verbRaw); - const targetRaw = stepTarget(step); - const targetPadded = - targetRaw.length > TARGET_W - ? targetRaw.slice(0, TARGET_W - 1) + '…' - : targetRaw.padEnd(TARGET_W); - const target = success ? theme.white(targetPadded) : theme.error(targetPadded); - const glyph = theme.muted(KIND_ICON[step.kind] ?? '●'); - const dur = theme.dim(fmtMs(ms).padStart(6)); - - console.log(` ${icon} ${num} ${verb}${target} ${glyph} ${dur}`); - if (message && message !== 'recorded') { - console.log(` ${success ? theme.dim('↳ ' + message) : theme.error('↳ ' + message)}`); - } -} - -function printStepSuccess(stepNum: number, step: FlowStep, message: string, ms?: number): void { - printPlaygroundStep(stepNum, step, true, message, ms); -} - -function printStepFail(stepNum: number, step: FlowStep, message: string, ms?: number): void { - printPlaygroundStep(stepNum, step, false, message, ms); -} - -/** - * Minimum matchScore (1-10) required to execute a tap in the playground. - * Below this threshold, vision found a loose match — show suggestion but don't execute. - * - * Re-exported from `src/flow/run-instruction.ts` so the SDK and playground share - * one source of truth (used to be defined twice, leading to drift risk). - */ -const MIN_MATCH_SCORE = DEFAULT_MIN_MATCH_SCORE; - -/** Convert step to YAML — preserve the user's original natural language input. */ -function stepToYaml(step: FlowStep): unknown { - // Playground steps always have verbatim (the exact text the user typed). - // Use it directly so the YAML reads like the user's instructions. - if (step.verbatim) return step.verbatim; - - // Fallback for steps without verbatim (shouldn't happen in playground) - switch (step.kind) { - case 'launchApp': - return 'launchApp'; - case 'openApp': - return `open ${step.query} app`; - case 'tap': - return `tap ${step.label}`; - case 'doubleTap': - return `double tap ${step.label}`; - case 'longPress': - return step.duration != null - ? `long press ${step.label} for ${step.duration}ms` - : `long press ${step.label}`; - case 'type': - return `type "${step.text}"`; - case 'swipe': - return `swipe ${step.direction}`; - case 'zoom': - return step.target - ? `zoom ${step.scale >= 1 ? 'in' : 'out'} ${step.scale}x on ${step.target}` - : `zoom ${step.scale >= 1 ? 'in' : 'out'} ${step.scale}x`; - case 'wait': - return `wait ${step.seconds} s`; - case 'waitUntil': - if (step.condition === 'screenLoaded') return 'wait until screen is loaded'; - if (step.condition === 'gone') return `wait until "${step.text}" is gone`; - return `wait until "${step.text}" is visible`; - case 'enter': - return 'press enter'; - case 'back': - return 'go back'; - case 'home': - return 'go home'; - case 'assert': - return `assert "${step.text}" is visible`; - case 'scrollAssert': - return `scroll ${step.direction} until "${step.text}" is visible`; - case 'getInfo': - return `getInfo: ${step.query}`; - case 'done': - return step.message ? `done: ${step.message}` : 'done'; - } -} - -function buildYamlString(): string { - const parts: string[] = []; - - if (state.meta.appId || state.meta.name || state.meta.platform) { - const metaObj: Record = {}; - if (state.meta.appId) metaObj.appId = state.meta.appId; - if (state.meta.name) metaObj.name = state.meta.name; - if (state.meta.platform) metaObj.platform = state.meta.platform; - parts.push(stringify(metaObj).trim()); - parts.push('---'); - } - - const yamlSteps = state.steps.map(stepToYaml); - - // Auto-append "done" if the last step isn't already a done step - const lastStep = state.steps[state.steps.length - 1]; - if (!lastStep || lastStep.kind !== 'done') { - yamlSteps.push('done'); - } - - parts.push(stringify({ steps: yamlSteps }).trim()); - - return parts.join('\n') + '\n'; -} - -/** - * Whether the given filename should be exported as a vitest spec (SDK test - * format) rather than the default YAML flow format. - */ -function isSdkTestFilename(name: string): boolean { - return /\.(?:test|spec)\.(?:m|c)?[jt]sx?$/i.test(name) || /\.(?:m|c)?ts$/i.test(name); -} - -/** - * Resolve the final on-disk path for an `/export` write — same rules as the - * CLI's `--export`. Bare filenames land in the configured directory (EXPORT_DIR); - * paths with a directory hint (./tests/foo.test.ts, /abs/...) are used verbatim. - * - * The configured directory differs by format: SDK tests go to EXPORT_DIR, YAML - * flows stay in cwd (mirrors the original playground behaviour). - */ -function resolvePlaygroundExportPath(filename: string, asSdkTest: boolean): string { - if (path.isAbsolute(filename)) return filename; - if (filename.includes('/') || filename.includes(path.sep)) { - return path.resolve(process.cwd(), filename); - } - if (asSdkTest) { - const dir = _deviceArgs.exportDir ?? loadConfig().EXPORT_DIR; - return path.resolve(process.cwd(), dir, filename); - } - return path.resolve(process.cwd(), filename); -} - -/** - * Build the vitest spec body for the current playground state. - * Each recorded step's `verbatim` (the user's original natural-language text) - * becomes one `await app.run(...)` call — no translation needed because the - * playground already accepts the same syntax that `AppClaw.run()` does. - */ -function buildSdkTestString(): string { - const instructions = state.steps - .map((s) => s.verbatim?.trim()) - .filter((v): v is string => !!v && v.length > 0); - return generateSdkTestFromInstructions({ - instructions, - config: { - describeName: state.meta.name || 'Recorded flow', - ...(state.meta.platform === 'ios' || state.meta.platform === 'android' - ? { platform: state.meta.platform } - : {}), - }, - }); -} - -/** Light syntax tint for a single code line — strings green, comments dim. */ -function highlightCodeLine(line: string): string { - const trimmed = line.trimStart(); - if (trimmed.startsWith('//') || trimmed.startsWith('*') || trimmed.startsWith('/*')) { - return theme.dim(line); - } - // eslint-disable-next-line no-useless-escape - return line.replace(/(['"`])(?:\\.|(?!\1).)*\1/g, (m) => chalk.hex('#22C55E')(m)); -} - -/** - * Render a syntax-tinted, line-numbered preview of generated export code in a - * bordered box. Used by `/export` (before writing) and `/preview`. - */ -function printCodePreview(body: string, filename: string, label: string): void { - const MAX_LINES = 60; - const lines = body.replace(/\n+$/, '').split('\n'); - const shown = lines.slice(0, MAX_LINES); - const gutter = String(shown.length).length; - const rendered = shown - .map((l, i) => `${theme.dim(String(i + 1).padStart(gutter))} ${highlightCodeLine(l)}`) - .join('\n'); - const more = - lines.length > MAX_LINES ? `\n${theme.dim(`… +${lines.length - MAX_LINES} more lines`)}` : ''; - console.log(); - printBox(rendered + more, { - title: `Preview · ${path.basename(filename)} · ${label}`, - titleAlignment: 'left', - borderColor: '#FC8EAC', - }); -} - -function printStepList(): void { - if (state.steps.length === 0) return; - - const title = state.meta.name - ? `${state.meta.name}${state.meta.appId ? ` (${state.meta.appId})` : ''}` - : state.meta.appId - ? state.meta.appId - : 'Flow'; - - const table = new Table({ - head: [ - chalk.hex('#9CA3AF')('#'), - chalk.hex('#9CA3AF')('Action'), - chalk.hex('#9CA3AF')('Target'), - chalk.hex('#9CA3AF')('Status'), - ], - style: { head: [], border: ['gray'] }, - chars: { - top: '─', - 'top-mid': '┬', - 'top-left': '╭', - 'top-right': '╮', - bottom: '─', - 'bottom-mid': '┴', - 'bottom-left': '╰', - 'bottom-right': '╯', - left: '│', - 'left-mid': '├', - mid: '─', - 'mid-mid': '┼', - right: '│', - 'right-mid': '┤', - middle: '│', - }, - colWidths: [5, 10, 40, 10], - wordWrap: true, - }); - - for (let i = 0; i < state.steps.length; i++) { - const step = state.steps[i]; - const action = stepAction(step); - const target = stepTarget(step); - const actionColored = chalk.hex('#9CC6F5').bold(action); - const statusColored = chalk.green('● pass'); - - table.push([ - chalk.hex('#FC8EAC')(`${i + 1}`), - actionColored, - chalk.white(target), - statusColored, - ]); - } - - console.log(); - console.log(` ${chalk.hex('#FC8EAC').bold(title)}`); - console.log(` ${table.toString().split('\n').join('\n ')}`); - console.log(); - console.log( - ` ${chalk.green('✓')} ${chalk.green.bold(`${state.steps.length}`)} ${chalk.dim(`step${state.steps.length === 1 ? '' : 's'} recorded`)} ${progressBar(state.steps.length, state.steps.length, 15)}` - ); - console.log(); -} - -// Step execution is delegated to runOneInstruction() in src/flow/run-instruction.js. -// The playground used to have its own runStepOnDevice() wrapper; that became dead -// code after the shared pipeline refactor. - -// ─── Memory inspection ────────────────────────────────── - -function runMemoryCommand(arg: string): void { - const sub = (arg || 'stats').toLowerCase(); - if (sub === 'stats') return printMemoryStats(); - if (sub === 'list') return printMemoryProcedureList(); - if (sub === 'paths') return printMemoryPaths(); - console.log(` ${theme.label('Usage:')} /memory stats | /memory list | /memory paths`); -} - -function printMemoryPaths(): void { - const trajPath = getTrajectoryStorePath(Config.EPISODIC_MEMORY_PATH || undefined); - const procPath = getProceduresStorePath(Config.PROCEDURAL_MEMORY_PATH || undefined); - console.log(); - console.log(` ${theme.label('Trajectories:')} ${theme.white(trajPath)}`); - console.log(` ${theme.label('Procedures: ')} ${theme.white(procPath)}`); - console.log(` ${theme.label('Namespace: ')} ${theme.white(Config.APPCLAW_MEMORY_NAMESPACE)}`); - console.log(); -} - -function printMemoryStats(): void { - const trajPath = getTrajectoryStorePath(Config.EPISODIC_MEMORY_PATH || undefined); - const procPath = getProceduresStorePath(Config.PROCEDURAL_MEMORY_PATH || undefined); - const trajStore = loadStore(Config.EPISODIC_MEMORY_PATH || undefined); - const procStore = loadProcedures(Config.PROCEDURAL_MEMORY_PATH || undefined); - const ns = Config.APPCLAW_MEMORY_NAMESPACE; - - const SEVEN_DAYS = 7 * 24 * 60 * 60 * 1000; - const trajNs = trajStore.entries.filter((e) => (e.namespace ?? 'default') === ns); - const procNs = procStore.entries.filter((e) => e.namespace === ns); - const trajStale = trajNs.filter( - (e) => e.successCount < 2 && Date.now() - e.timestamp > SEVEN_DAYS - ).length; - - const groupByApp = (arr: T[]): Array<[string, number]> => { - const m = new Map(); - for (const e of arr) m.set(e.appId, (m.get(e.appId) ?? 0) + 1); - return [...m.entries()].sort((a, b) => b[1] - a[1]); - }; - - console.log(); - console.log(hr('single', undefined, 'Memory stats')); - console.log(); - console.log(` ${theme.label('Namespace:')} ${theme.white(ns)}`); - console.log(); - - // Episodic - console.log( - ` ${theme.label('Episodic')} ${theme.dim(`(${trajPath})`)}\n` + - ` ${theme.label('Total:')} ${theme.white(String(trajStore.entries.length))}` + - ` ${theme.dim(`(${trajNs.length} in this namespace)`)}\n` + - ` ${theme.label('Stale-eligible:')} ${theme.white(String(trajStale))} ${theme.dim('(single-use, >7d — hidden from retrieval)')}` - ); - const trajByApp = groupByApp(trajNs); - if (trajByApp.length > 0) { - console.log(` ${theme.label('By app:')}`); - for (const [appId, n] of trajByApp.slice(0, 8)) { - console.log( - ` ${theme.dim('•')} ${theme.white(appId.padEnd(40))} ${theme.dim(String(n))}` - ); - } - } - console.log(); - - // Procedural - console.log( - ` ${theme.label('Procedural')} ${theme.dim(`(${procPath})`)}\n` + - ` ${theme.label('Total:')} ${theme.white(String(procStore.entries.length))}` + - ` ${theme.dim(`(${procNs.length} in this namespace)`)}` - ); - const procByApp = groupByApp(procNs); - if (procByApp.length > 0) { - console.log(` ${theme.label('By app:')}`); - for (const [appId, n] of procByApp.slice(0, 8)) { - console.log( - ` ${theme.dim('•')} ${theme.white(appId.padEnd(40))} ${theme.dim(String(n))}` - ); - } - } - console.log(); - console.log(theme.dim(` Type /memory list to see goal recipes, /memory paths for file paths.`)); - console.log(); -} - -function printMemoryProcedureList(): void { - const store = loadProcedures(Config.PROCEDURAL_MEMORY_PATH || undefined); - const ns = Config.APPCLAW_MEMORY_NAMESPACE; - const items = store.entries.filter((e) => e.namespace === ns); - - if (items.length === 0) { - console.log(); - console.log( - ` ${theme.dim(`No procedures yet in namespace "${ns}". Run a goal to record one.`)}` - ); - console.log(); - return; - } - - items.sort((a, b) => b.timestamp - a.timestamp); - - console.log(); - console.log(hr('single', undefined, `Procedures (${items.length} in namespace "${ns}")`)); - console.log(); - for (const p of items.slice(0, 20)) { - const ago = formatProcAgo(p.timestamp); - const goal = p.goalKeywords.join(' '); - const reuse = p.successCount > 1 ? ` ${theme.success(`×${p.successCount}`)}` : ''; - console.log( - ` ${theme.dim('•')} ${theme.white(goal)}${reuse}` + - ` ${theme.dim(`— ${p.appId}, ${p.steps.length} steps, ${ago}`)}` - ); - } - console.log(); -} - -function formatProcAgo(ts: number): string { - const days = Math.floor((Date.now() - ts) / (1000 * 60 * 60 * 24)); - if (days === 0) return 'today'; - if (days === 1) return '1d ago'; - if (days < 7) return `${days}d ago`; - if (days < 30) return `${Math.floor(days / 7)}w ago`; - return `${Math.floor(days / 30)}mo ago`; -} - -// ─── Slash commands ───────────────────────────────────── - -const COMMANDS: Record Promise | void }> = { - '/help': { - desc: 'Show available commands and supported step patterns', - run: () => printHelp(), - }, - '/list': { - desc: 'List all recorded steps', - run: () => { - if (state.steps.length === 0) { - console.log( - `\n ${theme.dim('No steps yet. Type a command like:')} ${theme.white('open youtube app')}\n` - ); - return; - } - printStepList(); - }, - }, - '/yaml': { - desc: 'Preview the YAML output', - run: () => { - if (state.steps.length === 0) { - console.log(`\n ${theme.dim('No steps to preview.')}\n`); - return; - } - const yamlStr = buildYamlString(); - console.log(); - // Print YAML with cyan coloring (not markdown — marked-terminal renders YAML as red) - for (const line of yamlStr.split('\n')) { - console.log(` ${chalk.cyan(line)}`); - } - console.log(` ${theme.dim('Use')} ${theme.info('/export ')} ${theme.dim('to save')}`); - console.log(); - }, - }, - '/preview': { - desc: 'Preview the generated code without saving (optionally pass a filename for format)', - run: (arg: string) => { - if (state.steps.length === 0) { - console.log(`\n ${theme.error('✗')} No steps to preview.\n`); - return; - } - const filename = arg.trim() || `flow-${Date.now()}.test.ts`; - const asSdkTest = isSdkTestFilename(filename); - const body = asSdkTest ? buildSdkTestString() : buildYamlString(); - const formatLabel = asSdkTest ? 'SDK test (vitest)' : 'YAML flow'; - printCodePreview(body, filename, formatLabel); - console.log( - ` ${theme.dim('Use')} ${theme.info(`/export ${arg.trim() || ''}`)} ${theme.dim('to save.')}\n` - ); - }, - }, - '/export': { - desc: - 'Export steps. SDK vitest test by default (e.g. /export my-flow.test.ts) ' + - 'or YAML flow by extension (.yaml/.yml).', - run: (arg: string) => { - if (state.steps.length === 0) { - console.log(`\n ${theme.error('✗')} No steps to export.\n`); - return; - } - const filename = arg.trim() || `flow-${Date.now()}.test.ts`; - const asSdkTest = isSdkTestFilename(filename); - const filepath = resolvePlaygroundExportPath(filename, asSdkTest); - const body = asSdkTest ? buildSdkTestString() : buildYamlString(); - const formatLabel = asSdkTest ? 'SDK test (vitest)' : 'YAML flow'; - - // Show the generated code before writing it. - printCodePreview(body, filepath, formatLabel); - - mkdirSync(path.dirname(filepath), { recursive: true }); - writeFileSync(filepath, body, 'utf-8'); - const runHint = asSdkTest - ? `vitest run ${path.relative(process.cwd(), filepath)}` - : `appclaw --flow ${path.relative(process.cwd(), filepath)}`; - const exportContent = [ - `${chalk.green.bold(`${state.steps.length}`)} ${chalk.dim('steps exported as')} ${chalk.white(formatLabel)}`, - '', - `${chalk.dim('File:')} ${chalk.white(filepath)}`, - `${chalk.dim('Run:')} ${chalk.cyan(runHint)}`, - ].join('\n'); - console.log(); - printBox(exportContent, { - title: 'Exported', - titleAlignment: 'left', - borderColor: '#22C55E', - }); - console.log(); - }, - }, - '/undo': { - desc: 'Remove the last step', - run: () => { - if (state.steps.length === 0) { - console.log(` ${theme.dim('Nothing to undo.')}`); - return; - } - const removed = state.steps.pop()!; - console.log(` ${theme.warn('↩')} Removed: ${theme.dim(removed.verbatim ?? removed.kind)}`); - if (state.steps.length > 0) { - printStepList(); - } else { - console.log(` ${theme.dim('All steps cleared.')}`); - } - }, - }, - '/clear': { - desc: 'Clear all steps and metadata', - run: () => { - const count = state.steps.length; - state.steps.length = 0; - state.meta = {}; - console.log(` ${theme.warn('↩')} Cleared ${count} steps.`); - }, - }, - '/meta': { - desc: 'Set flow metadata (e.g. /meta appId com.android.settings, /meta platform ios)', - run: (arg: string) => { - const parts = arg.trim().split(/\s+/); - const key = parts[0]; - const value = parts.slice(1).join(' '); - if (key === 'appId' && value) { - state.meta.appId = value; - console.log(` ${theme.success('✓')} appId = ${theme.white(value)}`); - } else if (key === 'name' && value) { - state.meta.name = value; - console.log(` ${theme.success('✓')} name = ${theme.white(value)}`); - } else if (key === 'platform') { - const p = value.toLowerCase(); - if (p === 'android' || p === 'ios') { - state.meta.platform = p; - console.log(` ${theme.success('✓')} platform = ${theme.white(p)}`); - } else { - console.log(` ${theme.label('Usage:')} /meta platform `); - } - } else { - console.log( - ` ${theme.label('Usage:')} /meta appId | /meta name | /meta platform ` - ); - if (state.meta.appId || state.meta.name || state.meta.platform) { - console.log(` ${theme.label('Current:')}`); - if (state.meta.appId) console.log(` appId: ${theme.white(state.meta.appId)}`); - if (state.meta.name) console.log(` name: ${theme.white(state.meta.name)}`); - if (state.meta.platform) console.log(` platform: ${theme.white(state.meta.platform)}`); - } - } - }, - }, - '/edit': { - desc: 'Edit a step by number (e.g. /edit 3 tap "Settings")', - run: (arg: string) => { - const match = arg.match(/^(\d+)\s+(.+)$/); - if (!match) { - console.log(` ${theme.label('Usage:')} /edit `); - return; - } - const idx = parseInt(match[1], 10) - 1; - if (idx < 0 || idx >= state.steps.length) { - console.log( - ` ${theme.error('✗')} Step ${idx + 1} does not exist (1–${state.steps.length}).` - ); - return; - } - const parsed = tryParseNaturalFlowLine(match[2]); - if (!parsed) { - console.log(` ${theme.error('✗')} Could not parse: ${theme.dim(match[2])}`); - return; - } - state.steps[idx] = parsed; - console.log(` ${theme.success('✓')} Updated step ${idx + 1}`); - printStepList(); - }, - }, - '/insert': { - desc: 'Insert a step at position (e.g. /insert 2 wait 3 s)', - run: (arg: string) => { - const match = arg.match(/^(\d+)\s+(.+)$/); - if (!match) { - console.log(` ${theme.label('Usage:')} /insert `); - return; - } - const idx = parseInt(match[1], 10) - 1; - if (idx < 0 || idx > state.steps.length) { - console.log(` ${theme.error('✗')} Position must be 1–${state.steps.length + 1}.`); - return; - } - const parsed = tryParseNaturalFlowLine(match[2]); - if (!parsed) { - console.log(` ${theme.error('✗')} Could not parse: ${theme.dim(match[2])}`); - return; - } - state.steps.splice(idx, 0, parsed); - console.log(` ${theme.success('✓')} Inserted at position ${idx + 1}`); - printStepList(); - }, - }, - '/delete': { - desc: 'Delete a step by number (e.g. /delete 3)', - run: (arg: string) => { - const idx = parseInt(arg.trim(), 10) - 1; - if (isNaN(idx) || idx < 0 || idx >= state.steps.length) { - console.log(` ${theme.error('✗')} Invalid step number. Use 1–${state.steps.length}.`); - return; - } - const removed = state.steps.splice(idx, 1)[0]; - console.log(` ${theme.warn('↩')} Deleted: ${theme.dim(removed.verbatim ?? removed.kind)}`); - if (state.steps.length > 0) { - printStepList(); - } - }, - }, - '/memory': { - desc: 'Inspect episodic + procedural memory (e.g. /memory stats, /memory list)', - run: (arg: string) => runMemoryCommand(arg.trim()), - }, -}; - -function printHelp(): void { - console.log(); - console.log(hr('single', undefined, 'Commands')); - console.log(); - printTable({ - headers: ['Command', 'Description'], - rows: [ - ...Object.entries(COMMANDS).map(([cmd, { desc }]) => [cmd, desc]), - ['/quit', 'Exit playground'], - ], - }); - console.log(); - console.log(hr('single', undefined, 'Examples')); - console.log(` ${theme.dim('Type natural commands — they run on the device instantly')}`); - console.log(); - - const examples: Array<{ category: string; lines: string[] }> = [ - { - category: 'Apps', - lines: ['open YouTube', 'launch Settings app', 'close YouTube', 'close the app'], - }, - { - category: 'Tap & Navigate', - lines: [ - 'tap on Login', - 'click Search button', - 'select English', - 'navigate to Settings screen', - ], - }, - { - category: 'Proximity (disambiguate by position)', - lines: [ - 'tap the login button below the password field', - 'tap the icon to the right of the title', - 'click the checkbox next to Terms', - 'type "pass" in the field below the email', - ], - }, - { - category: 'Long Press', - lines: [ - 'long press on first email', - 'long-press the image', - 'press and hold Delete button', - 'long press on file for 1500ms', - ], - }, - { - category: 'Type & Search', - lines: [ - 'type "hello world"', - 'type "john" in Username field', - 'search for appium 3.0', - 'press enter', - ], - }, - { - category: 'Scroll & Swipe', - lines: [ - 'scroll down', - 'swipe left', - 'scroll down 2 times until "Krishna" is visible', - 'scroll up to find "Notifications"', - ], - }, - { - category: 'Assert (recorded as a pass/fail step in your flow)', - lines: [ - 'assert "Welcome" is visible', - 'verify "Login" is displayed', - 'verify bell icon is present', - 'check red dot in the map', - ], - }, - { - category: 'Ask (inspect the screen — not recorded)', - lines: [ - 'Is there a bell icon on screen?', - 'What text is shown in the header?', - 'Is the map loaded?', - 'How many items are in the list?', - ], - }, - { - category: 'Wait & Sync', - lines: [ - 'wait 3 s', - 'wait until screen is loaded', - 'wait until "Search results" is visible', - 'wait until "Loading..." is gone', - 'wait 5s until search icon is visible', - 'wait 15s until screen is loaded', - ], - }, - { - category: 'Device Controls', - lines: ['go back', 'go home', 'toggle WiFi', 'close popup'], - }, - { - category: 'Flow', - lines: ['done', 'done: login flow finished'], - }, - ]; - - for (const section of examples) { - // Split "Title (hint)" into colored title + dimmed hint - const hintMatch = section.category.match(/^(.+?)(\s*\(.+\))$/); - if (hintMatch) { - console.log(` ${theme.step.bold(hintMatch[1])}${theme.dim(hintMatch[2])}`); - } else { - console.log(` ${theme.step.bold(section.category)}`); - } - for (const line of section.lines) { - console.log(` ${theme.info('›')} ${line}`); - } - console.log(); - } -} - -// ─── Header ───────────────────────────────────────────── - -function printPlaygroundHeader(): void { - const content = [ - appGradient('Execute commands live & export as an SDK test'), - '', - `${theme.dim('Commands run on device immediately.')}`, - `${theme.dim('Use')} ${theme.info('/yaml')} ${theme.dim('to preview and')} ${theme.info('/export')} ${theme.dim('to save.')}`, - `${theme.dim('Type')} ${theme.info('/help')} ${theme.dim('for all commands.')}`, - ].join('\n'); - - console.log(); - printBox(content, { title: 'AppClaw Playground', titleAlignment: 'left' }); - console.log(); -} - -// ─── Prompt ───────────────────────────────────────────── - -function getPrompt(): string { - return `\n ${chalk.hex('#FC8EAC').bold('›')} `; -} - -// ─── Device connection ────────────────────────────────── - -let _resolvedPlatform: 'android' | 'ios' = 'android'; - -async function connectToDevice(): Promise { - const config = loadConfig(); - - try { - ui.startSpinner(`Connecting to appium-mcp (${config.MCP_TRANSPORT})…`); - const mcpClient = await createMCPClient({ - transport: config.MCP_TRANSPORT, - host: config.MCP_HOST, - port: config.MCP_PORT, - url: config.MCP_URL || undefined, - }); - state.mcp = mcpClient; - ui.stopSpinner(); - ui.printSetupOk('Connected to appium-mcp'); - - // Full device setup pipeline (platform → device → iOS setup → session) - const deviceResult = await setupDevice(mcpClient, { - cliPlatform: _deviceArgs.platform ?? null, - cliDeviceType: _deviceArgs.deviceType ?? null, - cliUdid: _deviceArgs.udid ?? null, - cliDeviceName: _deviceArgs.deviceName ?? null, - config, - alwaysPickDevice: true, - }); - _resolvedPlatform = deviceResult.platform; - - // Auto-set platform in flow metadata so exported YAML includes it - if (!state.meta.platform) { - state.meta.platform = deviceResult.platform; - } - - // Initialize app resolver for "open X app" commands - ui.startSpinner('Loading installed apps…'); - const appResolver = new AppResolver(); - await appResolver.initialize(mcpClient, deviceResult.platform); - state.appResolver = appResolver; - ui.stopSpinner(); - ui.printSetupOk('App resolver ready'); - - // Surface the effective interaction mode so a silent DOM fallback is never a - // mystery. `isVisionMode()` (run-yaml-flow) requires BOTH AGENT_MODE=vision AND - // vision-locate being configured — if vision is requested but not configured, - // every command quietly runs against the DOM instead. - const visionLocate = isVisionLocateEnabled(); - if (Config.AGENT_MODE === 'vision' && visionLocate) { - ui.printSetupOk('Interaction mode: vision'); - } else if (Config.AGENT_MODE === 'vision' && !visionLocate) { - ui.printWarning( - 'AGENT_MODE=vision is set, but vision-locate is not configured — running in DOM mode. ' + - 'Set GEMINI_API_KEY / STARK_VISION_API_KEY / STARK_VISION_BASE_URL (or LLM_PROVIDER=gemini) to enable vision.' - ); - } else { - ui.printSetupOk('Interaction mode: dom'); - } - - const readyContent = [ - `${theme.dim('Type commands to execute on device.')}`, - '', - `${theme.dim('Examples:')}`, - ` ${theme.white('open youtube app')}`, - ` ${theme.white('click on Search')}`, - ` ${theme.white('type "hello"')}`, - ].join('\n'); - console.log(); - printBox(readyContent, { - title: 'Device connected', - titleAlignment: 'left', - borderColor: '#22C55E', - padding: { left: 2, right: 2, top: 1, bottom: 1 }, - }); - console.log(); - - return true; - } catch (err: any) { - ui.stopSpinner(); - // Always write to stderr so IDE extensions can see the error - process.stderr.write(`[playground] Connection failed: ${err?.message ?? err}\n`); - if (err?.stack) process.stderr.write(`[playground] ${err.stack}\n`); - ui.printError(`Failed to connect: ${err?.message ?? err}`); - // AppClaw drives Appium through the appium-mcp subprocess, which it starts itself — - // there is no separate "Appium server" to launch. A timeout (-32001) almost always - // means appium-mcp couldn't start/handshake in time (e.g. a cold `npx` download on a - // global install that doesn't bundle it), NOT that a server is missing. - const errMsg = String(err?.message ?? err); - const timedOut = errMsg.includes('-32001') || /timed out/i.test(errMsg); - if (timedOut) { - ui.printInfo( - 'AppClaw starts appium-mcp itself — no separate Appium server is needed. The MCP handshake ' + - 'timed out: on a first run appium-mcp may still be downloading via npx. Retry, or reinstall so ' + - "it's bundled (e.g. npm i -g appclaw@latest). Set MCP_DEBUG=1 to see appium-mcp's startup logs." - ); - } else { - ui.printInfo( - 'AppClaw starts appium-mcp itself — no separate Appium server is needed. Make sure a ' + - 'device/emulator is connected. Set MCP_DEBUG=1 to see appium-mcp’s startup logs.' - ); - } - console.log(); - return false; - } -} - -// ─── Main REPL ────────────────────────────────────────── - -export interface PlaygroundDeviceArgs { - platform?: 'android' | 'ios' | null; - deviceType?: 'simulator' | 'real' | null; - udid?: string | null; - deviceName?: string | null; - /** - * Override directory for bare-filename SDK-test exports (`--export-dir`). - * Takes precedence over the `EXPORT_DIR` config/env default. Ignored for - * paths that already include a directory hint or are absolute. - */ - exportDir?: string | null; -} - -/** Stash device args so connectToDevice can use them */ -let _deviceArgs: PlaygroundDeviceArgs = {}; - -/** - * JSON-mode playground — reads commands from stdin (one per line), - * emits NDJSON events to stdout. Used by IDE extensions. - */ -export async function runPlaygroundJson(deviceArgs?: PlaygroundDeviceArgs): Promise { - if (deviceArgs) _deviceArgs = deviceArgs; - - const { emitJson } = await import('@appclaw/core/json-emitter'); - - let connectError: string | undefined; - try { - const connected = await connectToDevice(); - if (!connected) { - connectError = 'connectToDevice returned false'; - } - } catch (err: any) { - connectError = err?.message ?? String(err); - } - - if (connectError) { - emitJson({ event: 'error', data: { message: `Failed to connect: ${connectError}` } }); - process.exit(1); - } - - emitJson({ event: 'connected', data: { transport: 'stdio' } }); - emitJson({ event: 'device_ready', data: { platform: _resolvedPlatform } }); - - // Graceful shutdown on SIGTERM (sent by VS Code extension bridge.stop()) - const gracefulShutdown = async () => { - await cleanup(); - emitJson({ event: 'done', data: { success: true, totalSteps: state.steps.length } }); - process.exit(0); - }; - process.on('SIGTERM', gracefulShutdown); - process.on('SIGINT', gracefulShutdown); - - if (process.stdin.isPaused()) process.stdin.resume(); - - const rl = readline.createInterface({ input: process.stdin }); - let processing = false; - - rl.on('line', async (input: string) => { - const line = input.trim(); - if (!line) return; - - if (processing) { - emitJson({ event: 'error', data: { message: 'Still processing previous command' } }); - return; - } - - processing = true; - - // Slash commands - if (line.startsWith('/')) { - if (line === '/quit' || line === '/exit' || line === '/q') { - await cleanup(); - emitJson({ event: 'done', data: { success: true, totalSteps: state.steps.length } }); - rl.close(); - processing = false; - return; - } - if (line === '/yaml') { - if (state.steps.length === 0) { - emitJson({ - event: 'flow_step', - data: { - step: 0, - total: 0, - kind: 'yaml', - target: 'No steps to preview', - status: 'failed', - }, - }); - } else { - const yamlStr = buildYamlString(); - emitJson({ - event: 'flow_step', - data: { - step: state.steps.length, - total: state.steps.length, - kind: 'yaml', - target: yamlStr, - status: 'passed', - }, - }); - } - processing = false; - return; - } - if (line.startsWith('/export')) { - const arg = line.slice(7).trim(); - const filename = arg || `flow-${Date.now()}.test.ts`; - const asSdkTest = isSdkTestFilename(filename); - const filepath = resolvePlaygroundExportPath(filename, asSdkTest); - if (state.steps.length === 0) { - emitJson({ - event: 'flow_step', - data: { - step: 0, - total: 0, - kind: 'export', - target: 'No steps to export', - status: 'failed', - }, - }); - } else { - const body = asSdkTest ? buildSdkTestString() : buildYamlString(); - mkdirSync(path.dirname(filepath), { recursive: true }); - writeFileSync(filepath, body, 'utf-8'); - emitJson({ - event: 'flow_step', - data: { - step: state.steps.length, - total: state.steps.length, - kind: 'export', - target: filepath, - status: 'passed', - }, - }); - } - processing = false; - return; - } - if (line === '/clear') { - state.steps.length = 0; - state.meta = {}; - emitJson({ - event: 'flow_step', - data: { step: 0, total: 0, kind: 'clear', target: 'All steps cleared', status: 'passed' }, - }); - processing = false; - return; - } - if (line === '/undo') { - if (state.steps.length === 0) { - emitJson({ - event: 'flow_step', - data: { step: 0, total: 0, kind: 'undo', target: 'Nothing to undo', status: 'failed' }, - }); - } else { - const removed = state.steps.pop()!; - emitJson({ - event: 'flow_step', - data: { - step: state.steps.length, - total: state.steps.length, - kind: 'undo', - target: removed.verbatim ?? removed.kind, - status: 'passed', - }, - }); - } - processing = false; - return; - } - if (line === '/list') { - const stepsInfo = state.steps.map((s, i) => `${i + 1}. ${s.verbatim ?? s.kind}`).join('\n'); - emitJson({ - event: 'flow_step', - data: { - step: state.steps.length, - total: state.steps.length, - kind: 'list', - target: stepsInfo || 'No steps yet', - status: state.steps.length > 0 ? 'passed' : 'failed', - }, - }); - processing = false; - return; - } - // Unknown slash command - emitJson({ - event: 'flow_step', - data: { - step: 0, - total: 0, - kind: 'info', - target: `Unknown command: ${line}. Available: /yaml /export /list /undo /clear /quit`, - status: 'failed', - }, - }); - processing = false; - return; - } - - const stepNum = state.steps.length + 1; - - // ── Per-line execution (JSON mode) ── - // - // Same pipeline as the interactive REPL — both delegate to runOneInstruction() - // so a fix to the instruction pipeline applies to all surfaces at once. - // Only the IO layer (emit JSON event vs print to terminal) differs here. - if (!state.mcp) { - emitJson({ - event: 'step', - data: { - step: stepNum, - action: 'error', - target: line, - success: false, - message: 'Not connected to device', - }, - }); - processing = false; - return; - } - - // Early-outs for bookkeeping-only steps that don't need device execution. - const earlyParse = tryParseNaturalFlowLine(line); - if (earlyParse?.kind === 'done') { - state.steps.push(earlyParse); - emitJson({ - event: 'step', - data: { step: stepNum, action: 'done', target: line, success: true, message: 'recorded' }, - }); - processing = false; - return; - } - if (earlyParse?.kind === 'getInfo') { - const infoAnswer = await handleGetInfo(earlyParse.query); - emitJson({ - event: 'step', - data: { - step: stepNum, - action: 'getInfo', - target: line, - success: true, - message: infoAnswer || 'No answer', - }, - }); - processing = false; - return; - } - - let outcome; - try { - outcome = await runOneInstruction(state.mcp, line, { - appResolver: state.appResolver ?? undefined, - minMatchScore: MIN_MATCH_SCORE, - }); - } catch (err: any) { - emitJson({ - event: 'step', - data: { - step: stepNum, - action: 'error', - target: line, - success: false, - message: err?.message ?? String(err), - }, - }); - processing = false; - return; - } - - if (outcome.isGetInfo) { - emitJson({ - event: 'step', - data: { - step: stepNum, - action: 'getInfo', - target: line, - success: true, - message: outcome.getInfoAnswer || outcome.result.message, - }, - }); - processing = false; - return; - } - if (outcome.step.kind === 'getInfo') { - const infoAnswer = await handleGetInfo(outcome.step.query); - emitJson({ - event: 'step', - data: { - step: stepNum, - action: 'getInfo', - target: line, - success: true, - message: infoAnswer || 'No answer', - }, - }); - processing = false; - return; - } - if (outcome.step.kind === 'done') { - state.steps.push(outcome.step); - emitJson({ - event: 'step', - data: { step: stepNum, action: 'done', target: line, success: true, message: 'recorded' }, - }); - processing = false; - return; - } - - if (outcome.result.success) { - state.steps.push(outcome.step); - } - const suggestion = - !outcome.result.success && outcome.step.kind === 'tap' && outcome.closestMatch - ? `Closest match: "${outcome.closestMatch}". Try: tap on ${outcome.closestMatch}` - : null; - emitJson({ - event: 'step', - data: { - step: stepNum, - action: outcome.step.kind, - target: line, - success: outcome.result.success, - message: suggestion ? `${outcome.result.message}\n${suggestion}` : outcome.result.message, - }, - }); - - processing = false; - }); - - rl.on('close', () => { - process.exit(0); - }); - - return new Promise((resolve) => { - rl.on('close', resolve); - }); -} - -export async function runPlayground(deviceArgs?: PlaygroundDeviceArgs): Promise { - if (deviceArgs) _deviceArgs = deviceArgs; - printPlaygroundHeader(); - - // If no platform specified, prompt the user before connecting - if (!_deviceArgs.platform) { - const { promptPlatformInline } = await import('@appclaw/core/device/platform-picker'); - const picked = await promptPlatformInline(); - if (picked) _deviceArgs.platform = picked; - } - - // Connect to device first - const connected = await connectToDevice(); - if (!connected) { - process.exit(1); - } - - // Ensure stdin is flowing before creating the REPL readline. - // Prior device-setup steps (spinners, MCP calls) can leave stdin paused. - if (process.stdin.isPaused()) process.stdin.resume(); - - // ── Ink REPL shell on interactive TTYs (pinned prompt, scrolling output) ── - const useInk = - !!process.stdout.isTTY && !!process.stdin.isTTY && process.env.APPCLAW_TUI !== 'off'; - if (useInk) { - const { runPlaygroundInk } = await import('../ui/ink/playground-runner.js'); - const cfg = loadConfig(); - await runPlaygroundInk({ - info: { - platform: _resolvedPlatform, - app: state.meta.appId, - model: cfg.LLM_MODEL || DEFAULT_MODELS[cfg.LLM_PROVIDER] || 'model', - mode: cfg.AGENT_MODE, - transport: cfg.MCP_TRANSPORT, - }, - onCommand: (line: string) => Promise.resolve(processLine(line)), - onQuit: cleanup, - getStepCount: () => state.steps.length, - }); - console.log(`\n ${theme.dim('Goodbye!')}\n`); - return; - } - - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - let processing = false; - - function prompt(): void { - rl.setPrompt(getPrompt()); - rl.prompt(); - } - - prompt(); - - rl.on('line', async (input: string) => { - const line = input.trim(); - - if (!line) { - prompt(); - return; - } - - // Prevent overlapping commands - if (processing) { - console.log(` ${theme.dim('Please wait for the current command to finish…')}`); - return; - } - - // Quit - if (line === '/quit' || line === '/exit' || line === '/q') { - if (state.steps.length > 0) { - console.log(); - console.log( - ` ${theme.warn('!')} ${state.steps.length} step${state.steps.length === 1 ? '' : 's'} not exported.` - ); - console.log( - ` ${theme.dim('Use')} ${theme.info('/export ')} ${theme.dim('to save, or type')} ${theme.info('/quit')} ${theme.dim('again to discard.')}` - ); - console.log(); - rl.once('line', async (confirm: string) => { - const c = confirm.trim(); - if (c === '/quit' || c === '/exit' || c === '/q' || c === 'y' || c === 'yes') { - await cleanup(); - rl.close(); - return; - } - await processLine(c); - prompt(); - }); - prompt(); - return; - } - await cleanup(); - rl.close(); - return; - } - - processing = true; - await processLine(line); - processing = false; - prompt(); - }); - - rl.on('close', () => { - console.log(`\n ${theme.dim('Goodbye!')}\n`); - }); - - return new Promise((resolve) => { - rl.on('close', resolve); - }); -} - -async function cleanup(): Promise { - if (state.mcp) { - try { - await state.mcp.callTool('appium_session_management', { action: 'delete' }); - } catch { - /* ignore — session may already be gone */ - } - try { - await state.mcp.close(); - } catch { - /* ignore */ - } - } -} - -// ─── Screen queries (via vision getInfo) ───────────── - -async function handleGetInfo(query: string): Promise { - if (!state.mcp) { - console.log(` ${theme.error('✗')} Not connected to device`); - return null; - } - - try { - ui.startSpinner('Analyzing screen', query); - const { screenshot } = await import('@appclaw/core/mcp/tools'); - const imageBase64 = await screenshot(state.mcp); - if (!imageBase64) { - ui.stopSpinner(); - console.log(` ${theme.error('✗')} Failed to capture screenshot`); - return null; - } - - const { - getStarkVisionApiKey, - getStarkVisionBaseUrl, - getStarkVisionCoordinateOrder, - getStarkVisionModel, - } = await import('@appclaw/core/vision/locate-enabled'); - const apiKey = getStarkVisionApiKey(); - const baseUrl = getStarkVisionBaseUrl(); - if (!apiKey && !baseUrl) { - ui.stopSpinner(); - console.log( - ` ${theme.error('✗')} getInfo requires vision (GEMINI_API_KEY or STARK_VISION_BASE_URL)` - ); - return null; - } - - const { default: starkVision } = await import('@appclaw/vision'); - const { StarkVisionClient } = starkVision; - const client = new StarkVisionClient({ - apiKey: apiKey || 'local', - model: getStarkVisionModel(), - disableThinking: true, - ...(baseUrl && { baseUrl }), - ...(baseUrl && { coordinateOrder: getStarkVisionCoordinateOrder() }), - }); - const response = await client.getElementInfo(imageBase64, query, true); - - let answer: string; - let explanation: string | undefined; - try { - const parsed = JSON.parse(response.replace(/(^```json\s*|```\s*$)/g, '').trim()); - answer = parsed.answer || response; - explanation = parsed.explanation; - } catch { - answer = response; - } - - ui.stopSpinner(); - console.log(); - const ansContent = explanation ? `${answer}\n\n${theme.dim(explanation)}` : answer; - printPanel({ title: 'Answer', content: ansContent }); - console.log(); - return answer; - } catch (err: any) { - ui.stopSpinner(); - console.log( - ` ${theme.error('✗')} Failed to get info: ${theme.error(err?.message ?? String(err))}` - ); - return null; - } -} - -async function processLine(line: string): Promise { - // Slash commands - if (line.startsWith('/')) { - const spaceIdx = line.indexOf(' '); - const cmd = spaceIdx === -1 ? line : line.slice(0, spaceIdx); - const arg = spaceIdx === -1 ? '' : line.slice(spaceIdx + 1); - - const handler = COMMANDS[cmd]; - if (handler) { - await handler.run(arg); - return; - } - console.log( - ` ${theme.error('✗')} Unknown command: ${theme.dim(cmd)} — type ${theme.info('/help')}` - ); - return; - } - - // ── Per-line execution ─────────────────────────────────── - // - // Pipeline (vision-first → regex → LLM → executeStep) lives in - // src/flow/run-instruction.ts so the SDK and playground stay in lockstep. - // Two early-outs for bookkeeping-only step kinds that don't need to touch - // the device (matches the playground's historical behaviour): - // - `done` → just records the step, no execution - // - `getInfo` → routed to handleGetInfo (a separate vision call) - if (!state.mcp) { - console.log(` ${theme.error('✗')} Not connected to device`); - return; - } - const earlyParse = tryParseNaturalFlowLine(line); - if (earlyParse?.kind === 'done') { - const stepNum = state.steps.length + 1; - state.steps.push(earlyParse); - printStepSuccess(stepNum, earlyParse, 'recorded'); - return; - } - if (earlyParse?.kind === 'getInfo') { - await handleGetInfo(earlyParse.query); - return; - } - - ui.startSpinner('Executing', line); - resetVisionTokens(); - let outcome; - const t0 = performance.now(); - try { - outcome = await runOneInstruction(state.mcp, line, { - appResolver: state.appResolver ?? undefined, - minMatchScore: MIN_MATCH_SCORE, - }); - } catch (err: any) { - ui.stopSpinner(); - console.log(` ${theme.error('✗')} ${theme.dim(`Failed: ${err?.message ?? String(err)}`)}`); - console.log( - ` ${theme.dim('Type')} ${theme.info('/help')} ${theme.dim('to see supported patterns')}` - ); - return; - } - ui.stopSpinner(); - const elapsedMs = Math.round(performance.now() - t0); - - const printVisionTokens = (): void => { - const vt = getVisionTokens(); - if (vt.totalTokens > 0) - ui.printStepTokens( - vt.inputTokens, - vt.outputTokens, - vt.cachedTokens || undefined, - visionCost(vt.inputTokens, vt.outputTokens), - 'vision' - ); - }; - - // Vision detected a "what's on screen" question — show the answer panel - // and skip the step-recording flow entirely (no device action happened). - if (outcome.isGetInfo) { - const ans = outcome.getInfoAnswer || outcome.result.message; - const ansBody = outcome.getInfoExplanation - ? `${ans}\n\n${theme.dim(outcome.getInfoExplanation)}` - : ans; - console.log(); - printPanel({ title: 'Answer', content: ansBody }); - printVisionTokens(); - console.log(); - return; - } - - // `done` and `getInfo` resolved via the LLM fallback (not by regex above). - if (outcome.step.kind === 'getInfo') { - await handleGetInfo(outcome.step.query); - return; - } - const stepNum = state.steps.length + 1; - if (outcome.step.kind === 'done') { - state.steps.push(outcome.step); - printStepSuccess(stepNum, outcome.step, 'recorded'); - return; - } - - if (outcome.result.success) { - state.steps.push(outcome.step); - printStepSuccess(stepNum, outcome.step, outcome.result.message, elapsedMs); - } else { - printStepFail(stepNum, outcome.step, outcome.result.message, elapsedMs); - if (outcome.step.kind === 'tap' && outcome.closestMatch) { - console.log( - ` ${theme.warn(`Closest match: "${outcome.closestMatch}". Try: tap on ${outcome.closestMatch}`)}` - ); - } - console.log(` ${theme.dim('Step not recorded. Fix and try again.')}`); - } - printVisionTokens(); -} diff --git a/packages/cli/src/step-recorder/flow-builder.ts b/packages/cli/src/step-recorder/flow-builder.ts new file mode 100644 index 0000000..25de576 --- /dev/null +++ b/packages/cli/src/step-recorder/flow-builder.ts @@ -0,0 +1,158 @@ +/** + * Flow-building helpers shared by every step-recording surface. + * + * Both recording surfaces — the TUI (`appclaw --tui`) and the headless JSON + * bridge (`appclaw --json --playground`) — accumulate the same `FlowStep[]` and + * must produce identical YAML / runner-spec output, so these take + * `(steps, meta)` explicitly instead of reaching into any surface's state. + * One implementation, two callers, no drift. + */ + +import path from 'node:path'; +import { stringify } from 'yaml'; + +import { loadConfig } from '@appclaw/core/config'; +import { generateSdkTestFromInstructions } from '@appclaw/core/sdk/goal-export'; +import { stepAction, stepTarget } from '@appclaw/core/ui/step-printer'; +import type { FlowStep, FlowMeta } from '@appclaw/core/flow/types'; + +/** Convert step to YAML — preserve the user's original natural language input. */ +export function stepToYaml(step: FlowStep): unknown { + // Recorded steps always have verbatim (the exact text the user typed). + // Use it directly so the YAML reads like the user's instructions. + if (step.verbatim) return step.verbatim; + + // Fallback for steps without verbatim (shouldn't happen when recording) + switch (step.kind) { + case 'launchApp': + return 'launchApp'; + case 'openApp': + return `open ${step.query} app`; + case 'tap': + return `tap ${step.label}`; + case 'doubleTap': + return `double tap ${step.label}`; + case 'longPress': + return step.duration != null + ? `long press ${step.label} for ${step.duration}ms` + : `long press ${step.label}`; + case 'type': + return `type "${step.text}"`; + case 'swipe': + return `swipe ${step.direction}`; + case 'zoom': + return step.target + ? `zoom ${step.scale >= 1 ? 'in' : 'out'} ${step.scale}x on ${step.target}` + : `zoom ${step.scale >= 1 ? 'in' : 'out'} ${step.scale}x`; + case 'wait': + return `wait ${step.seconds} s`; + case 'waitUntil': + if (step.condition === 'screenLoaded') return 'wait until screen is loaded'; + if (step.condition === 'gone') return `wait until "${step.text}" is gone`; + return `wait until "${step.text}" is visible`; + case 'enter': + return 'press enter'; + case 'back': + return 'go back'; + case 'home': + return 'go home'; + case 'assert': + return `assert "${step.text}" is visible`; + case 'scrollAssert': + return `scroll ${step.direction} until "${step.text}" is visible`; + case 'getInfo': + return `getInfo: ${step.query}`; + case 'done': + return step.message ? `done: ${step.message}` : 'done'; + } +} + +export function buildYamlString(steps: FlowStep[], meta: FlowMeta): string { + const parts: string[] = []; + + if (meta.appId || meta.name || meta.platform) { + const metaObj: Record = {}; + if (meta.appId) metaObj.appId = meta.appId; + if (meta.name) metaObj.name = meta.name; + if (meta.platform) metaObj.platform = meta.platform; + parts.push(stringify(metaObj).trim()); + parts.push('---'); + } + + const yamlSteps = steps.map(stepToYaml); + + // Auto-append "done" if the last step isn't already a done step + const lastStep = steps[steps.length - 1]; + if (!lastStep || lastStep.kind !== 'done') { + yamlSteps.push('done'); + } + + parts.push(stringify({ steps: yamlSteps }).trim()); + + return parts.join('\n') + '\n'; +} + +/** + * Whether the given filename should be exported as an @appclaw/runner spec + * format) rather than the default YAML flow format. + */ +export function isSdkTestFilename(name: string): boolean { + return /\.(?:test|spec)\.(?:m|c)?[jt]sx?$/i.test(name) || /\.(?:m|c)?ts$/i.test(name); +} + +/** + * Resolve the final on-disk path for an `/export` write — same rules as the + * CLI's `--export`. Bare filenames land in the configured directory (EXPORT_DIR); + * paths with a directory hint (./tests/foo.test.ts, /abs/...) are used verbatim. + * + * The configured directory differs by format: SDK tests go to EXPORT_DIR, YAML + * flows stay in cwd. + * + * `exportDir` is the `--export-dir` override; nullish falls back to EXPORT_DIR. + */ +export function resolveRecordedExportPath( + filename: string, + asSdkTest: boolean, + exportDir?: string | null +): string { + if (path.isAbsolute(filename)) return filename; + if (filename.includes('/') || filename.includes(path.sep)) { + return path.resolve(process.cwd(), filename); + } + if (asSdkTest) { + const dir = exportDir ?? loadConfig().EXPORT_DIR; + return path.resolve(process.cwd(), dir, filename); + } + return path.resolve(process.cwd(), filename); +} + +/** + * Build the @appclaw/runner spec body for a recorded step list. + * Each step's `verbatim` (the user's original natural-language text) becomes + * one `await app.run(...)` call — no translation needed because the recording + * surfaces accept the same syntax that `AppClaw.run()` does. + */ +export function buildSdkTestString(steps: FlowStep[], meta: FlowMeta): string { + const instructions = steps + .map((s) => s.verbatim?.trim()) + .filter((v): v is string => !!v && v.length > 0); + return generateSdkTestFromInstructions({ + instructions, + config: { + describeName: meta.name || 'Recorded flow', + ...(meta.platform === 'ios' || meta.platform === 'android' + ? { platform: meta.platform } + : {}), + }, + }); +} + +/** + * Compact one-line-per-step rendering, for surfaces that can't draw a table + * (the TUI logs these into its in-frame transcript). + */ +export function formatStepLines(steps: FlowStep[]): string[] { + return steps.map( + (s, i) => `${String(i + 1).padStart(2)}. ${stepAction(s).padEnd(8)}${stepTarget(s)}` + ); +} diff --git a/packages/cli/src/step-recorder/json-bridge.ts b/packages/cli/src/step-recorder/json-bridge.ts new file mode 100644 index 0000000..2ae219b --- /dev/null +++ b/packages/cli/src/step-recorder/json-bridge.ts @@ -0,0 +1,567 @@ +/** + * Headless step-recorder bridge — `appclaw --json --playground`. + * + * Reads one instruction per line from stdin and emits NDJSON events on stdout. + * The VS Code / Cursor extension drives this over a child process, so the + * protocol (event names, field names, ordering) is a shipped contract: change + * it only in lockstep with `vscode-extension/src/bridge.ts`. + * + * The interactive counterpart is `appclaw --tui`; this file deliberately keeps + * its own small copies of the session helpers rather than sharing a module with + * the TUI, so a TUI redesign can never shift the extension's wire behaviour. + */ + +import readline from 'node:readline'; +import { writeFileSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; + +import { loadConfig, Config } from '@appclaw/core/config'; +import { createMCPClient } from '@appclaw/core/mcp/client'; +import { setupDevice } from '@appclaw/core/device/index'; +import { AppResolver } from '@appclaw/core/agent/app-resolver'; +import { tryParseNaturalFlowLine } from '@appclaw/core/flow/natural-line'; +import { runOneInstruction, DEFAULT_MIN_MATCH_SCORE } from '@appclaw/core/flow/run-instruction'; +import { isVisionLocateEnabled } from '@appclaw/core/vision/locate-enabled'; +import type { FlowStep, FlowMeta } from '@appclaw/core/flow/types'; +import type { MCPClient } from '@appclaw/core/mcp/types'; +import { theme, printBox, printPanel } from '@appclaw/core/ui/terminal'; +import * as ui from '@appclaw/core/ui/terminal'; + +import { + buildYamlString as buildYaml, + buildSdkTestString as buildSdkTest, + isSdkTestFilename, + resolveRecordedExportPath, +} from './flow-builder.js'; +import { fetchScreenInfo } from './screen-info.js'; + +// ─── State ────────────────────────────────────────────── + +interface BridgeState { + steps: FlowStep[]; + meta: FlowMeta; + mcp: MCPClient | null; + appResolver: AppResolver | null; +} + +const state: BridgeState = { + steps: [], + meta: {}, + mcp: null, + appResolver: null, +}; + +/** + * Minimum matchScore (1-10) required to execute a tap. Below this threshold, + * vision found a loose match — surface the suggestion but don't execute. + * + * Sourced from `flow/run-instruction.ts` so the SDK, the TUI and this bridge + * share one threshold (it used to be defined per-surface, which drifted). + */ +const MIN_MATCH_SCORE = DEFAULT_MIN_MATCH_SCORE; + +/** YAML body for the current recording — delegates to the shared builder. */ +function buildYamlString(): string { + return buildYaml(state.steps, state.meta); +} + +/** Runner-spec body for the current recording — delegates to the shared builder. */ +function buildSdkTestString(): string { + return buildSdkTest(state.steps, state.meta); +} + +/** `/export` path resolution, with the `--export-dir` override applied. */ +function resolveBridgeExportPath(filename: string, asSdkTest: boolean): string { + return resolveRecordedExportPath(filename, asSdkTest, _deviceArgs.exportDir); +} + +// ─── Device connection ────────────────────────────────── + +let _resolvedPlatform: 'android' | 'ios' = 'android'; + +async function connectToDevice(): Promise { + const config = loadConfig(); + + try { + ui.startSpinner(`Connecting to appium-mcp (${config.MCP_TRANSPORT})…`); + const mcpClient = await createMCPClient({ + transport: config.MCP_TRANSPORT, + host: config.MCP_HOST, + port: config.MCP_PORT, + url: config.MCP_URL || undefined, + }); + state.mcp = mcpClient; + ui.stopSpinner(); + ui.printSetupOk('Connected to appium-mcp'); + + // Full device setup pipeline (platform → device → iOS setup → session) + const deviceResult = await setupDevice(mcpClient, { + cliPlatform: _deviceArgs.platform ?? null, + cliDeviceType: _deviceArgs.deviceType ?? null, + cliUdid: _deviceArgs.udid ?? null, + cliDeviceName: _deviceArgs.deviceName ?? null, + config, + alwaysPickDevice: true, + }); + _resolvedPlatform = deviceResult.platform; + + // Auto-set platform in flow metadata so exported YAML includes it + if (!state.meta.platform) { + state.meta.platform = deviceResult.platform; + } + + // Initialize app resolver for "open X app" commands + ui.startSpinner('Loading installed apps…'); + const appResolver = new AppResolver(); + await appResolver.initialize(mcpClient, deviceResult.platform); + state.appResolver = appResolver; + ui.stopSpinner(); + ui.printSetupOk('App resolver ready'); + + // Surface the effective interaction mode so a silent DOM fallback is never a + // mystery. `isVisionMode()` (run-yaml-flow) requires BOTH AGENT_MODE=vision AND + // vision-locate being configured — if vision is requested but not configured, + // every command quietly runs against the DOM instead. + const visionLocate = isVisionLocateEnabled(); + if (Config.AGENT_MODE === 'vision' && visionLocate) { + ui.printSetupOk('Interaction mode: vision'); + } else if (Config.AGENT_MODE === 'vision' && !visionLocate) { + ui.printWarning( + 'AGENT_MODE=vision is set, but vision-locate is not configured — running in DOM mode. ' + + 'Set GEMINI_API_KEY / STARK_VISION_API_KEY / STARK_VISION_BASE_URL (or LLM_PROVIDER=gemini) to enable vision.' + ); + } else { + ui.printSetupOk('Interaction mode: dom'); + } + + const readyContent = [ + `${theme.dim('Type commands to execute on device.')}`, + '', + `${theme.dim('Examples:')}`, + ` ${theme.white('open youtube app')}`, + ` ${theme.white('click on Search')}`, + ` ${theme.white('type "hello"')}`, + ].join('\n'); + console.log(); + printBox(readyContent, { + title: 'Device connected', + titleAlignment: 'left', + borderColor: '#22C55E', + padding: { left: 2, right: 2, top: 1, bottom: 1 }, + }); + console.log(); + + return true; + } catch (err: any) { + ui.stopSpinner(); + // Always write to stderr so IDE extensions can see the error + process.stderr.write(`[playground] Connection failed: ${err?.message ?? err}\n`); + if (err?.stack) process.stderr.write(`[playground] ${err.stack}\n`); + ui.printError(`Failed to connect: ${err?.message ?? err}`); + // AppClaw drives Appium through the appium-mcp subprocess, which it starts itself — + // there is no separate "Appium server" to launch. A timeout (-32001) almost always + // means appium-mcp couldn't start/handshake in time (e.g. a cold `npx` download on a + // global install that doesn't bundle it), NOT that a server is missing. + const errMsg = String(err?.message ?? err); + const timedOut = errMsg.includes('-32001') || /timed out/i.test(errMsg); + if (timedOut) { + ui.printInfo( + 'AppClaw starts appium-mcp itself — no separate Appium server is needed. The MCP handshake ' + + 'timed out: on a first run appium-mcp may still be downloading via npx. Retry, or reinstall so ' + + "it's bundled (e.g. npm i -g appclaw@latest). Set MCP_DEBUG=1 to see appium-mcp's startup logs." + ); + } else { + ui.printInfo( + 'AppClaw starts appium-mcp itself — no separate Appium server is needed. Make sure a ' + + 'device/emulator is connected. Set MCP_DEBUG=1 to see appium-mcp’s startup logs.' + ); + } + console.log(); + return false; + } +} + +async function cleanup(): Promise { + if (state.mcp) { + try { + await state.mcp.callTool('appium_session_management', { action: 'delete' }); + } catch { + /* ignore — session may already be gone */ + } + try { + await state.mcp.close(); + } catch { + /* ignore */ + } + } +} + +// ─── Screen queries (via vision getInfo) ───────────── + +async function handleGetInfo(query: string): Promise { + if (!state.mcp) { + console.log(` ${theme.error('✗')} Not connected to device`); + return null; + } + + const res = await fetchScreenInfo(state.mcp, query); + if (!res.ok) { + console.log( + res.reason === 'error' + ? ` ${theme.error('✗')} Failed to get info: ${theme.error(res.message)}` + : ` ${theme.error('✗')} ${res.message}` + ); + return null; + } + + console.log(); + const ansContent = res.explanation + ? `${res.answer}\n\n${theme.dim(res.explanation)}` + : res.answer; + printPanel({ title: 'Answer', content: ansContent }); + console.log(); + return res.answer; +} + +// ─── Entry point ──────────────────────────────────────── + +export interface PlaygroundDeviceArgs { + platform?: 'android' | 'ios' | null; + deviceType?: 'simulator' | 'real' | null; + udid?: string | null; + deviceName?: string | null; + /** + * Override directory for bare-filename SDK-test exports (`--export-dir`). + * Takes precedence over the `EXPORT_DIR` config/env default. Ignored for + * paths that already include a directory hint or are absolute. + */ + exportDir?: string | null; +} + +/** Stash device args so connectToDevice can use them */ +let _deviceArgs: PlaygroundDeviceArgs = {}; + +/** + * JSON-mode step recorder — reads commands from stdin (one per line), + * emits NDJSON events to stdout. Used by IDE extensions. + */ +export async function runPlaygroundJson(deviceArgs?: PlaygroundDeviceArgs): Promise { + if (deviceArgs) _deviceArgs = deviceArgs; + + const { emitJson } = await import('@appclaw/core/json-emitter'); + + let connectError: string | undefined; + try { + const connected = await connectToDevice(); + if (!connected) { + connectError = 'connectToDevice returned false'; + } + } catch (err: any) { + connectError = err?.message ?? String(err); + } + + if (connectError) { + emitJson({ event: 'error', data: { message: `Failed to connect: ${connectError}` } }); + process.exit(1); + } + + emitJson({ event: 'connected', data: { transport: 'stdio' } }); + emitJson({ event: 'device_ready', data: { platform: _resolvedPlatform } }); + + // Graceful shutdown on SIGTERM (sent by VS Code extension bridge.stop()) + const gracefulShutdown = async () => { + await cleanup(); + emitJson({ event: 'done', data: { success: true, totalSteps: state.steps.length } }); + process.exit(0); + }; + process.on('SIGTERM', gracefulShutdown); + process.on('SIGINT', gracefulShutdown); + + if (process.stdin.isPaused()) process.stdin.resume(); + + const rl = readline.createInterface({ input: process.stdin }); + let processing = false; + + rl.on('line', async (input: string) => { + const line = input.trim(); + if (!line) return; + + if (processing) { + emitJson({ event: 'error', data: { message: 'Still processing previous command' } }); + return; + } + + processing = true; + + // Slash commands + if (line.startsWith('/')) { + if (line === '/quit' || line === '/exit' || line === '/q') { + await cleanup(); + emitJson({ event: 'done', data: { success: true, totalSteps: state.steps.length } }); + rl.close(); + processing = false; + return; + } + if (line === '/yaml') { + if (state.steps.length === 0) { + emitJson({ + event: 'flow_step', + data: { + step: 0, + total: 0, + kind: 'yaml', + target: 'No steps to preview', + status: 'failed', + }, + }); + } else { + const yamlStr = buildYamlString(); + emitJson({ + event: 'flow_step', + data: { + step: state.steps.length, + total: state.steps.length, + kind: 'yaml', + target: yamlStr, + status: 'passed', + }, + }); + } + processing = false; + return; + } + if (line.startsWith('/export')) { + const arg = line.slice(7).trim(); + const filename = arg || `flow-${Date.now()}.spec.ts`; + const asSdkTest = isSdkTestFilename(filename); + const filepath = resolveBridgeExportPath(filename, asSdkTest); + if (state.steps.length === 0) { + emitJson({ + event: 'flow_step', + data: { + step: 0, + total: 0, + kind: 'export', + target: 'No steps to export', + status: 'failed', + }, + }); + } else { + const body = asSdkTest ? buildSdkTestString() : buildYamlString(); + mkdirSync(path.dirname(filepath), { recursive: true }); + writeFileSync(filepath, body, 'utf-8'); + emitJson({ + event: 'flow_step', + data: { + step: state.steps.length, + total: state.steps.length, + kind: 'export', + target: filepath, + status: 'passed', + }, + }); + } + processing = false; + return; + } + if (line === '/clear') { + state.steps.length = 0; + state.meta = {}; + emitJson({ + event: 'flow_step', + data: { step: 0, total: 0, kind: 'clear', target: 'All steps cleared', status: 'passed' }, + }); + processing = false; + return; + } + if (line === '/undo') { + if (state.steps.length === 0) { + emitJson({ + event: 'flow_step', + data: { step: 0, total: 0, kind: 'undo', target: 'Nothing to undo', status: 'failed' }, + }); + } else { + const removed = state.steps.pop()!; + emitJson({ + event: 'flow_step', + data: { + step: state.steps.length, + total: state.steps.length, + kind: 'undo', + target: removed.verbatim ?? removed.kind, + status: 'passed', + }, + }); + } + processing = false; + return; + } + if (line === '/list') { + const stepsInfo = state.steps.map((s, i) => `${i + 1}. ${s.verbatim ?? s.kind}`).join('\n'); + emitJson({ + event: 'flow_step', + data: { + step: state.steps.length, + total: state.steps.length, + kind: 'list', + target: stepsInfo || 'No steps yet', + status: state.steps.length > 0 ? 'passed' : 'failed', + }, + }); + processing = false; + return; + } + // Unknown slash command + emitJson({ + event: 'flow_step', + data: { + step: 0, + total: 0, + kind: 'info', + target: `Unknown command: ${line}. Available: /yaml /export /list /undo /clear /quit`, + status: 'failed', + }, + }); + processing = false; + return; + } + + const stepNum = state.steps.length + 1; + + // ── Per-line execution (JSON mode) ── + // + // Same pipeline as the TUI's step recorder — both delegate to + // runOneInstruction() so a fix to the instruction pipeline applies to all + // surfaces at once. Only the IO layer (emit JSON event vs draw a frame) + // differs here. + if (!state.mcp) { + emitJson({ + event: 'step', + data: { + step: stepNum, + action: 'error', + target: line, + success: false, + message: 'Not connected to device', + }, + }); + processing = false; + return; + } + + // Early-outs for bookkeeping-only steps that don't need device execution. + const earlyParse = tryParseNaturalFlowLine(line); + if (earlyParse?.kind === 'done') { + state.steps.push(earlyParse); + emitJson({ + event: 'step', + data: { step: stepNum, action: 'done', target: line, success: true, message: 'recorded' }, + }); + processing = false; + return; + } + if (earlyParse?.kind === 'getInfo') { + const infoAnswer = await handleGetInfo(earlyParse.query); + emitJson({ + event: 'step', + data: { + step: stepNum, + action: 'getInfo', + target: line, + success: true, + message: infoAnswer || 'No answer', + }, + }); + processing = false; + return; + } + + let outcome; + try { + outcome = await runOneInstruction(state.mcp, line, { + appResolver: state.appResolver ?? undefined, + minMatchScore: MIN_MATCH_SCORE, + }); + } catch (err: any) { + emitJson({ + event: 'step', + data: { + step: stepNum, + action: 'error', + target: line, + success: false, + message: err?.message ?? String(err), + }, + }); + processing = false; + return; + } + + if (outcome.isGetInfo) { + emitJson({ + event: 'step', + data: { + step: stepNum, + action: 'getInfo', + target: line, + success: true, + message: outcome.getInfoAnswer || outcome.result.message, + }, + }); + processing = false; + return; + } + if (outcome.step.kind === 'getInfo') { + const infoAnswer = await handleGetInfo(outcome.step.query); + emitJson({ + event: 'step', + data: { + step: stepNum, + action: 'getInfo', + target: line, + success: true, + message: infoAnswer || 'No answer', + }, + }); + processing = false; + return; + } + if (outcome.step.kind === 'done') { + state.steps.push(outcome.step); + emitJson({ + event: 'step', + data: { step: stepNum, action: 'done', target: line, success: true, message: 'recorded' }, + }); + processing = false; + return; + } + + if (outcome.result.success) { + state.steps.push(outcome.step); + } + const suggestion = + !outcome.result.success && outcome.step.kind === 'tap' && outcome.closestMatch + ? `Closest match: "${outcome.closestMatch}". Try: tap on ${outcome.closestMatch}` + : null; + emitJson({ + event: 'step', + data: { + step: stepNum, + action: outcome.step.kind, + target: line, + success: outcome.result.success, + message: suggestion ? `${outcome.result.message}\n${suggestion}` : outcome.result.message, + }, + }); + + processing = false; + }); + + rl.on('close', () => { + process.exit(0); + }); + + return new Promise((resolve) => { + rl.on('close', resolve); + }); +} diff --git a/packages/cli/src/step-recorder/memory-inspect.ts b/packages/cli/src/step-recorder/memory-inspect.ts new file mode 100644 index 0000000..19684e5 --- /dev/null +++ b/packages/cli/src/step-recorder/memory-inspect.ts @@ -0,0 +1,129 @@ +/** + * `/memory` inspector — episodic + procedural memory stats/listing. + * + * Reads only Config + the on-disk stores, so any surface can expose the same + * command. Output stays plain console.log, but the TUI captures it and renders + * it inside `OutputDialog` — a box far narrower than the terminal. So no line + * here may be sized to the terminal width: a full-width rule wrapped inside the + * box and came out as a second, stray underline. Section headings are plain + * text for that reason. + */ + +import { Config } from '@appclaw/core/config'; +import { theme } from '@appclaw/core/ui/terminal'; +import { loadStore, getTrajectoryStorePath } from '@appclaw/core/memory/store'; +import { loadProcedures, getProceduresStorePath } from '@appclaw/core/memory/procedures'; + +export function runMemoryCommand(arg: string): void { + const sub = (arg || 'stats').toLowerCase(); + if (sub === 'stats') return printMemoryStats(); + if (sub === 'list') return printMemoryProcedureList(); + if (sub === 'paths') return printMemoryPaths(); + console.log(` ${theme.label('Usage:')} /memory stats | /memory list | /memory paths`); +} + +function printMemoryPaths(): void { + const trajPath = getTrajectoryStorePath(Config.EPISODIC_MEMORY_PATH || undefined); + const procPath = getProceduresStorePath(Config.PROCEDURAL_MEMORY_PATH || undefined); + console.log(` ${theme.label('Trajectories:')} ${theme.white(trajPath)}`); + console.log(` ${theme.label('Procedures: ')} ${theme.white(procPath)}`); + console.log(` ${theme.label('Namespace: ')} ${theme.white(Config.APPCLAW_MEMORY_NAMESPACE)}`); +} + +function printMemoryStats(): void { + const trajPath = getTrajectoryStorePath(Config.EPISODIC_MEMORY_PATH || undefined); + const procPath = getProceduresStorePath(Config.PROCEDURAL_MEMORY_PATH || undefined); + const trajStore = loadStore(Config.EPISODIC_MEMORY_PATH || undefined); + const procStore = loadProcedures(Config.PROCEDURAL_MEMORY_PATH || undefined); + const ns = Config.APPCLAW_MEMORY_NAMESPACE; + + const SEVEN_DAYS = 7 * 24 * 60 * 60 * 1000; + const trajNs = trajStore.entries.filter((e) => (e.namespace ?? 'default') === ns); + const procNs = procStore.entries.filter((e) => e.namespace === ns); + const trajStale = trajNs.filter( + (e) => e.successCount < 2 && Date.now() - e.timestamp > SEVEN_DAYS + ).length; + + const groupByApp = (arr: T[]): Array<[string, number]> => { + const m = new Map(); + for (const e of arr) m.set(e.appId, (m.get(e.appId) ?? 0) + 1); + return [...m.entries()].sort((a, b) => b[1] - a[1]); + }; + + console.log(` ${theme.label('Memory stats')}`); + console.log(); + console.log(` ${theme.label('Namespace:')} ${theme.white(ns)}`); + console.log(); + + // Episodic + console.log( + ` ${theme.label('Episodic')} ${theme.dim(`(${trajPath})`)}\n` + + ` ${theme.label('Total:')} ${theme.white(String(trajStore.entries.length))}` + + ` ${theme.dim(`(${trajNs.length} in this namespace)`)}\n` + + ` ${theme.label('Stale-eligible:')} ${theme.white(String(trajStale))} ${theme.dim('(single-use, >7d — hidden from retrieval)')}` + ); + const trajByApp = groupByApp(trajNs); + if (trajByApp.length > 0) { + console.log(` ${theme.label('By app:')}`); + for (const [appId, n] of trajByApp.slice(0, 8)) { + console.log( + ` ${theme.dim('•')} ${theme.white(appId.padEnd(40))} ${theme.dim(String(n))}` + ); + } + } + console.log(); + + // Procedural + console.log( + ` ${theme.label('Procedural')} ${theme.dim(`(${procPath})`)}\n` + + ` ${theme.label('Total:')} ${theme.white(String(procStore.entries.length))}` + + ` ${theme.dim(`(${procNs.length} in this namespace)`)}` + ); + const procByApp = groupByApp(procNs); + if (procByApp.length > 0) { + console.log(` ${theme.label('By app:')}`); + for (const [appId, n] of procByApp.slice(0, 8)) { + console.log( + ` ${theme.dim('•')} ${theme.white(appId.padEnd(40))} ${theme.dim(String(n))}` + ); + } + } + console.log(); + console.log(theme.dim(` Type /memory list to see goal recipes, /memory paths for file paths.`)); +} + +function printMemoryProcedureList(): void { + const store = loadProcedures(Config.PROCEDURAL_MEMORY_PATH || undefined); + const ns = Config.APPCLAW_MEMORY_NAMESPACE; + const items = store.entries.filter((e) => e.namespace === ns); + + if (items.length === 0) { + console.log( + ` ${theme.dim(`No procedures yet in namespace "${ns}". Run a goal to record one.`)}` + ); + return; + } + + items.sort((a, b) => b.timestamp - a.timestamp); + + console.log(` ${theme.label('Procedures')} ${theme.dim(`(${items.length} in "${ns}")`)}`); + console.log(); + for (const p of items.slice(0, 20)) { + const ago = formatProcAgo(p.timestamp); + const goal = p.goalKeywords.join(' '); + const reuse = p.successCount > 1 ? ` ${theme.success(`×${p.successCount}`)}` : ''; + console.log( + ` ${theme.dim('•')} ${theme.white(goal)}${reuse}` + + ` ${theme.dim(`— ${p.appId}, ${p.steps.length} steps, ${ago}`)}` + ); + } +} + +function formatProcAgo(ts: number): string { + const days = Math.floor((Date.now() - ts) / (1000 * 60 * 60 * 24)); + if (days === 0) return 'today'; + if (days === 1) return '1d ago'; + if (days < 7) return `${days}d ago`; + if (days < 30) return `${Math.floor(days / 7)}w ago`; + return `${Math.floor(days / 30)}mo ago`; +} diff --git a/packages/cli/src/step-recorder/screen-info.ts b/packages/cli/src/step-recorder/screen-info.ts new file mode 100644 index 0000000..302ebce --- /dev/null +++ b/packages/cli/src/step-recorder/screen-info.ts @@ -0,0 +1,79 @@ +/** + * "What's on screen?" queries (getInfo) — screenshot + a single Stark vision call. + * + * Shared by the TUI and the headless JSON bridge. Only the data-fetching half + * lives here; each surface formats the answer and the failure reasons itself, + * so neither one's existing output changes. + */ + +import type { MCPClient } from '@appclaw/core/mcp/types'; +import * as ui from '@appclaw/core/ui/terminal'; + +export type ScreenInfoResult = + | { ok: true; answer: string; explanation?: string } + /** + * `reason` lets callers keep their own wording: 'error' is an unexpected + * throw (message is the raw error), the others are expected preconditions. + */ + | { ok: false; reason: 'screenshot' | 'no-vision' | 'error'; message: string }; + +/** + * Ask the vision model a question about the current screen. Drives the shared + * spinner (the TUI's renderer turns that into in-frame busy text), and always + * stops it before returning. + */ +export async function fetchScreenInfo(mcp: MCPClient, query: string): Promise { + try { + ui.startSpinner('Analyzing screen', query); + const { screenshot } = await import('@appclaw/core/mcp/tools'); + const imageBase64 = await screenshot(mcp); + if (!imageBase64) { + ui.stopSpinner(); + return { ok: false, reason: 'screenshot', message: 'Failed to capture screenshot' }; + } + + const { + getStarkVisionApiKey, + getStarkVisionBaseUrl, + getStarkVisionCoordinateOrder, + getStarkVisionModel, + } = await import('@appclaw/core/vision/locate-enabled'); + const apiKey = getStarkVisionApiKey(); + const baseUrl = getStarkVisionBaseUrl(); + if (!apiKey && !baseUrl) { + ui.stopSpinner(); + return { + ok: false, + reason: 'no-vision', + message: 'getInfo requires vision (GEMINI_API_KEY or STARK_VISION_BASE_URL)', + }; + } + + const { default: starkVision } = await import('@appclaw/vision'); + const { StarkVisionClient } = starkVision; + const client = new StarkVisionClient({ + apiKey: apiKey || 'local', + model: getStarkVisionModel(), + disableThinking: true, + ...(baseUrl && { baseUrl }), + ...(baseUrl && { coordinateOrder: getStarkVisionCoordinateOrder() }), + }); + const response = await client.getElementInfo(imageBase64, query, true); + + let answer: string; + let explanation: string | undefined; + try { + const parsed = JSON.parse(response.replace(/(^```json\s*|```\s*$)/g, '').trim()); + answer = parsed.answer || response; + explanation = parsed.explanation; + } catch { + answer = response; + } + + ui.stopSpinner(); + return { ok: true, answer, explanation }; + } catch (err: any) { + ui.stopSpinner(); + return { ok: false, reason: 'error', message: err?.message ?? String(err) }; + } +} diff --git a/packages/cli/src/tui/TuiApp.tsx b/packages/cli/src/tui/TuiApp.tsx new file mode 100644 index 0000000..3b74ec7 --- /dev/null +++ b/packages/cli/src/tui/TuiApp.tsx @@ -0,0 +1,99 @@ +import React, { useSyncExternalStore } from 'react'; +import { Box, useInput } from 'ink'; +import { subscribe, getSnapshot } from './store.js'; +import type { TuiActions } from './commands.js'; +import { WelcomeScreen } from './screens/WelcomeScreen.js'; +import { DevicePickerScreen } from './screens/DevicePickerScreen.js'; +import { MainScreen } from './screens/MainScreen.js'; +import { SettingsScreen } from './screens/SettingsScreen.js'; +import { HistoryScreen } from './screens/HistoryScreen.js'; +import { ProgressDialog } from './components/ProgressDialog.js'; +import { OutputDialog } from './components/OutputDialog.js'; +import { ConfirmDialog } from './components/ConfirmDialog.js'; +import { tuiStore } from './store.js'; + +export interface TuiAppProps { + actions: TuiActions; +} + +/** Root router for `appclaw --tui` — switches on store.screen. */ +export function TuiApp({ actions }: TuiAppProps) { + const ui = useSyncExternalStore(subscribe, getSnapshot); + + // With `exitOnCtrlC: false`, Ink delivers Ctrl+C here as a normal keypress + // (raw mode also suppresses tty SIGINT, so the process-level handler can't + // fire while the app is mounted). This always-mounted handler is the one + // reliable quit path — and it keeps raw mode held for the app's lifetime, + // so keystrokes never leak to the host shell while a goal is running. + useInput((input, key) => { + if (key.ctrl && input === 'c') void actions.quit(); + }); + + // Ahead of everything else — it's a blocking question, and leaving the + // screen behind it live would let the same keypress act twice. + if (ui.quitConfirmOpen) { + const count = `${ui.steps.length} step${ui.steps.length === 1 ? '' : 's'}`; + return ( + + { + tuiStore.setQuitConfirm(false); + void actions.quit(); + }} + onCancel={() => tuiStore.setQuitConfirm(false)} + /> + + ); + } + + // Rendered instead of the active screen (not layered over it) so the + // underlying screen unmounts and its useInput handlers stop competing for + // the same keystrokes. + if (ui.viewerOpen) { + return ( + + tuiStore.closeViewer()} + /> + + ); + } + + // Device setup owns the screen while it runs — it's a multi-step pipeline + // (discover → select → boot/WDA → create session) with no useful + // interaction available until it settles. + if (ui.connecting) { + return ( + + + + ); + } + + return ( + + {ui.screen === 'welcome' && } + {ui.screen === 'device-picker' && } + {ui.screen === 'main' && } + {ui.screen === 'settings' && } + {ui.screen === 'history' && } + + ); +} diff --git a/packages/cli/src/tui/capture-console.ts b/packages/cli/src/tui/capture-console.ts new file mode 100644 index 0000000..c2d833e --- /dev/null +++ b/packages/cli/src/tui/capture-console.ts @@ -0,0 +1,65 @@ +/** + * Run something that reports with `console.log` and collect what it wrote. + * + * Several bits of shared machinery (doctor, the memory inspector) print + * straight to the console because they were written for a plain CLI. Under the + * TUI that output is routed by Ink's `patchConsole` to the area *above* the + * rendered frame — which, in the alternate screen buffer with a full-height + * frame, is off screen. The result looked like the command silently did + * nothing. Capturing lets the caller put the output somewhere visible instead. + */ + +type ConsoleWriter = (...args: unknown[]) => void; + +function install(lines: string[]): { restore: () => void } { + // Restore to whatever console.log is *now* — under the TUI that is Ink's + // patched version, and putting back a pristine original would break + // patchConsole for everything afterwards. + const previous = console.log as ConsoleWriter; + console.log = (...args: unknown[]) => { + const text = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '); + lines.push(...text.split('\n')); + }; + return { + restore: () => { + console.log = previous; + }, + }; +} + +/** Capture a synchronous reporter's output. */ +export function captureConsole(body: () => void): string[] { + const lines: string[] = []; + const { restore } = install(lines); + try { + body(); + } catch (err) { + lines.push(`Failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + restore(); + } + return lines; +} + +/** + * Capture an async reporter's output. + * + * Note the console stays patched for the whole await, so anything else logging + * concurrently is captured too — acceptable here because these commands own the + * foreground while they run. + */ +export async function captureConsoleAsync( + body: () => Promise +): Promise<{ result: T | undefined; lines: string[] }> { + const lines: string[] = []; + const { restore } = install(lines); + let result: T | undefined; + try { + result = await body(); + } catch (err) { + lines.push(`Failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + restore(); + } + return { result, lines }; +} diff --git a/packages/cli/src/tui/commands.ts b/packages/cli/src/tui/commands.ts new file mode 100644 index 0000000..0f48f68 --- /dev/null +++ b/packages/cli/src/tui/commands.ts @@ -0,0 +1,551 @@ +/** + * Command palette — the slash commands shown in the left pane of the main + * TUI screen (wireframe: "Command pallet / All / commands for first time + * user"). Each command is a small declarative spec; `TuiActions` is the + * side-effecting surface a command is allowed to call into, implemented by + * the TUI entry point (packages/cli/src/tui/index.ts) so this module stays + * free of process/IO concerns and is easy to unit test / extend. + * + * The TUI is a step recorder first: a plain line runs + * ONE deterministic instruction and appends it to `store.steps`, and the flow + * commands below edit/preview/export that list. The autonomous agent loop is + * still available, but explicitly, behind `/goal`. + */ + +import { writeFileSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; + +import { tryParseNaturalFlowLine } from '@appclaw/core/flow/natural-line'; + +import { COLORS } from '../ui/ink/theme.js'; + +import { tuiStore, getSnapshot, type Platform, type DeviceSummary } from './store.js'; +import { + buildYamlString, + buildSdkTestString, + isSdkTestFilename, + resolveRecordedExportPath, + formatStepLines, +} from '../step-recorder/flow-builder.js'; +import { runMemoryCommand } from '../step-recorder/memory-inspect.js'; +import { currentSessionLog } from './session-log.js'; +import { captureConsole } from './capture-console.js'; + +export interface TuiActions { + /** Welcome screen: record the chosen platform and move to the device picker. */ + selectPlatform(platform: Platform): void; + /** Device picker screen: lock in a device, connect the Appium/MCP session, and move to main. */ + selectDevice(device: DeviceSummary): Promise; + goToDevicePicker(): void; + goToPlatformPicker(): void; + goToSettings(): void; + goToHistory(): void; + goToMain(): void; + saveSettings(): Promise; + /** `/stream` — mirror the device in the main screen's right-hand panel. */ + openStream(): Promise; + /** `/stream-close` — stop the mirror. */ + closeStream(): void; + runDoctor(): Promise; + /** One deterministic step, executed on device and recorded — the default for a plain line. */ + runInstruction(instruction: string): Promise; + /** Full autonomous agent loop (`/goal`) — nothing is recorded. */ + runGoal(goal: string): Promise; + quit(): Promise; +} + +export interface PaletteCommand { + id: string; + /** e.g. "/device" */ + name: string; + aliases?: string[]; + summary: string; + run(actions: TuiActions, args: string): void | Promise; +} + +/** Shared guard for the commands that need at least one recorded step. */ +function requireSteps(what: string): boolean { + if (getSnapshot().steps.length > 0) return true; + tuiStore.log( + 'warn', + `No steps to ${what}.`, + 'Type an instruction (e.g. tap on Login) to record one.' + ); + return false; +} + +/** ` ` argument shape shared by /edit and /insert. */ +function parseIndexedArg(args: string): { index: number; rest: string } | null { + const match = args.match(/^(\d+)\s+(.+)$/); + if (!match) return null; + return { index: parseInt(match[1], 10) - 1, rest: match[2] }; +} + +export const COMMANDS: PaletteCommand[] = [ + { + id: 'goal', + name: '/goal', + summary: 'Run the autonomous agent loop for a goal (not recorded)', + run: async (actions, args) => { + if (!args.trim()) { + tuiStore.setPaletteError('Usage: /goal '); + return; + } + await actions.runGoal(args.trim()); + }, + }, + { + id: 'list', + name: '/list', + summary: 'List all recorded steps', + run: () => { + const { steps, meta } = getSnapshot(); + if (steps.length === 0) { + tuiStore.log('info', 'No steps yet.', 'Type a command like: open youtube app'); + return; + } + const title = meta.name + ? `${meta.name}${meta.appId ? ` (${meta.appId})` : ''}` + : (meta.appId ?? 'Recorded flow'); + tuiStore.showViewer({ + title, + subtitle: `${steps.length} step${steps.length === 1 ? '' : 's'} recorded`, + lines: formatStepLines(steps), + }); + }, + }, + { + id: 'yaml', + name: '/yaml', + summary: 'Preview the YAML flow output', + run: () => { + if (!requireSteps('preview')) return; + const { steps, meta } = getSnapshot(); + tuiStore.showViewer({ + title: 'YAML flow', + subtitle: '/export .yaml to save', + lines: buildYamlString(steps, meta).split('\n'), + language: 'yaml', + }); + }, + }, + { + id: 'preview', + name: '/preview', + summary: 'Preview the generated code without saving (filename picks the format)', + run: (_actions, args) => { + if (!requireSteps('preview')) return; + const { steps, meta } = getSnapshot(); + const filename = args.trim() || `flow-${Date.now()}.spec.ts`; + const asSdkTest = isSdkTestFilename(filename); + const body = asSdkTest ? buildSdkTestString(steps, meta) : buildYamlString(steps, meta); + tuiStore.showViewer({ + title: path.basename(filename), + subtitle: asSdkTest ? 'runner spec — not saved' : 'YAML flow — not saved', + lines: body.split('\n'), + language: asSdkTest ? 'ts' : 'yaml', + }); + }, + }, + { + id: 'export', + name: '/export', + summary: 'Export steps as an @appclaw/runner spec (.spec.ts) or YAML flow (.yaml)', + run: (_actions, args) => { + if (!requireSteps('export')) return; + const { steps, meta, exportDir } = getSnapshot(); + const filename = args.trim() || `flow-${Date.now()}.spec.ts`; + const asSdkTest = isSdkTestFilename(filename); + const filepath = resolveRecordedExportPath(filename, asSdkTest, exportDir); + const body = asSdkTest ? buildSdkTestString(steps, meta) : buildYamlString(steps, meta); + try { + mkdirSync(path.dirname(filepath), { recursive: true }); + writeFileSync(filepath, body, 'utf-8'); + } catch (err) { + tuiStore.log( + 'error', + `Could not write ${filepath}`, + err instanceof Error ? err.message : String(err) + ); + return; + } + currentSessionLog()?.addExport(filepath); + const runHint = asSdkTest + ? // `appclaw test` (not the bare appclaw-runner bin) — it registers the + // tsx loader first, without which a .ts spec cannot be imported. It + // discovers specs under testDir and matches filters, not paths, so + // hint with the name rather than the path. + `appclaw test ${path.basename(filepath).replace(/\.[^.]+$/, '')}` + : `appclaw --flow ${path.relative(process.cwd(), filepath)}`; + // Show what was written, the way the CLI printed the body before + // saving — the transcript entry stays as the scrollback record. + tuiStore.showViewer({ + title: path.basename(filepath), + subtitle: `${steps.length} step${steps.length === 1 ? '' : 's'} · ${asSdkTest ? 'runner spec' : 'YAML flow'}`, + lines: body.split('\n'), + language: asSdkTest ? 'ts' : 'yaml', + status: { color: COLORS.green, text: `Saved to ${filepath}\nRun: ${runHint}` }, + }); + tuiStore.log( + 'result', + `${steps.length} step${steps.length === 1 ? '' : 's'} exported as ${asSdkTest ? 'runner spec' : 'YAML flow'}`, + `File: ${filepath}\nRun: ${runHint}` + ); + }, + }, + { + id: 'undo', + name: '/undo', + summary: 'Remove the last recorded step', + run: () => { + const removed = tuiStore.popStep(); + if (!removed) { + tuiStore.log('warn', 'Nothing to undo.'); + return; + } + tuiStore.log( + 'info', + `Removed: ${removed.verbatim ?? removed.kind}`, + `${getSnapshot().steps.length} step(s) left` + ); + }, + }, + { + id: 'edit', + name: '/edit', + summary: 'Replace a step by number (e.g. /edit 3 tap "Settings")', + run: (_actions, args) => { + const parsed = parseIndexedArg(args); + if (!parsed) { + tuiStore.setPaletteError('Usage: /edit '); + return; + } + const { steps } = getSnapshot(); + if (parsed.index < 0 || parsed.index >= steps.length) { + tuiStore.setPaletteError(`Step ${parsed.index + 1} does not exist (1–${steps.length}).`); + return; + } + const step = tryParseNaturalFlowLine(parsed.rest); + if (!step) { + tuiStore.setPaletteError(`Could not parse: ${parsed.rest}`); + return; + } + tuiStore.replaceStep(parsed.index, step); + tuiStore.log('info', `Updated step ${parsed.index + 1}`, parsed.rest); + }, + }, + { + id: 'insert', + name: '/insert', + summary: 'Insert a step at a position (e.g. /insert 2 wait 3 s)', + run: (_actions, args) => { + const parsed = parseIndexedArg(args); + if (!parsed) { + tuiStore.setPaletteError('Usage: /insert '); + return; + } + const { steps } = getSnapshot(); + // One past the end is legal — that's an append. + if (parsed.index < 0 || parsed.index > steps.length) { + tuiStore.setPaletteError(`Position must be 1–${steps.length + 1}.`); + return; + } + const step = tryParseNaturalFlowLine(parsed.rest); + if (!step) { + tuiStore.setPaletteError(`Could not parse: ${parsed.rest}`); + return; + } + tuiStore.insertStep(parsed.index, step); + tuiStore.log('info', `Inserted at position ${parsed.index + 1}`, parsed.rest); + }, + }, + { + id: 'delete', + name: '/delete', + summary: 'Delete a step by number (e.g. /delete 3)', + run: (_actions, args) => { + const { steps } = getSnapshot(); + const index = parseInt(args.trim(), 10) - 1; + if (isNaN(index) || index < 0 || index >= steps.length) { + tuiStore.setPaletteError(`Invalid step number. Use 1–${steps.length}.`); + return; + } + const removed = tuiStore.deleteStep(index)!; + tuiStore.log('info', `Deleted: ${removed.verbatim ?? removed.kind}`); + }, + }, + { + id: 'meta', + name: '/meta', + summary: 'Set flow metadata (/meta appId com.foo, /meta name Login, /meta platform ios)', + run: (_actions, args) => { + const parts = args.trim().split(/\s+/); + const key = parts[0]; + const value = parts.slice(1).join(' '); + if (key === 'appId' && value) { + tuiStore.setMeta({ appId: value }); + tuiStore.log('info', `appId = ${value}`); + return; + } + if (key === 'name' && value) { + tuiStore.setMeta({ name: value }); + tuiStore.log('info', `name = ${value}`); + return; + } + if (key === 'platform') { + const p = value.toLowerCase(); + if (p === 'android' || p === 'ios') { + tuiStore.setMeta({ platform: p }); + tuiStore.log('info', `platform = ${p}`); + } else { + tuiStore.setPaletteError('Usage: /meta platform '); + } + return; + } + const { meta } = getSnapshot(); + const current = [ + meta.appId ? `appId: ${meta.appId}` : null, + meta.name ? `name: ${meta.name}` : null, + meta.platform ? `platform: ${meta.platform}` : null, + ].filter((l): l is string => l !== null); + tuiStore.log( + 'warn', + 'Usage: /meta appId | /meta name | /meta platform ', + current.length > 0 ? `Current:\n${current.join('\n')}` : undefined + ); + }, + }, + { + id: 'clear', + name: '/clear', + summary: 'Clear all recorded steps and metadata', + run: () => { + const count = getSnapshot().steps.length; + tuiStore.clearSteps(); + tuiStore.log('info', `Cleared ${count} step${count === 1 ? '' : 's'} and metadata.`); + }, + }, + { + id: 'clear-log', + name: '/clear-log', + summary: 'Clear the transcript (recorded steps are kept)', + run: () => tuiStore.clearTranscript(), + }, + { + id: 'session', + name: '/session', + summary: 'Show where this session’s JSON log is being written', + run: () => { + const log = currentSessionLog(); + if (!log) { + tuiStore.log('warn', 'No session log for this run.'); + return; + } + tuiStore.log( + 'info', + 'Session log', + `${log.path}\nUpdated after every step, so a crash still leaves it readable.` + ); + }, + }, + { + id: 'memory', + name: '/memory', + summary: 'Inspect episodic + procedural memory (stats | list | paths)', + run: (_actions, args) => { + // Captured into the modal rather than left to console.log: Ink puts that + // above the frame, which is off screen under a full-height alt-screen + // layout, so the report was written but never visible. + const lines = captureConsole(() => runMemoryCommand(args.trim())); + tuiStore.showViewer({ + title: 'Memory', + subtitle: args.trim() ? `/memory ${args.trim()}` : 'episodic + procedural', + lines, + }); + }, + }, + { + id: 'device', + name: '/device', + summary: 'Pick a running Android emulator / iOS simulator', + run: (actions) => actions.goToDevicePicker(), + }, + { + id: 'platform', + name: '/platform', + summary: 'Switch between Android and iOS', + run: (actions) => actions.goToPlatformPicker(), + }, + { + id: 'stream', + name: '/stream', + summary: 'Mirror the device screen in the side panel (keeps typing live)', + run: (actions) => actions.openStream(), + }, + { + id: 'stream-close', + name: '/stream-close', + summary: 'Stop the device mirror', + run: (actions) => actions.closeStream(), + }, + { + id: 'settings', + name: '/settings', + aliases: ['/config'], + summary: 'View and edit AppClaw configuration', + run: (actions) => actions.goToSettings(), + }, + { + id: 'history', + name: '/history', + aliases: ['/runs'], + summary: 'Browse past run reports (.appclaw/runs)', + run: (actions) => actions.goToHistory(), + }, + { + id: 'doctor', + name: '/doctor', + summary: 'Run the environment preflight check', + run: (actions) => actions.runDoctor(), + }, + { + id: 'help', + name: '/help', + aliases: ['/?'], + summary: 'List available commands', + run: () => { + // The scrollable modal, not the transcript: the full list is longer than + // the transcript pane is tall, so logging it there overflowed the pane's + // height budget and pushed the status bar off screen. + const width = COMMANDS.reduce((w, c) => Math.max(w, c.name.length), 0); + tuiStore.showViewer({ + title: 'Commands', + subtitle: 'type a plain line to record a step · /goal to run the agent', + lines: COMMANDS.map((c) => `${c.name.padEnd(width + 2)}${c.summary}`), + }); + }, + }, + { + id: 'quit', + name: '/quit', + aliases: ['/exit', '/q'], + summary: 'Exit the TUI (warns once about unexported steps)', + run: async (actions) => { + await requestQuit(actions); + }, + }, +]; + +/** + * Quit, confirming first if there are unexported steps. Shared by `/quit` and + * the `q` key on the screens that have no text input — going straight to + * `actions.quit()` from a keypress would discard a recording with no warning. + * + * The confirmation is a modal rather than a transcript line: the screens that + * offer the `q` shortcut are exactly the ones that don't render a transcript, + * so a logged warning was invisible on all of them. + */ +export async function requestQuit(actions: TuiActions): Promise { + if (getSnapshot().steps.length > 0) { + tuiStore.setQuitConfirm(true); + return; + } + await actions.quit(); +} + +/** Case-insensitive prefix match against name + aliases, for the palette list. */ +export function matchCommands(query: string): PaletteCommand[] { + const q = query.trim().toLowerCase(); + if (!q || q === '/') return COMMANDS; + return COMMANDS.filter( + (c) => + c.name.toLowerCase().startsWith(q) || c.aliases?.some((a) => a.toLowerCase().startsWith(q)) + ); +} + +/** + * Tab completion for the prompt. Returns the line the input should become, or + * null when there is nothing to add. + * + * Completes to the longest prefix every candidate shares rather than jumping to + * the first match — `/stream` is also a prefix of `/stream-close`, so guessing + * either one would be wrong half the time. A single match completes fully and + * gains a trailing space, since every such command either takes arguments or is + * about to be submitted. + */ +export function completeCommand(line: string): string | null { + // Only the command word completes; once there's a space the rest is an + // argument (a filename, a step) that this cannot know anything about. + if (!line.startsWith('/') || line.includes(' ')) return null; + + // Complete against whatever actually matched — the alias if that's what was + // typed, not the canonical name. Otherwise "/co" would rewrite itself to + // "/settings", replacing the prefix rather than extending it. + const q = line.toLowerCase(); + const candidates: string[] = []; + for (const command of COMMANDS) { + if (command.name.toLowerCase().startsWith(q)) candidates.push(command.name); + for (const alias of command.aliases ?? []) { + if (alias.toLowerCase().startsWith(q)) candidates.push(alias); + } + } + if (candidates.length === 0) return null; + if (candidates.length === 1) return `${candidates[0]} `; + + let prefix = candidates[0]; + for (const candidate of candidates.slice(1)) { + while (!candidate.toLowerCase().startsWith(prefix.toLowerCase())) { + prefix = prefix.slice(0, -1); + if (!prefix) return null; + } + } + return prefix.length > line.length ? prefix : null; +} + +/** Resolve a full command line (e.g. "/device") to its spec, ignoring trailing args. */ +export function resolveCommand(line: string): PaletteCommand | undefined { + const head = line.trim().split(/\s+/, 1)[0]?.toLowerCase(); + if (!head) return undefined; + return COMMANDS.find( + (c) => c.name.toLowerCase() === head || c.aliases?.some((a) => a.toLowerCase() === head) + ); +} + +/** + * Execute one submitted line from the main screen's instruction box. + * A leading "/" dispatches to the matching palette command; anything else is + * one natural-language instruction, executed on device and recorded. + */ +export async function executeLine(line: string, actions: TuiActions): Promise { + const trimmed = line.trim(); + if (!trimmed) return; + + // Clear any stale feedback from a previous mistyped line before evaluating + // this one — it re-appears below if this line is also invalid. + tuiStore.setPaletteError(null); + + if (trimmed.startsWith('/')) { + const head = trimmed.split(/\s+/, 1)[0] ?? trimmed; + let cmd = resolveCommand(trimmed); + if (!cmd) { + // The palette advertises "type to filter, Enter to run", so honor a + // uniquely-matching prefix — e.g. "/dev" runs /device. + const candidates = matchCommands(head); + if (candidates.length === 1) cmd = candidates[0]; + } + if (!cmd) { + // Inline-only, right under the input — that's where the user is + // actually looking right after pressing enter. Also logging this to + // the transcript would duplicate it a second time further down. + tuiStore.setPaletteError(`Unknown command: ${trimmed} — try /help`); + return; + } + // Slice off what the user actually typed (alias or prefix), not the + // canonical name — their lengths can differ. + const args = trimmed.slice(head.length).trim(); + await cmd.run(actions, args); + return; + } + + await actions.runInstruction(trimmed); +} diff --git a/packages/cli/src/tui/components/CommandPalette.tsx b/packages/cli/src/tui/components/CommandPalette.tsx new file mode 100644 index 0000000..c27d6a8 --- /dev/null +++ b/packages/cli/src/tui/components/CommandPalette.tsx @@ -0,0 +1,175 @@ +import React from 'react'; +import { Box, Text } from 'ink'; +import TextInput from 'ink-text-input'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import { matchCommands } from '../commands.js'; + +export interface CommandPaletteProps { + /** Cell width of the column — computed by layout.ts so the stream panel beside it lands where the painter expects. */ + width: number; + /** How many commands to list — shrinks on short terminals so the frame still fits. */ + maxCommands: number; + query: string; + onQueryChange: (value: string) => void; + onSubmit: (value: string) => void; + disabled: boolean; + /** Feedback for the just-submitted line (e.g. an unknown command) — shown right under the input, not just in the transcript below. */ + error?: string | null; + /** Live progress text while disabled (from core's spinner hooks). */ + busyText?: string; + /** + * Whether the prompt owns the keyboard. When false the input stops consuming + * keys entirely (ink-text-input's own `focus` prop), which is what lets the + * transcript use the bare arrow keys without a modifier. + */ + focused: boolean; +} + +/** + * Left-column panel from the wireframe: a bordered "Command pallet" list + * (filtered live as the user types a leading "/") sitting above the + * "Type instruction" input, where plain text is treated as a goal. + */ +/** + * The cap on the visible command list (MAX_VISIBLE_COMMANDS) lives in + * stream/layout.ts: it decides this column's height, which is the floor for + * the whole two-column row, which is where the stream panel's picture starts. + */ +/** Width the command name is padded to before its summary. `/stream-close` is the longest. */ +const NAME_COLUMN_WIDTH = 15; + +export function CommandPalette({ + width, + maxCommands, + query, + onQueryChange, + onSubmit, + disabled, + error, + busyText, + focused, +}: CommandPaletteProps) { + /** + * A long instruction wraps the input box, and `ink-text-input` has no + * truncate option — clipping it would leave you typing blind past the first + * line. Instead the command list gives up a row for every extra line the + * input takes, so the column's total height stays what the layout budgeted + * and you can always see what you are typing. + */ + const inputInnerCols = Math.max(1, width - 2 /* border */ - 2 /* paddingX */ - 2 /* "❯ " */); + const wantedInputLines = Math.max(1, Math.ceil(Math.max(query.length, 1) / inputInnerCols)); + // The list can only give up so much: past this the input is clipped instead, + // because a genuinely pathological line must not be allowed to grow the + // column and corrupt the frame. + const inputLines = Math.min(wantedInputLines, Math.max(1, maxCommands)); + const commandBudget = Math.max(1, maxCommands - (inputLines - 1)); + + const matched = query.trim().startsWith('/') ? matchCommands(query) : matchCommands('/'); + const commands = matched.slice(0, commandBudget); + const hidden = matched.length - commands.length; + + return ( + // No marginRight — the parent column owns the gutter now that the + // transcript shares this column and must line up with the palette. + // marginTop is the gap below the transcript above it; the row is part of + // the transcript's height budget either way, it just renders here now. + + + {/* Every row here truncates rather than wraps. The palette's height is + budgeted as one row per command; a summary long enough to wrap + silently costs two, and the extra rows push the whole left column + past the frame — which Ink overlaps rather than clips. */} + + Command palette + + + All / commands — type to filter, Enter to run + + {/* Exactly `commandBudget` rows plus one overflow row, always — padded + with blanks when there is less to show. A filter that matches + nothing would otherwise render one line where ten stood, and the + column's height is budgeted, not measured. */} + + {commands.length === 0 ? ( + + No matching commands + + ) : ( + commands.map((c) => ( + + + {c.name.padEnd(NAME_COLUMN_WIDTH)} + + {c.summary} + + )) + )} + {Array.from({ + length: Math.max(0, commandBudget - Math.max(commands.length, 1)), + }).map((_, i) => ( + + ))} + + {hidden > 0 ? `… +${hidden} more — type / to filter` : ' '} + + + + + {/* Above the prompt rather than below it. The input is the last thing in + the column, so a message under it landed hard against the frame's + bottom edge and read as though it had escaped the layout. + + Always occupies a row, blank when there's nothing to say: rendering it + conditionally grew the column by a line the parent's height budget + hadn't reserved, which pushed the frame past the terminal. */} + {/* No marginTop: this row IS the gap between the list and the input when + it's empty, and adding a margin on top of it made that gap two rows + while every other gap in the column is one. + + Explicit width: `wrap="truncate"` truncates against the box it is in, + and a box left to size itself grows to fit the text instead of + clipping it — so a long message still cost extra rows. */} + + + {error || ' '} + + + + + + {symbols.prompt}{' '} + + {disabled ? ( + + {busyText || 'working…'} + + ) : ( + + )} + + + ); +} diff --git a/packages/cli/src/tui/components/ConfirmDialog.tsx b/packages/cli/src/tui/components/ConfirmDialog.tsx new file mode 100644 index 0000000..38d83b3 --- /dev/null +++ b/packages/cli/src/tui/components/ConfirmDialog.tsx @@ -0,0 +1,73 @@ +import React from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; + +export interface ConfirmDialogProps { + title: string; + message: string; + detail?: string; + confirmLabel: string; + cancelLabel: string; + onConfirm: () => void; + onCancel: () => void; +} + +/** + * Blocking yes/no modal. Rendered in place of the active screen so the screen + * underneath unmounts and its key handlers stop competing — important here, + * because the answer is a single keypress that several screens also bind. + */ +export function ConfirmDialog({ + title, + message, + detail, + confirmLabel, + cancelLabel, + onConfirm, + onCancel, +}: ConfirmDialogProps) { + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + + useInput((input, key) => { + if (input === 'y' || input === 'Y') onConfirm(); + // Enter deliberately does NOT confirm — this dialog appears on a keypress, + // and a stray return shouldn't be able to discard a recording. + else if (input === 'n' || input === 'N' || key.escape) onCancel(); + }); + + return ( + + + + {symbols.warning} {title} + + + + {message} + + {detail ? {detail} : null} + + + + + y + {' '} + {confirmLabel} ·{' '} + + n + {' '} + {cancelLabel} + + + + + ); +} diff --git a/packages/cli/src/tui/components/Header.tsx b/packages/cli/src/tui/components/Header.tsx new file mode 100644 index 0000000..4d5c0c0 --- /dev/null +++ b/packages/cli/src/tui/components/Header.tsx @@ -0,0 +1,19 @@ +import React from 'react'; +import { Box, Text } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; + +export interface HeaderProps { + subtitle?: string; +} + +/** Brand header reused across TUI screens — "AppClaw" plus an optional context line. */ +export function Header({ subtitle }: HeaderProps) { + return ( + + + {symbols.diamond} AppClaw + + {subtitle ? {subtitle} : null} + + ); +} diff --git a/packages/cli/src/tui/components/OutputDialog.tsx b/packages/cli/src/tui/components/OutputDialog.tsx new file mode 100644 index 0000000..3bb64c9 --- /dev/null +++ b/packages/cli/src/tui/components/OutputDialog.tsx @@ -0,0 +1,211 @@ +import React, { useMemo, useState } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import wrapAnsi from 'wrap-ansi'; +import { COLORS } from '../../ui/ink/theme.js'; +import { OrbitalSpinner } from '../../ui/ink/components/OrbitalSpinner.js'; +import { highlight, type Language } from '../highlight.js'; + +export interface OutputDialogProps { + title: string; + subtitle?: string; + lines: string[]; + running: boolean; + status?: { color: string; text: string }; + /** Colour for every body line; omit for text that already carries its own ANSI. */ + tint?: string; + /** Syntax-highlight the body; takes precedence over `tint`. */ + language?: Language; + onClose: () => void; +} + +/** SGR escapes carry no width, so they must come off before measuring. */ +const SGR = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g'); + +/** Terminal rows a line occupies once wrapped into `width` columns. */ +function rowsFor(line: string, width: number): number { + return Math.max(1, Math.ceil(line.replace(SGR, '').length / width)); +} + +/** + * Break one captured line into the terminal rows it will occupy, indenting + * every continuation two columns past where the line itself started. + * + * Wrapping is done here rather than left to Ink for two reasons: a hanging + * indent needs the first row and the rest to have different widths, which + * flexbox cannot express; and pre-wrapping keeps one array entry equal to one + * terminal row, so the viewport arithmetic below stays a plain slice. + * + * `wrap-ansi` rather than a hand-rolled split because these lines carry theme + * colour — cutting a string mid-escape corrupts the sequence, and the colour + * active at a break has to be reopened on the next row. + */ +function wrapWithHangingIndent(line: string, width: number): string[] { + const indent = /^ */.exec(line)?.[0].length ?? 0; + const body = line.slice(indent); + if (body === '') return [line]; + + // Never let the indent eat the line: a deeply indented line still needs + // usable columns for its text. + const hang = Math.min(indent + 2, Math.max(0, width - 8)); + const inner = Math.max(1, width - hang); + + // hard: break words with nowhere to wrap (a long path, a run of dashes) + // instead of letting them overflow the box. trim (the default) drops the + // space a break lands on, so continuations line up at exactly `hang` rather + // than one column further whenever the wrap happened to fall after a space. + const rows = wrapAnsi(body, inner, { hard: true }).split('\n'); + return rows.map((row, i) => ' '.repeat(i === 0 ? indent : hang) + row); +} + +/** + * Rows the dialog spends on everything except the scrolling body: the box + * border(2), the title(1), the body's marginTop(1), and the footer's + * marginTop(1) + line(1). + * + * Counted from the actual content rather than assumed, because the variable + * parts really do vary: /export's status is two lines ("Saved to …" then + * "Run: …") where doctor's is one, and a fixed estimate made the box taller + * than the terminal — which clipped its bottom border off the screen. + */ +function chromeRows(hasSubtitle: boolean, statusLines: number): number { + const border = 2; + const paddingY = 2; + const title = 1; + const subtitle = hasSubtitle ? 1 : 0; + const bodyMargin = 1; + const status = statusLines > 0 ? 1 + statusLines : 0; + const footer = 1 + 1; // its marginTop plus the line itself + return border + paddingY + title + subtitle + bodyMargin + status + footer; +} + +/** + * Scrollable modal for output the transcript pane can't hold — doctor's report + * and the generated YAML/spec bodies. Two reasons it exists rather than + * logging into the transcript: that pane is only a few rows tall (a file + * dumped there is truncated and buries the step log), and `console.log` + * output lands above the frame, off-screen under a full-height layout. + * + * Rendered in place of the active screen, so the screen underneath unmounts + * and its useInput handlers stop competing for the same keystrokes. + */ +export function OutputDialog({ + title, + subtitle, + lines, + running, + status, + tint, + language, + onClose, +}: OutputDialogProps) { + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + const columns = stdout.columns || 80; + const [offset, setOffset] = useState(0); + + const width = Math.min(columns - 6, 100); + // Usable text width inside the box: its border(2) plus paddingX={2} a side. + const contentWidth = Math.max(1, width - 6); + // A long status (an absolute export path) wraps inside the box, and every + // wrapped line is a row the body cannot also have. + const statusLines = status + ? status.text.split('\n').reduce((n, line) => n + rowsFor(line, contentWidth), 0) + : 0; + const viewport = Math.max(3, rows - chromeRows(Boolean(subtitle), statusLines)); + + // Body lines are wrapped, not truncated: doctor's config summary and absolute + // export paths run past the box, and an ellipsis there hides the very detail + // the dialog exists to show. Wrapping happens here so that one entry in + // `displayLines` is exactly one terminal row — which is what lets the + // viewport below stay a plain slice and the box close where it promises to. + const displayLines = useMemo( + () => lines.flatMap((line) => wrapWithHangingIndent(line, contentWidth)), + [lines, contentWidth] + ); + + // Highlight the whole body, not just the visible slice: block-comment state + // carries across lines, so colouring a window in isolation would mis-colour + // any body scrolled into from the middle of a comment. + const highlighted = useMemo( + () => (language ? highlight(displayLines, language) : null), + [displayLines, language] + ); + + const maxOffset = Math.max(0, displayLines.length - viewport); + const clamped = Math.min(offset, maxOffset); + const visible = displayLines.slice(clamped, clamped + viewport); + + useInput((input, key) => { + if (key.upArrow) setOffset((o) => Math.max(0, o - 1)); + else if (key.downArrow) setOffset((o) => Math.min(maxOffset, o + 1)); + else if (key.pageUp) setOffset((o) => Math.max(0, o - viewport)); + else if (key.pageDown) setOffset((o) => Math.min(maxOffset, o + viewport)); + else if (key.escape || input === 'q' || key.return) onClose(); + }); + + return ( + + + + {title} + + {subtitle ? {subtitle} : null} + + {running ? ( + + + Working… + + ) : ( + + {visible.length === 0 ? ( + (no output) + ) : ( + visible.map((line, i) => { + const segments = highlighted?.[clamped + i]; + if (!segments || segments.length === 0) { + return ( + + {line || ' '} + + ); + } + return ( + + {segments.map((seg, j) => ( + + {seg.text} + + ))} + + ); + }) + )} + + )} + + {status ? ( + + {status.text} + + ) : null} + + + + {maxOffset > 0 + ? `↑↓ scroll (${clamped + 1}-${clamped + visible.length}/${displayLines.length}) · ` + : ''} + esc close + + + + + ); +} diff --git a/packages/cli/src/tui/components/ProgressDialog.tsx b/packages/cli/src/tui/components/ProgressDialog.tsx new file mode 100644 index 0000000..660957f --- /dev/null +++ b/packages/cli/src/tui/components/ProgressDialog.tsx @@ -0,0 +1,57 @@ +import React from 'react'; +import { Box, Text, useStdout } from 'ink'; +import { COLORS } from '../../ui/ink/theme.js'; +import { OrbitalSpinner } from '../../ui/ink/components/OrbitalSpinner.js'; + +export interface ProgressDialogProps { + title: string; + /** Current step text, e.g. "Creating Appium session...". */ + message: string; + /** Optional secondary line under the message. */ + detail?: string; + /** Static hint pinned at the bottom of the dialog. */ + hint?: string; +} + +/** + * Centered modal progress box for long, multi-step work (device setup). + * + * Ink has no absolute positioning, so a "modal" here means this replaces the + * screen's content while it's up — which is also what keeps core's spinner + * output (routed in through the UIRenderer seam) from animating raw ANSI over + * the frame the way it did when it wrote straight to stdout. + */ +export function ProgressDialog({ title, message, detail, hint }: ProgressDialogProps) { + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + + return ( + + + + {title} + + + + + {message || 'Working…'} + + + {detail ? {detail} : null} + + {hint ? ( + + {hint} + + ) : null} + + + ); +} diff --git a/packages/cli/src/tui/components/StatusBar.tsx b/packages/cli/src/tui/components/StatusBar.tsx new file mode 100644 index 0000000..4de4c43 --- /dev/null +++ b/packages/cli/src/tui/components/StatusBar.tsx @@ -0,0 +1,42 @@ +import React from 'react'; +import { Box, Text } from 'ink'; +import { COLORS } from '../../ui/ink/theme.js'; + +export interface StatusBarProps { + breadcrumb: string; + hints: string[]; + message?: string; +} + +/** + * Fixed bottom hint bar — breadcrumb + keybinding hints. + * + * Its height is exactly STATUS_BAR_ROWS, always: `marginTop` + border(2) + + * the hint line + the message line. The message row is rendered even when + * empty because MainScreen budgets the rest of the frame around this number, + * and a bar that grew by a row whenever a status message appeared would push + * the frame past the terminal — which Ink overlaps rather than clips. + */ +export function StatusBar({ breadcrumb, hints, message }: StatusBarProps) { + return ( + + + + {breadcrumb} + + + {hints.join(' · ')} + + + + {message || ' '} + + + ); +} diff --git a/packages/cli/src/tui/components/StreamPanel.tsx b/packages/cli/src/tui/components/StreamPanel.tsx new file mode 100644 index 0000000..9f8d47a --- /dev/null +++ b/packages/cli/src/tui/components/StreamPanel.tsx @@ -0,0 +1,201 @@ +import React, { useEffect } from 'react'; +import { Box, Text, useStdout } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import { KITTY_IMAGE_ID, setStreamPanelVisible } from '../stream/frame-loop.js'; +import { imageRowWidth, panelImageCols, streamCells } from '../stream/layout.js'; +import { imageIdColor, MAX_PLACEHOLDER_INDEX, placeholderRow } from '../stream/placeholder.js'; +import { backendLabel } from '../stream/terminal-caps.js'; +import type { DeviceSummary, StreamState } from '../store.js'; + +export interface StreamPanelProps { + device: DeviceSummary | null; + stream: StreamState; + /** Cell width of the whole panel box — computed by layout.ts, not a percentage. */ + width: number; + /** Rows inside the border that belong to the picture. Must match layout.ts exactly. */ + imageRows: number; +} + +const STATUS_LABEL: Record = { + idle: 'Not started', + starting: 'Starting…', + running: 'Live', + error: 'Error', +}; + +const STATUS_COLOR: Record = { + idle: COLORS.dimmed, + starting: COLORS.cyan, + running: COLORS.green, + error: COLORS.red, +}; + +/** A row that exists only to be reserved — see the body comment. */ +function blank(key: string) { + return ; +} + +/** Round border glyph, matching `borderStyle="round"` on the panel and the frame. */ +const VERTICAL = '│'; + +/** + * One row of Unicode placeholder cells, plus the chrome it is about to eat. + * + * Ink bills each placeholder — a surrogate pair — as two cells of its output + * buffer while the terminal draws it as one, so this row overwrites the panel's + * right border, the frame's, and the padding between them. Re-emitting those + * characters here puts them back at the right *display* column; the miscounted + * cells run off the end of the line, where trailing spaces are trimmed anyway. + */ +function placeholderLine(key: string, row: number, cols: number, areaCols: number) { + const leftPad = Math.max(0, Math.floor((areaCols - cols) / 2)); + const rightPad = Math.max(0, areaCols - leftPad - cols); + return ( + + {' '.repeat(leftPad)} + {/* The terminal reads the image id out of this colour. */} + {placeholderRow(row, cols)} + {' '.repeat(rightPad)} {VERTICAL}{' '} + {VERTICAL} + + ); +} + +/** + * Exactly `imageRows` rows, always. + * + * For kitty those rows carry the placeholder cells the terminal turns into the + * device screen; for half-blocks they carry the frame as text. Either way the + * count is fixed, which keeps the panel's height from changing with the aspect + * ratio of the fitted image. + */ +function imageBody( + stream: StreamState, + imageRows: number, + areaCols: number, + cells: { cols: number; rows: number } | null +): React.ReactNode[] { + const rows: React.ReactNode[] = []; + const frame = stream.frameLines ?? []; + + if (cells) { + const above = Math.max(0, Math.floor((imageRows - cells.rows) / 2)); + for (let i = 0; i < above; i++) rows.push(blank(`pad-${i}`)); + for (let r = 0; r < cells.rows; r++) { + rows.push(placeholderLine(`img-${r}`, r, cells.cols, areaCols)); + } + } else if (frame.length > 0) { + const above = Math.max(0, Math.floor((imageRows - frame.length) / 2)); + for (let i = 0; i < above; i++) rows.push(blank(`pad-${i}`)); + frame.slice(0, imageRows - above).forEach((line, i) => { + // truncate rather than wrap: a row wider than the panel would otherwise + // reflow onto the next one and push the whole picture down. + rows.push( + + {line} + + ); + }); + } else { + const message = + stream.status === 'error' && stream.error + ? `${symbols.cross} ${stream.error}` + : stream.status === 'starting' + ? 'Waiting for the first frame…' + : '/stream mirrors the screen here'; + const above = Math.max(0, Math.floor((imageRows - 1) / 2)); + for (let i = 0; i < above; i++) rows.push(blank(`pad-${i}`)); + rows.push( + + {message} + + ); + } + + while (rows.length < imageRows) rows.push(blank(`fill-${rows.length}`)); + return rows.slice(0, imageRows); +} + +/** + * Right-hand column from the wireframe: two lines of status over the live + * device screen, sitting beside the command palette so the input stays usable + * while the picture updates. + */ +export function StreamPanel({ device, stream, width, imageRows }: StreamPanelProps) { + // TuiApp renders dialogs and other screens INSTEAD of MainScreen, so this + // panel — and with it every placeholder cell the picture is drawn into — can + // disappear while the loop is still running. Tell the loop, so it stops + // capturing frames nothing will render. + useEffect(() => { + setStreamPanelVisible(true); + return () => setStreamPanelVisible(false); + }, []); + + const { stdout } = useStdout(); + const termCols = stdout.columns || 80; + const termRows = stdout.rows || 24; + const areaCols = panelImageCols(termCols); + // The frame loop sizes the transmitted image with the same call on the same + // numbers, so the cell box and the placeholder grid always agree. + const fitted = stream.resolution + ? streamCells(termCols, termRows, stream.resolution.width, stream.resolution.height) + : null; + // Beyond the diacritic table a row cannot be addressed at all; no terminal is + // this tall, but a clamp beats throwing out of a render. + const cells = + fitted && stream.status === 'running' && stream.backend === 'kitty' + ? { cols: fitted.cols, rows: Math.min(fitted.rows, MAX_PLACEHOLDER_INDEX + 1) } + : null; + + const renderer = stream.backend ? backendLabel(stream.backend) : null; + const resolution = stream.resolution + ? `${stream.resolution.width}x${stream.resolution.height}` + : null; + // One line, built by parts — the row count above the picture is fixed at + // PANEL_STATUS_ROWS, so nothing here may ever become two lines. + const detail = + [device ? device.name : '(no device)', resolution, renderer] + .filter(Boolean) + .join(` ${symbols.dot} `) || '(no device)'; + + return ( + + {/* Exactly PANEL_STATUS_ROWS rows, unconditionally and never wrapping: + the painter's first image row is derived from that constant, so an + extra (or conditional) line here shifts the picture off its hole. */} + + + Device stream{' '} + + + {symbols.circle} {STATUS_LABEL[stream.status]} + + + + {detail} + + + {/* Wider than the panel on purpose while placeholders are up: the rows + have to reach the terminal's right edge to redraw the chrome their own + miscounted cells overwrite. flexShrink={0} stops Yoga from pulling it + back inside the panel. */} + + {imageBody(stream, imageRows, areaCols, cells)} + + + ); +} diff --git a/packages/cli/src/tui/highlight.ts b/packages/cli/src/tui/highlight.ts new file mode 100644 index 0000000..8426b7f --- /dev/null +++ b/packages/cli/src/tui/highlight.ts @@ -0,0 +1,259 @@ +/** + * Minimal syntax highlighter for the code shown in `OutputDialog` (/yaml, + * /preview, /export). + * + * Hand-rolled rather than pulling in a highlighter dependency: the only + * languages we ever render are the two we generate ourselves (a runner spec + * and a YAML flow), and a full grammar would be far more machinery than a + * read-only preview needs. It is a scanner, not a parser — it can't be + * confused into crashing on odd input, worst case a token is mis-coloured. + */ + +import { COLORS } from '../ui/ink/theme.js'; + +export type Language = 'ts' | 'yaml'; + +export interface Segment { + text: string; + color?: string; + bold?: boolean; +} + +const PALETTE = { + comment: COLORS.dimmed, + string: COLORS.green, + number: COLORS.yellow, + keyword: COLORS.brand, + builtin: COLORS.step, + punct: COLORS.label, + plain: COLORS.white, + key: COLORS.step, +}; + +const KEYWORDS = new Set([ + 'import', + 'export', + 'from', + 'as', + 'default', + 'const', + 'let', + 'var', + 'function', + 'return', + 'async', + 'await', + 'new', + 'class', + 'extends', + 'implements', + 'interface', + 'type', + 'if', + 'else', + 'for', + 'while', + 'do', + 'switch', + 'case', + 'break', + 'continue', + 'try', + 'catch', + 'finally', + 'throw', + 'typeof', + 'instanceof', + 'in', + 'of', + 'this', + 'super', + 'void', + 'delete', + 'yield', + 'static', + 'public', + 'private', + 'protected', + 'readonly', + 'null', + 'undefined', + 'true', + 'false', +]); + +/** Not language keywords, but the runner/AppClaw vocabulary these files are built from. */ +const BUILTINS = new Set([ + 'describe', + 'it', + 'test', + 'expect', + 'beforeAll', + 'afterAll', + 'beforeEach', + 'afterEach', + 'AppClaw', + 'app', + 'process', + 'console', +]); + +function wordSegment(word: string): Segment { + if (KEYWORDS.has(word)) return { text: word, color: PALETTE.keyword, bold: true }; + if (BUILTINS.has(word)) return { text: word, color: PALETTE.builtin }; + return { text: word, color: PALETTE.plain }; +} + +/** + * TypeScript/JavaScript. Block-comment state carries across lines, so this + * takes the whole body — the generated specs open with a multi-line JSDoc + * banner that would otherwise only colour its first line. + */ +function highlightTs(lines: string[]): Segment[][] { + let inBlockComment = false; + + return lines.map((line) => { + const segments: Segment[] = []; + let word = ''; + let i = 0; + + const flushWord = () => { + if (word) { + segments.push(wordSegment(word)); + word = ''; + } + }; + + while (i < line.length) { + if (inBlockComment) { + const end = line.indexOf('*/', i); + const stop = end === -1 ? line.length : end + 2; + segments.push({ text: line.slice(i, stop), color: PALETTE.comment }); + i = stop; + if (end !== -1) inBlockComment = false; + continue; + } + + const pair = line.slice(i, i + 2); + if (pair === '/*') { + flushWord(); + inBlockComment = true; + continue; + } + if (pair === '//') { + flushWord(); + segments.push({ text: line.slice(i), color: PALETTE.comment }); + i = line.length; + continue; + } + + const ch = line[i]; + + if (ch === '"' || ch === "'" || ch === '`') { + flushWord(); + let j = i + 1; + while (j < line.length) { + if (line[j] === '\\') { + j += 2; + continue; + } + if (line[j] === ch) { + j += 1; + break; + } + j += 1; + } + segments.push({ text: line.slice(i, j), color: PALETTE.string }); + i = j; + continue; + } + + if (/[A-Za-z_$]/.test(ch)) { + word += ch; + i += 1; + continue; + } + // A digit only starts a number when it isn't part of an identifier. + if (/[0-9]/.test(ch) && !word) { + let j = i; + while (j < line.length && /[0-9._]/.test(line[j])) j += 1; + segments.push({ text: line.slice(i, j), color: PALETTE.number }); + i = j; + continue; + } + if (/[0-9]/.test(ch)) { + word += ch; + i += 1; + continue; + } + + flushWord(); + segments.push({ text: ch, color: PALETTE.punct }); + i += 1; + } + + flushWord(); + return segments; + }); +} + +/** YAML: comments, `key:`, list dashes, quoted scalars. */ +function highlightYaml(lines: string[]): Segment[][] { + return lines.map((line) => { + const segments: Segment[] = []; + + const commentAt = line.indexOf('#'); + const code = commentAt === -1 ? line : line.slice(0, commentAt); + const comment = commentAt === -1 ? '' : line.slice(commentAt); + + if (code.trim() === '---') { + segments.push({ text: code, color: PALETTE.comment }); + } else { + // Leading indent + optional "- " marker, then an optional "key:". + const structure = code.match(/^(\s*)(-\s+)?([A-Za-z_][\w.-]*)(:)(.*)$/); + if (structure) { + const [, indent, dash, key, colon, rest] = structure; + if (indent) segments.push({ text: indent }); + if (dash) segments.push({ text: dash, color: PALETTE.punct }); + segments.push({ text: key, color: PALETTE.key }); + segments.push({ text: colon, color: PALETTE.punct }); + if (rest) segments.push(...yamlScalar(rest)); + } else { + const listItem = code.match(/^(\s*)(-\s+)(.*)$/); + if (listItem) { + const [, indent, dash, rest] = listItem; + if (indent) segments.push({ text: indent }); + segments.push({ text: dash, color: PALETTE.punct }); + segments.push(...yamlScalar(rest)); + } else if (code) { + segments.push(...yamlScalar(code)); + } + } + } + + if (comment) segments.push({ text: comment, color: PALETTE.comment }); + return segments; + }); +} + +/** A YAML value: quoted strings and bare numbers get their own colour. */ +function yamlScalar(text: string): Segment[] { + const quoted = text.match(/^(\s*)(["'].*["'])(\s*)$/); + if (quoted) { + const [, lead, body, trail] = quoted; + return [ + ...(lead ? [{ text: lead }] : []), + { text: body, color: PALETTE.string }, + ...(trail ? [{ text: trail }] : []), + ]; + } + if (/^\s*-?\d[\d._]*\s*$/.test(text)) return [{ text, color: PALETTE.number }]; + return [{ text, color: PALETTE.plain }]; +} + +/** Colour a whole body. Unknown languages fall through to plain text. */ +export function highlight(lines: string[], language?: Language): Segment[][] { + if (language === 'ts') return highlightTs(lines); + if (language === 'yaml') return highlightYaml(lines); + return lines.map((line) => [{ text: line }]); +} diff --git a/packages/cli/src/tui/index.ts b/packages/cli/src/tui/index.ts new file mode 100644 index 0000000..040cb99 --- /dev/null +++ b/packages/cli/src/tui/index.ts @@ -0,0 +1,755 @@ +/** + * `appclaw --tui` entry point. + * + * Mounts the multi-screen Ink app (TuiApp) and implements TuiActions — the + * side-effecting surface screens and commands call into: platform/device + * selection (which lazily opens one Appium/MCP session, reused across every + * line typed afterwards), settings read/write, run history, the device mirror + * (`/stream` shows frames in the main screen's right-hand panel), and the two + * execution modes below. + * + * Execution modes: a plain line is ONE deterministic instruction + * (`runInstruction` → runOneInstruction) that gets recorded into + * `store.steps` — the TUI is a step recorder, and + * `/list`, `/yaml`, `/export`… operate on that list. `/goal` opts into the + * autonomous agent loop instead. + * + * Scope note: `/goal` calls `runAgent` directly (one flat agent loop per + * submitted goal) rather than the CLI's full multi-sub-goal + * planner/orchestrator (decomposeGoal + screen-readiness reconciliation) — + * that keeps this surface simple for iterative, REPL-style use. For a single + * complex multi-step goal, `appclaw "goal"` outside the TUI still applies + * the full planner. + */ + +import React from 'react'; +import { render } from 'ink'; +import { existsSync } from 'node:fs'; +import { readFile, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { refreshConfig, Config, type AppClawConfig } from '@appclaw/core/config'; +import { createMCPClient } from '@appclaw/core/mcp/client'; +import { createLLMProvider } from '@appclaw/core/llm/provider'; +import { setupDevice } from '@appclaw/core/device/index'; +import { AppResolver } from '@appclaw/core/agent/app-resolver'; +import { runAgent } from '@appclaw/core/agent/loop'; +import { tryParseNaturalFlowLine } from '@appclaw/core/flow/natural-line'; +import { runOneInstruction, DEFAULT_MIN_MATCH_SCORE } from '@appclaw/core/flow/run-instruction'; +import { stepAction, stepTarget } from '@appclaw/core/ui/step-printer'; +import { listRunningDevices } from '@appclaw/core/device/emulator-list'; +import { loadRunIndex } from '@appclaw/core/report/writer'; +import { DEFAULT_MODELS } from '@appclaw/core/constants'; +import { setRenderer, type UIRenderer } from '@appclaw/core/ui/renderer'; +import type { FlowStep } from '@appclaw/core/flow/types'; + +import { COLORS, symbols } from '../ui/ink/theme.js'; + +import { TuiApp } from './TuiApp.js'; +import { + tuiStore, + getSnapshot, + subscribe, + type Platform, + type DeviceSummary, + type RunSummary, +} from './store.js'; +import { + createSessionLog, + setCurrentSessionLog, + listSessions, + sessionSucceeded, + SESSIONS_DIRNAME, +} from './session-log.js'; +import type { TuiActions } from './commands.js'; +import { getDeviceResolution } from './stream/capture.js'; +import { startStreamLoop, stopStreamLoop } from './stream/frame-loop.js'; +import { detectStreamBackend, backendLabel } from './stream/terminal-caps.js'; +import { fetchScreenInfo } from '../step-recorder/screen-info.js'; +import { captureConsoleAsync } from './capture-console.js'; +import { runDoctor } from '../cli/doctor.js'; + +export interface RunTuiOptions { + platform: Platform | null; + deviceType: 'simulator' | 'real' | null; + udid: string | null; + deviceName: string | null; + /** `--export-dir`: default directory for bare-filename `/export` writes. */ + exportDir?: string | null; +} + +/** + * Ink has no built-in full-screen mode — left alone, it renders inline in + * the normal scrollback, and any frame taller than the terminal scrolls off + * permanently (each redraw then stacks a new copy below the last). Entering + * the terminal's alternate screen buffer (same mechanism vim/htop/Claude + * Code use) gives the TUI a real, self-contained viewport instead, and + * restores whatever was on screen before when we leave it. + */ +function enterAltScreen(): void { + if (process.stdout.isTTY) process.stdout.write('\x1b[?1049h\x1b[?25l'); +} +function exitAltScreen(): void { + if (process.stdout.isTTY) process.stdout.write('\x1b[?25h\x1b[?1049l'); +} + +/** Config keys the settings screen exposes — a curated subset, not the full schema. */ +const SETTINGS_KEYS: Array<{ key: keyof AppClawConfig; description: string }> = [ + { key: 'LLM_PROVIDER', description: 'anthropic | openai | gemini | groq | ollama' }, + { key: 'LLM_MODEL', description: 'blank = provider default' }, + { key: 'AGENT_MODE', description: 'dom | vision' }, + { key: 'PLATFORM', description: 'android | ios | blank (prompt)' }, + { key: 'MAX_STEPS', description: 'per-goal step budget' }, + { key: 'WAIT_TIMEOUT', description: 'implicit wait, ms' }, +]; + +/** + * Routes core's imperative spinner calls into the store instead of letting + * them animate raw ANSI (cursor-hide, \r repaint) straight to stdout, which + * fights Ink for the same lines. Streaming is swallowed for the same reason. + */ +const tuiRenderer: Partial = { + startSpinner(message, detail) { + tuiStore.setBusy(message, detail); + }, + updateSpinner(message, detail) { + const s = getSnapshot(); + tuiStore.setBusy(message ?? s.busyMessage, detail ?? s.busyDetail); + }, + stopSpinner() { + tuiStore.clearBusy(); + }, + startStreaming() {}, + streamChunk() {}, + stopStreaming() {}, +}; + +/** Transcript row for a step that just got recorded: "✓ tap Login (0.8s)". */ +function logRecorded(step: FlowStep, message: string, ms?: number): void { + const duration = ms == null ? '' : ms < 1000 ? ` (${ms}ms)` : ` (${(ms / 1000).toFixed(1)}s)`; + tuiStore.log( + 'step', + `${symbols.check} ${stepAction(step)} ${stepTarget(step)}${duration}`, + message && message !== 'recorded' ? message : undefined + ); +} + +/** Run a getInfo query and put the answer in the transcript (never recorded as a step). */ +async function showScreenInfo(mcp: Parameters[0], query: string) { + const res = await fetchScreenInfo(mcp, query); + if (!res.ok) { + tuiStore.log( + 'error', + res.reason === 'error' ? `Failed to get info: ${res.message}` : res.message + ); + return; + } + tuiStore.log('result', res.answer, res.explanation); +} + +interface DeviceSession { + mcpClient: Awaited>; + scopedMcp: Awaited>; + llm: ReturnType; + appResolver: AppResolver; + platform: Platform; + modelName: string; +} + +export async function runTui(opts: RunTuiOptions): Promise { + const config = refreshConfig(); + tuiStore.reset(); + tuiStore.setExportDir(opts.exportDir ?? null); + + const sessionLog = createSessionLog(process.cwd()); + setCurrentSessionLog(sessionLog); + + // Mirror the recorded flow into the log whenever it changes, rather than + // calling setFlow from each of /undo, /edit, /insert, /delete and /clear — + // the store is the one place every mutation already passes through. + let loggedSteps = getSnapshot().steps; + let loggedMeta = getSnapshot().meta; + subscribe(() => { + const snapshot = getSnapshot(); + if (snapshot.steps === loggedSteps && snapshot.meta === loggedMeta) return; + loggedSteps = snapshot.steps; + loggedMeta = snapshot.meta; + sessionLog.setFlow(snapshot.steps, snapshot.meta); + }); + + let session: DeviceSession | null = null; + let quitting = false; + /** Generation token so a stale device-list response can't clobber a newer one. */ + let devicesReq = 0; + + /** + * Stop the frame loop (which also deletes the kitty image and its temp dir) + * and clear the panel state. Shared by /stream-close, device switches and + * quit, so no caller has to check whether a stream is running first. + */ + function resetStream(): void { + stopStreamLoop(); + tuiStore.setStream({ + status: 'idle', + error: undefined, + backend: undefined, + resolution: undefined, + frameLines: undefined, + }); + } + + /** Tear down the current Appium session + appium-mcp subprocess, if any. */ + async function closeSession(): Promise { + if (!session) return; + const old = session; + session = null; + try { + await old.mcpClient.callTool('appium_session_management', { action: 'delete' }); + } catch { + /* ignore */ + } + try { + await old.mcpClient.close(); + } catch { + /* ignore */ + } + } + + async function connect(platform: Platform, udid: string): Promise { + // A stream is bound to one udid — carrying it across a device switch would + // keep screencapping the device the user just left. + resetStream(); + // /device supports re-picking — close the previous session first, or each + // switch would orphan an appium-mcp subprocess and its Appium session. + await closeSession(); + tuiStore.setConnecting(true); + tuiStore.setBusy('Starting appium-mcp…'); + let mcpClient: Awaited> | null = null; + try { + mcpClient = await createMCPClient({ + transport: config.MCP_TRANSPORT, + host: config.MCP_HOST, + port: config.MCP_PORT, + url: config.MCP_URL || undefined, + }); + const availableTools = await mcpClient.listTools(); + const llm = createLLMProvider(config, availableTools); + tuiStore.setBusy('Setting up device…'); + const deviceResult = await setupDevice(mcpClient, { + cliPlatform: platform, + cliDeviceType: opts.deviceType ?? (platform === 'ios' ? 'simulator' : null), + cliUdid: udid, + cliDeviceName: null, + config, + }); + const appResolver = new AppResolver(); + tuiStore.setBusy('Loading installed apps…'); + await appResolver.initialize(deviceResult.scopedMcp, deviceResult.platform); + const modelName = config.LLM_MODEL || DEFAULT_MODELS[config.LLM_PROVIDER] || 'default'; + + session = { + mcpClient, + scopedMcp: deviceResult.scopedMcp, + llm, + appResolver, + platform: deviceResult.platform, + modelName, + }; + tuiStore.setConnecting(false); + tuiStore.setStatusMessage(''); + // Seed flow metadata so an exported YAML/spec carries the platform it was + // recorded against — /meta platform still wins if the user sets it. + if (!getSnapshot().meta.platform) tuiStore.setMeta({ platform: deviceResult.platform }); + sessionLog.setDevice(deviceResult.platform, { + name: deviceResult.deviceName, + udid: deviceResult.deviceUdid, + }); + sessionLog.setLlm(config.LLM_PROVIDER, modelName, config.AGENT_MODE); + tuiStore.log('info', `Connected — ${deviceResult.deviceName} (${deviceResult.platform})`); + tuiStore.goTo('main'); + } catch (err) { + // Don't leak the appium-mcp subprocess spawned before the failure. + if (mcpClient) { + try { + await mcpClient.close(); + } catch { + /* ignore */ + } + } + const message = err instanceof Error ? err.message : String(err); + tuiStore.setConnecting(false); + tuiStore.setStatusMessage(''); + // The picker doesn't render the transcript, so surface the failure in + // the error slot it does render — otherwise a failed connect looks like + // a silent no-op — and clear the not-actually-connected selection. + tuiStore.selectDevice(null); + tuiStore.setDevicesError(`Connection failed: ${message}`); + tuiStore.log('error', 'Device connection failed', message); + tuiStore.goTo('device-picker'); + } + } + + const actions: TuiActions = { + selectPlatform(platform) { + tuiStore.setPlatform(platform); + actions.goToDevicePicker(); + }, + + async selectDevice(device: DeviceSummary) { + tuiStore.selectDevice(device); + await connect(device.platform, device.udid); + }, + + goToDevicePicker() { + const platform = getSnapshot().platform; + if (!platform) { + tuiStore.log('warn', 'Pick a platform first', '/platform'); + return; + } + const req = ++devicesReq; + tuiStore.setDevicesLoading(true); + tuiStore.goTo('device-picker'); + listRunningDevices(platform) + .then((devices) => { + if (req !== devicesReq) return; // stale response (platform switched / refreshed) + tuiStore.setDevices(devices); + if (devices.length === 0) { + // listRunningDevices never rejects (missing adb/Xcode returns []), + // so give the empty state a diagnosis pointer. + tuiStore.setDevicesError( + platform === 'android' + ? // No key hints here — the picker has no command input, so it + // renders its own ("d run doctor · r refresh") beneath this. + 'Is adb on PATH and an emulator booted?' + : 'Is Xcode installed and a simulator booted?' + ); + } + }) + .catch((err) => { + if (req !== devicesReq) return; + tuiStore.setDevicesError(err instanceof Error ? err.message : String(err)); + }); + }, + + goToPlatformPicker() { + tuiStore.goTo('welcome'); + }, + + goToMain() { + tuiStore.goTo('main'); + }, + + goToSettings() { + tuiStore.setSettingsLoading(true); + tuiStore.goTo('settings'); + const fields = SETTINGS_KEYS.map(({ key, description }) => ({ + key: key as string, + value: String(Config[key] ?? ''), + description, + })); + tuiStore.setSettingsFields(fields); + }, + + async saveSettings() { + const { settingsFields } = getSnapshot(); + const envPath = resolve(process.cwd(), '.env'); + let lines: string[] = []; + if (existsSync(envPath)) { + lines = (await readFile(envPath, 'utf-8')).split('\n'); + } + for (const field of settingsFields) { + const idx = lines.findIndex((l) => l.startsWith(`${field.key}=`)); + const line = `${field.key}=${field.value}`; + if (idx >= 0) lines[idx] = line; + else lines.push(line); + // refreshConfig() re-parses process.env (dotenv only runs at startup), + // so mirror the new value there or the save wouldn't take effect until + // the next launch. + process.env[field.key] = field.value; + } + await writeFile(envPath, lines.join('\n'), 'utf-8'); + refreshConfig(); + tuiStore.markSettingsSaved(); + tuiStore.log( + 'info', + 'Settings saved to .env', + session ? 'Reconnect (/device) for provider/model changes to apply' : undefined + ); + }, + + goToHistory() { + tuiStore.setHistoryLoading(true); + tuiStore.goTo('history'); + // Flow runs and recording sessions are separate stores on disk; /history + // is the one place a user looks for "what have I run", so merge them. + // A failure to read either must not blank the other. + Promise.allSettled([loadRunIndex(process.cwd()), listSessions(process.cwd())]) + .then(([runsResult, sessionsResult]) => { + const flows: RunSummary[] = + runsResult.status === 'fulfilled' + ? runsResult.value.runs.map((r) => ({ + runId: r.runId, + source: 'flow' as const, + dir: `.appclaw/runs/${r.runId}`, + goal: r.flowName || r.flowFile, + success: r.success, + startedAt: r.startedAt, + durationMs: r.durationMs, + stepsExecuted: r.stepsExecuted, + stepsTotal: r.stepsTotal, + platform: r.platform, + })) + : []; + + const sessions: RunSummary[] = + sessionsResult.status === 'fulfilled' + ? sessionsResult.value.map((s) => { + const failures = s.events.filter((e) => e.ok === false).length; + const finished = s.finishedAt ? Date.parse(s.finishedAt) : null; + return { + runId: s.sessionId, + source: 'session' as const, + dir: `${SESSIONS_DIRNAME}/${s.sessionId}.json`, + goal: s.meta.name || `${s.steps.length} recorded step(s)`, + success: sessionSucceeded(s), + startedAt: s.startedAt, + durationMs: finished ? finished - Date.parse(s.startedAt) : undefined, + stepsExecuted: s.steps.length, + stepsTotal: s.steps.length, + platform: s.platform, + device: s.device?.name, + model: s.llm ? `${s.llm.provider}/${s.llm.model}` : undefined, + failures, + exports: s.exports, + live: !s.finishedAt, + }; + }) + : []; + + const failed = [runsResult, sessionsResult].find((r) => r.status === 'rejected'); + if (flows.length === 0 && sessions.length === 0 && failed) { + tuiStore.setHistoryError(String((failed as PromiseRejectedResult).reason)); + return; + } + tuiStore.setHistory( + [...flows, ...sessions].sort((a, b) => (a.startedAt < b.startedAt ? 1 : -1)) + ); + }) + .catch((err) => tuiStore.setHistoryError(err instanceof Error ? err.message : String(err))); + }, + + /** + * In-terminal mirror, shown in the main screen's right-hand panel so the + * command palette and instruction input stay usable while it runs. No + * browser and no second window — just adb screencap on a timer, fed to + * stream/frame-loop.ts, which works the same for an emulator, a physical + * device or a headless emulator. + */ + async openStream() { + const { device } = getSnapshot(); + if (!device) { + tuiStore.log('warn', 'No device selected', '/device'); + return; + } + if (device.platform !== 'android') { + // Capture is `adb exec-out screencap`; iOS simulators have no + // equivalent that streams into a pipe. + const message = + 'In-terminal streaming is Android-only (adb screencap). For an iOS simulator, use the Simulator app window.'; + tuiStore.setStream({ status: 'error', error: message }); + tuiStore.log('warn', 'Stream not available for iOS', message); + return; + } + + // Set before startStreamLoop: the panel has to be in its streaming + // layout (and therefore the right height) before the first frame is + // painted into it. + tuiStore.setStream({ + status: 'starting', + error: undefined, + frameLines: undefined, + }); + const backend = detectStreamBackend(); + try { + // Read once up front: the kitty path hands the PNG to the terminal + // without decoding it, so this is the only source of the aspect ratio + // the cell box is sized from. + const resolution = await getDeviceResolution(device.udid); + startStreamLoop({ + udid: device.udid, + deviceWidth: resolution.width, + deviceHeight: resolution.height, + backend, + // Half-blocks are text, so they go through the store and Ink renders + // them; the kitty backend never calls this (it writes pixels itself). + onFrame: (lines) => tuiStore.setStreamFrame(lines), + onError: (message) => { + // The loop has already stopped itself; the panel keeps showing the + // reason instead of flashing it past in the transcript. + tuiStore.setStream({ status: 'error', error: message, frameLines: undefined }); + tuiStore.log('error', 'Screen stream stopped', message); + }, + }); + tuiStore.setStream({ + status: 'running', + backend, + resolution, + error: undefined, + }); + tuiStore.log( + 'info', + `Streaming ${device.name} in the side panel (${backendLabel(backend)})`, + '/stream-close stops it — the input stays usable meanwhile' + ); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + stopStreamLoop(); + tuiStore.setStream({ + status: 'error', + error: message, + frameLines: undefined, + }); + tuiStore.log('error', 'Could not start the screen stream', message); + } + }, + + /** Stops the mirror and clears the panel. */ + closeStream() { + resetStream(); + }, + + async runDoctor() { + tuiStore.openViewerPending('appclaw doctor', 'environment preflight'); + const { result, lines } = await captureConsoleAsync(() => runDoctor([])); + const code = result ?? 1; + tuiStore.showViewer({ + title: 'appclaw doctor', + subtitle: 'environment preflight', + lines, + // No tint — doctor's lines already carry their own chalk colouring. + status: + code === 0 + ? { color: COLORS.green, text: `${symbols.check} All checks passed` } + : { color: COLORS.yellow, text: `${symbols.warning} Issues found — see above` }, + }); + tuiStore.log( + code === 0 ? 'result' : 'warn', + code === 0 ? 'doctor: all checks passed' : 'doctor: issues found' + ); + }, + + /** + * One deterministic instruction — the shared per-line pipeline + * (vision → regex → LLM → executeStep, via runOneInstruction), with the + * same two early-outs for bookkeeping-only kinds that never touch the + * device: `done` is recorded without executing, `getInfo` is answered by + * a separate vision call and is NOT recorded. + */ + async runInstruction(instruction: string) { + if (!session) { + tuiStore.log('warn', 'No device connected yet', '/device to pick one'); + return; + } + const active = session; + tuiStore.log('goal', instruction); + + const earlyParse = tryParseNaturalFlowLine(instruction); + if (earlyParse?.kind === 'done') { + tuiStore.pushStep(earlyParse); + logRecorded(earlyParse, 'recorded'); + return; + } + if (earlyParse?.kind === 'getInfo') { + await showScreenInfo(active.scopedMcp, earlyParse.query); + return; + } + + const t0 = performance.now(); + let outcome: Awaited>; + // The pipeline's own spinner calls come and go; seed the busy text so the + // disabled input reads "Executing " rather than a bare "working…". + tuiStore.setBusy('Executing', instruction); + try { + outcome = await runOneInstruction(active.scopedMcp, instruction, { + appResolver: active.appResolver, + minMatchScore: DEFAULT_MIN_MATCH_SCORE, + }); + } catch (err) { + tuiStore.log( + 'error', + `Failed: ${err instanceof Error ? err.message : String(err)}`, + 'Type /help to see supported patterns' + ); + return; + } finally { + tuiStore.clearBusy(); + } + const elapsedMs = Math.round(performance.now() - t0); + + // Vision read the line as a "what's on screen" question — no device + // action happened, so there is nothing to record. + if (outcome.isGetInfo) { + const answer = outcome.getInfoAnswer || outcome.result.message; + tuiStore.log('result', answer, outcome.getInfoExplanation); + return; + } + // `done` / `getInfo` resolved by the LLM fallback rather than the regex above. + if (outcome.step.kind === 'getInfo') { + await showScreenInfo(active.scopedMcp, outcome.step.query); + return; + } + if (outcome.step.kind === 'done') { + tuiStore.pushStep(outcome.step); + logRecorded(outcome.step, 'recorded'); + sessionLog.record({ + kind: 'instruction', + input: instruction, + ok: true, + step: outcome.step, + }); + return; + } + + if (outcome.result.success) { + tuiStore.pushStep(outcome.step); + logRecorded(outcome.step, outcome.result.message, elapsedMs); + sessionLog.record({ + kind: 'instruction', + input: instruction, + ok: true, + message: outcome.result.message, + durationMs: elapsedMs, + step: outcome.step, + }); + return; + } + const hint = + outcome.step.kind === 'tap' && outcome.closestMatch + ? `Closest match: "${outcome.closestMatch}". Try: tap on ${outcome.closestMatch}` + : null; + tuiStore.log( + 'error', + `${symbols.cross} ${stepAction(outcome.step)} ${stepTarget(outcome.step)}`, + [outcome.result.message, hint, 'Step not recorded. Fix and try again.'] + .filter((l): l is string => !!l) + .join('\n') + ); + sessionLog.record({ + kind: 'instruction', + input: instruction, + ok: false, + message: outcome.result.message, + durationMs: elapsedMs, + }); + }, + + async runGoal(goal: string) { + if (!session) { + tuiStore.log('warn', 'No device connected yet', '/device to pick one'); + return; + } + const active = session; + tuiStore.log('goal', goal); + try { + const result = await runAgent({ + goal, + displayGoal: goal, + mcp: active.scopedMcp, + llm: active.llm, + appResolver: active.appResolver, + maxSteps: config.MAX_STEPS, + stepDelay: config.STEP_DELAY, + maxElements: config.MAX_ELEMENTS, + visionMode: config.VISION_MODE, + modelName: active.modelName, + onStep: (event) => { + // The final "done" step's message is the same text as the + // Done/Failed summary logged right after runAgent resolves — + // showing both back-to-back is a duplicate, not new information. + if (event.decision.toolName === 'done') return; + tuiStore.log('step', `${event.step}. ${event.decision.toolName}`, event.result.message); + }, + }); + tuiStore.log( + result.success ? 'result' : 'error', + result.success ? `Done — ${result.reason}` : `Failed — ${result.reason}` + ); + sessionLog.record({ + kind: 'goal', + input: goal, + ok: result.success, + message: result.reason, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + tuiStore.log('error', 'Goal execution error', message); + sessionLog.record({ kind: 'goal', input: goal, ok: false, message }); + } finally { + active.llm.resetHistory(); + } + }, + + async quit() { + if (quitting) return; + quitting = true; + // Don't let a wedged Appium/MCP teardown trap the user in the TUI. + const hardExit = setTimeout(() => process.exit(0), 5000); + hardExit.unref(); + try { + sessionLog.finish(); + // Also clears the frame loop's temp dir and its lingering kitty image + // placement, which would otherwise outlive the process on screen. + resetStream(); + await closeSession(); + } finally { + clearTimeout(hardExit); + setRenderer(null); + instance.unmount(); + exitAltScreen(); + process.exit(0); + } + }, + }; + + enterAltScreen(); + setRenderer(tuiRenderer); + // Safety net for any exit path that skips actions.quit() (uncaught + // exception, Ink itself throwing) — otherwise the user's shell is left + // stuck showing the alt screen after the process is already gone. + process.once('exit', exitAltScreen); + + const instance = render(React.createElement(TuiApp, { actions }), { + // Route core's console output (doctor, agent-loop fallbacks) above the + // frame instead of interleaving with it. + patchConsole: true, + exitOnCtrlC: false, + }); + // Ctrl+C is handled inside TuiApp (raw mode suppresses tty SIGINT, so a + // process-level SIGINT handler alone can never fire while the app is + // mounted). These cover `kill -INT/-TERM/-HUP` — without them, Ink's + // signal-exit hook would re-raise and die with no session/child cleanup. + for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP'] as const) { + process.once(sig, () => { + void actions.quit(); + }); + } + + if (opts.platform) { + actions.selectPlatform(opts.platform); + if (opts.udid) { + const devices = await listRunningDevices(opts.platform); + const match = devices.find((d) => d.udid === opts.udid) ?? { + name: opts.deviceName || opts.udid, + udid: opts.udid, + state: 'unknown', + platform: opts.platform, + }; + await actions.selectDevice(match); + } + } + + // Keep runTui alive until the app unmounts for any reason (React render + // error, external unmount) and still run cleanup — quit() is idempotent. + await instance.waitUntilExit(); + await actions.quit(); +} diff --git a/packages/cli/src/tui/input-history.ts b/packages/cli/src/tui/input-history.ts new file mode 100644 index 0000000..b7d1223 --- /dev/null +++ b/packages/cli/src/tui/input-history.ts @@ -0,0 +1,25 @@ +/** + * Input history for the TUI's prompt — the `↑` recall a REPL is expected to + * have. + * + * Session-scoped and in memory only. A recording session is a self-contained + * piece of work against one app on one device, so yesterday's lines are noise + * to scroll past rather than shortcuts; and nothing here is written to disk, so + * the prompt leaves nothing behind. The session log already keeps the durable + * record of what was run. + */ + +/** Enough for a long session's recall without unbounded growth. */ +const MAX_ENTRIES = 500; + +/** + * Append one line. Consecutive duplicates collapse the way a shell's history + * does: holding `↑` through ten identical "tap on Login" entries is not recall, + * it's noise. + */ +export function appendHistory(history: string[], line: string): string[] { + const trimmed = line.trim(); + if (!trimmed) return history; + if (history[history.length - 1] === trimmed) return history; + return [...history, trimmed].slice(-MAX_ENTRIES); +} diff --git a/packages/cli/src/tui/screens/DevicePickerScreen.tsx b/packages/cli/src/tui/screens/DevicePickerScreen.tsx new file mode 100644 index 0000000..d87cbfa --- /dev/null +++ b/packages/cli/src/tui/screens/DevicePickerScreen.tsx @@ -0,0 +1,165 @@ +import React, { useState, useSyncExternalStore, useEffect } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import { subscribe, getSnapshot } from '../store.js'; +import { requestQuit, type TuiActions } from '../commands.js'; +import { Header } from '../components/Header.js'; +import { StatusBar } from '../components/StatusBar.js'; +import { useAvailableRows, windowFor } from '../useLayout.js'; + +export interface DevicePickerScreenProps { + actions: TuiActions; +} + +/** Header (2) + "Select a device" title (1) + border/padding (2) + StatusBar (3) + connecting line (1). */ +const RESERVED_ROWS = 2 + 1 + 2 + 3 + 1; + +/** Lists running Android emulators / iOS simulators for the chosen platform. */ +export function DevicePickerScreen({ actions }: DevicePickerScreenProps) { + const ui = useSyncExternalStore(subscribe, getSnapshot); + const [index, setIndex] = useState(0); + const [connecting, setConnecting] = useState(false); + const [errorDismissed, setErrorDismissed] = useState(false); + const maxVisible = useAvailableRows(RESERVED_ROWS); + const { items: visibleDevices, start } = windowFor(ui.devices, index, maxVisible); + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + + const showError = Boolean(ui.devicesError) && !errorDismissed; + + useEffect(() => { + setIndex(0); + }, [ui.devices]); + + // A newly-arrived error is always worth showing, even if the previous one + // was dismissed. + useEffect(() => { + if (ui.devicesError) setErrorDismissed(false); + }, [ui.devicesError]); + + useInput((input, key) => { + if (connecting) return; + // Escape closes the notice first; only a second press leaves the screen, + // so dismissing can't accidentally navigate away. + if (key.escape && showError) { + setErrorDismissed(true); + return; + } + if (input === 'd' && !key.ctrl && !key.meta) { + void actions.runDoctor(); + return; + } + if (input === 'q' && !key.ctrl && !key.meta) { + void requestQuit(actions); + return; + } + if (key.upArrow) setIndex((i) => Math.max(0, i - 1)); + else if (key.downArrow) setIndex((i) => Math.max(0, Math.min(ui.devices.length - 1, i + 1))); + else if (key.return && ui.devices[index]) { + setConnecting(true); + void actions + .selectDevice(ui.devices[index]) + .catch(() => { + /* connect() reports its own errors via the store */ + }) + .finally(() => setConnecting(false)); + } else if (key.escape) actions.goToPlatformPicker(); + else if (input === 'r' && !key.ctrl && !key.meta) actions.goToDevicePicker(); + }); + + const hints = showError + ? ['d doctor', 'r refresh', 'esc dismiss', 'q quit'] + : ['↑↓ select', 'enter connect', 'r refresh', 'd doctor', 'esc back', 'q quit']; + + return ( + + +
      + + {/* Centered in the remaining space — this screen holds a short list, + so pinning it to the top left it stranded in a mostly empty frame. */} + + + + Select a device + + + {ui.devicesLoading ? ( + + Searching for {ui.platform} devices… + + ) : showError ? ( + + + {symbols.cross} No devices found + + + {ui.devicesError} + + + d run doctor · r refresh · esc dismiss + + + ) : ui.devices.length === 0 ? ( + + + No {ui.platform === 'android' ? 'emulators' : 'simulators'} running — press r to + refresh. + + + ) : ( + // No explicit height — windowFor() already caps the row count, and + // forcing height={maxVisible} stretched this to the full screen + // when fewer devices were listed. + + {visibleDevices.map((d, i) => { + const trueIndex = start + i; + const up = + d.state.toLowerCase() === 'booted' || d.state.toLowerCase() === 'device'; + return ( + + {trueIndex === index ? `${symbols.prompt} ` : ' '} + {d.name} {d.state} + {d.hint ? · {d.hint} : null} + + ); + })} + + )} + + {connecting ? ( + + Connecting… + + ) : null} + + + + + + ); +} diff --git a/packages/cli/src/tui/screens/HistoryScreen.tsx b/packages/cli/src/tui/screens/HistoryScreen.tsx new file mode 100644 index 0000000..2f97b07 --- /dev/null +++ b/packages/cli/src/tui/screens/HistoryScreen.tsx @@ -0,0 +1,161 @@ +import React, { useSyncExternalStore } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import { subscribe, getSnapshot, tuiStore } from '../store.js'; +import { requestQuit, type TuiActions } from '../commands.js'; +import { Header } from '../components/Header.js'; +import { StatusBar } from '../components/StatusBar.js'; +import { useAvailableRows, windowFor } from '../useLayout.js'; + +export interface HistoryScreenProps { + actions: TuiActions; +} + +/** Header (2) + border/padding (2) + StatusBar (3). */ +const RESERVED_ROWS = 2 + 2 + 3; + +function formatDuration(ms?: number): string { + if (!ms) return '—'; + const s = Math.round(ms / 1000); + return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${s % 60}s`; +} + +/** Browses `.appclaw/runs/` (flow-run reports) via the shared runs.json index. */ +export function HistoryScreen({ actions }: HistoryScreenProps) { + const ui = useSyncExternalStore(subscribe, getSnapshot); + const runs = ui.history; + const selected = runs[ui.historySelected]; + const maxVisible = useAvailableRows(RESERVED_ROWS); + const { items: visibleRuns, start } = windowFor(runs, ui.historySelected, maxVisible); + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + + useInput((input, key) => { + if (input === 'q' && !key.ctrl && !key.meta) { + void requestQuit(actions); + return; + } + if (key.upArrow) tuiStore.setHistorySelected(Math.max(0, ui.historySelected - 1)); + else if (key.downArrow) + tuiStore.setHistorySelected(Math.min(runs.length - 1, ui.historySelected + 1)); + else if (key.escape) actions.goToMain(); + }); + + return ( + + +
      + + + {ui.historyLoading ? ( + Loading… + ) : ui.historyError ? ( + {ui.historyError} + ) : runs.length === 0 ? ( + + Nothing yet — record steps in the TUI, or run a YAML flow. + + ) : ( + + {visibleRuns.map((r, i) => { + const trueIndex = start + i; + return ( + + {trueIndex === ui.historySelected ? `${symbols.prompt} ` : ' '} + + {r.success ? symbols.check : symbols.cross} + {' '} + {/* The two kinds live in different places on disk and mean + different things, so the row says which it is. */} + + {r.source === 'session' ? 'session' : 'flow '} + {' '} + {r.goal} + {r.live ? · live : null} + + ); + })} + + )} + + + + Detail + + {selected ? ( + + {selected.goal} + {selected.startedAt} + + {selected.platform ?? ''} ·{' '} + {selected.live ? 'running' : formatDuration(selected.durationMs)} ·{' '} + {selected.stepsExecuted ?? 0}/{selected.stepsTotal ?? 0} steps + + {selected.device ? ( + + {selected.device} + + ) : null} + {selected.model ? ( + + {selected.model} + + ) : null} + {/* Failed instructions never became steps, so the step count + alone hides them — the whole reason the log keeps them. */} + {selected.failures ? ( + + {selected.failures} failed instruction + {selected.failures === 1 ? '' : 's'} + + ) : null} + {selected.exports?.length ? ( + + {selected.exports.length} export + {selected.exports.length === 1 ? '' : 's'} + + ) : null} + + + {selected.dir} + + + + ) : ( + Select a run + )} + + + + + + ); +} diff --git a/packages/cli/src/tui/screens/MainScreen.tsx b/packages/cli/src/tui/screens/MainScreen.tsx new file mode 100644 index 0000000..55a373b --- /dev/null +++ b/packages/cli/src/tui/screens/MainScreen.tsx @@ -0,0 +1,364 @@ +import React, { useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import { COLORS } from '../../ui/ink/theme.js'; +import { subscribe, getSnapshot, tuiStore, type TranscriptEntry } from '../store.js'; +import { executeLine, completeCommand, matchCommands, type TuiActions } from '../commands.js'; +import { appendHistory } from '../input-history.js'; +import { Header } from '../components/Header.js'; +import { CommandPalette } from '../components/CommandPalette.js'; +import { StreamPanel } from '../components/StreamPanel.js'; +import { StatusBar } from '../components/StatusBar.js'; +import { + columnWidths, + panelImageRows, + transcriptRows, + visibleCommandCount, + MIN_MAIN_ROWS, +} from '../stream/layout.js'; + +const KIND_COLOR: Record = { + info: COLORS.white, + warn: COLORS.yellow, + error: COLORS.red, + step: COLORS.step, + goal: COLORS.brand, + result: COLORS.green, + command: COLORS.dimmed, +}; + +interface TranscriptRow { + key: string; + text: string; + color?: string; + /** Detail lines sit indented under the entry they belong to. */ + indent: boolean; +} + +/** + * Flatten entries into one object per rendered line. + * + * Scrolling needs a stable, countable unit, and an entry is not one — a single + * entry can render as many lines (a multi-line `detail`, plus the blank + * separator before a new instruction). Windowing over rows makes the scroll + * maths exact instead of the estimate the tail-only view could get away with. + */ +function toRows(entries: TranscriptEntry[]): TranscriptRow[] { + const rows: TranscriptRow[] = []; + entries.forEach((entry, i) => { + if (entry.kind === 'goal' && i > 0) { + rows.push({ key: `${entry.id}:gap`, text: '', indent: false }); + } + rows.push({ + key: `${entry.id}:text`, + text: entry.text, + color: KIND_COLOR[entry.kind], + indent: false, + }); + if (entry.detail) { + entry.detail.split('\n').forEach((line, j) => { + rows.push({ key: `${entry.id}:d${j}`, text: line, color: COLORS.dimmed, indent: true }); + }); + } + }); + return rows; +} + +function Transcript({ + rows, + budget, + scrolledBy, + total, + focused, +}: { + rows: TranscriptRow[]; + budget: number; + scrolledBy: number; + total: number; + focused: boolean; +}) { + const scrollable = total > budget; + return ( + + + + Transcript + + 0 ? COLORS.yellow : COLORS.dimmed}> + {focused + ? scrolledBy > 0 + ? `↑${scrolledBy} · ↑↓ scroll · ⇧tab back` + : 'scrolling · ↑↓ · ⇧tab back' + : scrollable + ? 'live · ⇧tab to scroll' + : ''} + + + {rows.length === 0 ? ( + Nothing yet — try /help or type a goal. + ) : ( + rows.map((row) => ( + // marginLeft on a Box rather than a leading space in the text — a + // space only offsets the first wrapped line, leaving continuation + // lines flush left and ragged. + + {row.text || ' '} + + )) + )} + + ); +} + +export interface MainScreenProps { + actions: TuiActions; +} + +/** Wireframe's main window: header, left command palette + instruction box, right device stream panel. */ +export function MainScreen({ actions }: MainScreenProps) { + const ui = useSyncExternalStore(subscribe, getSnapshot); + const [value, setValue] = useState(''); + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + const cols = stdout.columns || 80; + + // The columns are full height and independent now, so the picture's size no + // longer depends on whether a stream is running — only the panel's contents do. + const widths = columnWidths(cols); + const imageRows = panelImageRows(rows); + const maxCommands = visibleCommandCount(rows); + const transcriptBudget = transcriptRows(rows); + + /** Which pane the keyboard belongs to. Shift+Tab moves between them. */ + const [focus, setFocus] = useState<'input' | 'transcript'>('input'); + /** Lines submitted in this session only — nothing is carried in from disk. */ + const [history, setHistory] = useState([]); + /** + * How far back through history we are: 0 is the live draft, 1 the most recent + * entry. The draft is stashed on the first `↑` so coming back down restores + * what was half-typed rather than clearing it. + */ + const [historyIndex, setHistoryIndex] = useState(0); + const draftRef = useRef(''); + + function recallHistory(direction: -1 | 1): void { + if (history.length === 0) return; + const next = Math.min(history.length, Math.max(0, historyIndex - direction)); + if (next === historyIndex) return; + if (historyIndex === 0) draftRef.current = value; + setHistoryIndex(next); + setValue(next === 0 ? draftRef.current : history[history.length - next]); + } + + const allRows = useMemo(() => toRows(ui.transcript), [ui.transcript]); + /** Rows scrolled back from the newest line; 0 follows the tail live. */ + const [scrolledBy, setScrolledBy] = useState(0); + const maxScroll = Math.max(0, allRows.length - transcriptBudget); + const scroll = Math.min(scrolledBy, maxScroll); + const visibleRows = allRows.slice(maxScroll - scroll, maxScroll - scroll + transcriptBudget); + + // While the user is reading history, grow the offset by however many rows + // arrived so the viewport stays put instead of sliding under them. At 0 the + // offset is left alone, which is what keeps the live tail following. + const previousRowCount = useRef(allRows.length); + useEffect(() => { + const added = allRows.length - previousRowCount.current; + previousRowCount.current = allRows.length; + if (added > 0 && scrolledBy > 0) setScrolledBy((o) => o + added); + }, [allRows.length, scrolledBy]); + + /** + * Focus decides what the bare arrow keys mean, so neither the prompt nor the + * transcript needs a modifier: at the prompt they recall history, in the + * transcript they scroll. Shift+Tab moves between the two, and typing a + * printable character jumps straight back to the prompt with that character, + * so getting out of scrolling never costs a keystroke. + * + * While the transcript has focus is given `focus={false}`, which + * makes it ignore input entirely — that is what frees the arrows without the + * component inserting stray characters. + */ + useInput((input, key) => { + const page = Math.max(1, transcriptBudget - 1); + + // Shift+Tab moves between the panes, in both directions — the conventional + // "focus previous" key, and free because early-returns on it. + // Plain Tab stays with completion. + if (key.tab && key.shift) { + if (focus === 'input') { + if (maxScroll > 0) setFocus('transcript'); + } else { + setFocus('input'); + } + return; + } + + if (focus === 'transcript') { + if (key.upArrow) setScrolledBy((o) => Math.min(maxScroll, o + 1)); + else if (key.downArrow) setScrolledBy((o) => Math.max(0, o - 1)); + else if (key.pageUp) setScrolledBy((o) => Math.min(maxScroll, o + page)); + else if (key.pageDown) setScrolledBy((o) => Math.max(0, o - page)); + // Shift+Tab and Enter both return; Esc is deliberately unbound here, so + // it stays free for something that genuinely means "cancel". + else if (key.return) setFocus('input'); + else if (input && !key.ctrl && !key.meta && !key.tab) { + // Carry the keystroke into the prompt rather than swallowing it. + setFocus('input'); + setValue((v) => v + input); + } + return; + } + + if (key.tab) { + const completed = completeCommand(value); + if (completed) { + setValue(completed); + tuiStore.setPaletteError(null); + return; + } + // Tab that changes nothing has to say why. Silence is indistinguishable + // from a broken key, and the common cases — an ambiguous prefix, or a + // word that is already complete — both produce no edit. + const partial = value.trim(); + if (!partial.startsWith('/') || partial.includes(' ')) return; + const matches = matchCommands(partial); + tuiStore.setPaletteError( + matches.length === 0 + ? `No command matches ${partial}` + : matches.length === 1 + ? `${matches[0].name} — already complete` + : matches.map((m) => m.name).join(' ') + ); + return; + } + + if (key.upArrow) recallHistory(-1); + else if (key.downArrow) recallHistory(1); + // Page keys still scroll from the prompt, for anyone who reaches for them. + else if (key.pageUp) setScrolledBy((o) => Math.min(maxScroll, o + page)); + else if (key.pageDown) setScrolledBy((o) => Math.max(0, o - page)); + }); + + function updateQuery(next: string): void { + setValue(next); + // Typing puts you back on the live draft — otherwise the next ↓ would jump + // to a history entry instead of the line being edited. + if (historyIndex !== 0) setHistoryIndex(0); + // Clear a previous "unknown command" note as soon as the user starts + // fixing it, rather than leaving it up until they resubmit. + if (ui.paletteError) tuiStore.setPaletteError(null); + } + + async function submit(raw: string): Promise { + if (ui.running) return; // keep the typed text if the submit is ignored + setValue(''); + setHistory((h) => appendHistory(h, raw)); + setHistoryIndex(0); + draftRef.current = ''; + setScrolledBy(0); // jump back to live so the result is visible + tuiStore.setRunning(true); + try { + await executeLine(raw, actions); + } catch (err) { + tuiStore.log('error', err instanceof Error ? err.message : String(err)); + } finally { + tuiStore.setRunning(false); + } + } + + // The recorded-step count is the core feedback loop of a recording session, + // so it rides along with the device context rather than hiding in /list. + const recorded = `${ui.steps.length} step${ui.steps.length === 1 ? '' : 's'} recorded`; + const subtitle = ui.platform + ? `${ui.platform}${ui.device ? ` · ${ui.device.name}` : ' · no device selected'} · ${recorded}` + : recorded; + + // Below this the frame cannot fit, and Ink does not clip an over-tall frame + // — it overlaps rows, printing two lines of text onto one. Saying so beats + // rendering something corrupted. + if (rows < MIN_MAIN_ROWS) { + return ( + + + Terminal too small + + + {rows} rows available, {MIN_MAIN_ROWS} needed — resize and it redraws. + + + ); + } + + return ( + + {/* Outer frame box — the wireframe's outer rectangle, wrapping the + palette, instruction input, stream panel and transcript. */} + +
      + {/* Two full-height columns. The transcript sits under the palette + rather than spanning the frame, which is what frees the whole right + column — and therefore the whole frame height — for the picture. */} + + {/* Transcript on top, input at the bottom of the column — history + reads downward into the prompt you type next, and the input sits + where the cursor already is rather than mid-column. */} + + + + + + + + + + ); +} diff --git a/packages/cli/src/tui/screens/SettingsScreen.tsx b/packages/cli/src/tui/screens/SettingsScreen.tsx new file mode 100644 index 0000000..2e7ebed --- /dev/null +++ b/packages/cli/src/tui/screens/SettingsScreen.tsx @@ -0,0 +1,123 @@ +import React, { useState, useSyncExternalStore } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import TextInput from 'ink-text-input'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import { subscribe, getSnapshot, tuiStore } from '../store.js'; +import { requestQuit, type TuiActions } from '../commands.js'; +import { Header } from '../components/Header.js'; +import { StatusBar } from '../components/StatusBar.js'; + +export interface SettingsScreenProps { + actions: TuiActions; +} + +/** View/edit the curated config subset, written back to .env on save. */ +export function SettingsScreen({ actions }: SettingsScreenProps) { + const ui = useSyncExternalStore(subscribe, getSnapshot); + const [index, setIndex] = useState(0); + const [editing, setEditing] = useState(false); + const [draft, setDraft] = useState(''); + const [saving, setSaving] = useState(false); + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + + useInput( + (input, key) => { + const fields = ui.settingsFields; + // Safe to bind here because this handler is inactive while editing — + // otherwise "q" would be swallowed instead of typed into a value. + if (input === 'q' && !key.ctrl && !key.meta) { + void requestQuit(actions); + return; + } + if (key.upArrow) setIndex((i) => Math.max(0, i - 1)); + else if (key.downArrow) setIndex((i) => Math.max(0, Math.min(fields.length - 1, i + 1))); + else if (key.return && fields[index]) { + setDraft(fields[index].value); + setEditing(true); + } else if (input === 's' && !key.ctrl && !key.meta) { + // plain "s" only — Ink reports Ctrl+S as input "s" + key.ctrl, and a + // flow-control reflex shouldn't silently write .env + setSaving(true); + void actions + .saveSettings() + .catch((err) => + tuiStore.log( + 'error', + 'Could not save settings', + err instanceof Error ? err.message : String(err) + ) + ) + .finally(() => setSaving(false)); + } else if (key.escape) { + actions.goToMain(); + } + }, + { isActive: !editing } + ); + + // While editing, Escape cancels back to browsing (ink-text-input has no + // escape handling of its own — without this, Enter/commit is the only exit). + useInput( + (_input, key) => { + if (key.escape) setEditing(false); + }, + { isActive: editing } + ); + + function commit(value: string): void { + const field = ui.settingsFields[index]; + if (field) tuiStore.updateSettingField(field.key, value); + setEditing(false); + } + + return ( + + +
      + + {ui.settingsLoading ? ( + Loading… + ) : ( + ui.settingsFields.map((f, i) => ( + + + {i === index ? `${symbols.prompt} ` : ' '} + {f.key.padEnd(14)} + {i === index && editing ? ( + + ) : ( + {f.value || '(unset)'} + )} + + {f.description && i === index ? ( + {f.description} + ) : null} + + )) + )} + + + {saving ? ( + Saving… + ) : ui.settingsSaved ? ( + {symbols.check} Saved + ) : ui.settingsDirty ? ( + Unsaved changes — press s to save + ) : null} + + + + + ); +} diff --git a/packages/cli/src/tui/screens/WelcomeScreen.tsx b/packages/cli/src/tui/screens/WelcomeScreen.tsx new file mode 100644 index 0000000..68fe509 --- /dev/null +++ b/packages/cli/src/tui/screens/WelcomeScreen.tsx @@ -0,0 +1,116 @@ +import React, { useState, useSyncExternalStore } from 'react'; +import { Box, Text, useInput, useStdout } from 'ink'; +import { COLORS, symbols } from '../../ui/ink/theme.js'; +import type { Platform } from '../store.js'; +import { requestQuit, type TuiActions } from '../commands.js'; +import { StatusBar } from '../components/StatusBar.js'; +import { subscribe, getSnapshot } from '../store.js'; + +const OPTIONS: Platform[] = ['android', 'ios']; + +const LABELS: Record = { android: 'Android', ios: 'iOS' }; + +/** + * Block-letter wordmark. Terminals can't scale a font, so "bigger" has to be + * drawn — one glyph per 5 rows of block characters. + */ +const BANNER = [ + ' █████ ██████ ██████ ██████ ██ █████ ██ ██', + '██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██', + '███████ ██████ ██████ ██ ██ ███████ ██ █ ██', + '██ ██ ██ ██ ██ ██ ██ ██ ██ ███ ██', + '██ ██ ██ ██ ██████ ███████ ██ ██ ███ ███ ', +]; + +/** Widest banner row plus the frame's border+padding, below which we fall back to plain text. */ +const BANNER_MIN_COLUMNS = 57 + 8; + +export interface WelcomeScreenProps { + actions: TuiActions; +} + +/** + * Splash screen: the wordmark and platform picker centered together inside the + * full-screen frame. No separate top-left header here — the banner is the + * branding, so repeating it in the corner would just be noise. + */ +export function WelcomeScreen({ actions }: WelcomeScreenProps) { + const [index, setIndex] = useState(0); + const ui = useSyncExternalStore(subscribe, getSnapshot); + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + const columns = stdout.columns || 80; + const showBanner = columns >= BANNER_MIN_COLUMNS; + + useInput((input, key) => { + if (input === 'q' && !key.ctrl && !key.meta) { + void requestQuit(actions); + return; + } + if (key.upArrow) setIndex((i) => (i > 0 ? i - 1 : OPTIONS.length - 1)); + else if (key.downArrow) setIndex((i) => (i < OPTIONS.length - 1 ? i + 1 : 0)); + else if (key.return) actions.selectPlatform(OPTIONS[index]); + }); + + return ( + + + {showBanner ? ( + + {BANNER.map((line, i) => ( + + {line} + + ))} + + ) : ( + + {symbols.diamond} AppClaw + + )} + + + Agentic mobile automation + + + + Select a platform + + + + {OPTIONS.map((opt, i) => ( + // A blank row between options so the box reads as a deliberate + // menu rather than two cramped lines. + + + {i === index ? `${symbols.prompt} ` : ' '} + {LABELS[opt].padEnd(9)} + + + ))} + + + + + ); +} diff --git a/packages/cli/src/tui/session-log.ts b/packages/cli/src/tui/session-log.ts new file mode 100644 index 0000000..140856f --- /dev/null +++ b/packages/cli/src/tui/session-log.ts @@ -0,0 +1,195 @@ +/** + * JSON session log for `appclaw --tui`. + * + * A TUI session is a recording session: the user types instructions, each runs + * on the device and appends a step. The log is that history in machine-readable + * form — what was typed, what it resolved to, whether it worked, how long it + * took — so a session can be reviewed, diffed or turned into a flow after the + * fact, not just while the transcript is still on screen. + * + * Deliberately NOT reusing `RunManifest` from the report writer: that type is + * built around a YAML flow file (`flowFile`, `stepsTotal` known up front) and a + * REPL session has neither, so it could only be stored by inventing values. + * + * The file is rewritten after every event rather than appended once at the end, + * because the most interesting sessions are the ones that end in a crash or a + * kill — a log that only exists on a clean exit would miss them. + */ + +import { mkdirSync, renameSync, writeFileSync } from 'node:fs'; +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { FlowStep, FlowMeta } from '@appclaw/core/flow/types'; + +export type SessionEventKind = 'instruction' | 'goal' | 'command' | 'export' | 'error'; + +export interface SessionEvent { + at: string; + kind: SessionEventKind; + /** Exactly what the user typed. */ + input: string; + ok?: boolean; + message?: string; + durationMs?: number; + /** The step this instruction recorded, when it recorded one. */ + step?: FlowStep; +} + +export interface SessionLogData { + sessionId: string; + startedAt: string; + finishedAt?: string; + platform?: string; + device?: { name: string; udid: string }; + llm?: { provider: string; model: string; mode: string }; + events: SessionEvent[]; + /** The recorded flow as it stood when the session ended. */ + steps: FlowStep[]; + meta: FlowMeta; + /** Paths written by /export during the session. */ + exports: string[]; +} + +/** `20260825T124312-4f2` — sorts chronologically, unique enough per session. */ +function newSessionId(now: Date, random: number): string { + const stamp = now + .toISOString() + .replace(/[-:]/g, '') + .replace(/\.\d+Z$/, ''); + const suffix = Math.floor(random * 0xfff) + .toString(16) + .padStart(3, '0'); + return `${stamp}-${suffix}`; +} + +export interface SessionLog { + readonly path: string; + setDevice(platform: string, device: { name: string; udid: string }): void; + setLlm(provider: string, model: string, mode: string): void; + record(event: Omit): void; + /** Mirror the recorded flow, which /undo, /edit and /clear can mutate after the fact. */ + setFlow(steps: FlowStep[], meta: FlowMeta): void; + addExport(filePath: string): void; + finish(): void; +} + +/** + * The active session's log. A module-level holder so command handlers can + * append to it without `TuiActions` growing a method per event type. + */ +let current: SessionLog | null = null; + +export function setCurrentSessionLog(log: SessionLog | null): void { + current = log; +} + +export function currentSessionLog(): SessionLog | null { + return current; +} + +export function createSessionLog(projectRoot: string, now = new Date()): SessionLog { + const sessionId = newSessionId(now, Math.random()); + const dir = path.join(projectRoot, '.appclaw', 'sessions'); + const file = path.join(dir, `${sessionId}.json`); + + const data: SessionLogData = { + sessionId, + startedAt: now.toISOString(), + events: [], + steps: [], + meta: {}, + exports: [], + }; + + let failed = false; + + function flush(): void { + // A logging failure must never take the session down with it — report once + // and then stay quiet rather than throwing on every subsequent event. + if (failed) return; + try { + mkdirSync(dir, { recursive: true }); + // Write-then-rename: a kill midway through leaves the previous complete + // file rather than a truncated one. + const tmp = `${file}.tmp`; + writeFileSync(tmp, `${JSON.stringify(data, null, 2)}\n`, 'utf-8'); + renameSync(tmp, file); + } catch { + failed = true; + } + } + + flush(); + + return { + path: file, + setDevice(platform, device) { + data.platform = platform; + data.device = device; + flush(); + }, + setLlm(provider, model, mode) { + data.llm = { provider, model, mode }; + flush(); + }, + record(event) { + data.events.push({ at: new Date().toISOString(), ...event }); + flush(); + }, + setFlow(steps, meta) { + data.steps = steps; + data.meta = meta; + flush(); + }, + addExport(filePath) { + data.exports.push(filePath); + flush(); + }, + finish() { + data.finishedAt = new Date().toISOString(); + flush(); + }, + }; +} + +export const SESSIONS_DIRNAME = path.join('.appclaw', 'sessions'); + +/** + * Every readable session log, newest first. + * + * A malformed or half-written file is skipped rather than failing the whole + * listing: the log is rewritten live, so the session currently running can be + * read mid-write, and one bad file shouldn't hide every other session. + */ +export async function listSessions(projectRoot: string): Promise { + const dir = path.join(projectRoot, SESSIONS_DIRNAME); + let names: string[]; + try { + names = await readdir(dir); + } catch { + return []; // no sessions yet + } + + const sessions = await Promise.all( + names + .filter((n) => n.endsWith('.json')) + .map(async (name) => { + try { + const raw = await readFile(path.join(dir, name), 'utf-8'); + const parsed = JSON.parse(raw) as SessionLogData; + return parsed && typeof parsed.sessionId === 'string' ? parsed : null; + } catch { + return null; + } + }) + ); + + return sessions + .filter((s): s is SessionLogData => s !== null) + .sort((a, b) => (a.startedAt < b.startedAt ? 1 : -1)); +} + +/** Did every recorded instruction in this session succeed? */ +export function sessionSucceeded(session: SessionLogData): boolean { + return !session.events.some((e) => e.ok === false); +} diff --git a/packages/cli/src/tui/store.ts b/packages/cli/src/tui/store.ts new file mode 100644 index 0000000..e52acf9 --- /dev/null +++ b/packages/cli/src/tui/store.ts @@ -0,0 +1,430 @@ +/** + * TUI store — observable state for the `appclaw --tui` shell. + * + * A small custom pub/sub (subscribe + snapshot, + * consumed via React's useSyncExternalStore) so the two Ink surfaces stay + * consistent. This store additionally owns screen routing, since the TUI is + * multi-screen, so one store backs several screens. + */ + +import type { FlowStep, FlowMeta } from '@appclaw/core/flow/types'; +import type { Language } from './highlight.js'; + +export type TuiScreen = 'welcome' | 'device-picker' | 'main' | 'settings' | 'history'; + +export type Platform = 'android' | 'ios'; + +export interface DeviceSummary { + name: string; + udid: string; + /** "Booted" | "Shutdown" | "device" | "offline" | "unauthorized" | ... */ + state: string; + platform: Platform; + /** e.g. iOS runtime version */ + hint?: string; +} + +export type TranscriptKind = 'info' | 'warn' | 'error' | 'step' | 'goal' | 'result' | 'command'; + +export interface TranscriptEntry { + id: number; + kind: TranscriptKind; + text: string; + detail?: string; + ts: number; +} + +export type StreamStatus = 'idle' | 'starting' | 'running' | 'error'; + +/** `/stream` — the device mirror shown in the main screen's right-hand panel. */ +export interface StreamState { + status: StreamStatus; + error?: string; + /** + * The current half-block frame, one string per cell row, rendered by + * as ordinary Ink content. Only the half-block backend uses + * this: its output is text, so letting Ink own those cells costs nothing and + * makes the picture immune to repaints. The kitty backend has real pixels, + * which no component output can express — it writes to stdout instead and + * leaves this undefined. + */ + frameLines?: string[]; + /** Which renderer the terminal turned out to support — shown in the panel's status line. */ + backend?: 'kitty' | 'halfblock'; + resolution?: { width: number; height: number }; +} + +/** + * One row in /history. Two things land here: YAML flow runs from + * `.appclaw/runs/` and TUI recording sessions from `.appclaw/sessions/`. + * They're different enough to label but similar enough to browse together. + */ +export interface RunSummary { + runId: string; + source: 'flow' | 'session'; + dir: string; + goal: string; + success: boolean; + startedAt: string; + durationMs?: number; + stepsExecuted?: number; + stepsTotal?: number; + platform?: string; + reportPath?: string; + /** Sessions only: device, model and what went wrong, for the detail pane. */ + device?: string; + model?: string; + failures?: number; + exports?: string[]; + /** Sessions only: still open (no finishedAt) — usually the run you're in now. */ + live?: boolean; +} + +export interface TuiState { + screen: TuiScreen; + platform: Platform | null; + device: DeviceSummary | null; + devices: DeviceSummary[]; + devicesLoading: boolean; + devicesError: string | null; + + /** + * The recorded flow — the TUI is a step recorder first, so a + * plain instruction line appends here and every /list, /yaml, /export, + * /edit… command reads or edits this list. + */ + steps: FlowStep[]; + meta: FlowMeta; + /** + * `--export-dir` override for bare-filename SDK-test exports. Stored rather + * than threaded through TuiActions because only /export reads it. + */ + exportDir: string | null; + /** + * Quitting with unexported steps asks first. A modal rather than the + * "type /quit again" convention: the warning used to go to the + * transcript, which the screens offering the `q` shortcut don't even render. + */ + quitConfirmOpen: boolean; + + transcript: TranscriptEntry[]; + running: boolean; + paletteQuery: string; + /** Inline feedback shown right under the instruction input (e.g. a typo'd command) — separate from the transcript, which scrolls out of view. */ + paletteError: string | null; + + stream: StreamState; + + settingsFields: Array<{ key: string; value: string; description?: string }>; + settingsLoading: boolean; + settingsDirty: boolean; + settingsSaved: boolean; + + history: RunSummary[]; + historyLoading: boolean; + historyError: string | null; + historySelected: number; + + statusMessage: string; + + /** + * Live progress text, fed by the registered UIRenderer's spinner hooks — + * core's device-setup pipeline reports through those (e.g. "Creating Appium + * session..."), which would otherwise animate raw ANSI over the Ink frame. + */ + busyMessage: string; + busyDetail?: string; + /** Show that progress as a modal dialog (device setup) rather than inline. */ + connecting: boolean; + + /** + * Full-screen scrollable output modal, shared by everything whose output is + * too big for the transcript pane: `appclaw doctor`, and the generated + * YAML/spec bodies from /yaml, /preview and /export. The transcript is a + * handful of rows tall, so dumping a file into it truncated the content and + * buried the step log. + */ + viewerOpen: boolean; + viewerTitle: string; + viewerSubtitle?: string; + viewerLines: string[]; + /** Still producing output (doctor) — shows a spinner instead of the body. */ + viewerRunning: boolean; + /** Closing verdict line, e.g. doctor's pass/fail or an export's file path. */ + viewerStatus?: { color: string; text: string }; + /** Colour applied to every body line; omit for output that carries its own ANSI (doctor). */ + viewerTint?: string; + /** Syntax-highlight the body (generated YAML / runner specs). */ + viewerLanguage?: Language; +} + +function initial(): TuiState { + return { + screen: 'welcome', + platform: null, + device: null, + devices: [], + devicesLoading: false, + devicesError: null, + + steps: [], + meta: {}, + exportDir: null, + quitConfirmOpen: false, + + transcript: [], + running: false, + paletteQuery: '', + paletteError: null, + + stream: { status: 'idle' }, + + settingsFields: [], + settingsLoading: false, + settingsDirty: false, + settingsSaved: false, + + history: [], + historyLoading: false, + historyError: null, + historySelected: 0, + + statusMessage: '', + + busyMessage: '', + busyDetail: undefined, + connecting: false, + + viewerOpen: false, + viewerTitle: '', + viewerSubtitle: undefined, + viewerLines: [], + viewerRunning: false, + viewerStatus: undefined, + viewerTint: undefined, + viewerLanguage: undefined, + }; +} + +const MAX_TRANSCRIPT = 500; + +let state: TuiState = initial(); +let nextId = 1; +const listeners = new Set<() => void>(); + +function emit(): void { + for (const l of listeners) l(); +} +function set(patch: Partial): void { + state = { ...state, ...patch }; + emit(); +} + +export function subscribe(l: () => void): () => void { + listeners.add(l); + return () => listeners.delete(l); +} +export function getSnapshot(): TuiState { + return state; +} + +export const tuiStore = { + reset(): void { + state = initial(); + nextId = 1; + emit(); + }, + + goTo(screen: TuiScreen): void { + set({ screen }); + }, + + setPlatform(platform: Platform): void { + // Switching platform invalidates the previous platform's device list, + // selection, and any stale listing error. + set({ platform, devices: [], device: null, devicesError: null }); + }, + + setDevices(devices: DeviceSummary[]): void { + set({ devices, devicesLoading: false, devicesError: null }); + }, + setDevicesLoading(loading: boolean): void { + set({ devicesLoading: loading, devicesError: loading ? null : state.devicesError }); + }, + setDevicesError(error: string): void { + set({ devicesError: error, devicesLoading: false }); + }, + selectDevice(device: DeviceSummary | null): void { + set({ device }); + }, + + // ── Recorded flow ── + // Every mutator replaces the array rather than mutating in place; + // useSyncExternalStore compares by reference, so an in-place push would + // never repaint the step count. + pushStep(step: FlowStep): void { + set({ steps: [...state.steps, step] }); + }, + /** Drop the last step and return it (null when empty) — powers /undo. */ + popStep(): FlowStep | null { + if (state.steps.length === 0) return null; + const steps = state.steps.slice(); + const removed = steps.pop()!; + set({ steps }); + return removed; + }, + insertStep(index: number, step: FlowStep): void { + const steps = state.steps.slice(); + steps.splice(index, 0, step); + set({ steps }); + }, + replaceStep(index: number, step: FlowStep): void { + set({ steps: state.steps.map((s, i) => (i === index ? step : s)) }); + }, + /** Remove the step at `index` and return it (null when out of range). */ + deleteStep(index: number): FlowStep | null { + if (index < 0 || index >= state.steps.length) return null; + const steps = state.steps.slice(); + const [removed] = steps.splice(index, 1); + set({ steps }); + return removed; + }, + clearSteps(): void { + set({ steps: [], meta: {} }); + }, + setMeta(patch: Partial): void { + set({ meta: { ...state.meta, ...patch } }); + }, + setExportDir(exportDir: string | null): void { + set({ exportDir }); + }, + setQuitConfirm(quitConfirmOpen: boolean): void { + set({ quitConfirmOpen }); + }, + + log(kind: TranscriptKind, text: string, detail?: string): void { + const entry: TranscriptEntry = { id: nextId++, kind, text, detail, ts: Date.now() }; + const transcript = [...state.transcript, entry]; + // Long-lived REPL: only the tail is ever rendered, so cap retention. + set({ + transcript: + transcript.length > MAX_TRANSCRIPT ? transcript.slice(-MAX_TRANSCRIPT) : transcript, + }); + }, + clearTranscript(): void { + set({ transcript: [] }); + }, + setRunning(running: boolean): void { + set({ running }); + }, + + setPaletteQuery(paletteQuery: string): void { + set({ paletteQuery }); + }, + setPaletteError(paletteError: string | null): void { + set({ paletteError }); + }, + + setStream(stream: Partial): void { + set({ stream: { ...state.stream, ...stream } }); + }, + /** + * Separate from setStream because it runs on every captured frame: it drops + * the update unless a stream is actually live, so a frame that arrives just + * after /stream-close can't resurrect a closed panel. + */ + setStreamFrame(frameLines: string[]): void { + if (state.stream.status === 'idle') return; + set({ stream: { ...state.stream, frameLines } }); + }, + + setSettingsFields(settingsFields: TuiState['settingsFields']): void { + set({ settingsFields, settingsLoading: false, settingsDirty: false, settingsSaved: false }); + }, + setSettingsLoading(settingsLoading: boolean): void { + set({ settingsLoading }); + }, + updateSettingField(key: string, value: string): void { + set({ + settingsFields: state.settingsFields.map((f) => (f.key === key ? { ...f, value } : f)), + settingsDirty: true, + settingsSaved: false, + }); + }, + markSettingsSaved(): void { + set({ settingsDirty: false, settingsSaved: true }); + }, + + setHistory(history: RunSummary[]): void { + set({ history, historyLoading: false, historyError: null, historySelected: 0 }); + }, + setHistoryLoading(historyLoading: boolean): void { + set({ historyLoading, historyError: historyLoading ? null : state.historyError }); + }, + setHistoryError(historyError: string): void { + set({ historyError, historyLoading: false }); + }, + setHistorySelected(historySelected: number): void { + set({ historySelected }); + }, + + setStatusMessage(statusMessage: string): void { + set({ statusMessage }); + }, + + setBusy(busyMessage: string, busyDetail?: string): void { + set({ busyMessage, busyDetail }); + }, + clearBusy(): void { + set({ busyMessage: '', busyDetail: undefined }); + }, + setConnecting(connecting: boolean): void { + set(connecting ? { connecting } : { connecting, busyMessage: '', busyDetail: undefined }); + }, + + /** Open the modal in its "still working" state (doctor). */ + openViewerPending(viewerTitle: string, viewerSubtitle?: string): void { + set({ + viewerOpen: true, + viewerRunning: true, + viewerTitle, + viewerSubtitle, + viewerLines: [], + viewerStatus: undefined, + viewerTint: undefined, + viewerLanguage: undefined, + }); + }, + /** Open (or fill) the modal with finished output. */ + showViewer(opts: { + title: string; + subtitle?: string; + lines: string[]; + status?: { color: string; text: string }; + tint?: string; + language?: Language; + }): void { + set({ + viewerOpen: true, + viewerRunning: false, + viewerTitle: opts.title, + viewerSubtitle: opts.subtitle, + viewerLines: opts.lines, + viewerStatus: opts.status, + viewerTint: opts.tint, + viewerLanguage: opts.language, + }); + }, + closeViewer(): void { + set({ + viewerOpen: false, + viewerRunning: false, + viewerTitle: '', + viewerSubtitle: undefined, + viewerLines: [], + viewerStatus: undefined, + viewerTint: undefined, + viewerLanguage: undefined, + }); + }, +}; diff --git a/packages/cli/src/tui/stream/capture.ts b/packages/cli/src/tui/stream/capture.ts new file mode 100644 index 0000000..c484f88 --- /dev/null +++ b/packages/cli/src/tui/stream/capture.ts @@ -0,0 +1,107 @@ +/** + * Frame capture for the in-terminal device stream. + * + * Two shapes, one per render backend: + * - `capturePng` (`screencap -p`) for the Kitty path, which hands the PNG + * file to the terminal and never decodes it here. + * - `captureRaw` (`screencap`, no `-p`) for the half-block path — a raw RGBA + * framebuffer, so downsampling is plain arithmetic with no PNG decoder and + * therefore no new dependency. + * + * Every adb invocation passes argv as an array: the udid comes from device + * discovery and must never be interpolated into a shell string. + */ + +import { execFile } from 'node:child_process'; +import { writeFile } from 'node:fs/promises'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** A raw 1080x2400 frame is ~10MB; execFile's 1MB default would truncate it. */ +const MAX_CAPTURE_BYTES = 64 * 1024 * 1024; + +/** A capture that outlives its frame slot is already stale — fail instead of queueing. */ +const CAPTURE_TIMEOUT_MS = 8000; + +export interface RawFrame { + width: number; + height: number; + /** Tightly packed RGBA, `width * height * 4` bytes. */ + pixels: Buffer; +} + +export interface DeviceResolution { + width: number; + height: number; +} + +async function adb(udid: string, args: string[]): Promise { + const { stdout } = await execFileAsync('adb', ['-s', udid, ...args], { + encoding: 'buffer', + maxBuffer: MAX_CAPTURE_BYTES, + timeout: CAPTURE_TIMEOUT_MS, + }); + return stdout; +} + +/** Grab one PNG frame straight to `outPath` (the terminal reads the file itself). */ +export async function capturePng(udid: string, outPath: string): Promise { + const png = await adb(udid, ['exec-out', 'screencap', '-p']); + if (png.length === 0) throw new Error('adb screencap returned no data — is the device still up?'); + await writeFile(outPath, png); +} + +/** Grab one raw RGBA frame. */ +export async function captureRaw(udid: string): Promise { + return parseRawScreencap(await adb(udid, ['exec-out', 'screencap'])); +} + +/** Header sizes to try, oldest first: newer Android appended a colorspace field. */ +const HEADER_SIZES = [12, 16] as const; + +/** + * Parse `screencap`'s raw output: little-endian uint32 width, height, format, + * and (newer Android only) colorspace, followed by RGBA pixels. + * + * Which header size applies is not discoverable from a version string, so it's + * inferred from the payload length. If neither fits, the frame is rejected + * rather than rendered — a wrong offset produces plausible-looking garbage, + * which is far worse to debug than an error message. + */ +export function parseRawScreencap(buf: Buffer): RawFrame { + if (buf.length < 16) { + throw new Error(`adb screencap returned ${buf.length} bytes — too short to be a framebuffer`); + } + const width = buf.readUInt32LE(0); + const height = buf.readUInt32LE(4); + if (width <= 0 || height <= 0 || width > 20000 || height > 20000) { + throw new Error(`adb screencap header looks invalid (${width}x${height})`); + } + for (const headerSize of HEADER_SIZES) { + if (buf.length - headerSize === width * height * 4) { + return { width, height, pixels: buf.subarray(headerSize) }; + } + } + throw new Error( + `Unreadable screencap framebuffer: ${buf.length} bytes for ${width}x${height} ` + + '(expected a 12- or 16-byte header followed by RGBA pixels)' + ); +} + +/** + * Device resolution, read once when the stream opens. The Kitty path never + * decodes the PNG, so this is the only source of the aspect ratio it needs to + * size the cell box with. + */ +export async function getDeviceResolution(udid: string): Promise { + const out = (await adb(udid, ['shell', 'wm', 'size'])).toString('utf-8'); + // "Physical size: 1080x2400" — optionally followed by "Override size: …", + // which is what the framebuffer actually is, so the LAST match wins. + const matches = [...out.matchAll(/(\d+)x(\d+)/g)]; + const last = matches[matches.length - 1]; + if (!last) { + throw new Error(`Could not read device resolution from: ${out.trim() || '(no adb output)'}`); + } + return { width: Number(last[1]), height: Number(last[2]) }; +} diff --git a/packages/cli/src/tui/stream/frame-loop.ts b/packages/cli/src/tui/stream/frame-loop.ts new file mode 100644 index 0000000..fc0509c --- /dev/null +++ b/packages/cli/src/tui/stream/frame-loop.ts @@ -0,0 +1,184 @@ +/** + * The capture → encode → present loop behind `/stream`. + * + * Deliberately outside React, because the two backends need different things: + * + * - halfblock frames are just text with SGR colours, so they are handed to the + * store via `onFrame` and rendered as ordinary Ink content. The capture + * interval doubles as the store's update rate — every frame is an Ink + * re-render, so it must stay coarse. + * - kitty frames are real pixels, which no component output can express. This + * module ships them to the terminal and stops there: it never places them. + * Placement is done by the U+10EEEE placeholder cells renders, + * so Ink decides where the picture goes and every Ink repaint redraws it. + * + * That split is why nothing here touches the cursor, wraps `process.stdout.write` + * or knows a terminal coordinate. + */ + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { capturePng, captureRaw } from './capture.js'; +import { renderHalfBlocks } from './halfblock.js'; +import { kittyDeleteImage, kittyTransmitVirtual } from './kitty.js'; +import { streamCells } from './layout.js'; +import type { StreamBackend } from './terminal-caps.js'; + +/** ~5fps. `adb exec-out screencap` costs ~150ms, so a shorter tick would only queue. */ +export const STREAM_FRAME_INTERVAL_MS = 200; + +/** + * Arbitrary but stable, and the number encodes into the + * foreground colour of every placeholder cell. Reusing one id is what makes + * each frame replace the last. + */ +export const KITTY_IMAGE_ID = 7301; + +export interface StreamLoopOptions { + udid: string; + deviceWidth: number; + deviceHeight: number; + backend: StreamBackend; + /** halfblock only: one ready-to-render text row per cell row, for the store. */ + onFrame(lines: string[]): void; + /** Called once, after the loop has already stopped itself. */ + onError(message: string): void; +} + +/** + * How many captures in a row may fail before the stream gives up. + * + * `screencap` fails transiently for reasons that have nothing to do with the + * stream being unviable — the compositor is mid-transition while an app opens + * or closes, or adb is briefly busy. Stopping on the first error meant closing + * an app could kill the mirror, which is exactly when you want to watch it. A + * genuinely dead device fails every tick and still stops within a second. + */ +const MAX_CONSECUTIVE_FAILURES = 4; + +interface LoopState extends StreamLoopOptions { + timer: NodeJS.Timeout; + tmpDir: string; + /** A capture is in flight — skip this tick rather than stacking adb calls. */ + busy: boolean; + /** Ping-pong index for the two PNG paths. */ + slot: 0 | 1; + /** An image is loaded in the terminal and has to be deleted on teardown. */ + transmitted: boolean; + /** Consecutive failed captures; any success resets it. */ + failures: number; +} + +let loop: LoopState | null = null; + +/** + * Whether the panel is currently on screen. TuiApp renders dialogs and other + * screens INSTEAD of MainScreen, so the placeholder cells can vanish while the + * loop is still running, and capturing for a picture nobody renders is pure + * cost. + */ +let visible = true; + +export function isStreamLoopRunning(): boolean { + return loop !== null; +} + +/** Driven by 's mount/unmount — the panel is what renders the picture. */ +export function setStreamPanelVisible(next: boolean): void { + visible = next; +} + +export function startStreamLoop(options: StreamLoopOptions): void { + stopStreamLoop(); + const state: LoopState = { + ...options, + // Placeholder: replaced immediately below. setInterval needs `state` to + // exist, and `state` needs the handle, so one of the two has to be filled + // in after construction. + timer: undefined as unknown as NodeJS.Timeout, + tmpDir: mkdtempSync(join(tmpdir(), 'appclaw-stream-')), + busy: false, + slot: 0, + transmitted: false, + failures: 0, + }; + state.timer = setInterval(() => void tick(state), STREAM_FRAME_INTERVAL_MS); + // The Ink app already keeps the process alive; this timer must not be what + // holds it open after the TUI is gone. + state.timer.unref(); + loop = state; + void tick(state); +} + +export function stopStreamLoop(): void { + const state = loop; + if (!state) return; + // Null first: an in-flight tick checks identity and bails instead of + // transmitting into a panel that is already closing. + loop = null; + clearInterval(state.timer); + if (state.transmitted) { + write(kittyDeleteImage(KITTY_IMAGE_ID)); + state.transmitted = false; + } + try { + rmSync(state.tmpDir, { recursive: true, force: true }); + } catch { + /* best effort — a leftover temp dir is harmless */ + } +} + +async function tick(state: LoopState): Promise { + if (loop !== state || state.busy || !visible) return; + state.busy = true; + try { + // Read the terminal size every tick so a resize re-fits the image without + // any resize plumbing of its own. runs the same function + // against the same numbers, so the cell box always matches the placeholders. + const cells = streamCells( + process.stdout.columns || 80, + process.stdout.rows || 24, + state.deviceWidth, + state.deviceHeight + ); + + if (state.backend === 'kitty') { + // Alternate paths: the terminal reads the file itself, asynchronously + // from our point of view, so overwriting the one it may still be reading + // would tear the frame. + state.slot = state.slot === 0 ? 1 : 0; + const file = join(state.tmpDir, `frame-${state.slot}.png`); + await capturePng(state.udid, file); + if (loop !== state) return; + write(kittyTransmitVirtual(file, { id: KITTY_IMAGE_ID, ...cells })); + state.transmitted = true; + } else { + const frame = await captureRaw(state.udid); + if (loop !== state) return; + state.onFrame(renderHalfBlocks(frame, cells.cols, cells.rows)); + } + state.failures = 0; + } catch (err) { + if (loop !== state) return; + state.failures += 1; + // Ride out a transient failure; only a sustained one means the stream is + // actually unviable (device gone, adb missing), and that fails every tick + // so it still stops promptly. + if (state.failures < MAX_CONSECUTIVE_FAILURES) return; + stopStreamLoop(); + state.onError(err instanceof Error ? err.message : String(err)); + } finally { + state.busy = false; + } +} + +/** + * Graphics commands carry no cursor movement and paint nothing, so they can be + * interleaved with Ink's frames without saving or restoring anything. + */ +function write(body: string): void { + if (!process.stdout.isTTY) return; + process.stdout.write(body); +} diff --git a/packages/cli/src/tui/stream/halfblock.ts b/packages/cli/src/tui/stream/halfblock.ts new file mode 100644 index 0000000..79a608f --- /dev/null +++ b/packages/cli/src/tui/stream/halfblock.ts @@ -0,0 +1,68 @@ +/** + * ANSI half-block renderer — the backend for terminals with no graphics + * protocol. + * + * Each cell prints U+2580 UPPER HALF BLOCK with a 24-bit foreground (the top + * pixel) over a 24-bit background (the bottom pixel), so one text cell carries + * two vertical pixels. Input is the raw RGBA framebuffer from + * `adb exec-out screencap`, which is why this needs no PNG decoder. + */ + +import type { RawFrame } from './capture.js'; + +const HALF_BLOCK = '▀'; +const RESET = '\x1b[0m'; + +/** + * Nearest-neighbour downsample of `frame` to a `cols x rows` cell grid + * (`cols x rows*2` pixels). Returns one ready-to-write string per row, + * each self-terminating with a colour reset so a short row can't bleed its + * background across the rest of the line. + * + * Colour escapes are emitted only when the colour actually changes: large flat + * regions (status bars, app backgrounds) are the common case, and repeating + * both SGR sequences per cell roughly triples the bytes written per frame. + */ +export function renderHalfBlocks(frame: RawFrame, cols: number, rows: number): string[] { + const { width, height, pixels } = frame; + const lines: string[] = []; + if (cols <= 0 || rows <= 0 || width <= 0 || height <= 0) return lines; + + // Pixel rows per cell row is 2 — the whole point of the half block. + const pixelRows = rows * 2; + + for (let r = 0; r < rows; r++) { + // Sample from the middle of each source block rather than its corner, so a + // 1px line doesn't disappear purely because it landed on a boundary. + const topY = clamp(Math.floor(((r * 2 + 0.5) * height) / pixelRows), height - 1); + const bottomY = clamp(Math.floor(((r * 2 + 1.5) * height) / pixelRows), height - 1); + const topRowOffset = topY * width; + const bottomRowOffset = bottomY * width; + + let line = ''; + let lastFg = ''; + let lastBg = ''; + for (let c = 0; c < cols; c++) { + const x = clamp(Math.floor(((c + 0.5) * width) / cols), width - 1); + const t = (topRowOffset + x) * 4; + const b = (bottomRowOffset + x) * 4; + const fg = `${pixels[t]};${pixels[t + 1]};${pixels[t + 2]}`; + const bg = `${pixels[b]};${pixels[b + 1]};${pixels[b + 2]}`; + if (fg !== lastFg) { + line += `\x1b[38;2;${fg}m`; + lastFg = fg; + } + if (bg !== lastBg) { + line += `\x1b[48;2;${bg}m`; + lastBg = bg; + } + line += HALF_BLOCK; + } + lines.push(line + RESET); + } + return lines; +} + +function clamp(value: number, max: number): number { + return value < 0 ? 0 : value > max ? max : value; +} diff --git a/packages/cli/src/tui/stream/kitty.ts b/packages/cli/src/tui/stream/kitty.ts new file mode 100644 index 0000000..c1a7bc0 --- /dev/null +++ b/packages/cli/src/tui/stream/kitty.ts @@ -0,0 +1,63 @@ +/** + * Kitty graphics protocol escape sequences. + * + * Wire format is `ESC _ G ; ESC \`, where the payload + * is always base64. Frames are sent by PATH (`t=f`, payload = the base64'd + * filename) rather than by inlining the image bytes: a 180KB PNG is ~245KB of + * base64 on every tick, and pushing that through stdout five times a second + * competes with Ink for the same pipe. + * + * Nothing here positions anything. Placement is the job of the U+10EEEE + * placeholder cells Ink renders (see placeholder.ts) — this module only hands + * the terminal the pixels those cells refer to. + */ + +const ESC = '\x1b'; +const GRAPHICS_START = `${ESC}_G`; +const GRAPHICS_END = `${ESC}\\`; + +export interface KittyPlacement { + /** Stable image id, so each frame replaces the last instead of stacking. */ + id: number; + cols: number; + rows: number; +} + +/** + * Transmit the PNG at `filePath` and declare a VIRTUAL placement (`U=1`) of + * `cols x rows` cells for it. + * + * Virtual means invisible: the terminal draws nothing, moves no cursor, and + * just remembers that image `id` is `cols x rows` cells big — which is what the + * placeholder cells then reference. `c`/`r` make the terminal do the scaling, + * which is why nothing here needs an image resizer. + * + * The virtual placement is re-declared on every frame because re-transmitting + * data for an existing id deletes the image AND all of its placements; a + * transmit that did not recreate it would leave the placeholders pointing at + * nothing. `q=2` suppresses both the OK and the error replies — they would + * otherwise arrive on stdin and be read as keystrokes by Ink. + */ +export function kittyTransmitVirtual(filePath: string, placement: KittyPlacement): string { + const payload = Buffer.from(filePath, 'utf-8').toString('base64'); + const control = [ + 'a=T', // transmit and create the placement... + 'U=1', // ...but a virtual one: nothing is drawn at the cursor + 'f=100', // payload is a PNG + 't=f', // ...referenced by file path + `i=${placement.id}`, + `c=${placement.cols}`, + `r=${placement.rows}`, + 'q=2', + ].join(','); + return `${GRAPHICS_START}${control};${payload}${GRAPHICS_END}`; +} + +/** + * Forget the image entirely. Without this the terminal holds the last frame's + * pixels for the rest of the session, and any stray placeholder cell left on + * screen would still render it. + */ +export function kittyDeleteImage(id: number): string { + return `${GRAPHICS_START}a=d,d=i,i=${id},q=2${GRAPHICS_END}`; +} diff --git a/packages/cli/src/tui/stream/layout.ts b/packages/cli/src/tui/stream/layout.ts new file mode 100644 index 0000000..3852a6e --- /dev/null +++ b/packages/cli/src/tui/stream/layout.ts @@ -0,0 +1,222 @@ +/** + * Geometry for MainScreen's two-column row. + * + * Every constant below mirrors a prop somewhere in TuiApp / MainScreen / + * CommandPalette / StreamPanel, and they live together because several of them + * have to add up: the two columns must fill the frame exactly, and the panel's + * rows must be the ones the transcript budgeted around. + * + * Note what is NOT here any more: absolute terminal coordinates. The device + * picture is placed by the Unicode placeholder cells renders, so + * Ink positions it — which is what makes it immune to `patchConsole` output + * pushing the frame down the screen. + */ + +/** TuiApp wraps every screen in `paddingX={1}`, so column 1 is never content. */ +export const APP_PADDING_COLS = 1; +/** MainScreen's outer frame box: `borderStyle="round"` — one cell on each side. */ +export const FRAME_BORDER = 1; +/** ...plus its `paddingX={1}`. */ +export const FRAME_PADDING_COLS = 1; +/**
      : title line + subtitle line + `marginBottom={1}`. */ +export const HEADER_ROWS = 3; +/** CommandPalette's `marginRight={1}` — the gutter between the two columns. */ +export const COLUMN_GAP_COLS = 1; +/** StreamPanel's own `borderStyle="round"`. */ +export const PANEL_BORDER = 1; +/** ...plus its `paddingX={1}`. */ +export const PANEL_PADDING_COLS = 1; +/** Text rows StreamPanel spends above the picture (status + device line). */ +export const PANEL_STATUS_ROWS = 2; + +/** + * Longest command list the palette shows before folding the rest into a + * "+N more" line. Lives here rather than in the component because the palette + * column's height is the floor for the whole row's height, and therefore part + * of this module's arithmetic. + */ +export const MAX_VISIBLE_COMMANDS = 10; + +/** + * Rows the palette spends no matter how many commands it lists. + * List box: border(2) + title(1) + subtitle(1) + marginTop(1) + the "+N more" + * overflow line(1). Feedback row: 1 — always rendered, blank when idle, and it + * doubles as the single-row gap before the input. Input box: border(2) + 1 line. + */ +const PALETTE_FIXED_ROWS = 2 + 1 + 1 + 1 + 1 + 1 + (2 + 1); + +/** + * : its `marginTop` + border(2) + the hint line + the message line. + * The message row is always rendered (blank when idle) so this stays constant — + * it was budgeted at 3 while rendering 4, and that single missing row made the + * whole frame over-tall, which Ink resolves by overlapping rows rather than + * clipping them (two commands printed on one line, box borders colliding). + */ +export const STATUS_BAR_ROWS = 1 + 2 + 1 + 1; + +/** + * Rows spent outside the two columns entirely: frame border(2) + header + + * StatusBar. Everything else belongs to one of the two columns. + */ +const CHROME_ROWS = 2 + HEADER_ROWS + STATUS_BAR_ROWS; +/** The transcript box's own marginTop(1) + border(2) + title(1). */ +export const TRANSCRIPT_CHROME_ROWS = 4; +/** A transcript shorter than this stops being useful. */ +const MIN_TRANSCRIPT_ROWS = 4; +/** Below this the two-column layout is nonsense anyway; clamp so the maths stays positive. */ +const MIN_CONTENT_COLS = 20; +/** + * The palette's share of the content width — the rest goes to the picture. + * + * Close to half, and that costs the stream nothing: `fitCells` derives the + * picture's width from its height and the device aspect ratio, so for a tall + * phone the rows are the binding constraint and the spare columns on the right + * would only have been blank. Giving them to the left column is what keeps + * command summaries readable instead of truncated. + */ +const LEFT_COLUMN_SHARE = 0.45; +/** Narrow enough and the command list becomes unreadable, so the split stops here. */ +const MIN_PALETTE_COLS = 34; + +/** + * Rows available to the two columns. + * + * The layout is two full-height columns, not a row of panels above a + * full-width transcript: the palette and transcript stack on the left, and the + * stream panel owns the whole right column. That is what lets the picture use + * the full height of the frame — the device screen is tall and thin, so height + * is the only dimension that makes it bigger. + */ +export function contentRows(termRows: number): number { + return Math.max(1, termRows - CHROME_ROWS); +} + +/** + * How many commands the palette can list without pushing the frame past the + * terminal. A fixed count made the chrome a fixed 33 rows, so anything shorter + * than ~36 rows rendered a frame taller than the screen — which Ink does not + * clip, it overlaps, producing two rows of text on one line. + */ +export function visibleCommandCount(termRows: number): number { + const spare = + contentRows(termRows) - MIN_TRANSCRIPT_ROWS - TRANSCRIPT_CHROME_ROWS - PALETTE_FIXED_ROWS; + return Math.max(1, Math.min(MAX_VISIBLE_COMMANDS, spare)); +} + +/** The palette's height at the top of the left column. */ +export function paletteRows(termRows: number): number { + return PALETTE_FIXED_ROWS + visibleCommandCount(termRows); +} + +/** + * Shortest terminal the layout still fits in: chrome + the smallest useful + * palette + the smallest useful transcript. Below this MainScreen shows a "too + * small" notice instead of a frame that would render corrupted. + */ +export const MIN_MAIN_ROWS = + CHROME_ROWS + PALETTE_FIXED_ROWS + 1 + TRANSCRIPT_CHROME_ROWS + MIN_TRANSCRIPT_ROWS; + +/** + * Scrolling rows in the transcript — whatever the palette above it leaves. + * Unlike before, this does not shrink when a stream starts: the transcript is + * beside the picture now, not above it, so the two no longer compete for rows. + */ +export function transcriptRows(termRows: number): number { + return Math.max( + MIN_TRANSCRIPT_ROWS, + contentRows(termRows) - paletteRows(termRows) - TRANSCRIPT_CHROME_ROWS + ); +} + +/** Rows inside StreamPanel's border that belong to the picture — the full column. */ +export function panelImageRows(termRows: number): number { + return Math.max(1, contentRows(termRows) - 2 * PANEL_BORDER - PANEL_STATUS_ROWS); +} + +/** + * Width of each column. Explicit integers rather than Ink's `width="50%"`: + * a percentage is resolved by the layout engine's own rounding, which the + * painter cannot reproduce, and two 50% columns plus a 1-cell gutter overflow + * and get shrunk by an amount that depends on the terminal width. + */ +export function columnWidths(termCols: number): { left: number; right: number } { + const content = Math.max( + MIN_CONTENT_COLS, + termCols - 2 * (APP_PADDING_COLS + FRAME_BORDER + FRAME_PADDING_COLS) + ); + // A third to the palette and transcript, two thirds to the picture. The left + // column holds text that reads fine narrow; the right holds a phone screen + // whose width follows its height, so extra columns there are not wasted. + const left = Math.max( + MIN_PALETTE_COLS, + Math.floor((content - COLUMN_GAP_COLS) * LEFT_COLUMN_SHARE) + ); + return { left, right: Math.max(1, content - COLUMN_GAP_COLS - left) }; +} + +/** Cells inside StreamPanel's border and padding that the picture may use. */ +export function panelImageCols(termCols: number): number { + const { right } = columnWidths(termCols); + return Math.max(1, right - 2 * (PANEL_BORDER + PANEL_PADDING_COLS)); +} + +/** + * Chrome between the panel's content and the right edge of the terminal: + * the panel's own padding and border, the frame's, and the app's padding. + * + * A row of placeholder cells has to redraw all of it. Ink's output buffer bills + * a placeholder (a surrogate pair) as two cells while the terminal draws it as + * one, so the write runs past the panel and overwrites whatever chrome sits to + * its right. Since the panel is the last column, the overspill lands off the + * end of the row — harmless — as long as the row itself carries the chrome it + * clobbered. + */ +export const IMAGE_ROW_TRAILING_COLS = + PANEL_PADDING_COLS + PANEL_BORDER + FRAME_PADDING_COLS + FRAME_BORDER + APP_PADDING_COLS; + +/** Display width of one placeholder row: the panel's content plus that chrome. */ +export function imageRowWidth(termCols: number): number { + return panelImageCols(termCols) + IMAGE_ROW_TRAILING_COLS; +} + +/** + * Largest `cols x rows` cell box inside `maxCols x maxRows` that preserves the + * device's aspect ratio. Terminal cells are roughly twice as tall as they are + * wide, so a cell box matching `deviceW:deviceH` needs + * `cols = rows * (deviceW/deviceH) * 2`. + */ +export function fitCells( + deviceWidth: number, + deviceHeight: number, + maxCols: number, + maxRows: number +): { cols: number; rows: number } { + if (deviceWidth <= 0 || deviceHeight <= 0) { + return { cols: Math.max(1, maxCols), rows: Math.max(1, maxRows) }; + } + const aspect = deviceWidth / deviceHeight; + let rows = Math.max(1, maxRows); + let cols = Math.round(rows * aspect * 2); + if (cols > maxCols) { + cols = Math.max(1, maxCols); + rows = Math.round(cols / (aspect * 2)); + } + return { + cols: Math.max(1, Math.min(maxCols, cols)), + rows: Math.max(1, Math.min(maxRows, rows)), + }; +} + +/** + * The cell box one frame occupies. Both sides of the stream call this — the + * loop to tell the terminal how big the image is, to know how + * many placeholder cells to render — so they cannot disagree. + */ +export function streamCells( + termCols: number, + termRows: number, + deviceWidth: number, + deviceHeight: number +): { cols: number; rows: number } { + return fitCells(deviceWidth, deviceHeight, panelImageCols(termCols), panelImageRows(termRows)); +} diff --git a/packages/cli/src/tui/stream/placeholder.ts b/packages/cli/src/tui/stream/placeholder.ts new file mode 100644 index 0000000..e415029 --- /dev/null +++ b/packages/cli/src/tui/stream/placeholder.ts @@ -0,0 +1,96 @@ +/** + * Kitty Unicode placeholders — the encoding that lets Ink, not us, decide where + * a graphics-protocol image lands. + * + * The alternative is to write the image at absolute terminal coordinates from + * outside React, which loses twice over: Ink is mounted with `patchConsole`, so + * any console line above the frame shifts every one of those coordinates; and + * Ink blanks the image's cells on every repaint, so the picture has to be + * restored after each keystroke — an erase-then-redraw that reads as flicker. + * + * With placeholders the image is bound to TEXT: a cell holding U+10EEEE with + * the image id in its foreground colour is drawn by the terminal as the + * corresponding cell of the image. Ink emits those cells as ordinary component + * output, so Ink positions the picture and every repaint re-draws it. + */ + +/** + * The placeholder character. A cell containing it renders as one cell of the + * image identified by the cell's foreground colour. + */ +export const PLACEHOLDER_CHAR = String.fromCodePoint(0x10eeee); + +/** + * Ordered diacritic table: entry N encodes the number N. + * + * Verbatim first 256 entries of kitty's `rowcolumn-diacritics.txt` + * (https://sw.kovidgoyal.net/kitty/graphics-protocol/ → "Unicode placeholders"; + * the file is generated from UnicodeData.txt 6.0.0, combining class 230). 256 is + * far more than the rows or columns any terminal panel can have, so the table is + * truncated there rather than carrying all 297 entries. + */ +export const ROWCOLUMN_DIACRITICS: readonly number[] = [ + 0x0305, 0x030d, 0x030e, 0x0310, 0x0312, 0x033d, 0x033e, 0x033f, 0x0346, 0x034a, 0x034b, 0x034c, + 0x0350, 0x0351, 0x0352, 0x0357, 0x035b, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, 0x0368, 0x0369, + 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, 0x0592, + 0x0593, 0x0594, 0x0595, 0x0597, 0x0598, 0x0599, 0x059c, 0x059d, 0x059e, 0x059f, 0x05a0, 0x05a1, + 0x05a8, 0x05a9, 0x05ab, 0x05ac, 0x05af, 0x05c4, 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, + 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065a, 0x065b, 0x065d, 0x065e, 0x06d6, 0x06d7, 0x06d8, + 0x06d9, 0x06da, 0x06db, 0x06dc, 0x06df, 0x06e0, 0x06e1, 0x06e2, 0x06e4, 0x06e7, 0x06e8, 0x06eb, + 0x06ec, 0x0730, 0x0732, 0x0733, 0x0735, 0x0736, 0x073a, 0x073d, 0x073f, 0x0740, 0x0741, 0x0743, + 0x0745, 0x0747, 0x0749, 0x074a, 0x07eb, 0x07ec, 0x07ed, 0x07ee, 0x07ef, 0x07f0, 0x07f1, 0x07f3, + 0x0816, 0x0817, 0x0818, 0x0819, 0x081b, 0x081c, 0x081d, 0x081e, 0x081f, 0x0820, 0x0821, 0x0822, + 0x0823, 0x0825, 0x0826, 0x0827, 0x0829, 0x082a, 0x082b, 0x082c, 0x082d, 0x0951, 0x0953, 0x0954, + 0x0f82, 0x0f83, 0x0f86, 0x0f87, 0x135d, 0x135e, 0x135f, 0x17dd, 0x193a, 0x1a17, 0x1a75, 0x1a76, + 0x1a77, 0x1a78, 0x1a79, 0x1a7a, 0x1a7b, 0x1a7c, 0x1b6b, 0x1b6d, 0x1b6e, 0x1b6f, 0x1b70, 0x1b71, + 0x1b72, 0x1b73, 0x1cd0, 0x1cd1, 0x1cd2, 0x1cda, 0x1cdb, 0x1ce0, 0x1dc0, 0x1dc1, 0x1dc3, 0x1dc4, + 0x1dc5, 0x1dc6, 0x1dc7, 0x1dc8, 0x1dc9, 0x1dcb, 0x1dcc, 0x1dd1, 0x1dd2, 0x1dd3, 0x1dd4, 0x1dd5, + 0x1dd6, 0x1dd7, 0x1dd8, 0x1dd9, 0x1dda, 0x1ddb, 0x1ddc, 0x1ddd, 0x1dde, 0x1ddf, 0x1de0, 0x1de1, + 0x1de2, 0x1de3, 0x1de4, 0x1de5, 0x1de6, 0x1dfe, 0x20d0, 0x20d1, 0x20d4, 0x20d5, 0x20d6, 0x20d7, + 0x20db, 0x20dc, 0x20e1, 0x20e7, 0x20e9, 0x20f0, 0x2cef, 0x2cf0, 0x2cf1, 0x2de0, 0x2de1, 0x2de2, + 0x2de3, 0x2de4, 0x2de5, 0x2de6, 0x2de7, 0x2de8, 0x2de9, 0x2dea, 0x2deb, 0x2dec, 0x2ded, 0x2dee, + 0x2def, 0x2df0, 0x2df1, 0x2df2, 0x2df3, 0x2df4, 0x2df5, 0x2df6, 0x2df7, 0x2df8, 0x2df9, 0x2dfa, + 0x2dfb, 0x2dfc, 0x2dfd, 0x2dfe, 0x2dff, 0xa66f, 0xa67c, 0xa67d, 0xa6f0, 0xa6f1, 0xa8e0, 0xa8e1, + 0xa8e2, 0xa8e3, 0xa8e4, 0xa8e5, +]; + +/** Largest row index `placeholderRow` can encode. */ +export const MAX_PLACEHOLDER_INDEX = ROWCOLUMN_DIACRITICS.length - 1; + +/** The diacritic that encodes `index`, as a string. */ +export function diacritic(index: number): string { + const codePoint = ROWCOLUMN_DIACRITICS[index]; + if (codePoint === undefined) { + throw new RangeError(`No row/column diacritic for index ${index}`); + } + return String.fromCodePoint(codePoint); +} + +/** + * One row of an image, as `cols` placeholder cells. + * + * Only the first cell carries diacritics (its row, then column 0). The protocol + * lets the rest inherit: a placeholder with no diacritics takes its row and its + * image id from the cell on its left and adds one to that cell's column — which + * holds as long as the whole run shares one foreground colour, and it does, + * because a single paints it. + */ +export function placeholderRow(row: number, cols: number): string { + if (cols <= 0) return ''; + return PLACEHOLDER_CHAR + diacritic(row) + diacritic(0) + PLACEHOLDER_CHAR.repeat(cols - 1); +} + +/** + * The image id as a hex colour, for Ink's `color` prop. + * + * The terminal reads the id out of the cell's foreground colour, so the id has + * to survive the trip through Ink and chalk as 24-bit RGB — which caps usable + * ids at 0xFFFFFF and needs a truecolor-capable terminal (chalk downgrading to + * 256 colours would hand the terminal a different number). + */ +export function imageIdColor(id: number): string { + if (!Number.isInteger(id) || id <= 0 || id > 0xffffff) { + throw new RangeError(`Image id ${id} does not fit a 24-bit foreground colour`); + } + return `#${id.toString(16).padStart(6, '0')}`; +} diff --git a/packages/cli/src/tui/stream/terminal-caps.ts b/packages/cli/src/tui/stream/terminal-caps.ts new file mode 100644 index 0000000..698b8af --- /dev/null +++ b/packages/cli/src/tui/stream/terminal-caps.ts @@ -0,0 +1,33 @@ +/** + * Picks the in-terminal image backend for `/stream`. + * + * Detection is env-based rather than a terminal capability query: the Kitty + * graphics protocol's "are you there" probe is a round trip that answers on + * stdin, and stdin is already held in raw mode by Ink's input handling — the + * reply would be delivered as keystrokes instead, or swallow the user's. + */ + +export type StreamBackend = 'kitty' | 'halfblock'; + +/** + * `kitty` for terminals that implement the Kitty graphics protocol (real + * pixels), `halfblock` for everything else (24-bit colour text, always + * available). `APPCLAW_STREAM_BACKEND` forces one, mainly so the fallback path + * can be exercised on a terminal that supports graphics. + */ +export function detectStreamBackend(env: NodeJS.ProcessEnv = process.env): StreamBackend { + const override = env.APPCLAW_STREAM_BACKEND?.trim().toLowerCase(); + if (override === 'kitty' || override === 'halfblock') return override; + + // Ghostty and WezTerm implement the protocol but don't advertise it in TERM. + if (env.TERM_PROGRAM?.toLowerCase() === 'ghostty') return 'kitty'; + if (env.WEZTERM_EXECUTABLE) return 'kitty'; + if (env.KITTY_WINDOW_ID) return 'kitty'; + if ((env.TERM ?? '').toLowerCase().includes('kitty')) return 'kitty'; + + return 'halfblock'; +} + +export function backendLabel(backend: StreamBackend): string { + return backend === 'kitty' ? 'kitty graphics' : 'ANSI half-blocks'; +} diff --git a/packages/cli/src/tui/useLayout.ts b/packages/cli/src/tui/useLayout.ts new file mode 100644 index 0000000..d894de9 --- /dev/null +++ b/packages/cli/src/tui/useLayout.ts @@ -0,0 +1,26 @@ +import { useStdout } from 'ink'; + +/** + * Rows left for scrollable content after `reserved` fixed rows (header, + * status bar, borders). Ink has no alt-screen concept of its own — without + * bounding content to the real terminal height, a tall frame just scrolls + * the terminal, and once a frame scrolls off it can never be redrawn again + * (that's what produced the duplicated stacked frames in scrollback). + */ +export function useAvailableRows(reserved: number): number { + const { stdout } = useStdout(); + const rows = stdout.rows || 24; + return Math.max(3, rows - reserved); +} + +/** Slice a list to a scrolling window that always keeps `selected` in view. `start` is the original index of `items[0]`, for re-deriving each row's true index. */ +export function windowFor( + items: T[], + selected: number, + maxVisible: number +): { items: T[]; start: number } { + if (items.length <= maxVisible) return { items, start: 0 }; + let start = Math.max(0, selected - maxVisible + 1); + start = Math.min(start, items.length - maxVisible); + return { items: items.slice(start, start + maxVisible), start }; +} diff --git a/packages/cli/src/ui/ink/PlaygroundApp.tsx b/packages/cli/src/ui/ink/PlaygroundApp.tsx deleted file mode 100644 index be1dade..0000000 --- a/packages/cli/src/ui/ink/PlaygroundApp.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import React, { useState, useSyncExternalStore } from 'react'; -import { Box, Text, useApp } from 'ink'; -import TextInput from 'ink-text-input'; -import { COLORS, symbols } from './theme.js'; -import { OrbitalSpinner } from './components/OrbitalSpinner.js'; -import { PlaygroundBottomBar, type PlaygroundInfo } from './components/PlaygroundBottomBar.js'; -import { subscribe, getSnapshot } from './playground-store.js'; - -export interface PlaygroundAppProps { - info: PlaygroundInfo; - /** Execute one REPL line (slash command or natural instruction). */ - onCommand: (line: string) => Promise; - /** Tear down the session before exit. */ - onQuit: () => Promise; - /** Current recorded step count (read after each command). */ - getStepCount: () => number; - /** Notify when the line count changes so the prompt counter updates. */ - refreshStepCount: () => void; -} - -const QUIT = new Set(['/quit', '/exit', '/q']); - -/** - * Ink playground REPL shell. Owns raw-mode stdin via (no readline), - * pins a styled prompt + status bar at the bottom, and lets command output - * scroll above it via Ink's patchConsole. - */ -export function PlaygroundApp({ info, onCommand, onQuit, refreshStepCount }: PlaygroundAppProps) { - const { exit } = useApp(); - const ui = useSyncExternalStore(subscribe, getSnapshot); - const [value, setValue] = useState(''); - - async function submit(raw: string): Promise { - const line = raw.trim(); - setValue(''); - if (!line) return; - if (ui.processing) return; - - const { pgStore } = await import('./playground-store.js'); - - // Quit handling with unsaved-steps confirmation. - if (QUIT.has(line)) { - if (ui.stepCount > 0 && !ui.pendingQuit) { - pgStore.setPendingQuit(true); - return; - } - await onQuit(); - exit(); - return; - } - - if (ui.pendingQuit) pgStore.setPendingQuit(false); - - pgStore.setProcessing(true); - try { - await onCommand(line); - } catch (err) { - console.log(` ${symbols.cross} ${err instanceof Error ? err.message : String(err)}`); - } finally { - pgStore.clearStatus(); - pgStore.setProcessing(false); - refreshStepCount(); - } - } - - return ( - - {ui.processing ? ( - - - - {' '} - {ui.status || 'Working…'} - - {ui.detail ? · {ui.detail} : null} - - ) : ( - - - {symbols.prompt}{' '} - - - - )} - - - - ); -} diff --git a/packages/cli/src/ui/ink/components/PlaygroundBottomBar.tsx b/packages/cli/src/ui/ink/components/PlaygroundBottomBar.tsx deleted file mode 100644 index 15c0c0e..0000000 --- a/packages/cli/src/ui/ink/components/PlaygroundBottomBar.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import React from 'react'; -import { Box, Text, useStdout } from 'ink'; -import { COLORS, symbols } from '../theme.js'; - -export interface PlaygroundInfo { - platform: string; - app?: string; - model: string; - mode: string; - transport: string; -} - -interface Props extends PlaygroundInfo { - stepCount: number; - pendingQuit: boolean; -} - -const Sep = () => │; - -/** - * Pinned status bar for the playground REPL — session context on one line, - * key hints below. Inspired by the kane-cli BottomBar. - */ -export function PlaygroundBottomBar({ - platform, - app, - model, - mode, - transport, - stepCount, - pendingQuit, -}: Props) { - const { stdout } = useStdout(); - const width = Math.min(stdout?.columns ?? 80, 80); - - return ( - - {'─'.repeat(width)} - - ● - {platform} - - model - {model} - - mode - {mode} - - mcp - - {transport === 'sse' ? 'remote' : 'local'} - - {app ? ( - <> - - app - {app} - - ) : null} - - steps - - {stepCount} - - - - {pendingQuit ? ( - - {symbols.warning} unsaved steps — /export to save, or /quit again to discard - - ) : ( - - ↵ run · /help · /yaml · /preview · /export <file> · /undo · /quit - - )} - - - ); -} diff --git a/packages/cli/src/ui/ink/playground-runner.tsx b/packages/cli/src/ui/ink/playground-runner.tsx deleted file mode 100644 index d4ed6b9..0000000 --- a/packages/cli/src/ui/ink/playground-runner.tsx +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Ink playground launcher. Mounts the shell and registers a - * minimal renderer override so the agent's ANSI spinner (used by processLine) - * becomes status-bar text instead of fighting Ink for the cursor. All other - * command output stays on console.log and renders above via patchConsole. - */ -import React from 'react'; -import { render } from 'ink'; -import { PlaygroundApp } from './PlaygroundApp.js'; -import type { PlaygroundInfo } from './components/PlaygroundBottomBar.js'; -import { pgStore, getSnapshot } from './playground-store.js'; -import { setRenderer, type UIRenderer } from '@appclaw/core/ui/renderer'; - -const pgRenderer: Partial = { - startSpinner(message, detail) { - pgStore.setStatus(message, detail); - }, - updateSpinner(message, detail) { - const s = getSnapshot(); - pgStore.setStatus(message ?? s.status, detail ?? s.detail); - }, - stopSpinner() { - pgStore.clearStatus(); - }, - // streaming is unused in the playground — swallow to avoid ANSI writes - startStreaming() {}, - streamChunk() {}, - stopStreaming() {}, -}; - -export function runPlaygroundInk(opts: { - info: PlaygroundInfo; - onCommand: (line: string) => Promise; - onQuit: () => Promise; - getStepCount: () => number; -}): Promise { - pgStore.reset(); - pgStore.setStepCount(opts.getStepCount()); - setRenderer(pgRenderer); - - const instance = render( - pgStore.setStepCount(opts.getStepCount())} - />, - { patchConsole: true, exitOnCtrlC: true } - ); - - return instance.waitUntilExit().finally(() => setRenderer(null)); -} diff --git a/packages/cli/src/ui/ink/playground-store.ts b/packages/cli/src/ui/ink/playground-store.ts deleted file mode 100644 index 80c760a..0000000 --- a/packages/cli/src/ui/ink/playground-store.ts +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Playground REPL store — small observable state for the Ink playground shell. - * - * The playground reuses all existing command output (help, tables, step - * results) via console.log + Ink's patchConsole; this store only carries the - * live status bar / input state that the Ink shell renders at the bottom. - */ - -export interface PlaygroundUIState { - /** A command is executing — input is hidden, spinner shown. */ - processing: boolean; - /** Spinner label while processing (fed by the spinner renderer override). */ - status: string; - detail?: string; - /** Recorded step count (drives the prompt counter). */ - stepCount: number; - /** /quit pressed with unsaved steps — awaiting confirmation. */ - pendingQuit: boolean; -} - -function initial(): PlaygroundUIState { - return { processing: false, status: '', stepCount: 0, pendingQuit: false }; -} - -let state: PlaygroundUIState = initial(); -const listeners = new Set<() => void>(); - -function emit(): void { - for (const l of listeners) l(); -} -function set(patch: Partial): void { - state = { ...state, ...patch }; - emit(); -} - -export function subscribe(l: () => void): () => void { - listeners.add(l); - return () => listeners.delete(l); -} -export function getSnapshot(): PlaygroundUIState { - return state; -} - -export const pgStore = { - reset(): void { - state = initial(); - emit(); - }, - setStatus(status: string, detail?: string): void { - set({ status, detail }); - }, - clearStatus(): void { - set({ status: '', detail: undefined }); - }, - setProcessing(processing: boolean): void { - set({ processing }); - }, - setStepCount(stepCount: number): void { - set({ stepCount }); - }, - setPendingQuit(pendingQuit: boolean): void { - set({ pendingQuit }); - }, -}; diff --git a/packages/core/src/agent/app-resolver.ts b/packages/core/src/agent/app-resolver.ts index ca832c9..e311269 100644 --- a/packages/core/src/agent/app-resolver.ts +++ b/packages/core/src/agent/app-resolver.ts @@ -142,7 +142,7 @@ export class AppResolver { * Re-fetch the installed-app list from the device and rebuild the lookup. * * The list is otherwise cached at init time, so an app installed *after* the - * session started (the common playground case: launch → sideload app → "open X") + * session started (the common step-recorder case: launch → sideload app → "open X") * would never resolve. Callers use this to retry once on a resolution miss. * Returns true if the list was refreshed, false if no device handle is available. */ diff --git a/packages/core/src/agent/loop.ts b/packages/core/src/agent/loop.ts index f09359a..7bbe5fd 100644 --- a/packages/core/src/agent/loop.ts +++ b/packages/core/src/agent/loop.ts @@ -1027,7 +1027,7 @@ async function executeMetaTool( ): Promise { /** * Scale LLM-provided 0-1000 normalized coordinates to device space. - * Uses the same scaleCoordinates() from df-vision that the playground uses — + * Uses the same scaleCoordinates() from df-vision that the step recorders use — * guarantees identical coordinate handling across both paths. * * Note: df-vision convention is [y, x] order for coordinates. @@ -1073,7 +1073,7 @@ async function executeMetaTool( const skipFastTapCoords = shouldPreferVisionLocateTap(selector); // Fast path: LLM provided 0-1000 normalized coordinates — skip vision locate entirely - // Uses same scaleCoordinates() from df-vision as the playground + // Uses same scaleCoordinates() from df-vision as the step recorders if (tapX != null && tapY != null && !skipFastTapCoords) { const scaled = await scaleLLMCoords(tapX, tapY); const tapped = await tapAtCoordinates(mcp, scaled.x, scaled.y); diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 35a13ed..3e18624 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -85,12 +85,17 @@ const envSchema = z.object({ LOG_DIR: z.string().default('logs'), /** - * Default directory for exported SDK test specs (from `--export` and the - * playground's `/export .test.ts`). Bare filenames land here; paths - * with a directory component (e.g. `./tests/foo.test.ts` or `/abs/path`) - * are used verbatim. Override per-run via the `--export-dir` CLI flag. + * Default directory for exported test specs (from `--export` and `/export + * .test.ts`). Bare filenames land here; paths with a directory + * component (e.g. `./tests/foo.test.ts` or `/abs/path`) are used verbatim. + * Override per-run via the `--export-dir` CLI flag. + * + * Defaults to the runner's own `testDir` so an export is runnable where it + * lands: `appclaw-runner` discovers specs under `testDir` and takes filters, + * not paths, so a file written outside it could never be run without being + * moved first. */ - EXPORT_DIR: z.string().default('.appclaw/exports'), + EXPORT_DIR: z.string().default('tests'), /** Gemini API key for Stark vision (optional if GEMINI_API_KEY is set). */ STARK_VISION_API_KEY: z.string().default(''), diff --git a/packages/core/src/device/emulator-list.ts b/packages/core/src/device/emulator-list.ts new file mode 100644 index 0000000..3c5f028 --- /dev/null +++ b/packages/core/src/device/emulator-list.ts @@ -0,0 +1,107 @@ +/** + * Direct device/emulator discovery for UI surfaces that need a device list + * BEFORE an Appium/MCP session exists (the TUI's device picker, `appclaw + * doctor`). Shells out to `adb` / `xcrun simctl` directly — no MCP round + * trip — mirroring the checks in `appclaw doctor` but returning structured + * data instead of printing. + * + * For the actual Appium session, `discoverAndSelectDevice` in + * device-picker.ts remains the source of truth (it also locks the choice in + * via the `select_device` MCP tool). This module is read-only. + */ + +import { exec as execCallback } from 'node:child_process'; +import { promisify } from 'node:util'; +import { homedir } from 'node:os'; +import { delimiter, join } from 'node:path'; +import { listIOSSimulators } from './ios-simulators.js'; + +const exec = promisify(execCallback); + +export interface RunningDevice { + name: string; + udid: string; + /** "device" | "offline" | "unauthorized" | "Booted" | "Shutdown" */ + state: string; + platform: 'android' | 'ios'; + hint?: string; +} + +/** List Android devices/emulators via `adb devices -l`. Empty array if adb is unavailable. */ +export async function listAndroidDevices(): Promise { + // adb is commonly not on the user's interactive PATH — mirror the augmented + // PATH mcp/client.ts builds for the appium-mcp subprocess, so the picker + // sees the same devices the agent will. + const androidHome = + process.env.ANDROID_HOME || + process.env.ANDROID_SDK_ROOT || + join(homedir(), 'Library', 'Android', 'sdk'); + let stdout: string; + try { + ({ stdout } = await exec('adb devices -l', { + timeout: 10_000, + env: { + ...process.env, + PATH: [ + join(androidHome, 'platform-tools'), + join(androidHome, 'emulator'), + process.env.PATH ?? '', + ].join(delimiter), + }, + })); + } catch { + return []; + } + + // Filter structurally rather than dropping line 0: adb can prepend daemon + // startup notices ("* daemon not running; starting now…"), which would push + // the real header down and turn "List of devices attached" into a phantom + // device entry. + const lines = stdout + .split('\n') + .map((l) => l.trim()) + .filter(Boolean) + .filter((l) => !l.startsWith('*') && !/^(List of devices|adb server version)/i.test(l)); + + const devices: RunningDevice[] = []; + for (const line of lines) { + const parts = line.split(/\s+/); + if (parts.length < 2) continue; + const [serial, ...rest] = parts; + // Two-word states exist, e.g. "no permissions (user not in plugdev group)". + const state = rest[0] === 'no' && rest.length > 1 ? `${rest[0]} ${rest[1]}` : rest[0]; + const modelMatch = line.match(/model:(\S+)/); + const name = modelMatch ? modelMatch[1].replace(/_/g, ' ') : serial; + devices.push({ name, udid: serial, state, platform: 'android' }); + } + return devices; +} + +/** List iOS simulators via `xcrun simctl`. Empty array on non-macOS or if Xcode CLT is missing. */ +export async function listIOSDevices(): Promise { + if (process.platform !== 'darwin') return []; + try { + const sims = await listIOSSimulators(); + return sims.map((s) => ({ + name: s.name, + udid: s.udid, + state: s.state, + platform: 'ios' as const, + hint: `iOS ${s.iosVersion}`, + })); + } catch { + return []; + } +} + +/** List devices for one platform, booted/online first. */ +export async function listRunningDevices(platform: 'android' | 'ios'): Promise { + const devices = platform === 'android' ? await listAndroidDevices() : await listIOSDevices(); + const isUp = (d: RunningDevice) => + d.state.toLowerCase() === 'booted' || d.state.toLowerCase() === 'device'; + return [...devices].sort((a, b) => { + const upDiff = Number(isUp(b)) - Number(isUp(a)); + if (upDiff !== 0) return upDiff; + return a.name.localeCompare(b.name); + }); +} diff --git a/packages/core/src/device/index.ts b/packages/core/src/device/index.ts index 64e4628..5842ffe 100644 --- a/packages/core/src/device/index.ts +++ b/packages/core/src/device/index.ts @@ -30,7 +30,7 @@ export interface DeviceSetupArgs { config: AppClawConfig; /** * Always show the device picker even when a single device is available or the platform - * is pre-selected. Used by playground mode so the user always gets to choose a device. + * is pre-selected. Used by the step-recorder surfaces so the user always picks a device. */ alwaysPickDevice?: boolean; /** diff --git a/packages/core/src/flow/run-instruction.ts b/packages/core/src/flow/run-instruction.ts index adf835e..47138c2 100644 --- a/packages/core/src/flow/run-instruction.ts +++ b/packages/core/src/flow/run-instruction.ts @@ -1,9 +1,9 @@ /** * Shared "run one natural-language instruction" pipeline. * - * Both the SDK's `StepRunner` and the playground's per-line handler used to + * Both the SDK's `StepRunner` and the CLI's per-line step recorders used to * implement this pipeline inline. That duplication was the cause of bugs - * where one surface (e.g. playground) supported a behaviour that the other + * where one surface (e.g. the TUI) supported a behaviour that the other * (e.g. SDK) silently lacked — most recently, element-targeted swipes. * * Now both surfaces call `runOneInstruction()` and add only their own UI / @@ -45,7 +45,7 @@ import { locatorCacheStorage, type LocatorCacheCtx } from '../sdk/locator-cache. /** * Minimum matchScore (1-10) for a vision tap to be considered "found". Below * this, the tap is rejected even if Gemini returned coordinates. Shared by - * SDK + playground so both surfaces agree on what counts as a real match. + * SDK + step recorders so every surface agrees on what counts as a real match. */ export const DEFAULT_MIN_MATCH_SCORE = 4; @@ -101,7 +101,7 @@ export async function runOneInstruction( // through every `await` via AsyncLocalStorage so helpers in run-yaml-flow.ts // can read it without us having to thread a new parameter through // executeStep + 6 helper signatures. `undefined` is the normal case for - // non-SDK callers (playground, YAML, replayer) → no behavior change. + // non-SDK callers (TUI, YAML, replayer) → no behavior change. return locatorCacheStorage.run(options?.locatorCache, () => runOneInstructionInner(mcp, instruction, options) ); diff --git a/packages/core/src/flow/run-yaml-flow.ts b/packages/core/src/flow/run-yaml-flow.ts index 16e602f..5b432ed 100644 --- a/packages/core/src/flow/run-yaml-flow.ts +++ b/packages/core/src/flow/run-yaml-flow.ts @@ -876,7 +876,7 @@ function isVisionMode(): boolean { // ─── SDK locator cache helpers ────────────────────────────────────────────── // // These run only when an SDK caller opted into `locatorCache`. For YAML-flow, -// playground, and goal-agent paths the active context is always undefined and +// step-recorder, and goal-agent paths the active context is always undefined and // every helper below short-circuits — behavior unchanged. /** Build a cache key from the current page source + the action being performed. */ @@ -2267,7 +2267,7 @@ async function executeStepUnredacted( scroll?: ScrollControl ): Promise { // Vision mode: natural language steps (verbatim set) use visionExecute with the original - // instruction — same path as the playground — for precise context-aware element location. + // instruction — same path as the step recorders — for precise context-aware element location. // Explicit YAML steps (no verbatim) fall through to the normal DOM/vision-locate paths. if ( isVisionMode() && @@ -2373,7 +2373,7 @@ async function executeStepUnredacted( } else { // No name: close the current foreground app. Read it from the live page // source (its package), falling back to session state — so this works even - // when the locator cache isn't active (playground / SDK). + // when the locator cache isn't active (step recorder / SDK). const pageSource = await getPageSource(mcp).catch(() => ''); pkg = extractAppIdFromDom(pageSource) ?? diff --git a/packages/core/src/flow/vision-execute.ts b/packages/core/src/flow/vision-execute.ts index 43b84f7..a59e67e 100644 --- a/packages/core/src/flow/vision-execute.ts +++ b/packages/core/src/flow/vision-execute.ts @@ -1,5 +1,5 @@ /** - * Hybrid single-call vision executor for the playground. + * Hybrid single-call vision executor for the step recorders. * * In vision mode (AGENT_MODE=vision), takes a screenshot + raw instruction * and sends ONE LLM call to Stark Vision's `understandAndLocate()`. @@ -460,7 +460,7 @@ export interface VisionExecuteResult { /** * How directly the user's instruction described the located element (1–10). * Comes from the LLM's matchScore in the combinedInstructionPrompt response. - * Low score = instruction was vague/sloppy → playground should refine the YAML verbatim. + * Low score = instruction was vague/sloppy → the caller should refine the YAML verbatim. * High score = instruction was intentional/accurate → keep as-is. */ matchScore?: number; diff --git a/packages/core/src/sdk/goal-export.ts b/packages/core/src/sdk/goal-export.ts index 6163fd1..654e375 100644 --- a/packages/core/src/sdk/goal-export.ts +++ b/packages/core/src/sdk/goal-export.ts @@ -4,7 +4,7 @@ * After `app.runGoal(goal)` finishes, the agent leaves behind a `history` of * tool-call decisions (find_and_click, find_and_type, launch_app, …). This * module translates those decisions back into the natural-language form that - * `app.run(...)` accepts, then renders a complete vitest spec file. + * `app.run(...)` accepts, then renders a complete @appclaw/runner spec file. * * Why translate back to natural language instead of dumping raw tool calls? * The SDK's `app.run()` is the supported public surface — `find_and_click` is @@ -20,13 +20,16 @@ export interface GenerateSdkTestConfig { provider?: string; platform?: string; agentMode?: string; - /** Module path used in the generated `import { AppClaw } from '<...>'`. Default: 'appclaw'. */ + /** Module path for the generated `import { test } from '<...>'`. Default: '@appclaw/runner'. */ sdkImport?: string; - /** `describe(...)` block title. Default: 'Goal replay'. */ + /** Prefixed onto the test title when set (e.g. from `/meta name`). */ describeName?: string; - /** `it(...)` test title. Default: the goal text. */ + /** `test(...)` title. Default: derived from the recorded steps. */ testName?: string; - /** vitest timeout in ms. Default: 120000. */ + /** + * Unused. The runner takes its per-test timeout from `appclaw.config.ts` or + * `--timeout`, so a literal in the spec would be ignored. + */ timeoutMs?: number; } @@ -105,7 +108,7 @@ export function keepOnlyFinalAttempt(history: StepRecord[]): StepRecord[] { } /** - * Build a descriptive `it(...)` test name from the recorded instructions. + * Build a descriptive `test(...)` name from the recorded instructions. * Prefers anchors that tell a story: the launched app + the final user action, * e.g. "launches YouTube and taps search icon". Falls back to "executes * recorded steps" when the trajectory doesn't have a clean anchor. @@ -148,9 +151,9 @@ export function instructionsFromHistory(history: StepRecord[]): string[] { } /** - * Render a complete vitest spec file that replays the agent's trajectory via - * `app.run(...)` calls. The output is ready to write to disk and run with - * `vitest run path/to/file`. + * Render a complete @appclaw/runner spec that replays the agent's trajectory + * via `app.run(...)` calls. Write it under the runner's `testDir` and it is + * picked up by `appclaw test`. */ export function generateSdkTest(opts: { goal: string; @@ -166,9 +169,9 @@ export function generateSdkTest(opts: { } /** - * Render a vitest spec from a flat list of natural-language instructions. + * Render a runner spec from a flat list of natural-language instructions. * - * Used by the playground (`/export some.test.ts`) where steps are already in + * Used by `/export some.test.ts` where steps are already in * `app.run()`-ready form — no agent history to translate. The optional `goal` * is purely for the header comment; pass empty string if there's no concept * of a goal (interactive sessions). @@ -193,32 +196,36 @@ function renderSdkTest(opts: { }): string { const cfg = opts.config ?? {}; const { instructions, goal } = opts; - const sdkImport = cfg.sdkImport ?? 'appclaw'; - const describeName = cfg.describeName ?? 'Recorded flow'; + const runnerImport = cfg.sdkImport ?? '@appclaw/runner'; // Default test name describes what the steps DO, not what the goal asked for. // The original goal is preserved in the file header comment for traceability. const testName = cfg.testName ?? defaultTestNameFromSteps(instructions); - const timeoutMs = cfg.timeoutMs ?? 120000; + // The runner has no `describe` requirement, so the group name (when the user + // set one via /meta name) is folded into the title rather than wrapping the + // file in a block that exists only to hold one test. + const describeName = cfg.describeName ?? ''; + const testTitle = + describeName && describeName !== 'Recorded flow' ? `${describeName} — ${testName}` : testName; const agentStepsUsed = opts.agentStepsUsed; - const optionLines: string[] = []; - if (cfg.provider) optionLines.push(` provider: ${JSON.stringify(cfg.provider)},`); - optionLines.push(` apiKey: process.env.LLM_API_KEY,`); - if (cfg.platform) optionLines.push(` platform: ${JSON.stringify(cfg.platform)},`); - if (cfg.agentMode) optionLines.push(` agentMode: ${JSON.stringify(cfg.agentMode)},`); + // Provider, apiKey and agentMode are not written into the spec any more: the + // runner owns the session, and those belong in appclaw.config.ts where one + // change covers every test. Platform is the exception — it selects which + // devices a test is eligible for, so it stays per-test. + const testOptions = cfg.platform ? `{ platform: ${JSON.stringify(cfg.platform)} }, ` : ''; - const runLines = instructions.map((i) => ` await app.run(${JSON.stringify(i)});`); + const runLines = instructions.map((i) => ` await app.run(${JSON.stringify(i)});`); const fromAgent = agentStepsUsed !== undefined; const goalLine = goal.trim() ? `Original goal: ${goal.replace(/\*\//g, '*\\/')}` - : 'Source: AppClaw playground'; + : 'Source: AppClaw recording session'; const stepsLine = fromAgent ? `Steps recorded: ${instructions.length} (from ${agentStepsUsed} agent step${agentStepsUsed === 1 ? '' : 's'})` : `Steps recorded: ${instructions.length}`; // Caveats vary by source: agent-mode exports inherit non-determinism from the - // LLM trajectory; playground exports are user-validated steps and only need a + // LLM trajectory; recorded exports are user-validated steps and only need a // brief reminder that selectors may drift across UI revisions. const caveats = fromAgent ? [ @@ -238,19 +245,19 @@ function renderSdkTest(opts: { ' cross-check against the AgentResult.history.', '', '4. Edit freely. Treat this file as a draft: rename the test, add assertions', - ' (`app.verify(...)`), tighten selectors, split into multiple `it()` blocks.', + ' (`app.verify(...)`), tighten selectors, split into multiple `test()` blocks.', ] : [ - '1. Each step is the verbatim text you typed in the playground — exactly', - ' what `AppClaw.run()` will receive. There is no translator in between,', - ' so the replay should behave identically to the playground session.', + '1. Each step is the verbatim text you typed — exactly what `app.run()`', + ' will receive. There is no translator in between, so the replay should', + ' behave identically to the session it was recorded in.', '', '2. Selectors are still locator strings, not stable IDs. If the app UI', ' changes (icons relabelled, layout shifts), the natural-language', ' selectors above may need updating.', '', '3. Edit freely. Treat this file as a draft: rename the test, add', - ' assertions (`app.verify(...)`), split into multiple `it()` blocks.', + ' assertions (`app.verify(...)`), split into multiple `test()` blocks.', ]; const caveatBlock = caveats.map((l) => ` * ${l}`.replace(/ +$/, '')).join('\n'); @@ -265,20 +272,10 @@ function renderSdkTest(opts: { * ${caveatBlock} */ -import { AppClaw } from ${JSON.stringify(sdkImport)}; -import { describe, it } from 'vitest'; -import 'dotenv/config'; - -describe(${JSON.stringify(describeName)}, () => { - it(${JSON.stringify(testName)}, async () => { - const app = new AppClaw({ -${optionLines.join('\n')} - }); +import { test } from ${JSON.stringify(runnerImport)}; +test(${JSON.stringify(testTitle)}, ${testOptions}async ({ app }) => { ${runLines.join('\n')} - - await app.teardown(); - }, ${timeoutMs}); }); `; } diff --git a/packages/core/src/sdk/index.ts b/packages/core/src/sdk/index.ts index ecc027b..1a09590 100644 --- a/packages/core/src/sdk/index.ts +++ b/packages/core/src/sdk/index.ts @@ -106,7 +106,7 @@ export class AppClaw { // `silent` controls per-step log lines (✓ #1 tap "label" ...). Default is // FALSE — most SDK consumers want to see what's happening on the device, - // matching the playground's UX. Pass `silent: true` for a quiet test run. + // matching the TUI's UX. Pass `silent: true` for a quiet test run. // The legacy --json-mode terminal suppression still runs when JSON mode is // enabled regardless, so JSON consumers stay clean. this.silent = options.silent === true; @@ -181,7 +181,7 @@ export class AppClaw { /** * Execute a single natural-language instruction on the device. * - * Equivalent to the playground's per-command execution: the instruction is + * Equivalent to the TUI's per-command execution: the instruction is * interpreted (regex → LLM fallback) and executed immediately as one step. * Each call is captured as a step in the auto-generated report. * diff --git a/packages/core/src/sdk/step-runner.ts b/packages/core/src/sdk/step-runner.ts index 8bdc2ff..ad25b54 100644 --- a/packages/core/src/sdk/step-runner.ts +++ b/packages/core/src/sdk/step-runner.ts @@ -3,7 +3,7 @@ * adapts the result for the SDK's report + console-print conventions. * * The execution pipeline itself (vision-first → regex → LLM → executeStep) - * lives in `src/flow/run-instruction.ts` and is shared with the playground. + * lives in `src/flow/run-instruction.ts` and is shared with the CLI's step recorders. * This module's job is only: * - mark the step start on the report collector * - record a step row + screenshot in the report @@ -41,7 +41,7 @@ export class StepRunner { /** * When true, suppress the per-step `✓ #N tap "label"` log line. * Default false — SDK consumers want to see what's happening on the device, - * matching the playground's visibility. Pass true to silence (e.g. for noisy + * matching the TUI's visibility. Pass true to silence (e.g. for noisy * CI logs where the test framework already reports per-test outcomes). */ private readonly silent: boolean = false, @@ -69,7 +69,7 @@ export class StepRunner { setPreActionCapture(!!this.collector); // All "instruction → step → executed" logic lives in runOneInstruction so - // the SDK and playground stay in lockstep. See src/flow/run-instruction.ts. + // the SDK and the step recorders stay in lockstep. See src/flow/run-instruction.ts. const { step, result } = await runOneInstruction(this.mcp, instruction, { appResolver: this.appResolver, tapPoll: this.tapPoll, diff --git a/packages/core/src/sdk/types.ts b/packages/core/src/sdk/types.ts index 08261e3..f9b25f9 100644 --- a/packages/core/src/sdk/types.ts +++ b/packages/core/src/sdk/types.ts @@ -109,7 +109,7 @@ export interface AppClawOptions { /** * Suppress per-step log lines (`✓ #1 tap "search icon"` etc.). * Defaults to `false` — SDK consumers see device activity by default, - * matching the playground's UX. Set `true` for a quiet test run where the + * matching the TUI's UX. Set `true` for a quiet test run where the * surrounding framework (vitest/jest) already reports per-test outcomes. */ silent?: boolean; diff --git a/packages/core/src/ui/step-printer.ts b/packages/core/src/ui/step-printer.ts index d77a9fc..c8d99e1 100644 --- a/packages/core/src/ui/step-printer.ts +++ b/packages/core/src/ui/step-printer.ts @@ -1,5 +1,5 @@ /** - * Per-step result printer shared by the playground and the SDK. + * Per-step result printer shared by the CLI's step recorders and the SDK. * * Renders a single executed step in the compact two-line form: * @@ -7,7 +7,7 @@ * ● Tapped "search icon" at [432, 421] * * Extracted here so callers don't have to reimplement the formatting. The - * playground uses it for interactive REPL feedback; `src/sdk/step-runner.ts` + * CLI uses it for interactive step feedback; `src/sdk/step-runner.ts` * uses it so SDK consumers see what's happening on the device without having * to enable verbose logging. */ @@ -121,7 +121,7 @@ export function stepTarget(step: FlowStep): string { } /** - * Print one step's result in the two-line compact form used by the playground + * Print one step's result in the two-line compact form used by the step recorders * and the SDK. Goes to stdout — no spinners, no progress bars, safe for CI logs. */ export function printStepResult( diff --git a/packages/core/src/ui/terminal.ts b/packages/core/src/ui/terminal.ts index b8b804a..bc750de 100644 --- a/packages/core/src/ui/terminal.ts +++ b/packages/core/src/ui/terminal.ts @@ -576,7 +576,8 @@ export function printInteractiveHeader(): void { `${theme.info('--record')} ${theme.muted('Record actions for replay')}`, `${theme.info('--replay')} ${theme.muted('Replay a recorded flow')}`, `${theme.info('--flow')} ${theme.muted('Run steps from a YAML file')}`, - `${theme.info('--playground')} ${theme.muted('Build YAML flows interactively')}`, + `${theme.info('--tui')} ${theme.muted('Terminal Studio: record steps, mirror device')}`, + `${theme.info('--playground')} ${theme.muted('Alias for --tui')}`, `${theme.info('--plan')} ${theme.muted('Decompose complex goals')}`, `${theme.info('--explore')} ${theme.muted('Generate flows from a PRD')}`, ].join('\n'); diff --git a/tests/sdk/goal-export.test.ts b/tests/sdk/goal-export.test.ts new file mode 100644 index 0000000..8b6bfc5 --- /dev/null +++ b/tests/sdk/goal-export.test.ts @@ -0,0 +1,76 @@ +/** + * The generated spec is a user-facing artifact — it is what `/export` and + * `--export` write to disk, and it has to be runnable by `appclaw-runner` + * without edits. Nothing covered its shape before, which is how it kept + * emitting vitest boilerplate after the runner became the target. + */ +import { describe, expect, test } from 'vitest'; +import { generateSdkTest, generateSdkTestFromInstructions } from '@appclaw/core/sdk/goal-export'; +import type { AgentResult } from '@appclaw/core/agent/loop'; + +const INSTRUCTIONS = ['open youtube', 'click on search', 'type hello']; + +function spec(config?: Parameters[0]['config']): string { + return generateSdkTestFromInstructions({ instructions: INSTRUCTIONS, config }); +} + +describe('generated spec targets @appclaw/runner', () => { + test('imports the runner, not vitest or the SDK class', () => { + const out = spec(); + expect(out).toContain(`import { test } from "@appclaw/runner"`); + expect(out).not.toContain('vitest'); + expect(out).not.toContain('new AppClaw'); + // The runner owns the session, so the spec must not manage its lifecycle. + expect(out).not.toContain('teardown'); + expect(out).not.toContain('dotenv'); + }); + + test('uses the runner call shape: test(title, async ({ app }) => …)', () => { + const out = spec(); + expect(out).toMatch(/test\(".*", async \(\{ app \}\) => \{/); + for (const instruction of INSTRUCTIONS) { + expect(out).toContain(`await app.run(${JSON.stringify(instruction)});`); + } + // No describe wrapper around a single test. + expect(out).not.toContain('describe('); + }); + + test('platform rides along as a test option, not constructor config', () => { + const out = spec({ platform: 'android' }); + expect(out).toContain('{ platform: "android" }'); + // Credentials and mode belong in appclaw.config.ts, not in every spec. + expect(out).not.toContain('apiKey'); + expect(out).not.toContain('agentMode'); + }); + + test('omits the option object entirely when no platform is known', () => { + expect(spec()).toMatch(/test\(".*", async \(/); + }); + + test('a /meta name is folded into the title rather than wrapping in describe', () => { + const out = spec({ describeName: 'Login flow' }); + expect(out).toContain('Login flow — '); + expect(out).not.toContain('describe('); + }); + + test('agent exports keep the goal in the header and translate history', () => { + const result = { + success: true, + reason: 'done', + stepsUsed: 2, + history: [ + { decision: { toolName: 'launch_app', args: { appName: 'YouTube' } }, result: 'ok' }, + { + decision: { toolName: 'find_and_click', args: { selector: 'search icon' } }, + result: 'ok', + }, + ], + } as unknown as AgentResult; + + const out = generateSdkTest({ goal: 'search youtube', result }); + expect(out).toContain('Original goal: search youtube'); + expect(out).toContain('await app.run("open YouTube app");'); + expect(out).toContain('await app.run("tap search icon");'); + expect(out).toContain(`import { test } from "@appclaw/runner"`); + }); +}); diff --git a/tests/ui/command-palette.test.tsx b/tests/ui/command-palette.test.tsx new file mode 100644 index 0000000..93288ec --- /dev/null +++ b/tests/ui/command-palette.test.tsx @@ -0,0 +1,82 @@ +/** + * The palette's height is budgeted exactly by stream/layout.ts, so its row + * count must not depend on what it is showing. Both regressions it has had were + * this: a feedback row that only rendered when there was an error, and command + * summaries long enough to wrap onto a second line. + */ +import { describe, expect, test } from 'vitest'; +import { render } from 'ink-testing-library'; +import React from 'react'; +import { CommandPalette } from '@appclaw/cli/tui/components/CommandPalette'; +import { columnWidths } from '@appclaw/cli/tui/stream/layout'; + +const ANSI = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g'); + +function rowsOf(element: React.ReactElement): string[] { + const { lastFrame } = render(element); + return (lastFrame() ?? '').replace(ANSI, '').replace(/\n$/, '').split('\n'); +} + +function palette(overrides: Partial> = {}) { + return ( + {}} + onSubmit={() => {}} + disabled={false} + error={null} + focused + {...overrides} + /> + ); +} + +describe('CommandPalette', () => { + test('the row count does not change when a message appears', () => { + const quiet = rowsOf(palette()); + const noisy = rowsOf(palette({ error: 'Unknown command: /qutd — try /help' })); + expect(noisy.length).toBe(quiet.length); + }); + + test('the message sits above the prompt, not below it', () => { + const rows = rowsOf(palette({ error: 'Unknown command: /qutd — try /help' })); + const message = rows.findIndex((r) => r.includes('Unknown command')); + const prompt = rows.findIndex((r) => r.includes('Type a step')); + expect(message).toBeGreaterThanOrEqual(0); + expect(prompt).toBeGreaterThanOrEqual(0); + // Below the input the message landed against the frame's bottom edge. + expect(message).toBeLessThan(prompt); + }); + + test('content it cannot control never costs it a row', () => { + // A very long message, at the narrowest column the layout allows and a + // roomy one: neither may wrap, because the column's height is budgeted. + for (const cols of [80, 120, 200]) { + const width = columnWidths(cols).left; + const baseline = rowsOf(palette({ width })).length; + expect(rowsOf(palette({ width, error: 'a'.repeat(400) })).length).toBe(baseline); + expect(rowsOf(palette({ width, query: '/x'.repeat(200) })).length).toBe(baseline); + } + }); + + test('never renders wider than its column', () => { + for (const cols of [80, 120, 200]) { + const width = columnWidths(cols).left; + const rows = rowsOf(palette({ width, error: 'a'.repeat(400) })); + expect(Math.max(...rows.map((r) => [...r].length))).toBeLessThanOrEqual(width); + } + }); + + test('a long busy message truncates instead of wrapping', () => { + const rows = rowsOf(palette({ disabled: true, busyText: 'Executing '.repeat(40) })); + expect(rows.length).toBe(rowsOf(palette()).length); + }); + + test('losing focus does not change its height', () => { + // Focus only re-colours the border and stops the input consuming keys; if + // it changed the row count it would resize the column underneath it. + expect(rowsOf(palette({ focused: false })).length).toBe(rowsOf(palette()).length); + }); +}); diff --git a/tests/ui/complete-command.test.ts b/tests/ui/complete-command.test.ts new file mode 100644 index 0000000..7395d71 --- /dev/null +++ b/tests/ui/complete-command.test.ts @@ -0,0 +1,42 @@ +/** + * Tab completion for the TUI prompt. + * + * The interesting cases are the ambiguous ones: `/stream` is also a prefix of + * `/stream-close`, and `/exit` is an alias of `/quit` that collides with + * `/export` — so "complete to the first match" would guess wrong more often + * than not. + */ +import { describe, expect, test } from 'vitest'; +import { completeCommand } from '@appclaw/cli/tui/commands'; + +describe('completeCommand', () => { + test('a unique prefix completes and gains a trailing space', () => { + expect(completeCommand('/exp')).toBe('/export '); + expect(completeCommand('/mem')).toBe('/memory '); + expect(completeCommand('/g')).toBe('/goal '); + }); + + test('an ambiguous prefix extends only as far as every candidate agrees', () => { + // /stream and /stream-close — no trailing space, since the word is not + // finished. + expect(completeCommand('/st')).toBe('/stream'); + }); + + test('nothing to add returns null rather than a no-op edit', () => { + expect(completeCommand('/stream')).toBeNull(); // already at the shared prefix + expect(completeCommand('/ex')).toBeNull(); // /export vs /exit + expect(completeCommand('/zzz')).toBeNull(); // no match + }); + + test('completes the alias that was typed, not the canonical name', () => { + // /co is a prefix of /config, an alias of /settings. Completing to + // "/settings" would replace what was typed instead of extending it. + expect(completeCommand('/co')).toBe('/config '); + }); + + test('leaves plain instructions and command arguments alone', () => { + expect(completeCommand('tap on Login')).toBeNull(); + expect(completeCommand('/export flow')).toBeNull(); + expect(completeCommand('')).toBeNull(); + }); +}); diff --git a/tests/ui/ink-ui.test.tsx b/tests/ui/ink-ui.test.tsx index d26f80b..025262a 100644 --- a/tests/ui/ink-ui.test.tsx +++ b/tests/ui/ink-ui.test.tsx @@ -10,9 +10,12 @@ import { RunScreen, tailSlice } from '@appclaw/cli/ui/ink/RunScreen'; import type { TimelineEntry, JourneySummaryData } from '@appclaw/cli/ui/ink/store'; import { store } from '@appclaw/cli/ui/ink/store'; import { askUserViaInk } from '@appclaw/cli/ui/ink/InkRenderer'; -import { PlaygroundApp } from '@appclaw/cli/ui/ink/PlaygroundApp'; -import { pgStore } from '@appclaw/cli/ui/ink/playground-store'; import { FinalSummary } from '@appclaw/cli/ui/ink/components/FinalSummary'; +import { Box, Text } from 'ink'; +import stringWidth from 'string-width'; +import { StreamPanel } from '@appclaw/cli/tui/components/StreamPanel'; +import { columnWidths, panelImageRows } from '@appclaw/cli/tui/stream/layout'; +import { PLACEHOLDER_CHAR } from '@appclaw/cli/tui/stream/placeholder'; const tick = (ms = 30) => new Promise((r) => setTimeout(r, ms)); @@ -214,72 +217,6 @@ describe('Ink RunScreen', () => { }); }); -describe('Ink PlaygroundApp', () => { - test('renders prompt with label + step count, runs a command', async () => { - pgStore.reset(); - let steps = 0; - const ran: string[] = []; - const { lastFrame, stdin, unmount } = render( - { - ran.push(line); - steps += 1; - }} - onQuit={async () => {}} - getStepCount={() => steps} - refreshStepCount={() => pgStore.setStepCount(steps)} - /> - ); - pgStore.setStepCount(0); - await tick(); - expect(lastFrame()).toContain('android'); - expect(lastFrame()).toContain('steps 0'); - - stdin.write('tap on Login'); - await tick(); - stdin.write('\r'); - await tick(60); - - expect(ran).toEqual(['tap on Login']); - expect(lastFrame()).toContain('steps 1'); - unmount(); - }); - - test('quit requires confirmation when steps are unsaved', async () => { - pgStore.reset(); - let quit = false; - const { stdin, unmount } = render( - {}} - onQuit={async () => { - quit = true; - }} - getStepCount={() => 3} - refreshStepCount={() => {}} - /> - ); - pgStore.setStepCount(3); - await tick(); - - // first /quit → confirmation, not quit yet - stdin.write('/quit'); - await tick(); - stdin.write('\r'); - await tick(40); - expect(quit).toBe(false); - - // second /quit → quits - stdin.write('/quit'); - await tick(); - stdin.write('\r'); - await tick(40); - expect(quit).toBe(true); - unmount(); - }); -}); - describe('Ink FinalSummary', () => { const journeyData = ( overrides: Partial = {}, @@ -335,3 +272,76 @@ describe('Ink FinalSummary', () => { unmount(); }); }); + +/** + * The whole point of Unicode placeholders is that Ink lays the picture out, so + * the thing worth pinning down is that a row of them still measures like text. + * + * It nearly doesn't: Ink's output buffer bills each placeholder (a surrogate + * pair) as two cells while the terminal draws it as one, so a placeholder row + * runs past the panel and eats the borders to its right. redraws + * them; this asserts the redraw lands in the right column. + */ +describe('StreamPanel unicode placeholders', () => { + const termCols = 100; // ink-testing-library's stdout width + const termRows = 24; + + function frameOf(stream: unknown) { + const { left, right } = columnWidths(termCols); + const imageRows = panelImageRows(termRows, true); + const { lastFrame, unmount } = render( + + + + + {Array.from({ length: imageRows + 2 }, (_, i) => ( + palette + ))} + + + + + + ); + const out = lastFrame() ?? ''; + unmount(); + return out; + } + + const live = { + status: 'running', + backend: 'kitty', + resolution: { width: 1080, height: 2400 }, + }; + + test('every rendered row is the same display width, borders included', () => { + const lines = frameOf(live).split('\n'); + const widths = new Set(lines.map((line) => stringWidth(line))); + expect(widths.size).toBe(1); + }); + + test('the rows carry one addressed placeholder cell per image row', () => { + const out = frameOf(live); + const rows = out.split('\n').filter((line) => line.includes(PLACEHOLDER_CHAR)); + expect(rows.length).toBeGreaterThan(0); + // Row 0's leading cell: placeholder + row diacritic 0 + column diacritic 0. + expect(rows[0]).toContain(`${PLACEHOLDER_CHAR}̅̅`); + // Colour is asserted in tui-stream.test.ts — chalk emits none under vitest. + }); + + test('a half-block stream still renders as plain text rows', () => { + const out = frameOf({ + status: 'running', + backend: 'halfblock', + resolution: { width: 1080, height: 2400 }, + frameLines: ['▀▀▀▀'], + }); + expect(out).not.toContain(PLACEHOLDER_CHAR); + expect(out).toContain('▀▀▀▀'); + }); +}); diff --git a/tests/ui/main-screen-layout.test.tsx b/tests/ui/main-screen-layout.test.tsx new file mode 100644 index 0000000..09a8f57 --- /dev/null +++ b/tests/ui/main-screen-layout.test.tsx @@ -0,0 +1,76 @@ +/** + * The whole-frame layout test. + * + * Ink does not clip a frame taller than the terminal — it overlaps rows, so one + * unbudgeted row shows up as two commands printed on one line, box borders + * colliding, and the first line of a box silently vanishing. Component-level + * tests could not catch that: every piece was individually correct and the + * chrome constant was wrong. This renders the real screen at real sizes and + * asserts the frame closes properly. + */ +import { describe, expect, test } from 'vitest'; +import { render } from 'ink'; +import React from 'react'; +import { PassThrough } from 'node:stream'; +import { MainScreen } from '@appclaw/cli/tui/screens/MainScreen'; +import { tuiStore } from '@appclaw/cli/tui/store'; +import type { TuiActions } from '@appclaw/cli/tui/commands'; + +const ANSI = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g'); +const actions = new Proxy({}, { get: () => async () => {} }) as TuiActions; + +async function frameAt(columns: number, rows: number): Promise { + const frames: string[] = []; + const stdout = new PassThrough() as unknown as NodeJS.WriteStream; + (stdout as unknown as { write: (s: string) => boolean }).write = (s: string) => { + frames.push(s); + return true; + }; + Object.defineProperty(stdout, 'columns', { value: columns }); + Object.defineProperty(stdout, 'rows', { value: rows }); + Object.assign(stdout, { isTTY: true }); + + const stdin = new PassThrough() as unknown as NodeJS.ReadStream; + Object.assign(stdin, { isTTY: true, setRawMode: () => stdin, ref: () => {}, unref: () => {} }); + + tuiStore.reset(); + tuiStore.goTo('main'); + const instance = render(, { + stdout, + stdin, + patchConsole: false, + exitOnCtrlC: false, + }); + await new Promise((r) => setTimeout(r, 40)); + const painted = frames.filter((f) => f.includes('╭')); + instance.unmount(); + return (painted[0] ?? '').replace(ANSI, '').replace(/\n$/, '').split('\n'); +} + +describe('MainScreen frame', () => { + for (const [cols, rows] of [ + [200, 46], + [120, 40], + [100, 30], + ] as const) { + test(`${cols}x${rows}: fills the terminal without overlapping`, async () => { + const frame = await frameAt(cols, rows); + expect(frame.length).toBeLessThanOrEqual(rows); + + // Both titles survive: an over-tall frame overwrites the first line of + // whichever boxes overflow, and these are the two that showed it. + expect(frame.some((l) => l.includes('AppClaw'))).toBe(true); + expect(frame.some((l) => l.includes('Command palette'))).toBe(true); + + // The first command keeps its own row rather than merging with the next. + const goal = frame.find((l) => l.includes('/goal')); + expect(goal).toBeDefined(); + expect(goal).not.toContain('/list'); + + // The outer frame's bottom border is alone on its row — when the frame + // overflowed it shared one with the input box's border ("╰─╰───"). + const closing = frame.filter((l) => l.trimStart().startsWith('╰')); + expect(closing.every((l) => !l.replace(/[╰─╯\s]/g, ''))).toBe(true); + }); + } +}); diff --git a/tests/ui/output-dialog.test.tsx b/tests/ui/output-dialog.test.tsx new file mode 100644 index 0000000..f986fc8 --- /dev/null +++ b/tests/ui/output-dialog.test.tsx @@ -0,0 +1,158 @@ +/** + * The dialog's height is derived, not assumed — and it has been wrong twice: + * once for forgetting the box's own paddingY, once for a status line that + * wrapped. Both showed up the same way, as a bottom border pushed off the + * screen, so these assert the box actually closes rather than checking the + * arithmetic that produces it. + */ +import { describe, expect, test } from 'vitest'; +import { render } from 'ink-testing-library'; +import React from 'react'; +import { OutputDialog } from '@appclaw/cli/tui/components/OutputDialog'; + +const ANSI = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g'); + +function frameOf(element: React.ReactElement): string[] { + const { lastFrame } = render(element); + return (lastFrame() ?? '').replace(ANSI, '').replace(/\n$/, '').split('\n'); +} + +const BODY = Array.from({ length: 41 }, (_, i) => `line ${i + 1} of a generated spec`); + +/** The real /export payload: a two-line status whose first line is long enough to wrap. */ +const EXPORT_STATUS = { + color: '#22C55E', + text: + 'Saved to /Users/someone/Documents/git/AppClaw/tests/flow-1787647975314.test.ts\n' + + 'Run: appclaw test flow-1787647975314', +}; + +describe('OutputDialog', () => { + const cases: Array<[string, React.ReactElement]> = [ + [ + 'export: subtitle + wrapping two-line status', + {}} + />, + ], + [ + 'doctor: subtitle + single-line status', + {}} + />, + ], + [ + 'yaml: no status at all', + {}} + />, + ], + [ + 'no subtitle, no status', + {}} />, + ], + ]; + + for (const [name, element] of cases) { + test(`${name} — the box opens and closes inside the terminal`, () => { + const lines = frameOf(element); + expect(lines.filter((l) => l.includes('╭'))).toHaveLength(1); + expect(lines.filter((l) => l.includes('╰'))).toHaveLength(1); + // ink-testing-library reports no stdout.rows, so the component falls back + // to 24 — the shortest terminal we claim to support here. + expect(lines.length).toBeLessThanOrEqual(24); + }); + } + + test('an over-wide body line wraps in full without bursting the box', () => { + // Captured console output is sized to the terminal, not to this much + // narrower box. Truncating it loses the detail the dialog exists to show + // (doctor's config summary ended in an ellipsis), but wrapping costs extra + // rows the height budget has to know about — or the bottom border is + // pushed off screen. Both halves are asserted here. + const wide = 'x'.repeat(400); + const lines = frameOf( + {}} /> + ); + expect(lines.filter((l) => l.includes('╰'))).toHaveLength(1); + expect(lines.length).toBeLessThanOrEqual(24); + // Every one of the 400 characters survives, spread across wrapped rows. + const xs = lines.join('').match(/x/g)?.length ?? 0; + expect(xs).toBe(400); + expect(lines.some((l) => l.includes('…'))).toBe(false); + }); + + test('continuation rows hang two columns past the line they belong to', () => { + // Without this a wrapped doctor line resumed at the box's left edge, so the + // remainder read like a new entry rather than the tail of the one above. + const long = ` ✓ ${'word '.repeat(40).trim()}`; + const rows = frameOf( + {}} /> + ) + // Strip the border so the measurement is relative to the box interior. + .filter((l) => l.includes('│')) + .map((l) => l.slice(l.indexOf('│') + 1)) + .filter((l) => l.trim()); + + const first = rows.findIndex((l) => l.includes('✓')); + expect(first).toBeGreaterThanOrEqual(0); + const indentOf = (l: string) => l.length - l.trimStart().length; + // The line itself is indented 2; its continuations sit at 2 + 2. + expect(indentOf(rows[first + 1])).toBe(indentOf(rows[first]) + 2); + // And it really did continue rather than start something new. + expect(rows[first + 1].trimStart().startsWith('word')).toBe(true); + }); + + test('an unbreakable run is hard-broken rather than overflowing', () => { + const path = ` ✓ /Users/someone/${'a/'.repeat(60)}file.json`; + const rows = frameOf( + {}} /> + ); + expect(rows.filter((l) => l.includes('╰'))).toHaveLength(1); + // Every row is the same width — nothing punched out through the border. + const widths = new Set( + rows.filter((l) => l.includes('│') || l.includes('╰')).map((l) => l.length) + ); + expect(widths.size).toBe(1); + }); + + test('scrolling accounts for wrapped rows, not just line count', () => { + // 12 lines that each wrap to 3 rows cannot all fit a 24-row terminal, so + // the dialog must offer to scroll even though 12 < the viewport in lines. + const fat = Array.from({ length: 12 }, (_, i) => `${i}:${'y'.repeat(200)}`); + const lines = frameOf( + {}} /> + ); + expect(lines.length).toBeLessThanOrEqual(24); + expect(lines.filter((l) => l.includes('╰'))).toHaveLength(1); + expect(lines.join('\n')).toContain('scroll'); + }); + + test('a long body scrolls rather than growing the box', () => { + const short = frameOf( + {}} /> + ); + const long = frameOf( + {}} /> + ); + expect(long.length).toBeLessThanOrEqual(24); + expect(long.length).toBeGreaterThanOrEqual(short.length); + expect(long.join('\n')).toContain('scroll'); + }); +}); diff --git a/tests/ui/tui-stream.test.ts b/tests/ui/tui-stream.test.ts new file mode 100644 index 0000000..103a4f2 --- /dev/null +++ b/tests/ui/tui-stream.test.ts @@ -0,0 +1,302 @@ +/** + * In-terminal device stream — the pure pieces behind `/stream`. + * + * Everything covered here is deliberately free of adb, React and stdout, which + * is why the capture/encode logic lives in its own modules rather than inside + * the Ink component. + */ +import { describe, test, expect } from 'vitest'; +import { parseRawScreencap } from '@appclaw/cli/tui/stream/capture'; +import { renderHalfBlocks } from '@appclaw/cli/tui/stream/halfblock'; +import { kittyTransmitVirtual, kittyDeleteImage } from '@appclaw/cli/tui/stream/kitty'; +import { + APP_PADDING_COLS, + COLUMN_GAP_COLS, + FRAME_BORDER, + FRAME_PADDING_COLS, + IMAGE_ROW_TRAILING_COLS, + PANEL_BORDER, + PANEL_PADDING_COLS, + PANEL_STATUS_ROWS, + columnWidths, + contentRows, + fitCells, + imageRowWidth, + paletteRows, + panelImageCols, + panelImageRows, + streamCells, + transcriptRows, + TRANSCRIPT_CHROME_ROWS, +} from '@appclaw/cli/tui/stream/layout'; +import { + diacritic, + imageIdColor, + MAX_PLACEHOLDER_INDEX, + PLACEHOLDER_CHAR, + placeholderRow, + ROWCOLUMN_DIACRITICS, +} from '@appclaw/cli/tui/stream/placeholder'; +import { detectStreamBackend } from '@appclaw/cli/tui/stream/terminal-caps'; + +/** A screencap payload: little-endian w/h/format (+ optional colorspace), then RGBA. */ +function framebuffer(width: number, height: number, headerSize: 12 | 16): Buffer { + const header = Buffer.alloc(headerSize); + header.writeUInt32LE(width, 0); + header.writeUInt32LE(height, 4); + header.writeUInt32LE(1, 8); + const pixels = Buffer.alloc(width * height * 4); + for (let i = 0; i < width * height; i++) { + pixels[i * 4] = i; // R varies per pixel so downsampling is observable + pixels[i * 4 + 1] = 10; + pixels[i * 4 + 2] = 20; + pixels[i * 4 + 3] = 255; + } + return Buffer.concat([header, pixels]); +} + +describe('parseRawScreencap', () => { + test('reads a 12-byte header (older Android)', () => { + const frame = parseRawScreencap(framebuffer(4, 3, 12)); + expect(frame.width).toBe(4); + expect(frame.height).toBe(3); + expect(frame.pixels.length).toBe(4 * 3 * 4); + }); + + test('reads a 16-byte header (colorspace field present)', () => { + const frame = parseRawScreencap(framebuffer(4, 3, 16)); + expect(frame.width).toBe(4); + expect(frame.pixels.length).toBe(4 * 3 * 4); + }); + + test('rejects a payload matching neither header size instead of rendering garbage', () => { + const bad = Buffer.concat([framebuffer(4, 3, 12), Buffer.alloc(7)]); + expect(() => parseRawScreencap(bad)).toThrow(/Unreadable screencap framebuffer/); + }); + + test('rejects a nonsense header', () => { + const bad = Buffer.alloc(64); // width/height both 0 + expect(() => parseRawScreencap(bad)).toThrow(/looks invalid/); + }); +}); + +describe('renderHalfBlocks', () => { + test('emits one line per cell row, each ending in a colour reset', () => { + const frame = parseRawScreencap(framebuffer(8, 8, 12)); + const lines = renderHalfBlocks(frame, 4, 3); + expect(lines).toHaveLength(3); + for (const line of lines) { + expect(line.endsWith('\x1b[0m')).toBe(true); + // One upper-half-block per column — two source pixels per cell. + expect([...line].filter((c) => c === '▀')).toHaveLength(4); + } + }); + + test('carries 24-bit foreground and background colours', () => { + const frame = parseRawScreencap(framebuffer(8, 8, 12)); + const [first] = renderHalfBlocks(frame, 4, 3); + expect(first).toMatch(/\x1b\[38;2;\d+;\d+;\d+m/); + expect(first).toMatch(/\x1b\[48;2;\d+;\d+;\d+m/); + }); + + test('repeats a colour escape only when the colour actually changes', () => { + // A uniform frame should cost exactly one fg + one bg escape per row. + const width = 4; + const height = 4; + const header = Buffer.alloc(12); + header.writeUInt32LE(width, 0); + header.writeUInt32LE(height, 4); + const flat = Buffer.alloc(width * height * 4, 0x40); + const frame = parseRawScreencap(Buffer.concat([header, flat])); + const [line] = renderHalfBlocks(frame, 4, 2); + expect(line.match(/\x1b\[38;2;/g)).toHaveLength(1); + expect(line.match(/\x1b\[48;2;/g)).toHaveLength(1); + }); +}); + +describe('kitty escape sequences', () => { + test('transmits by path, base64-encoded, sized in cells', () => { + const seq = kittyTransmitVirtual('/tmp/frame-0.png', { id: 7, cols: 40, rows: 50 }); + expect(seq.startsWith('\x1b_G')).toBe(true); + expect(seq.endsWith('\x1b\\')).toBe(true); + const [control, payload] = seq.slice(3, -2).split(';'); + expect(control).toContain('a=T'); + expect(control).toContain('f=100'); + expect(control).toContain('t=f'); + expect(control).toContain('i=7'); + expect(control).toContain('c=40'); + expect(control).toContain('r=50'); + // q=2 suppresses the terminal's reply, which would otherwise land on the + // stdin Ink is reading in raw mode. + expect(control).toContain('q=2'); + expect(Buffer.from(payload, 'base64').toString('utf-8')).toBe('/tmp/frame-0.png'); + }); + + test('the placement is virtual, so nothing is drawn at the cursor', () => { + // U=1 is the whole point: the image is only a prototype for the U+10EEEE + // cells Ink renders. Without it the terminal would paint at the cursor, + // which is exactly the absolute-coordinate painting this replaced. + const seq = kittyTransmitVirtual('/tmp/frame-0.png', { id: 7, cols: 4, rows: 5 }); + expect(seq.slice(3, -2).split(';')[0]).toContain('U=1'); + // No cursor movement anywhere in the sequence. + expect(seq).not.toMatch(/\x1b\[\d+;\d+H/); + }); + + test('deletes the image by id', () => { + expect(kittyDeleteImage(7)).toBe('\x1b_Ga=d,d=i,i=7,q=2\x1b\\'); + }); +}); + +describe('unicode placeholders', () => { + const D = (n: number) => String.fromCodePoint(ROWCOLUMN_DIACRITICS[n]!); + + test('the placeholder is U+10EEEE', () => { + expect(PLACEHOLDER_CHAR.codePointAt(0)).toBe(0x10eeee); + expect([...PLACEHOLDER_CHAR]).toHaveLength(1); + }); + + test('the diacritic table starts where kitty says it does', () => { + // From rowcolumn-diacritics.txt: 0 -> U+0305, 1 -> U+030D, 2 -> U+030E. + expect(diacritic(0)).toBe('̅'); + expect(diacritic(1)).toBe('̍'); + expect(diacritic(2)).toBe('̎'); + expect(ROWCOLUMN_DIACRITICS).toHaveLength(256); + expect(new Set(ROWCOLUMN_DIACRITICS).size).toBe(256); + }); + + test('a row is one diacritic-carrying cell followed by bare placeholders', () => { + // kitty's own example: `\U10EEEE\U0305\U0305` is row 0, column 0. + expect(placeholderRow(0, 1)).toBe(`${PLACEHOLDER_CHAR}̅̅`); + expect(placeholderRow(3, 4)).toBe( + `${PLACEHOLDER_CHAR}${D(3)}${D(0)}${PLACEHOLDER_CHAR.repeat(3)}` + ); + }); + + test('every row has exactly one placeholder per cell', () => { + for (const cols of [1, 2, 17, 64]) { + const row = placeholderRow(5, cols); + expect([...row].filter((c) => c === PLACEHOLDER_CHAR)).toHaveLength(cols); + // Only the leading cell is addressed; the rest inherit row and column. + expect([...row].filter((c) => ROWCOLUMN_DIACRITICS.includes(c.codePointAt(0)!))).toHaveLength( + 2 + ); + } + expect(placeholderRow(0, 0)).toBe(''); + }); + + test('rows past the table are refused rather than silently mis-addressed', () => { + expect(() => placeholderRow(MAX_PLACEHOLDER_INDEX, 1)).not.toThrow(); + expect(() => placeholderRow(MAX_PLACEHOLDER_INDEX + 1, 1)).toThrow(RangeError); + }); + + test('the image id round-trips through a 24-bit foreground colour', () => { + const id = 7301; + expect(imageIdColor(id)).toBe('#001c85'); + const [, r, g, b] = /^#(..)(..)(..)$/.exec(imageIdColor(id))!; + expect((parseInt(r!, 16) << 16) | (parseInt(g!, 16) << 8) | parseInt(b!, 16)).toBe(id); + // 0 is not a valid image id, and anything past 24 bits cannot be carried. + expect(() => imageIdColor(0)).toThrow(RangeError); + expect(() => imageIdColor(0x1000000)).toThrow(RangeError); + }); +}); + +describe('layout', () => { + test('fitCells preserves the device aspect over ~1:2 cells', () => { + // 1080x2400 in a tall, wide area: height-bound, so cols ≈ rows * 0.45 * 2. + const { cols, rows } = fitCells(1080, 2400, 200, 50); + expect(rows).toBe(50); + expect(cols).toBe(45); + }); + + test('fitCells falls back to width when the area is too narrow', () => { + const { cols, rows } = fitCells(1080, 2400, 20, 50); + expect(cols).toBe(20); + expect(rows).toBeLessThanOrEqual(50); + expect(rows).toBe(22); + }); + + test('the two columns fill the frame exactly, with one gutter cell', () => { + const termCols = 120; + const { left, right } = columnWidths(termCols); + const chrome = 2 * (APP_PADDING_COLS + FRAME_BORDER + FRAME_PADDING_COLS); + expect(left + COLUMN_GAP_COLS + right).toBe(termCols - chrome); + }); + + test('the image area is the inside of the right panel', () => { + const { right } = columnWidths(120); + expect(panelImageCols(120)).toBe(right - 2 * (PANEL_BORDER + PANEL_PADDING_COLS)); + }); + + test('a placeholder row reaches from the panel content to the terminal edge', () => { + // Panel content start + this width must be exactly the terminal width, or + // the chrome the row redraws would land in the wrong column. + const { left, right } = columnWidths(120); + const contentStart = + APP_PADDING_COLS + + FRAME_BORDER + + FRAME_PADDING_COLS + + left + + COLUMN_GAP_COLS + + PANEL_BORDER + + PANEL_PADDING_COLS; + expect(contentStart + imageRowWidth(120)).toBe(120); + expect(imageRowWidth(120)).toBe(panelImageCols(120) + IMAGE_ROW_TRAILING_COLS); + expect(right).toBeGreaterThan(0); + }); + + test('streamCells fits the device into the panel, in cells only', () => { + const cells = streamCells(120, 40, 1080, 2400); + expect(cells).toEqual(fitCells(1080, 2400, panelImageCols(120), panelImageRows(40, true))); + // No absolute coordinates survive: Ink positions the picture now. + expect(Object.keys(cells).sort()).toEqual(['cols', 'rows']); + }); + + test('the picture gets the full content height, independent of the palette', () => { + for (const termRows of [30, 40, 60]) { + // The columns are independent now: the panel spans the whole content + // area rather than matching the palette's height. + expect(panelImageRows(termRows)).toBe( + contentRows(termRows) - 2 * PANEL_BORDER - PANEL_STATUS_ROWS + ); + expect(panelImageRows(termRows)).toBeGreaterThan(paletteRows(termRows) - PANEL_STATUS_ROWS); + } + }); + + test('the left column fills the content height exactly', () => { + for (const termRows of [30, 40, 60, 80]) { + const used = paletteRows(termRows) + TRANSCRIPT_CHROME_ROWS + transcriptRows(termRows); + // Palette + transcript must fill their column exactly, or the frame + // over/under-runs the terminal — Ink overlaps rows rather than clipping. + expect(used).toBe(contentRows(termRows)); + } + }); + + test('the palette column never gets so narrow the command list is unreadable', () => { + for (const termCols of [80, 120, 200, 400]) { + const { left, right } = columnWidths(termCols); + expect(left).toBeGreaterThanOrEqual(34); + // The picture takes the larger share on any reasonably wide terminal. + if (termCols >= 120) expect(right).toBeGreaterThan(left); + } + }); +}); + +describe('detectStreamBackend', () => { + test('kitty for Ghostty, kitty, WezTerm', () => { + expect(detectStreamBackend({ TERM_PROGRAM: 'ghostty' })).toBe('kitty'); + expect(detectStreamBackend({ KITTY_WINDOW_ID: '1' })).toBe('kitty'); + expect(detectStreamBackend({ TERM: 'xterm-kitty' })).toBe('kitty'); + expect(detectStreamBackend({ WEZTERM_EXECUTABLE: '/usr/bin/wezterm' })).toBe('kitty'); + }); + + test('half-blocks for everything else', () => { + expect(detectStreamBackend({ TERM: 'xterm-256color' })).toBe('halfblock'); + expect(detectStreamBackend({})).toBe('halfblock'); + }); + + test('APPCLAW_STREAM_BACKEND overrides detection', () => { + expect( + detectStreamBackend({ TERM_PROGRAM: 'ghostty', APPCLAW_STREAM_BACKEND: 'halfblock' }) + ).toBe('halfblock'); + }); +}); diff --git a/vscode-extension/README.md b/vscode-extension/README.md index 27a6192..faafd84 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -46,15 +46,15 @@ AI-powered mobile automation agent — control Android & iOS devices from VS Cod ## Commands -| Command | Description | -| ---------------------------- | --------------------------------- | -| `AppClaw: Run Goal` | Enter a natural language goal | -| `AppClaw: Run Flow File` | Run the current YAML flow file | -| `AppClaw: Run This Step` | Run a single step from a flow | -| `AppClaw: Open Device Panel` | Open the device preview panel | -| `AppClaw: Start Playground` | Interactive playground mode | -| `AppClaw: Take Screenshot` | Capture the current device screen | -| `AppClaw: Stop Execution` | Stop the running agent | +| Command | Description | +| -------------------------------- | ----------------------------------------- | +| `AppClaw: Run Goal` | Enter a natural language goal | +| `AppClaw: Run Flow File` | Run the current YAML flow file | +| `AppClaw: Run This Step` | Run a single step from a flow | +| `AppClaw: Open Device Panel` | Open the device preview panel | +| `AppClaw: Start Terminal Studio` | Step recorder, device picker, run history | +| `AppClaw: Take Screenshot` | Capture the current device screen | +| `AppClaw: Stop Execution` | Stop the running agent | ## Configuration diff --git a/vscode-extension/package.json b/vscode-extension/package.json index d1f93d9..ed3b57a 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -80,7 +80,7 @@ }, { "command": "appclaw.playground", - "title": "AppClaw: Start Playground", + "title": "AppClaw: Start Terminal Studio", "icon": "$(terminal)" }, { diff --git a/vscode-extension/src/extension.ts b/vscode-extension/src/extension.ts index b3df7a2..43c679f 100644 --- a/vscode-extension/src/extension.ts +++ b/vscode-extension/src/extension.ts @@ -548,18 +548,21 @@ export function activate(context: vscode.ExtensionContext): void { }) ); - // Playground — opens an interactive REPL in the integrated terminal + // Terminal Studio — the full-screen Ink app, in the integrated terminal. Not + // to be confused with the device panel's "Playground" mode, which is the + // headless `--json --playground` NDJSON bridge in bridge.ts. The command id + // stays `appclaw.playground` so existing keybindings keep working. context.subscriptions.push( vscode.commands.registerCommand('appclaw.playground', () => { const { command, baseArgs } = getCliCommand(); const env = getEnvFromSettings(); const terminal = vscode.window.createTerminal({ - name: 'AppClaw Playground', + name: 'AppClaw Terminal Studio', env, }); terminal.show(); - terminal.sendText(`${command} ${baseArgs.join(' ')} --playground`.trim()); + terminal.sendText(`${command} ${baseArgs.join(' ')} --tui`.trim()); }) ); From ec1630569ecf9e98595d086156051c41a4c949b6 Mon Sep 17 00:00:00 2001 From: delta456 Date: Wed, 26 Aug 2026 14:20:44 +0530 Subject: [PATCH 3/4] chore: fix ci --- tests/ui/main-screen-layout.test.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/ui/main-screen-layout.test.tsx b/tests/ui/main-screen-layout.test.tsx index 09a8f57..ef7f690 100644 --- a/tests/ui/main-screen-layout.test.tsx +++ b/tests/ui/main-screen-layout.test.tsx @@ -35,9 +35,15 @@ async function frameAt(columns: number, rows: number): Promise { tuiStore.reset(); tuiStore.goTo('main'); + // debug: true is load-bearing, not a leftover. Ink's onRender consults + // `is-in-ci` and, when set, buffers the frame instead of writing it — so on + // GitHub Actions nothing reaches `stdout` until unmount and every assertion + // below sees an empty frame. The debug branch is checked first and writes on + // every render, which is also how ink-testing-library stays CI-safe. const instance = render(, { stdout, stdin, + debug: true, patchConsole: false, exitOnCtrlC: false, }); From 26bd11974549be0a09f94a96887cce3504d84d33 Mon Sep 17 00:00:00 2001 From: Swastik Baranwal Date: Thu, 27 Aug 2026 00:14:21 +0530 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/cli/src/step-recorder/flow-builder.ts | 2 +- packages/cli/src/tui/components/CommandPalette.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/step-recorder/flow-builder.ts b/packages/cli/src/step-recorder/flow-builder.ts index 25de576..2097e5d 100644 --- a/packages/cli/src/step-recorder/flow-builder.ts +++ b/packages/cli/src/step-recorder/flow-builder.ts @@ -94,7 +94,7 @@ export function buildYamlString(steps: FlowStep[], meta: FlowMeta): string { /** * Whether the given filename should be exported as an @appclaw/runner spec - * format) rather than the default YAML flow format. + * format rather than the default YAML flow format. */ export function isSdkTestFilename(name: string): boolean { return /\.(?:test|spec)\.(?:m|c)?[jt]sx?$/i.test(name) || /\.(?:m|c)?ts$/i.test(name); diff --git a/packages/cli/src/tui/components/CommandPalette.tsx b/packages/cli/src/tui/components/CommandPalette.tsx index c27d6a8..3f8b7c9 100644 --- a/packages/cli/src/tui/components/CommandPalette.tsx +++ b/packages/cli/src/tui/components/CommandPalette.tsx @@ -26,9 +26,9 @@ export interface CommandPaletteProps { } /** - * Left-column panel from the wireframe: a bordered "Command pallet" list + * Left-column panel from the wireframe: a bordered "Command palette" list * (filtered live as the user types a leading "/") sitting above the - * "Type instruction" input, where plain text is treated as a goal. + * instruction input; plain text runs one deterministic step and records it. */ /** * The cap on the visible command list (MAX_VISIBLE_COMMANDS) lives in