diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdc7ffa..c834a79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,9 @@ jobs: - name: Restored agents start one at a time if: runner.os == 'Windows' run: node qa/restore-pacing.mjs + - name: Images tied to their terminal rows + if: runner.os == 'Windows' + run: node qa/image-rows.mjs - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() with: @@ -141,4 +144,5 @@ jobs: qa/shots/perf-gate/ qa/shots/images/ qa/shots/images-remote/ + qa/shots/image-rows/ qa/shots/claude-update/ diff --git a/README.md b/README.md index f26ff78..04d712f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ See what needs attention, resume the right conversation, and work on a paired PC -> **1.39.0** shows the screenshots and images a Claude session worked with โ€” Claude Desktop shows them inline; DevDeck puts them in a strip beside the terminal, also for sessions on another PC โ€” adds ๐Ÿ“Ž / drag-and-drop attachments (images travel to a remote PC), **Update all** for Claude Code on Windows, staggered session restores that no longer stall a PC, and ends the daily false "Login expired" ([release notes](docs/releases/1.39.0.md)). 1.38.0 made closing a session reliable across paired PCs ([notes](docs/releases/1.38.0.md)). See [the maintenance review](docs/quality-review-2026-09-08.md) and [session synchronization review](docs/session-sync-review-2026-09-08.md). +> **1.40.0** ties each screenshot to the terminal row it appeared on โ€” hover a highlighted `[Image]` for a thumbnail, click for the full view, and jump from the viewer back to the row โ€” says what led to each image, previews new screenshots, copies images, and adds Ctrl+โ†‘/โ†“ between your prompts ([release notes](docs/releases/1.40.0.md)). 1.39.0 shows the screenshots and images a Claude session worked with โ€” Claude Desktop shows them inline; DevDeck puts them in a strip beside the terminal, also for sessions on another PC โ€” adds ๐Ÿ“Ž / drag-and-drop attachments (images travel to a remote PC), **Update all** for Claude Code on Windows, staggered session restores that no longer stall a PC, and ends the daily false "Login expired" ([release notes](docs/releases/1.39.0.md)). 1.38.0 made closing a session reliable across paired PCs ([notes](docs/releases/1.38.0.md)). See [the maintenance review](docs/quality-review-2026-09-08.md) and [session synchronization review](docs/session-sync-review-2026-09-08.md).
DevDeck project overview with repository and session navigation diff --git a/docs/quality.md b/docs/quality.md index 8c330c9..dc6e995 100644 --- a/docs/quality.md +++ b/docs/quality.md @@ -32,7 +32,7 @@ Results and screenshots are written to `qa/shots/maintenance/` and `qa/shots/res The CI performance gate is `qa:perf-gate` (Windows, every push): a fixture agent instead of a real one, and units that do not depend on the runner's speed โ€” running CSS animations and DOM mutation records on an idle deck (both must be ~0), DOM mutations outside the terminals while two tiles repaint at 10fps (a per-chunk sidebar rebuild is ~8,000 against a limit of 400), and the main process's longest event-loop stall while working and while 20,000 lines stream. Each limit was checked by injecting the regression it guards against: an infinite CSS animation, a sidebar render per PTY chunk, and a 700ms synchronous block in the main process all fail it. Results go to `qa/shots/perf-gate/results.json`. -`qa:images` (Windows) drives the cockpit image strip, full-size viewer and ๐Ÿ“Ž attach against an isolated home holding a transcript with a tool screenshot, a pasted image and a written image file; its fixture `claude` echoes what it is sent, so an attached path is read back off the terminal. `qa:update` (Windows) checks "Update all": a fixture `claude` prints Claude Code's failed-update footer, and the harness asserts that `claude update` ran with no session process alive and that every conversation was resumed by id. `qa:images-remote` (Windows) pairs a host and a viewer over real TLS and checks the same from the viewer: thumbnails and the full view cross the link, an attached JPEG lands on the host as PNG with the host path typed, and a non-image named `.png` or an unindexed id is refused over the link. `qa:restore-pacing` (Windows) checks that restored agents start one at a time: a fixture that draws for 2s while booting must not see the next one launched before it finished (measured before the fix: 26โ€“49ms apart). None of these starts a real agent. +`qa:images` (Windows) drives the cockpit image strip, full-size viewer and ๐Ÿ“Ž attach against an isolated home holding a transcript with a tool screenshot, a pasted image and a written image file; its fixture `claude` echoes what it is sent, so an attached path is read back off the terminal. `qa:update` (Windows) checks "Update all": a fixture `claude` prints Claude Code's failed-update footer, and the harness asserts that `claude update` ran with no session process alive and that every conversation was resumed by id. `qa:images-remote` (Windows) pairs a host and a viewer over real TLS and checks the same from the viewer: thumbnails and the full view cross the link, an attached JPEG lands on the host as PNG with the host path typed, and a non-image named `.png` or an unindexed id is refused over the link. `qa:restore-pacing` (Windows) checks that restored agents start one at a time: a fixture that draws for 2s while booting must not see the next one launched before it finished (measured before the fix: 26โ€“49ms apart). `qa:image-rows` (Windows) ties images to the terminal: a fixture `claude` draws a screen the way Claude Code does (`โ— server - tool (MCP)` calls, `โŽฟ [Image]` results, shaded `โฏ` prompts, a composer between rules) over a transcript holding the same screenshots, and the harness checks that each placeholder is marked and opens its own image, that the viewer shows what led to an image and scrolls the terminal to its row, that copying puts an image on the clipboard (the person's clipboard is restored afterwards), that a new screenshot is previewed while the strip is closed, and that Ctrl+Up/Down steps between sent prompts without entering the composer. None of these starts a real agent. Do not run these against a live development conversation. Do not equate a short loopback test with sleep/resume, VPN changes or multi-day field validation. diff --git a/docs/releases/1.40.0.md b/docs/releases/1.40.0.md new file mode 100644 index 0000000..04aa0b6 --- /dev/null +++ b/docs/releases/1.40.0.md @@ -0,0 +1,20 @@ +# DevDeck 1.40.0 + +## Which step was this screenshot? + +- **Image rows in the terminal.** Where Claude Code printed `[Image]` (a tool's screenshot) or `[Image #N]` (an image you pasted), the placeholder is now highlighted. Hover it for a thumbnail, click it for the full view. Each placeholder is checked against the tool named on the call line above it, so a row is never given another step's image. If the screen and the record don't line up, for example where Claude Code folded the output away, the row is simply left unmarked. +- **Show where it appeared.** The full-size viewer has a button that closes it and scrolls the terminal to the row the image was shown on, with the call that made it just above and the row briefly highlighted. The button is greyed out once that row has left the scrollback. +- **What led to it.** Thumbnails and the viewer now say what the tool was pointed at (an element, a file, a URL), the page the browser was on for a browser screenshot, and the last thing Claude said before taking it. A pasted image shows the words you sent with it. All of this comes from the session's own record, so it is there even when the terminal row is long gone. + +## Smaller things + +- **New screenshot preview.** While the image strip is closed, a new screenshot appears briefly in the corner of the terminal. Several in a row show as one preview with a count. Click it to open the image. It stays while the pointer is on it. Images you pasted yourself are not announced. +- **Copy image.** The viewer can put the image on the clipboard, ready to paste into another app or another session. +- **Ctrl+โ†‘ / Ctrl+โ†“ between your prompts.** In a Claude session, jump to the previous or next prompt you sent. The landing row is briefly highlighted. Going past the last prompt returns to the bottom. Claude Code's own input box is never a stop, and the keys are never sent to the agent. + +## Under the hood + +- A new automated check (`qa:image-rows`) drives a fixture screen drawn the way Claude Code draws it. It covers marking and opening image rows, the viewer's context and "show in terminal", copying, the arrival preview and prompt jumps. It runs on every change on Windows. +- The terminal now enables xterm's marker and decoration API. Image rows are re-paired at most every 2.5 seconds, and only for the selected session when it has images; the performance gate is unchanged (no idle animation or DOM churn, no main-process stall while streaming). + +The context lines need 1.40.0 on the PC running the session. With an older host the strip and viewer keep working without them. diff --git a/package-lock.json b/package-lock.json index 49cdaf4..d98bfce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "devdeck", - "version": "1.39.0", + "version": "1.40.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "devdeck", - "version": "1.39.0", + "version": "1.40.0", "license": "MIT", "dependencies": { "@fontsource-variable/geist": "5.3.0", diff --git a/package.json b/package.json index a38251f..6e53ae1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "devdeck", - "version": "1.39.0", + "version": "1.40.0", "description": "Project command deck โ€” at-a-glance state + Claude Code, Codex, and Antigravity resume", "main": "dist/main/main.js", "type": "commonjs", @@ -42,6 +42,7 @@ "check:docs": "node scripts/check-docs.mjs", "qa:session-sync": "npm run build && node qa/session-sync.mjs", "qa:images": "npm run build && node qa/images.mjs", + "qa:image-rows": "npm run build && node qa/image-rows.mjs", "qa:images-remote": "npm run build && node qa/images-remote.mjs", "qa:update": "npm run build && node qa/claude-update.mjs", "qa:restore-pacing": "npm run build && node qa/restore-pacing.mjs" diff --git a/qa/image-rows.mjs b/qa/image-rows.mjs new file mode 100644 index 0000000..92f5e8c --- /dev/null +++ b/qa/image-rows.mjs @@ -0,0 +1,227 @@ +// Windows harness: an image ties back to the terminal row it was shown on, and back again โ€” plus the +// arrival preview, copying an image, and Ctrl+Up/Down between prompts. A fixture `claude` draws a +// screen the way Claude Code does (`โ— server - tool (MCP)` / `โŽฟ [Image]` / shaded `โฏ prompt` rows) +// over an isolated HOME whose transcript holds the same images. Never starts a real agent. +// Run with npm run qa:image-rows. +import { _electron as electron } from 'playwright'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(process.argv[2] ?? dirname(dirname(fileURLToPath(import.meta.url)))); +if (process.platform !== 'win32') { console.log('SKIP: Windows native PTY required'); process.exit(0); } +const shots = resolve(process.argv[3] ?? join(root, 'qa', 'shots', 'image-rows')); +mkdirSync(shots, { recursive: true }); +const { closeElectron } = await import('file:///' + join(root, 'qa', 'electron-lifecycle.mjs').replace(/\\/g, '/')); + +const sandbox = mkdtempSync(join(tmpdir(), 'devdeck-image-rows-')); +const repo = join(sandbox, 'rows-workspace'); +const bin = join(sandbox, 'bin'); +const profile = join(sandbox, 'profile'); +const home = join(profile, 'home'); +for (const p of [repo, bin, home]) mkdirSync(p, { recursive: true }); +execFileSync('git', ['init', '-q', '-b', 'qa-rows'], { cwd: repo }); + +const sid = '7a1c2b3d-1111-4222-8333-944455556666'; +const tdir = join(home, '.claude', 'projects', repo.replace(/[^a-zA-Z0-9]/g, '-')); +mkdirSync(tdir, { recursive: true }); +const transcript = join(tdir, sid + '.jsonl'); +const png = (f) => readFileSync(join(root, f)).toString('base64'); +const ts = new Date().toISOString(); +const said = (text) => ({ type: 'assistant', timestamp: ts, message: { role: 'assistant', content: [{ type: 'text', text }] } }); +const call = (id, name, input = {}) => ({ type: 'assistant', timestamp: ts, message: { role: 'assistant', content: [{ type: 'tool_use', id, name, input }] } }); +const shot = (id, data) => ({ type: 'user', timestamp: ts, message: { role: 'user', content: [{ tool_use_id: id, type: 'tool_result', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data } }] }] } }); +const prompt = (text) => ({ type: 'user', timestamp: ts, cwd: repo, sessionId: sid, message: { role: 'user', content: text } }); +const lines = [ + prompt('๋กœ๊ทธ์ธ ํ™”๋ฉด ํ™•์ธํ•ด์ค˜'), + said('๋กœ๊ทธ์ธ ํŽ˜์ด์ง€๋ฅผ ์—ด์–ด ๋ณด๊ฒ ์Šต๋‹ˆ๋‹ค.'), + call('n1', 'mcp__playwright__browser_navigate', { url: 'http://localhost:3000/login' }), + call('s1', 'mcp__playwright__browser_take_screenshot', { element: 'Submit button' }), + shot('s1', png('docs/demo/poster.png')), + prompt('๋‹ค์Œ ๋‹จ๊ณ„'), + said('๋ฐ”ํƒ•ํ™”๋ฉด์„ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค.'), + call('w1', 'mcp__windows-mcp__Screenshot'), + shot('w1', png('build/icon.png')), +]; +writeFileSync(transcript, lines.map((l) => JSON.stringify(l)).join('\n') + '\n'); +// What the fixture appends when it is sent SHOT: one more browser screenshot. +writeFileSync(join(bin, 'more.jsonl'), [ + said('ํ—ค๋”๋„ ์ฐ์–ด ๋‘ก๋‹ˆ๋‹ค.'), + call('s2', 'mcp__playwright__browser_take_screenshot', { element: 'Header' }), + shot('s2', png('build/icon.png')), +].map((l) => JSON.stringify(l)).join('\n') + '\n'); + +const SHADE = '\x1b[48;5;237m'; +const RESET = '\x1b[0m'; +writeFileSync(join(bin, 'agent.js'), ` +const fs = require('fs'); +const out = (s) => process.stdout.write(s.replace(/\\n/g, '\\r\\n')); +const filler = (tag, n) => Array.from({ length: n }, (_, i) => ' ' + tag + ' output line ' + i).join('\\n') + '\\n'; +out(${JSON.stringify(SHADE)} + 'โฏ ๋กœ๊ทธ์ธ ํ™”๋ฉด ํ™•์ธํ•ด์ค˜' + ${JSON.stringify(RESET)} + '\\n'); +out('โ— ๋กœ๊ทธ์ธ ํŽ˜์ด์ง€๋ฅผ ์—ด์–ด ๋ณด๊ฒ ์Šต๋‹ˆ๋‹ค.\\n'); +out('โ— playwright - browser_navigate (MCP)(url: "http://localhost:3000/login")\\n โŽฟ Navigated\\n'); +out('โ— playwright - browser_take_screenshot (MCP)(element: "Submit button")\\n โŽฟ [Image]\\n'); +out(filler('first', 70)); +out(${JSON.stringify(SHADE)} + 'โฏ ๋‹ค์Œ ๋‹จ๊ณ„' + ${JSON.stringify(RESET)} + '\\n'); +out('โ— windows-mcp - Screenshot (MCP)\\n โŽฟ [Image]\\n'); +out(filler('second', 70)); +const composer = () => out('โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\\nโฏ \\nโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\\n'); +composer(); +out('FIXTURE_READY\\n'); +let typed = ''; +if (process.stdin.isTTY) process.stdin.setRawMode(true); +process.stdin.on('data', (d) => { + typed += d.toString(); + if (!typed.includes('SHOT')) return; + typed = ''; + out('โ— playwright - browser_take_screenshot (MCP)(element: "Header")\\n โŽฟ [Image]\\n'); + composer(); + fs.appendFileSync(${JSON.stringify(transcript)}, fs.readFileSync(${JSON.stringify(join(bin, 'more.jsonl'))})); +}); +setInterval(() => {}, 1 << 30); +`); +writeFileSync(join(bin, 'claude.cmd'), '@echo off\r\nnode "%~dp0agent.js" %*\r\n'); + +writeFileSync(join(profile, 'state.json'), JSON.stringify({ projects: {}, settings: { + language: 'ko', folders: [{ path: repo, kind: 'repo' }], viewMode: 'list', + cockpitSessions: [{ tileId: 'tile-rows-1', projectPath: repo, name: 'rows-workspace', sessionId: sid, agentId: 'claude', label: 'Rows QA', lastActiveMs: Date.now() }], +} })); +const env = { ...process.env, HOME: home, USERPROFILE: home, CLAUDECODE: '', CLAUDE_CODE_SSE_PORT: '', CLAUDE_CODE_ENTRYPOINT: '' }; +const pathKey = Object.keys(env).find((k) => k.toLowerCase() === 'path') || 'PATH'; +env[pathKey] = bin + ';' + env[pathKey]; + +const app = await electron.launch({ args: ['.', '--user-data-dir=' + profile, '--no-sandbox', '--disable-gpu'], cwd: root, env }); +const errors = []; +let failed = false; +const check = async (name, fn) => { await fn(); console.log('PASS ' + name); }; +// The person's clipboard is put back afterwards: this harness copies an image onto it. +const savedClip = await app.evaluate(({ clipboard }) => ({ text: clipboard.readText(), image: clipboard.readImage().toDataURL() })); +try { + const win = await app.firstWindow(); + win.setDefaultTimeout(15000); + win.on('pageerror', (e) => errors.push(String(e))); + win.on('console', (m) => { if (m.type() === 'error') errors.push(m.text()); }); + + const screenText = () => win.evaluate(() => [...document.querySelectorAll('.ck-term.show .xterm-rows > div')].map((r) => r.textContent)); + const topRow = async () => (await screenText())[0] ?? ''; + /** The `[Image]` placeholder spans on screen, with the colour they are drawn in. */ + // A decorated run can be split into several spans, so the placeholder is found in the row's text + // and measured over whichever spans cover it. + const imageSpans = () => win.evaluate(() => [...document.querySelectorAll('.ck-term.show .xterm-rows > div')] + .filter((row) => row.textContent.includes('[Image]')) + .map((row) => { + const at = row.textContent.indexOf('[Image]'); + let pos = 0; const covering = []; + for (const s of row.querySelectorAll('span')) { + const end = pos + s.textContent.length; + if (end > at && pos < at + 7) covering.push(s); + pos = end; + } + const rects = covering.map((s) => s.getBoundingClientRect()); + const x0 = Math.min(...rects.map((r) => r.left)); const x1 = Math.max(...rects.map((r) => r.right)); + return { x: (x0 + x1) / 2, y: rects[0].top + rects[0].height / 2, color: covering[0] ? getComputedStyle(covering[0]).color : '', row: row.textContent }; + })); + + await check('session opens with the fixture screen', async () => { + await win.waitForSelector('#shell-session-groups .shell-session', { timeout: 30000 }); + await win.locator('#shell-session-groups .shell-session').first().click(); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].some((r) => r.textContent.includes('FIXTURE_READY')), null, { timeout: 20000 }); + await win.waitForFunction(() => document.querySelector('#ck-header .ck-img-count')?.textContent === '2'); + }); + + await check('Ctrl+Up steps back through the prompts the person sent, Ctrl+Down returns', async () => { + await win.locator('.ck-term.show .xterm-screen').click(); + await win.keyboard.press('Control+ArrowUp'); + await win.waitForFunction(() => document.querySelector('.ck-term.show .xterm-rows > div')?.textContent.includes('๋‹ค์Œ ๋‹จ๊ณ„')); + await win.keyboard.press('Control+ArrowUp'); + await win.waitForFunction(() => document.querySelector('.ck-term.show .xterm-rows > div')?.textContent.includes('๋กœ๊ทธ์ธ ํ™”๋ฉด ํ™•์ธํ•ด์ค˜')); + await win.screenshot({ path: join(shots, '1-prompt-jump.png') }); + await win.keyboard.press('Control+ArrowDown'); + await win.waitForFunction(() => document.querySelector('.ck-term.show .xterm-rows > div')?.textContent.includes('๋‹ค์Œ ๋‹จ๊ณ„')); + await win.keyboard.press('Control+ArrowDown'); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows > div')].some((r) => r.textContent.includes('FIXTURE_READY'))); + // The composer's own `โฏ` is not a sent prompt, and nothing reached the agent. + assert.ok(!(await topRow()).includes('โ”€โ”€โ”€โ”€'), 'never jumped into the composer'); + }); + + await check('placeholders are marked on screen and open the image they stand for', async () => { + await win.keyboard.press('Control+ArrowUp'); + await win.waitForFunction(() => document.querySelector('.ck-term.show .xterm-rows > div')?.textContent.includes('๋‹ค์Œ ๋‹จ๊ณ„')); + let spans = []; + for (const until = Date.now() + 8000; Date.now() < until; await win.waitForTimeout(250)) { + spans = await imageSpans(); + if (spans[0]?.color === 'rgb(168, 205, 255)') break; + } + assert.equal(spans.length, 1, 'the windows screenshot row is on screen'); + assert.equal(spans[0].color, 'rgb(168, 205, 255)', 'drawn in the mark colour'); + await win.mouse.move(spans[0].x, spans[0].y); + await win.waitForSelector('.ck-img-hover img[src^="data:image"]'); + await win.screenshot({ path: join(shots, '2-hover.png') }); + await win.mouse.click(spans[0].x, spans[0].y); + await win.waitForFunction(() => document.querySelector('.ck-img-viewer .ck-img-stage img')?.getAttribute('src')?.startsWith('data:image')); + const caption = await win.locator('.ck-img-caption').textContent(); + assert.match(caption, /windows-mcp ยท Screenshot/); + assert.match(caption, /๋ฐ”ํƒ•ํ™”๋ฉด์„ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค/); + await win.keyboard.press('Escape'); + await win.waitForSelector('.ck-img-viewer', { state: 'detached' }); + assert.equal(await win.locator('.ck-img-hover').count(), 0, 'hover preview gone after the click'); + }); + + await check('the viewer says what led to an image and goes to its row in the terminal', async () => { + await win.locator('#ck-header button[aria-pressed]').click(); + await win.waitForFunction(() => document.querySelectorAll('#ck-images .ck-img-thumb').length === 2); + await win.locator('#ck-images .ck-img-thumb').nth(1).click(); // oldest: the login screenshot + await win.waitForFunction(() => document.querySelector('.ck-img-viewer .ck-img-stage img')?.getAttribute('src')?.startsWith('data:image')); + const context = await win.locator('.ck-img-context').textContent(); + assert.match(context, /Submit button/); + assert.match(context, /localhost:3000\/login/); + assert.match(context, /๋กœ๊ทธ์ธ ํŽ˜์ด์ง€๋ฅผ ์—ด์–ด ๋ณด๊ฒ ์Šต๋‹ˆ๋‹ค/); + await win.screenshot({ path: join(shots, '3-viewer-context.png') }); + + await win.locator('.ck-img-bar button[aria-label="์ด๋ฏธ์ง€ ๋ณต์‚ฌ"]').click(); + await win.waitForFunction(() => document.body.textContent.includes('์ด๋ฏธ์ง€๋ฅผ ํด๋ฆฝ๋ณด๋“œ์— ๋ณต์‚ฌํ–ˆ์Šต๋‹ˆ๋‹ค')); + assert.equal(await app.evaluate(({ clipboard }) => clipboard.readImage().isEmpty()), false, 'an image is on the clipboard'); + + await win.locator('.ck-img-bar button[aria-label="ํ„ฐ๋ฏธ๋„์—์„œ ์œ„์น˜ ๋ณด๊ธฐ"]').click(); + await win.waitForSelector('.ck-img-viewer', { state: 'detached' }); + // The scroll is drawn on the next frame; on a slow runner that is after the viewer is gone. + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows > div')].some((r) => r.textContent.includes('(element: "Submit button")'))); + const spans = await imageSpans(); + assert.ok(spans.some((s) => s.row.includes('[Image]')), 'the login screenshot row is on screen'); + assert.ok((await screenText()).some((r) => r.includes('browser_take_screenshot (MCP)(element: "Submit button")')), 'with its call above it'); + await win.screenshot({ path: join(shots, '4-revealed.png') }); + await win.locator('#ck-header button[aria-pressed]').click(); // close the strip again + await win.waitForFunction(() => document.querySelector('#ck-images')?.classList.contains('hidden')); + }); + + await check('a new screenshot is previewed over the terminal while the strip is closed', async () => { + await win.locator('.ck-term.show .xterm-screen').click(); + await win.keyboard.type('SHOT'); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].some((r) => r.textContent.includes('(element: "Header")'))); + // Re-selecting re-reads the list at once instead of waiting for the next tick. + await win.locator('#shell-session-groups .shell-session').first().click(); + await win.waitForSelector('.ck-img-preview img[src^="data:image"]', { timeout: 40000 }); + await win.screenshot({ path: join(shots, '5-preview.png') }); + await win.locator('.ck-img-preview').click(); + await win.waitForFunction(() => document.querySelector('.ck-img-context')?.textContent.includes('Header')); + await win.locator('.ck-img-bar button[aria-label="ํ„ฐ๋ฏธ๋„์—์„œ ์œ„์น˜ ๋ณด๊ธฐ"]').click(); + await win.waitForSelector('.ck-img-viewer', { state: 'detached' }); + assert.equal(await win.locator('.ck-img-preview').count(), 0, 'the preview went with the click'); + }); + + assert.deepEqual(errors, [], 'no renderer errors'); +} catch (e) { + console.log('FAIL', e?.message ?? e); + failed = true; + try { const w = await app.firstWindow(); await w.screenshot({ path: join(shots, 'failure.png') }); } catch { /* window gone */ } +} finally { + await app.evaluate(({ clipboard, nativeImage }, saved) => { + if (saved.image && saved.image !== 'data:image/png;base64,') clipboard.writeImage(nativeImage.createFromDataURL(saved.image)); + else clipboard.writeText(saved.text); + }, savedClip).catch(() => {}); + await closeElectron(app).catch(() => {}); +} +if (failed) process.exitCode = 1; diff --git a/src/main/api/deckApi.ts b/src/main/api/deckApi.ts index f74d5a8..40d3c2a 100644 --- a/src/main/api/deckApi.ts +++ b/src/main/api/deckApi.ts @@ -1,5 +1,5 @@ import { createSessionLabels } from '../sessionLabels'; -import { dialog, shell, app, clipboard, type BrowserWindow } from 'electron'; +import { dialog, shell, app, clipboard, nativeImage, type BrowserWindow } from 'electron'; import { homedir, release, tmpdir } from 'node:os'; import path, { join } from 'node:path'; import { readFile, realpath, stat } from 'node:fs/promises'; @@ -77,6 +77,8 @@ const REPO_URL = 'https://github.com/writingdeveloper/devdeck'; /** Paste-image ceiling. A screenshot is around a megabyte; this is generous and still far below the * link's frame limit once base64 expansion is counted. */ const MAX_PASTE_IMAGE_BYTES = 5 * 1024 * 1024; +/** A full-size view is at most 5MB of image, so ~7MB as a data URL; this leaves room and no more. */ +const MAX_CLIPBOARD_IMAGE_CHARS = 12 * 1024 * 1024; const REMOTE_IMAGE_BYTES_PER_HOUR = 200 * 1024 * 1024; /** * Image reads decode and re-encode on THIS process. Opening the strip asks for dozens of thumbnails @@ -1095,6 +1097,14 @@ export function createDeckApi(cfg: DeckApiConfig): DeckApiBundle { // navigator.clipboard reliably). Used so Ctrl+C copies a selection instead of sending SIGINT. send('clipboard:writeText', localOnly, (text: string) => clipboard.writeText(String(text ?? ''))); invoke('clipboard:readText', localOnly, () => clipboard.readText()); + /** Put an image the viewer is showing on the clipboard. Only image data URLs; never a path. */ + invoke('clipboard:writeImage', localOnly, (dataUrl: string) => { + if (typeof dataUrl !== 'string' || !/^data:image\/(?:png|jpeg);base64,/.test(dataUrl) || dataUrl.length > MAX_CLIPBOARD_IMAGE_CHARS) return false; + const img = nativeImage.createFromDataURL(dataUrl); + if (img.isEmpty()) return false; + clipboard.writeImage(img); + return true; + }); // Paste a CLIPBOARD IMAGE (e.g. a screenshot) into the terminal: Claude Code can't read the OS // clipboard (esp. native Windows), but it DOES read an image off a file path. So on Ctrl+V with an // image on the clipboard, write it to a temp PNG and return the path for the renderer to inject as diff --git a/src/main/sessionImages.test.ts b/src/main/sessionImages.test.ts index 4ade352..86a6053 100644 --- a/src/main/sessionImages.test.ts +++ b/src/main/sessionImages.test.ts @@ -93,3 +93,41 @@ describe('mergeImageRefs', () => { expect(mergeImageRefs([], [...b, ...c.add(pasted(), 30)], 1).map((r) => r.id)).toEqual(['30.0']); }); }); + +describe('image context', () => { + const said = (text: string) => JSON.stringify({ type: 'assistant', timestamp: TS, message: { role: 'assistant', content: [{ type: 'text', text }] } }); + it('says what led to each image: the call\'s target, the page its browser was on, and the agent\'s last words', () => { + const c = new SessionImageCollector(); + const lines = [ + said('Let me open the login page.\nThen check it.'), + toolCall('n1', 'mcp__playwright__browser_navigate', { url: 'http://localhost:3000/login' }), + said('Now the button state:'), + toolCall('s1', 'mcp__playwright__browser_take_screenshot', { element: 'Submit button', ref: 'e12' }), + toolResultWithImage('s1'), + toolCall('w1', 'mcp__windows-mcp__Screenshot'), + toolResultWithImage('w1'), + ]; + const refs = lines.flatMap((l, i) => c.add(l, i * 1000)); + expect(refs.map((r) => [r.detail, r.page, r.said])).toEqual([ + ['Submit button', 'http://localhost:3000/login', 'Now the button state:'], + // Another server's screenshot is not of that page. + [null, null, 'Now the button state:'], + ]); + }); + + it('gives a pasted image the words the person sent with it, without the placeholder', () => { + const c = new SessionImageCollector(); + const line = JSON.stringify({ type: 'user', timestamp: TS, message: { role: 'user', content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: PNG_B64 } }, + { type: 'text', text: '[Image #1] ์ด ๋ฒ„ํŠผ ์œ„์น˜ ์ด์ƒํ•ด์š”' }, + ] } }); + expect(c.add(line, 0).map((r) => r.said)).toEqual(['์ด ๋ฒ„ํŠผ ์œ„์น˜ ์ด์ƒํ•ด์š”']); + }); + + it('never parses a huge call just for context, and still names its tool', () => { + const c = new SessionImageCollector(); + c.add(toolCall('big', 'mcp__x__shot', { text: 'y'.repeat(200_000) }), 0); + const [ref] = c.add(toolResultWithImage('big'), 1); + expect([ref.tool, ref.detail]).toEqual(['mcp__x__shot', null]); + }); +}); diff --git a/src/preload/preload.ts b/src/preload/preload.ts index d6c396a..a22e43f 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -60,6 +60,7 @@ contextBridge.exposeInMainWorld('devdeck', { writeText: (text: string) => ipcRenderer.send('clipboard:writeText', text), readText: (): Promise => ipcRenderer.invoke('clipboard:readText'), readImage: (): Promise => ipcRenderer.invoke('clipboard:readImage'), + writeImage: (dataUrl: string): Promise => ipcRenderer.invoke('clipboard:writeImage', dataUrl), readImageBytes: (): Promise<{ tooLarge: boolean; bytes: string | null } | null> => ipcRenderer.invoke('clipboard:readImageBytes'), }, // Diagnostics: one file per machine, readable by a person or an agent sitting at it. diff --git a/src/renderer/cockpitView.ts b/src/renderer/cockpitView.ts index 00d2036..93d7c3e 100644 --- a/src/renderer/cockpitView.ts +++ b/src/renderer/cockpitView.ts @@ -27,7 +27,8 @@ import { ptyCompatFor } from './ptyCompat'; import { followPtySize, type TerminalDims } from '../shared/terminalSize'; import { basename } from '../shared/paths'; import type { PtySessionInfo as RunningSession } from '../main/ptyHost'; -import { imageStripState, initImageStrip, openSingleImage, refreshImages, showImagesFor, toggleImageStrip, type ImageSource } from './imageStrip'; +import { currentImageRefs, currentImageSourceKey, imageStripState, imageThumbUrl, initImageStrip, openImageById, openSingleImage, refreshImages, showImagesFor, toggleImageStrip, type ImageSource } from './imageStrip'; +import { jumpToPrompt, marksFor } from './termMarks'; import { attachFiles, pathsAsPrompt } from './attachments'; /** What cockpit:sessionMeta answers with: the log-derived facts plus the ready-made summary line. */ @@ -158,7 +159,7 @@ export function mountCockpit(): void { mainEl = document.querySelector('#view-cockpit .ck-main')!; const strip = document.getElementById('ck-images'); // The strip opening or closing changes the terminal's height, and its count lives in the header. - if (strip) initImageStrip(strip, () => { if (!editingId) renderHeader(); scheduleLayout(); }); + if (strip) initImageStrip(strip, () => { if (!editingId) renderHeader(); scheduleLayout(); }, { popupIn: termsEl, refsChanged: markSelectedTerminal }); // Mid-turn, new screenshots show up as they are taken rather than at the end of the turn; between // turns a slow re-read still catches images written some other way (another process, a resume). // Only the selected session is ever read, and an unchanged transcript costs one stat. @@ -168,7 +169,11 @@ export function mountCockpit(): void { const sel = selectedId ? live.get(selectedId) : null; if (!sel || sel.session.agentId !== 'claude') return; imageTick = (imageTick + 1) % 6; - if (sel.session.activity === 'working' || imageTick === 0) void refreshImages(); + if (sel.session.activity === 'working' || imageTick === 0) { + void refreshImages(); + // The screen can print an image's row after the transcript listed it; re-pair while it runs. + markSelectedTerminal(); + } }, 5000); // The toolbar's Refresh is the button users already press when a terminal looks wrong; make it // mean that. The deck reload is wired separately, in projectsView โ€” both listeners fire. @@ -740,19 +745,59 @@ function pickAttachments(machineId: string, term: Terminal): void { } /** Where the image strip reads a session's images from: the machine running it. */ -function imageSourceFor(l: { session: CockpitSession; machineId: string; openedSessionId: string | null }): ImageSource | null { +function imageSourceFor(l: { session: CockpitSession; machineId: string; openedSessionId: string | null; term?: Terminal }): ImageSource | null { const sid = l.openedSessionId; if (!sid || l.session.agentId !== 'claude') return null; const deck = deckFor(l.machineId); const projectPath = l.session.projectPath; + const term = l.term; return { key: `${l.machineId}|${projectPath}|${sid}`, list: () => deck.cockpit.sessionImages(projectPath, sid, 'claude'), get: (id, size) => deck.cockpit.sessionImage(projectPath, sid, id, size), openFile: l.machineId === LOCAL_MACHINE_ID ? (path) => { void window.devdeck.cockpit.openFile(projectPath, path); } : undefined, + canReveal: term ? (id) => marksFor(term).locate(id) !== null : undefined, + reveal: term ? (id) => { marksFor(term).reveal(id); term.focus(); } : undefined, }; } +/** Pair the selected session's images with the rows of its terminal they were printed on. */ +function markSelectedTerminal(): void { + const l = selectedId ? live.get(selectedId) : null; + if (!l) return; + // The strip follows the selection, but its list may still be the previous session's for a moment. + const src = imageSourceFor(l); + marksFor(l.term).update(src && currentImageSourceKey() === src.key ? currentImageRefs() : []); +} + +const MARK_SETTLE_MS = 2500; +let markTimer: ReturnType | null = null; +function scheduleMarkSelected(): void { + if (markTimer || currentImageRefs().length === 0) return; + markTimer = setTimeout(() => { markTimer = null; markSelectedTerminal(); }, MARK_SETTLE_MS); +} + +/** Hover preview for an image row in the terminal: a thumbnail beside the pointer. */ +let hoverPreview: HTMLElement | null = null; +let hoverSeq = 0; +function showHoverPreview(id: string, e: MouseEvent): void { + hideHoverPreview(); + const seq = ++hoverSeq; + const el = document.createElement('div'); el.className = 'ck-img-hover'; + const img = document.createElement('img'); img.alt = ''; + el.appendChild(img); + el.style.left = `${Math.min(e.clientX + 14, window.innerWidth - 260)}px`; + // Above the pointer when there is room, below it near the top โ€” never over the row being hovered. + el.style.top = `${e.clientY > 200 ? e.clientY - 180 : e.clientY + 18}px`; + void imageThumbUrl(id).then((url) => { + if (seq !== hoverSeq || !url) return; + img.src = url; + document.body.appendChild(el); + hoverPreview = el; + }); +} +function hideHoverPreview(): void { hoverSeq++; hoverPreview?.remove(); hoverPreview = null; } + async function pasteImageToRemote(machineId: string, term: Terminal): Promise { let payload: { tooLarge: boolean; bytes: string | null } | null = null; try { payload = await window.devdeck.clipboard.readImageBytes(); } catch { payload = null; } @@ -808,7 +853,7 @@ async function buildTile(p: OpenReq): Promise { el.classList.add('show'); // No cursor blink: it is a repaint twice a second per visible terminal for the life of the app, // and the spinner beside the input already says whether the agent is busy. - const term = new Terminal({ fontFamily: 'Cascadia Mono, Consolas, monospace', fontSize: 12, theme: { background: '#0a0b0e' }, cursorBlink: false, scrollback: 10_000, windowsPty }); + const term = new Terminal({ fontFamily: 'Cascadia Mono, Consolas, monospace', fontSize: 12, theme: { background: '#0a0b0e' }, cursorBlink: false, scrollback: 10_000, windowsPty, allowProposedApi: true }); // markers and decorations (termMarks) are "proposed" API const fit = new FitAddon(); term.loadAddon(fit); term.open(el); fit.fit(); el.dataset.termRenderer = 'dom'; // read by the perf/link harnesses and by anyone debugging a slow machine // The size this terminal is actually drawing at. Written on every resize because a pty is shared: @@ -870,6 +915,9 @@ async function buildTile(p: OpenReq): Promise { range: toRange(h), text: h.url, activate: (_e: MouseEvent, text: string) => { void window.devdeck.cockpit.openLink(text); }, })), + // An image row: the thumbnail on hover, the image on click. Which image it is was worked out + // when the rows were marked (termMarks); a row nothing was paired with links to nothing. + ...imageRowLinks(rows[rel]?.text ?? '', hoveredIdx), // Local file paths the agent printed (e.g. "> [image] assets\a.png", "โ€บ [file] SFX\bell.wav") // โ€” click opens the OS default app (inert-content extensions only, re-checked in main). ...findFilePathLinks(rows).filter(onRow).map((h) => ({ @@ -890,6 +938,23 @@ async function buildTile(p: OpenReq): Promise { callback(links.length ? links : undefined); }, }; + /** Links for the `[Image]` placeholders on one row, when they were paired with a known image. */ + function imageRowLinks(text: string, row: number): { range: { start: { x: number; y: number }; end: { x: number; y: number } }; text: string; activate: (e: MouseEvent, text: string) => void; hover: (e: MouseEvent) => void; leave: () => void }[] { + if (!text.includes('[Image')) return []; + const out: ReturnType = []; + for (const m of text.matchAll(/\[Image(?: #\d{1,4})?\]/g)) { + const col = m.index ?? 0; + const id = marksFor(term).idAt(row, col); + if (!id) continue; + out.push({ + range: { start: { x: col + 1, y: row + 1 }, end: { x: col + m[0].length, y: row + 1 } }, text: m[0], + activate: () => { hideHoverPreview(); openImageById(id); }, + hover: (e) => showHoverPreview(id, e), + leave: () => hideHoverPreview(), + }); + } + return out; + } term.registerLinkProvider(linkProvider); // Files dropped on the terminal are handed to the agent by path, as ๐Ÿ“Ž does (see attachments.ts). // preventDefault on dragover is what makes this a drop target at all โ€” without it Electron would @@ -926,12 +991,20 @@ async function buildTile(p: OpenReq): Promise { return false; } if (action === 'find') { e.preventDefault(); openFindBar(); return false; } // Ctrl+F searches scrollback, never reaches the PTY + if (action === 'prompt-prev' || action === 'prompt-next') { + e.preventDefault(); + if (!jumpToPrompt(term, action === 'prompt-prev' ? -1 : 1)) toast(tr('cockpit.prompt_none')); + return false; + } // Swallowed here so the PTY never sees it; the shell's own document listener does the focusing // (the DOM event still bubbles โ€” returning false only stops xterm from forwarding it). if (action === 'quickopen') { e.preventDefault(); return false; } return true; }); const search = new SearchAddon(); term.loadAddon(search); + // Image rows are re-paired once the screen settles after new output โ€” only for the selected + // tile, and only when it has images at all, so a streaming session costs one scan every few seconds. + term.onWriteParsed(() => { if (selectedId && live.get(selectedId)?.term === term) scheduleMarkSelected(); }); // Copy-on-select: selecting text copies it right away (like Claude Code's own auto-copy / classic // Windows quick-edit). This matters most while a TUI has mouse tracking ON โ€” a plain drag goes to the // TUI, so the user selects with Shift+drag, and requiring another Ctrl+C afterwards was exactly the @@ -1589,6 +1662,7 @@ function select(id: string): void { void refreshMeta(id); void refreshGit(id); void refreshImages(); // renderHeader pointed the strip at this session; a re-select re-reads it too + markSelectedTerminal(); // Selecting a tile is when its provider is most visible (header mark + usage footer) โ€” re-check it // here too instead of waiting up to 30s for the tick. Main's listing is cached, so this is cheap. void refreshProvider(id); diff --git a/src/renderer/features/cockpit/cockpit.css b/src/renderer/features/cockpit/cockpit.css index c698c4a..193d059 100644 --- a/src/renderer/features/cockpit/cockpit.css +++ b/src/renderer/features/cockpit/cockpit.css @@ -238,8 +238,85 @@ .ck-img-caption { flex: 1; min-width: 0; + display: flex; + flex-direction: column; + gap: 2px; + font-size: 12.5px; +} + +.ck-img-headline, +.ck-img-context { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; - font-size: 12.5px; +} + +.ck-img-context { + font-size: 11.5px; + color: #9aa0ab; +} + +.ck-img-context:empty { + display: none; +} + +/* A new image, previewed over the terminal while the strip is closed. */ +.ck-img-preview { + position: absolute; + right: 14px; + bottom: 14px; + z-index: 6; + display: flex; + flex-direction: column; + gap: 4px; + padding: 6px; + border: 1px solid color-mix(in srgb, var(--dd-focus) 45%, transparent); + border-radius: 8px; + background: var(--dd-surface-raised); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.45); + cursor: pointer; + animation: ck-img-preview-in 160ms ease-out; +} + +.ck-img-preview img { + display: block; + max-width: 220px; + max-height: 140px; + border-radius: 4px; + object-fit: contain; + background: #0a0b0e; +} + +.ck-img-preview-cap { + font-size: 11.5px; + color: #c9ccd4; + text-align: left; +} + +@keyframes ck-img-preview-in { + from { opacity: 0; transform: translateY(6px); } + to { opacity: 1; transform: none; } +} + +@media (prefers-reduced-motion: reduce) { + .ck-img-preview { animation: none; } +} + +/* Hovering an [Image] row in the terminal. */ +.ck-img-hover { + position: fixed; + z-index: 50; + padding: 4px; + border: 1px solid rgba(255, 255, 255, 0.14); + border-radius: 6px; + background: var(--dd-surface-raised); + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.5); + pointer-events: none; +} + +.ck-img-hover img { + display: block; + max-width: 240px; + max-height: 160px; + border-radius: 3px; } diff --git a/src/renderer/global.d.ts b/src/renderer/global.d.ts index 41849da..9747425 100644 --- a/src/renderer/global.d.ts +++ b/src/renderer/global.d.ts @@ -122,6 +122,7 @@ declare global { writeText(text: string): void; readText(): Promise; readImage(): Promise; + writeImage(dataUrl: string): Promise; readImageBytes(): Promise<{ tooLarge: boolean; bytes: string | null } | null>; }; cockpit: { diff --git a/src/renderer/icons.ts b/src/renderer/icons.ts index 7255c7e..0379f78 100644 --- a/src/renderer/icons.ts +++ b/src/renderer/icons.ts @@ -2,7 +2,7 @@ export type IconName = | 'search' | 'projects' | 'sessions' | 'tasks' | 'usage' | 'settings' | 'refresh' | 'more' | 'play' | 'panel-left' | 'folder' | 'plus' | 'restart' | 'close' | 'pin' | 'edit' | 'brain' | 'clock' - | 'trash' | 'chevron-left' | 'chevron-right' | 'chevron-down' | 'minimize' | 'maximize' | 'restore' | 'moon' | 'power' | 'globe' | 'machine' | 'image' | 'paperclip' | 'download'; + | 'trash' | 'chevron-left' | 'chevron-right' | 'chevron-down' | 'minimize' | 'maximize' | 'restore' | 'moon' | 'power' | 'globe' | 'machine' | 'image' | 'paperclip' | 'download' | 'copy' | 'terminal'; const paths: Record = { search: '', @@ -41,6 +41,8 @@ const paths: Record = { image: '', paperclip: '', download: '', + copy: '', + terminal: '', }; export function iconMarkup(name: IconName): string { diff --git a/src/renderer/imageStrip.ts b/src/renderer/imageStrip.ts index 5e48342..40ea8ee 100644 --- a/src/renderer/imageStrip.ts +++ b/src/renderer/imageStrip.ts @@ -8,6 +8,7 @@ import type { SessionImageRef } from '../shared/sessionImages'; import type { ViewImage } from '../main/imageView'; import { tr } from './i18n-runtime'; import { createIcon } from './icons'; +import { toast } from './loadError'; /** Everything the strip needs to know about one session โ€” where its images come from. */ export interface ImageSource { @@ -17,6 +18,10 @@ export interface ImageSource { get(id: string, size: 'thumb' | 'full'): Promise; /** Open an image FILE in this machine's viewer; absent for a session on another machine. */ openFile?(path: string): void; + /** Whether the row this image was shown on is still in the session's terminal. */ + canReveal?(id: string): boolean; + /** Scroll the session's terminal to the row this image was shown on. */ + reveal?(id: string): void; } const THUMB_CACHE_LIMIT = 240; @@ -34,7 +39,9 @@ let open = readOpenPref(); const seenId = new Map(); const thumbs = new Map>(); let onCountChange: () => void = () => {}; +let onRefsChange: () => void = () => {}; let refreshSeq = 0; +let popupHost: HTMLElement | null = null; function readOpenPref(): boolean { try { return localStorage.getItem(OPEN_PREF) === '1'; } catch { return false; } @@ -43,18 +50,47 @@ function writeOpenPref(v: boolean): void { try { localStorage.setItem(OPEN_PREF, v ? '1' : '0'); } catch { /* a private window simply forgets */ } } -export function initImageStrip(el: HTMLElement, countChanged: () => void): void { +/** + * `popupIn` is where a new image is previewed while the strip is closed (over the terminal); + * `refsChanged` fires whenever the selected session's list changes, for whoever marks the terminal. + */ +export function initImageStrip(el: HTMLElement, countChanged: () => void, opts: { popupIn?: HTMLElement; refsChanged?: () => void } = {}): void { stripEl = el; onCountChange = countChanged; + popupHost = opts.popupIn ?? null; + if (opts.refsChanged) onRefsChange = opts.refsChanged; render(); } +/** Which session the strip's list belongs to right now. */ +export function currentImageSourceKey(): string | null { return source?.key ?? null; } + +/** The selected session's images, oldest first. */ +export function currentImageRefs(): readonly SessionImageRef[] { return refs; } + +/** Open the viewer on one of the selected session's images. */ +export function openImageById(id: string): boolean { + const src = source; if (!src) return false; + const at = refs.findIndex((r) => r.id === id); + if (at < 0) return false; + openViewer(src, refs, at); + return true; +} + +/** A small thumbnail of one of the selected session's images, for a hover preview. */ +export function imageThumbUrl(id: string): Promise { + const src = source; const ref = refs.find((r) => r.id === id); + return src && ref ? thumbUrl(src, ref) : Promise.resolve(null); +} + /** Point the strip at a session (or none). Cheap to call on every selection change. */ export function showImagesFor(next: ImageSource | null): void { if (next?.key === source?.key) { source = next; return; } source = next; refs = []; + dismissPreview(); // it was about the session being left render(); + onRefsChange(); void refreshImages(); } @@ -67,14 +103,62 @@ export async function refreshImages(): Promise { // git refresh, and a list arriving after one of those is still this session's list. if (seq !== refreshSeq || src.key !== source?.key) return; const changed = list.length !== refs.length || list[list.length - 1]?.id !== refs[refs.length - 1]?.id; + const known = refs.length > 0 ? new Set(refs.map((r) => r.id)) : null; refs = list; const newest = list.length ? list[list.length - 1].id : null; if (!seenId.has(src.key)) seenId.set(src.key, newest); // what was there before we looked is not "new" if (open) seenId.set(src.key, newest); - if (changed) render(); + if (changed) { + render(); + onRefsChange(); + // Only what arrived while this session was being watched: the first list of a session is history. + // The person's own pastes are not news to them. + const arrived = known ? list.filter((r) => !known.has(r.id) && r.kind !== 'pasted') : []; + if (!open && arrived.length) previewArrival(src, arrived); + } onCountChange(); } +// ---- arrival preview: a new screenshot shows up briefly over the terminal while the strip is closed ---- +const PREVIEW_MS = 4000; +let previewEl: HTMLElement | null = null; +let previewTimer: ReturnType | null = null; + +function previewArrival(src: ImageSource, arrived: SessionImageRef[]): void { + if (!popupHost) return; + const ref = arrived[arrived.length - 1]; + const extra = Number(previewEl?.dataset.count ?? 0) + arrived.length; + previewEl?.remove(); + const b = document.createElement('button'); b.type = 'button'; b.className = 'ck-img-preview'; + b.dataset.count = String(extra); + const label = describe(ref); + b.title = `${label}\n${tr('cockpit.image_preview_hint')}`; b.setAttribute('aria-label', label); + const img = document.createElement('img'); img.alt = ''; + const cap = document.createElement('span'); cap.className = 'ck-img-preview-cap'; + cap.textContent = extra > 1 ? tr('cockpit.image_preview_many', { count: String(extra) }) : tr('cockpit.image_preview_one'); + b.append(img, cap); + void thumbUrl(src, ref).then((url) => { if (url) img.src = url; else b.remove(); }); + const id = ref.id; + b.addEventListener('click', () => { dismissPreview(); if (source?.key === src.key) openViewer(src, refs, refs.findIndex((r) => r.id === id)); }); + // Kept while the pointer is on it, so it can be reached before it goes. + b.addEventListener('mouseenter', () => { if (previewTimer) clearTimeout(previewTimer); }); + b.addEventListener('mouseleave', () => armPreview()); + popupHost.appendChild(b); + previewEl = b; + armPreview(); +} + +function armPreview(): void { + if (previewTimer) clearTimeout(previewTimer); + previewTimer = setTimeout(dismissPreview, PREVIEW_MS); +} + +function dismissPreview(): void { + if (previewTimer) clearTimeout(previewTimer); + previewTimer = null; + previewEl?.remove(); previewEl = null; +} + export function imageStripState(): { total: number; unseen: number; open: boolean } { const total = refs.length; if (!source || !seenId.has(source.key)) return { total, unseen: 0, open }; @@ -87,6 +171,7 @@ export function imageStripState(): { total: number; unseen: number; open: boolea export function toggleImageStrip(): void { open = !open; writeOpenPref(open); + if (open) dismissPreview(); if (open && source) seenId.set(source.key, refs.length ? refs[refs.length - 1].id : null); render(); onCountChange(); @@ -116,7 +201,7 @@ function render(): void { for (const ref of shown) { const b = document.createElement('button'); b.type = 'button'; b.className = `ck-img-thumb kind-${ref.kind}`; const label = describe(ref); - b.title = label; b.setAttribute('aria-label', label); + b.title = [label, ...contextLines(ref)].join('\n'); b.setAttribute('aria-label', label); const img = document.createElement('img'); img.alt = ''; img.decoding = 'async'; img.loading = 'lazy'; b.appendChild(img); void thumbUrl(src, ref).then((url) => { @@ -137,6 +222,18 @@ function describe(ref: SessionImageRef): string { return when ? `${what} ยท ${when}` : what; } +/** + * What was going on when the image was made, from the transcript: what the tool was pointed at, the + * page its browser was on, and what the agent had just said (or the person, for a paste). + */ +function contextLines(ref: SessionImageRef): string[] { + const out: string[] = []; + if (ref.detail && ref.detail !== ref.path) out.push(`โ–ธ ${ref.detail}`); + if (ref.page && ref.page !== ref.detail) out.push(`๐ŸŒ ${ref.page}`); + if (ref.said) out.push(`โ€œ${ref.said}โ€`); + return out; +} + /** `mcp__playwright__browser_take_screenshot` โ†’ `playwright ยท browser_take_screenshot`. */ function prettyTool(tool: string | null): string { if (!tool) return ''; @@ -154,6 +251,9 @@ function openViewer(src: ImageSource, list: SessionImageRef[], startIndex: numbe stage.append(img, status); const bar = document.createElement('div'); bar.className = 'ck-img-bar'; const caption = document.createElement('span'); caption.className = 'ck-img-caption'; + const headline = document.createElement('span'); headline.className = 'ck-img-headline'; + const context = document.createElement('span'); context.className = 'ck-img-context'; + caption.append(headline, context); const counter = document.createElement('span'); counter.className = 'ck-img-counter'; const mk = (icon: Parameters[0], label: string, fn: () => void): HTMLButtonElement => { const b = document.createElement('button'); b.type = 'button'; b.className = 'ck-act'; b.title = label; b.setAttribute('aria-label', label); @@ -167,17 +267,37 @@ function openViewer(src: ImageSource, list: SessionImageRef[], startIndex: numbe const a = document.createElement('a'); a.href = img.src; a.download = fileNameFor(list[index], img.src); a.click(); }); const openBtn = mk('folder', tr('cockpit.image_open_file'), () => { const r = list[index]; if (r.path && src.openFile) src.openFile(r.path); }); + const copy = mk('copy', tr('cockpit.image_copy'), () => { + if (!img.src) return; + void toPngDataUrl(img).then((url) => window.devdeck.clipboard.writeImage(url)) + .then((ok) => toast(tr(ok ? 'cockpit.image_copied' : 'cockpit.image_copy_failed'))) + .catch(() => toast(tr('cockpit.image_copy_failed'))); + }); + // Where in the conversation this was: the row of the terminal it was shown on. + const reveal = mk('terminal', tr('cockpit.image_reveal'), () => { + const id = list[index].id; + if (!src.canReveal?.(id)) return; + done(); + src.reveal?.(id); + }); const close = mk('close', tr('cockpit.close'), () => done()); - bar.append(prev, counter, next, caption, save, openBtn, close); + bar.append(prev, counter, next, caption, reveal, copy, save, openBtn, close); overlay.append(stage, bar); let seq = 0; const show = (): void => { const ref = list[index]; const my = ++seq; - caption.textContent = describe(ref); + headline.textContent = describe(ref); + const lines = contextLines(ref); + context.textContent = lines.join(' ยท '); context.title = lines.join('\n'); counter.textContent = `${list.length - index} / ${list.length}`; // counted newest-first, like the strip openBtn.hidden = !(ref.kind === 'file' && src.openFile); + const locatable = !!src.canReveal?.(ref.id); + reveal.hidden = !src.canReveal; // a lone image opened from a path has no place in a conversation + reveal.disabled = !locatable; + reveal.title = tr(locatable ? 'cockpit.image_reveal' : 'cockpit.image_reveal_gone'); + reveal.setAttribute('aria-label', reveal.title); prev.disabled = index >= list.length - 1; next.disabled = index <= 0; // prev = newer, next = older status.textContent = tr('cockpit.image_loading'); img.removeAttribute('src'); void src.get(ref.id, 'full').then((v) => { @@ -207,6 +327,15 @@ function openViewer(src: ImageSource, list: SessionImageRef[], startIndex: numbe show(); } +/** The shown image in a form Electron decodes (PNG or JPEG pass through; anything else is redrawn as PNG). */ +async function toPngDataUrl(img: HTMLImageElement): Promise { + if (/^data:image\/(?:png|jpeg);/.test(img.src)) return img.src; + await img.decode().catch(() => undefined); + const c = document.createElement('canvas'); c.width = img.naturalWidth; c.height = img.naturalHeight; + c.getContext('2d')?.drawImage(img, 0, 0); + return c.toDataURL('image/png'); +} + function fileNameFor(ref: SessionImageRef, dataUrl: string): string { if (ref.path) return ref.path.split(/[\\/]/).pop() || 'image.png'; const ext = /^data:image\/(\w+)/.exec(dataUrl)?.[1]?.replace('jpeg', 'jpg') ?? 'png'; diff --git a/src/renderer/locales/en.json b/src/renderer/locales/en.json index 589a4e1..7efb9b9 100644 --- a/src/renderer/locales/en.json +++ b/src/renderer/locales/en.json @@ -480,6 +480,15 @@ "cockpit.image_prev": "Newer", "cockpit.image_next": "Older", "cockpit.image_save": "Save image", + "cockpit.image_copy": "Copy image", + "cockpit.image_copied": "Image copied to the clipboard", + "cockpit.image_copy_failed": "Could not copy the image", + "cockpit.image_reveal": "Show where it appeared in the terminal", + "cockpit.image_reveal_gone": "Its row is no longer in the terminal", + "cockpit.image_preview_one": "New image", + "cockpit.image_preview_many": "{count} new images", + "cockpit.image_preview_hint": "Click to view", + "cockpit.prompt_none": "No earlier prompts on this screen", "cockpit.image_open_file": "Open file", "cockpit.image_loading": "Loadingโ€ฆ", "cockpit.update_all": "Update all ({count})", diff --git a/src/renderer/locales/ja.json b/src/renderer/locales/ja.json index 4bdb44d..a37632c 100644 --- a/src/renderer/locales/ja.json +++ b/src/renderer/locales/ja.json @@ -480,6 +480,15 @@ "cockpit.image_prev": "ๆ–ฐใ—ใ„ๆ–น", "cockpit.image_next": "ๅคใ„ๆ–น", "cockpit.image_save": "็”ปๅƒใ‚’ไฟๅญ˜", + "cockpit.image_copy": "็”ปๅƒใ‚’ใ‚ณใƒ”ใƒผ", + "cockpit.image_copied": "็”ปๅƒใ‚’ใ‚ฏใƒชใƒƒใƒ—ใƒœใƒผใƒ‰ใซใ‚ณใƒ”ใƒผใ—ใพใ—ใŸ", + "cockpit.image_copy_failed": "็”ปๅƒใ‚’ใ‚ณใƒ”ใƒผใงใใพใ›ใ‚“ใงใ—ใŸ", + "cockpit.image_reveal": "ใ‚ฟใƒผใƒŸใƒŠใƒซใงไฝ็ฝฎใ‚’่กจ็คบ", + "cockpit.image_reveal_gone": "ใ“ใฎ็”ปๅƒใฎ่กŒใฏใ‚‚ใ†ใ‚ฟใƒผใƒŸใƒŠใƒซใซใ‚ใ‚Šใพใ›ใ‚“", + "cockpit.image_preview_one": "ๆ–ฐใ—ใ„็”ปๅƒ", + "cockpit.image_preview_many": "ๆ–ฐใ—ใ„็”ปๅƒ {count} ไปถ", + "cockpit.image_preview_hint": "ใ‚ฏใƒชใƒƒใ‚ฏใ—ใฆ่กจ็คบ", + "cockpit.prompt_none": "ใ“ใฎ็”ป้ขใซไปฅๅ‰ใฎใƒ—ใƒญใƒณใƒ—ใƒˆใฏใ‚ใ‚Šใพใ›ใ‚“", "cockpit.image_open_file": "ใƒ•ใ‚กใ‚คใƒซใ‚’้–‹ใ", "cockpit.image_loading": "่ชญใฟ่พผใฟไธญโ€ฆ", "cockpit.update_all": "ใ™ในใฆๆ›ดๆ–ฐ ({count})", diff --git a/src/renderer/locales/ko.json b/src/renderer/locales/ko.json index fdc5d88..0580fa1 100644 --- a/src/renderer/locales/ko.json +++ b/src/renderer/locales/ko.json @@ -480,6 +480,15 @@ "cockpit.image_prev": "๋” ์ตœ๊ทผ", "cockpit.image_next": "์ด์ „", "cockpit.image_save": "์ด๋ฏธ์ง€ ์ €์žฅ", + "cockpit.image_copy": "์ด๋ฏธ์ง€ ๋ณต์‚ฌ", + "cockpit.image_copied": "์ด๋ฏธ์ง€๋ฅผ ํด๋ฆฝ๋ณด๋“œ์— ๋ณต์‚ฌํ–ˆ์Šต๋‹ˆ๋‹ค", + "cockpit.image_copy_failed": "์ด๋ฏธ์ง€๋ฅผ ๋ณต์‚ฌํ•˜์ง€ ๋ชปํ–ˆ์Šต๋‹ˆ๋‹ค", + "cockpit.image_reveal": "ํ„ฐ๋ฏธ๋„์—์„œ ์œ„์น˜ ๋ณด๊ธฐ", + "cockpit.image_reveal_gone": "์ด ์ด๋ฏธ์ง€๊ฐ€ ๋‚˜์˜จ ์ค„์ด ํ„ฐ๋ฏธ๋„์— ๋” ์ด์ƒ ์—†์Šต๋‹ˆ๋‹ค", + "cockpit.image_preview_one": "์ƒˆ ์ด๋ฏธ์ง€", + "cockpit.image_preview_many": "์ƒˆ ์ด๋ฏธ์ง€ {count}๊ฐœ", + "cockpit.image_preview_hint": "ํด๋ฆญํ•ด์„œ ๋ณด๊ธฐ", + "cockpit.prompt_none": "์ด ํ™”๋ฉด์— ์ด์ „ ํ”„๋กฌํ”„ํŠธ๊ฐ€ ์—†์Šต๋‹ˆ๋‹ค", "cockpit.image_open_file": "ํŒŒ์ผ ์—ด๊ธฐ", "cockpit.image_loading": "๋ถˆ๋Ÿฌ์˜ค๋Š” ์ค‘โ€ฆ", "cockpit.update_all": "๋ชจ๋‘ ์—…๋ฐ์ดํŠธ ({count})", diff --git a/src/renderer/locales/zh.json b/src/renderer/locales/zh.json index cace641..c6a034e 100644 --- a/src/renderer/locales/zh.json +++ b/src/renderer/locales/zh.json @@ -480,6 +480,15 @@ "cockpit.image_prev": "่พƒๆ–ฐ", "cockpit.image_next": "่พƒๆ—ง", "cockpit.image_save": "ไฟๅญ˜ๅ›พ็‰‡", + "cockpit.image_copy": "ๅคๅˆถๅ›พ็‰‡", + "cockpit.image_copied": "ๅทฒๅฐ†ๅ›พ็‰‡ๅคๅˆถๅˆฐๅ‰ช่ดดๆฟ", + "cockpit.image_copy_failed": "ๆ— ๆณ•ๅคๅˆถๅ›พ็‰‡", + "cockpit.image_reveal": "ๅœจ็ปˆ็ซฏไธญๆ˜พ็คบไฝ็ฝฎ", + "cockpit.image_reveal_gone": "่ฟ™ๅผ ๅ›พ็‰‡ๆ‰€ๅœจ็š„่กŒๅทฒไธๅœจ็ปˆ็ซฏไธญ", + "cockpit.image_preview_one": "ๆ–ฐๅ›พ็‰‡", + "cockpit.image_preview_many": "{count} ๅผ ๆ–ฐๅ›พ็‰‡", + "cockpit.image_preview_hint": "็‚นๅ‡ปๆŸฅ็œ‹", + "cockpit.prompt_none": "ๆญคๅฑๅน•ไธŠๆฒกๆœ‰ไน‹ๅ‰็š„ๆ็คบ", "cockpit.image_open_file": "ๆ‰“ๅผ€ๆ–‡ไปถ", "cockpit.image_loading": "ๅŠ ่ฝฝไธญโ€ฆ", "cockpit.update_all": "ๅ…จ้ƒจๆ›ดๆ–ฐ ({count})", diff --git a/src/renderer/termMarks.ts b/src/renderer/termMarks.ts new file mode 100644 index 0000000..c4064da --- /dev/null +++ b/src/renderer/termMarks.ts @@ -0,0 +1,123 @@ +// src/renderer/termMarks.ts +// Ties the selected session's images to the terminal rows they were shown on, and finds the person's +// earlier prompts โ€” so "which step was this screenshot?" and "where did I ask that?" are one click +// or one key away instead of a scroll through thousands of rows. +// +// The rows are found by what Claude Code prints (see shared/terminalMarks); each is then held by an +// xterm marker, which follows its row as output scrolls it up and is disposed when the scrollback +// drops it. +import type { IDecoration, IMarker, Terminal } from '@xterm/xterm'; +import type { SessionImageRef } from '../shared/sessionImages'; +import { findImagePlaceholders, findPromptRows, nextPromptRow, pairPlaceholders, type PromptRowInput } from '../shared/terminalMarks'; + +const MARK_BG = '#1f3352'; +const MARK_FG = '#a8cdff'; +const FLASH_BG = '#4a4220'; +const FLASH_MS = 1600; +/** Rows of context kept above a row scrolled to, so the call that led to it is on screen too. */ +const REVEAL_CONTEXT_ROWS = 4; + +interface Mark { id: string; marker: IMarker; deco: IDecoration | undefined; col: number; len: number } + +class TermMarks { + private marks: Mark[] = []; + private sig = ''; + constructor(private readonly term: Terminal) {} + + /** Re-find the placeholders on screen and pair them with `refs`. Cheap to call when nothing moved. */ + update(refs: readonly SessionImageRef[]): void { + const buf = this.term.buffer.active; + if (buf.type !== 'normal' || refs.length === 0) { this.clear(); return; } + const rows = readRows(this.term); + const found = findImagePlaceholders(rows); + const pairs = pairPlaceholders(refs, found, (r) => rows[r] ?? ''); + const wanted = [...pairs].map(([mi, id]) => ({ ...found[mi], id })); + const sig = wanted.map((w) => `${w.row}:${w.col}:${w.id}`).join('|'); + if (sig === this.sig && this.marks.every((m) => !m.marker.isDisposed)) return; + this.clear(); + this.sig = sig; + const cursorRow = buf.baseY + buf.cursorY; + for (const w of wanted) { + const marker = this.term.registerMarker(w.row - cursorRow); + if (!marker) continue; + const deco = this.term.registerDecoration({ marker, x: w.col, width: w.len, backgroundColor: MARK_BG, foregroundColor: MARK_FG, layer: 'top' }); + // Colour only: the link under it has to keep getting the pointer. + deco?.onRender((el) => { el.style.pointerEvents = 'none'; }); + this.marks.push({ id: w.id, marker, deco, col: w.col, len: w.len }); + } + } + + /** The image shown at buffer cell (row, col), 0-based. */ + idAt(row: number, col: number): string | null { + for (const m of this.marks) if (!m.marker.isDisposed && m.marker.line === row && col >= m.col && col < m.col + m.len) return m.id; + return null; + } + + locate(id: string): number | null { + const m = this.marks.find((x) => x.id === id && !x.marker.isDisposed); + return m ? m.marker.line : null; + } + + reveal(id: string): boolean { + const row = this.locate(id); + if (row === null) return false; + this.term.scrollToLine(Math.max(0, row - REVEAL_CONTEXT_ROWS)); + flashRow(this.term, row); + return true; + } + + clear(): void { + for (const m of this.marks) { m.deco?.dispose(); m.marker.dispose(); } + this.marks = []; this.sig = ''; + } +} + +const byTerm = new WeakMap(); +export function marksFor(term: Terminal): TermMarks { + let m = byTerm.get(term); + if (!m) { m = new TermMarks(term); byTerm.set(term, m); } + return m; +} + +function readRows(term: Terminal): string[] { + const buf = term.buffer.active; + const rows = new Array(buf.length); + for (let i = 0; i < buf.length; i++) rows[i] = buf.getLine(i)?.translateToString(true) ?? ''; + return rows; +} + +/** Highlight a row for a moment, so the eye lands on it after a jump. */ +export function flashRow(term: Terminal, row: number): void { + const buf = term.buffer.active; + const marker = term.registerMarker(row - (buf.baseY + buf.cursorY)); + if (!marker) return; + const deco = term.registerDecoration({ marker, x: 0, width: term.cols, backgroundColor: FLASH_BG, layer: 'bottom' }); + setTimeout(() => { deco?.dispose(); marker.dispose(); }, FLASH_MS); +} + +/** + * Scroll to the previous (-1) or next (+1) prompt the person sent, from the top of the viewport. + * Past the last one is the bottom. Returns false when the screen holds no prompt to go to. + */ +export function jumpToPrompt(term: Terminal, dir: -1 | 1): boolean { + const buf = term.buffer.active; + if (buf.type !== 'normal') return false; + const rows: PromptRowInput[] = new Array(buf.length); + for (let i = 0; i < buf.length; i++) { + const line = buf.getLine(i); + const text = line?.translateToString(true) ?? ''; + // The pointer is the first non-blank cell; its background is what tells a sent prompt from text. + const at = text.search(/\S/); + const cell = at >= 0 ? line?.getCell(at) : undefined; + rows[i] = { text, shaded: !!cell && !cell.isBgDefault() }; + } + const prompts = findPromptRows(rows); + if (prompts.length === 0) return false; + // At the bottom, the newest prompt may be on screen already โ€” going back starts from it, not above it. + const from = dir < 0 && buf.viewportY >= buf.baseY ? buf.length : buf.viewportY; + const to = nextPromptRow(prompts, from, dir); + if (to === null) { term.scrollToBottom(); return true; } + term.scrollToLine(to); + flashRow(term, to); + return true; +} diff --git a/src/shared/sessionImages.ts b/src/shared/sessionImages.ts index 87f7cce..8b42603 100644 --- a/src/shared/sessionImages.ts +++ b/src/shared/sessionImages.ts @@ -19,6 +19,15 @@ export interface SessionImageRef { tool: string | null; /** For `file`: the path exactly as the tool named it. */ path: string | null; + /** + * What the tool was asked to do, as one line โ€” the element, file or URL it was pointed at. Absent + * from hosts older than 1.40.0, so read as optional. + */ + detail?: string | null; + /** The page that tool's browser was last sent to, for a screenshot from a browser tool. */ + page?: string | null; + /** The agent's last words before this image โ€” or, for a pasted image, what the person wrote with it. */ + said?: string | null; } const IMAGE_FILE_RE = /\.(?:png|jpe?g|gif|webp|bmp)$/i; @@ -27,13 +36,61 @@ const PATH_FIELDS = ['file_path', 'path', 'filePath', 'output_path', 'outputPath /** How long a tool name is worth remembering for: tool results follow their call within a few lines. */ const TOOL_NAME_MEMORY = 256; const TOOL_USE_RE = /"type":"tool_use","id":"([^"]{1,100})","name":"([^"]{1,200})"/g; +/** + * Lines past this are not parsed just for context: they are the ones carrying a whole file in a + * Write call or a screenshot, and JSON-parsing megabytes per tool call would dwarf the index itself. + */ +const CONTEXT_PARSE_MAX = 64 * 1024; +const CONTEXT_MAX_CHARS = 160; +/** Tool-input fields worth showing as "what it was asked", most telling first. */ +const DETAIL_FIELDS = ['url', 'element', 'selector', 'filename', 'file_path', 'path', 'filePath', 'name', 'description', 'query', 'text']; /** Cheap pre-filter on the raw bytes of a line, so most lines are never JSON-parsed at all. */ export function lineMayHoldImage(line: string): boolean { return line.includes('"type":"image"') || (line.includes('"tool_use"') && /\.(?:png|jpe?g|gif|webp|bmp)"/i.test(line)); } -interface Block { type?: unknown; source?: unknown; content?: unknown; tool_use_id?: unknown; id?: unknown; name?: unknown; input?: unknown } +interface Block { type?: unknown; source?: unknown; content?: unknown; tool_use_id?: unknown; id?: unknown; name?: unknown; input?: unknown; text?: unknown } + +/** One line of at most CONTEXT_MAX_CHARS: the first non-empty line of `text`, whitespace collapsed. */ +export function oneLine(text: string): string | null { + const first = text.split(/\r?\n/).map((l) => l.replace(/\s+/g, ' ').trim()).find((l) => l.length > 0); + if (!first) return null; + return first.length > CONTEXT_MAX_CHARS ? first.slice(0, CONTEXT_MAX_CHARS - 1) + 'โ€ฆ' : first; +} + +/** The input field that best says what a tool call was aimed at. */ +export function toolDetail(input: unknown): string | null { + if (!input || typeof input !== 'object') return null; + const rec = input as Record; + for (const field of DETAIL_FIELDS) { + const v = rec[field]; + if (typeof v === 'string' && v.trim() && v.length <= 2000) return oneLine(v); + } + return null; +} + +/** `mcp__playwright__browser_navigate` โ†’ `playwright`; a built-in tool has no server. */ +export function toolServer(tool: string | null): string | null { + return tool ? /^mcp__(.+?)__/.exec(tool)?.[1] ?? null : null; +} + +function textOf(content: unknown[]): string | null { + let out: string | null = null; + for (const raw of content) { + const b = (raw ?? {}) as Block; + if (b.type === 'text' && typeof b.text === 'string') { + const t = oneLine(stripImageNoise(b.text)); + if (t) out = t; // the LAST thing said is the one that leads into what follows + } + } + return out; +} + +/** The placeholders an agent writes where an image was, which say nothing on their own. */ +function stripImageNoise(text: string): string { + return text.replace(/\[Image\b[^\]]*\]/gi, ' ').replace(/\S*devdeck-paste-[0-9a-f-]+\.(?:png|jpe?g|gif|webp)/gi, ' '); +} function isEmbeddedImage(b: Block): boolean { if (b.type !== 'image' || !b.source || typeof b.source !== 'object') return false; @@ -71,47 +128,71 @@ export function embeddedImageBlocks(entry: unknown): { block: { media_type: stri * recent calls are remembered (boundedly) as their lines go by. */ export class SessionImageCollector { - private toolNames = new Map(); + private toolNames = new Map(); + /** Per MCP server, the page its browser was last sent to. */ + private pages = new Map(); + private lastSaid: string | null = null; /** `line` is one complete transcript line; `offset` is the byte offset it starts at. */ add(line: string, offset: number): SessionImageRef[] { - // Tool calls are named on every line that makes one โ€” most never hold an image, so this is a - // regex over the raw text rather than a parse. - if (line.includes('"tool_use"')) { - for (const m of line.matchAll(TOOL_USE_RE)) this.remember(m[1], m[2]); - } - if (!lineMayHoldImage(line)) return []; + const mayHoldImage = lineMayHoldImage(line); + // Context comes from lines that hold no image: the calls that led up to one and what the agent + // said meanwhile. Only small lines are parsed for it; the rest are named by a regex alone. + const small = line.length <= CONTEXT_PARSE_MAX; let entry: unknown; - try { entry = JSON.parse(line); } catch { return []; } - const tsRaw = (entry as { timestamp?: unknown }).timestamp; + if (mayHoldImage || (small && (line.includes('"tool_use"') || line.includes('"type":"text"')))) { + try { entry = JSON.parse(line); } catch { entry = undefined; } + } + if (entry === undefined && line.includes('"tool_use"')) { + for (const m of line.matchAll(TOOL_USE_RE)) this.remember(m[1], m[2], null); + } + const content = (entry as { message?: { role?: unknown; content?: unknown } } | undefined)?.message?.content; + const role = (entry as { message?: { role?: unknown } } | undefined)?.message?.role; + const blocks = Array.isArray(content) ? content : []; + const refs: SessionImageRef[] = []; + const tsRaw = (entry as { timestamp?: unknown } | undefined)?.timestamp; const parsedTs = typeof tsRaw === 'string' ? Date.parse(tsRaw) : NaN; const ts = Number.isFinite(parsedTs) ? parsedTs : null; - const refs: SessionImageRef[] = []; - const content = (entry as { message?: { content?: unknown } })?.message?.content; - if (Array.isArray(content)) { - for (const raw of content) { - const b = (raw ?? {}) as Block; - if (b.type !== 'tool_use' || typeof b.name !== 'string') continue; - if (typeof b.id === 'string') this.remember(b.id, b.name); - const input = (b.input ?? {}) as Record; - for (const field of PATH_FIELDS) { - const p = input[field]; - if (typeof p === 'string' && p.length < 1024 && IMAGE_FILE_RE.test(p)) { - refs.push({ id: `f:${p}`, kind: 'file', ts, tool: b.name, path: p }); - break; - } + for (const raw of blocks) { + const b = (raw ?? {}) as Block; + if (b.type !== 'tool_use' || typeof b.name !== 'string') continue; + const detail = toolDetail(b.input); + if (typeof b.id === 'string') this.remember(b.id, b.name, detail); + const input = (b.input ?? {}) as Record; + const server = toolServer(b.name); + if (server && typeof input.url === 'string' && /^https?:\/\//i.test(input.url)) this.pages.set(server, oneLine(input.url) ?? input.url); + if (!mayHoldImage) continue; + for (const field of PATH_FIELDS) { + const p = input[field]; + if (typeof p === 'string' && p.length < 1024 && IMAGE_FILE_RE.test(p)) { + refs.push({ id: `f:${p}`, kind: 'file', ts, tool: b.name, path: p, detail, page: null, said: this.lastSaid }); + break; } } } - embeddedImageBlocks(entry).forEach((img, n) => { - refs.push({ id: `${offset}.${n}`, kind: img.kind, ts, tool: img.toolUseId ? this.toolNames.get(img.toolUseId) ?? null : null, path: null }); - }); + // What the agent said; a user line's text belongs to that user's own images, read below. + const said = blocks.length ? textOf(blocks) : null; + if (role === 'assistant' && said) this.lastSaid = said; + + if (mayHoldImage && entry !== undefined) { + embeddedImageBlocks(entry).forEach((img, n) => { + const call = img.toolUseId ? this.toolNames.get(img.toolUseId) : undefined; + const tool = call?.name ?? null; + const server = toolServer(tool); + refs.push({ + id: `${offset}.${n}`, kind: img.kind, ts, tool, path: null, + detail: call?.detail ?? null, + page: server && /browser|chrome|playwright|puppeteer/i.test(tool ?? '') ? this.pages.get(server) ?? null : null, + said: img.kind === 'pasted' ? said : this.lastSaid, + }); + }); + } return refs; } - private remember(id: string, name: string): void { - this.toolNames.set(id, name); + private remember(id: string, name: string, detail: string | null): void { + this.toolNames.set(id, { name, detail }); if (this.toolNames.size > TOOL_NAME_MEMORY) this.toolNames.delete(this.toolNames.keys().next().value as string); } } diff --git a/src/shared/terminalKeys.test.ts b/src/shared/terminalKeys.test.ts index 0d3b289..83ed34d 100644 --- a/src/shared/terminalKeys.test.ts +++ b/src/shared/terminalKeys.test.ts @@ -57,6 +57,12 @@ describe('decideKeyAction', () => { expect(decideKeyAction(key({ ctrlKey: true, key: 'f' }), false)).toBe('find'); expect(decideKeyAction(key({ ctrlKey: true, key: 'F' }), true)).toBe('find'); }); + it('Ctrl+Up / Ctrl+Down step between prompts; plain and shifted arrows still reach the agent', () => { + expect(decideKeyAction(key({ ctrlKey: true, key: 'ArrowUp' }), false)).toBe('prompt-prev'); + expect(decideKeyAction(key({ ctrlKey: true, key: 'ArrowDown' }), true)).toBe('prompt-next'); + expect(decideKeyAction(key({ key: 'ArrowUp' }), false)).toBe('pass'); + expect(decideKeyAction(key({ ctrlKey: true, shiftKey: true, key: 'ArrowUp' }), false)).toBe('pass'); + }); it('Alt+F / plain f still pass through', () => { expect(decideKeyAction(key({ ctrlKey: true, altKey: true, key: 'f' }), false)).toBe('pass'); expect(decideKeyAction(key({ key: 'f' }), false)).toBe('pass'); diff --git a/src/shared/terminalKeys.ts b/src/shared/terminalKeys.ts index 00f543b..ca55454 100644 --- a/src/shared/terminalKeys.ts +++ b/src/shared/terminalKeys.ts @@ -1,4 +1,4 @@ -export type KeyAction = 'copy' | 'paste' | 'find' | 'quickopen' | 'pass'; +export type KeyAction = 'copy' | 'paste' | 'find' | 'quickopen' | 'prompt-prev' | 'prompt-next' | 'pass'; /** The keyboard-event shape we need โ€” a subset of DOM KeyboardEvent, so it's unit-testable without a DOM. */ export interface KeyLike { @@ -24,6 +24,8 @@ export interface KeyLike { * - Ctrl+Shift+P -> quickopen (the sidebar's search). Shift does not change the control character a * terminal derives from a letter, so without this the chord would reach the PTY as a plain Ctrl+P * (readline's "previous history") while the user was trying to jump to another session. + * - Ctrl+Up / Ctrl+Down -> prompt-prev / prompt-next: scroll to the previous or next prompt the person + * sent. Claude Code binds neither; a shell would read them as plain arrows, which they are not. * - Everything else passes through to the PTY unchanged. * * Alt is excluded so Alt+C / Alt+V (rare app bindings) are never hijacked. @@ -48,5 +50,7 @@ export function decideKeyAction(e: KeyLike, hasSelection: boolean): KeyAction { if (k === 'c' && hasSelection) return 'copy'; if (k === 'v') return 'paste'; if (k === 'f') return 'find'; + if (!e.shiftKey && e.key === 'ArrowUp') return 'prompt-prev'; + if (!e.shiftKey && e.key === 'ArrowDown') return 'prompt-next'; return 'pass'; } diff --git a/src/shared/terminalMarks.test.ts b/src/shared/terminalMarks.test.ts new file mode 100644 index 0000000..fea50cf --- /dev/null +++ b/src/shared/terminalMarks.test.ts @@ -0,0 +1,80 @@ +// src/shared/terminalMarks.test.ts +import { describe, it, expect } from 'vitest'; +import { findImagePlaceholders, findPromptRows, nextPromptRow, pairPlaceholders } from './terminalMarks'; +import type { SessionImageRef } from './sessionImages'; + +const ref = (id: string, kind: SessionImageRef['kind'], tool: string | null): SessionImageRef => ({ id, kind, ts: null, tool, path: null }); + +describe('findImagePlaceholders', () => { + it('finds tool images and pasted ones where Claude Code printed them', () => { + const rows = ['โ— playwright - browser_take_screenshot (MCP)(element: "x")', ' โŽฟ [Image]', 'โฏ [Image #2] ์ด๊ฑฐ ๋ด์ค˜', 'see [Imagenet]']; + expect(findImagePlaceholders(rows, 100)).toEqual([ + { row: 101, col: 5, len: 7, kind: 'screenshot' }, + { row: 102, col: 2, len: 10, kind: 'pasted' }, + ]); + }); +}); + +describe('pairPlaceholders', () => { + const screen = [ + 'โ— playwright - browser_navigate (MCP)(url: "http://a")', // 0 + 'โ— playwright - browser_take_screenshot (MCP)', // 1 + ' โŽฟ [Image]', // 2 + 'โ— windows-mcp - Screenshot (MCP)', // 3 + ' โŽฟ [Image]', // 4 + 'โ— playwright - browser_take_screenshot (MCP)', // 5 + ' โŽฟ [Image]', // 6 + ]; + const text = (r: number): string => screen[r] ?? ''; + const marks = findImagePlaceholders(screen); + + it('pairs from the newest end, checking each against the tool named above it', () => { + const refs = [ref('a', 'screenshot', 'mcp__playwright__browser_take_screenshot'), ref('b', 'screenshot', 'mcp__windows-mcp__Screenshot'), ref('c', 'screenshot', 'mcp__playwright__browser_take_screenshot')]; + expect([...pairPlaceholders(refs, marks, text)].sort()).toEqual([[0, 'a'], [1, 'b'], [2, 'c']]); + }); + + it('skips a transcript image the screen does not show instead of shifting every older one', () => { + // 'x' (a blender look) is in the transcript but its output was folded away on screen. + const refs = [ref('a', 'screenshot', 'mcp__playwright__browser_take_screenshot'), ref('b', 'screenshot', 'mcp__windows-mcp__Screenshot'), ref('x', 'screenshot', 'mcp__blender__look'), ref('c', 'screenshot', 'mcp__playwright__browser_take_screenshot')]; + expect([...pairPlaceholders(refs, marks, text)].sort()).toEqual([[0, 'a'], [1, 'b'], [2, 'c']]); + }); + + it('leaves a placeholder unpaired rather than giving it a neighbour\'s image', () => { + const refs = [ref('p', 'screenshot', 'mcp__playwright__browser_take_screenshot')]; + // Only one playwright image is known: it is the newest placeholder's, and the windows one gets none. + const pairs = pairPlaceholders(refs, marks, text); + expect([...pairs]).toEqual([[2, 'p']]); + }); + + it('checks only the call a placeholder belongs to, not the one before it', () => { + const top = screen.slice(0, 5); // newest image is the windows one, right under a playwright call + const pairs = pairPlaceholders([ref('p1', 'screenshot', 'mcp__playwright__browser_take_screenshot'), ref('p2', 'screenshot', 'mcp__playwright__browser_take_screenshot')], findImagePlaceholders(top), (r) => top[r] ?? ''); + expect([...pairs]).toEqual([[0, 'p2']]); + }); + + it('pairs pasted images by order', () => { + const rows = ['โฏ [Image #1] first', 'ok', 'โฏ [Image #2] second']; + const pairs = pairPlaceholders([ref('p1', 'pasted', null), ref('p2', 'pasted', null)], findImagePlaceholders(rows), (r) => rows[r]); + expect([...pairs].sort()).toEqual([[0, 'p1'], [1, 'p2']]); + }); +}); + +describe('findPromptRows', () => { + const row = (text: string, shaded = false) => ({ text, shaded }); + it('finds shaded prompt rows above the composer and ignores the composer itself', () => { + const rows = [ + row('โฏ ์ฒซ ์งˆ๋ฌธ', true), row('โ— ๋‹ต๋ณ€'), row('> quoted, not shaded'), row('> ๋‘๋ฒˆ์งธ', true), row('โ— ๋‹ต'), + row('โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€'), row('โฏ typingโ€ฆ', true), row('โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€'), row(' โตโต auto mode'), + ]; + expect(findPromptRows(rows, 10)).toEqual([10, 13]); + }); + + it('steps between prompts from the viewport top, and past the last one to the bottom', () => { + const prompts = [10, 40, 90]; + expect(nextPromptRow(prompts, 50, -1)).toBe(40); + expect(nextPromptRow(prompts, 40, -1)).toBe(10); + expect(nextPromptRow(prompts, 5, -1)).toBe(10); + expect(nextPromptRow(prompts, 40, 1)).toBe(90); + expect(nextPromptRow(prompts, 90, 1)).toBeNull(); + }); +}); diff --git a/src/shared/terminalMarks.ts b/src/shared/terminalMarks.ts new file mode 100644 index 0000000..3767af3 --- /dev/null +++ b/src/shared/terminalMarks.ts @@ -0,0 +1,128 @@ +// src/shared/terminalMarks.ts +// Finding things in a Claude Code screen that the transcript also knows about: the rows where an +// image was shown, and the rows where the person's own prompts are. +// +// Claude Code prints `[Image]` where a tool returned an image and `[Image #N]` where the person +// pasted one, and draws each earlier prompt as `โฏ text` (`>` where the pointer glyph is not used) on +// its own background. Pure over row text so it can be tested without a terminal. +import { toolServer, type SessionImageRef } from './sessionImages'; + +export interface ImagePlaceholder { + /** Buffer row (0-based, absolute). */ + row: number; + /** Column where `[Image` starts, and how many cells the placeholder takes. */ + col: number; + len: number; + kind: 'screenshot' | 'pasted'; +} + +const PLACEHOLDER_RE = /\[Image( #\d{1,4})?\]/g; + +/** Every image placeholder on `rows` (row `i` is buffer row `firstRow + i`), top to bottom. */ +export function findImagePlaceholders(rows: readonly string[], firstRow = 0): ImagePlaceholder[] { + const out: ImagePlaceholder[] = []; + rows.forEach((text, i) => { + if (!text.includes('[Image')) return; + for (const m of text.matchAll(PLACEHOLDER_RE)) { + out.push({ row: firstRow + i, col: m.index ?? 0, len: m[0].length, kind: m[1] ? 'pasted' : 'screenshot' }); + } + }); + return out; +} + +/** How far above a placeholder the call that produced it is looked for (its arguments can wrap). */ +const CALL_CONTEXT_ROWS = 12; +/** How many transcript images may be missing from the screen (folded output) between two that show. */ +const MAX_SKIP = 4; +/** A tool call's first line: Claude Code's bullet (`โ—`, `โบ` on macOS), then the tool. */ +const CALL_LINE_RE = /^\s*[โ—โบโ€ข]\s/; + +const squash = (s: string): string => s.toLowerCase().replace(/[^a-z0-9]/g, ''); + +/** The words a placeholder's call line should contain for `ref` to be the image it stands for. */ +function callWords(ref: SessionImageRef): string[] { + if (!ref.tool) return []; + const server = toolServer(ref.tool); + return [squash(server ?? ref.tool)].filter(Boolean); +} + +/** + * Pair each placeholder with the image it shows, newest first, the way they are on screen. + * + * The screen and the transcript list the same images in the same order, but either can be missing + * some: the scrollback only reaches so far back, the transcript index only so many images, and + * Claude Code folds long tool output. So the pairing walks both from the newest end, and a + * placeholder is paired only when the call line above it names the same tool โ€” a ref the screen + * does not show is skipped (up to MAX_SKIP), and a placeholder nothing matches is left unpaired + * rather than given a neighbour's image. + */ +export function pairPlaceholders(refs: readonly SessionImageRef[], marks: readonly ImagePlaceholder[], rowText: (row: number) => string): Map { + const paired = new Map(); // index into marks -> ref id + for (const kind of ['screenshot', 'pasted'] as const) { + const ks = refs.filter((r) => r.kind === kind); + let ri = ks.length - 1; + for (let mi = marks.length - 1; mi >= 0 && ri >= 0; mi--) { + const mark = marks[mi]; + if (mark.kind !== kind) continue; + if (kind === 'pasted') { paired.set(mi, ks[ri].id); ri--; continue; } // nothing to check a paste against + // From the call's own bullet line down to the placeholder โ€” not further, or the call before it + // would vouch for this one. + let context = ''; + for (let r = mark.row - 1; r >= Math.max(0, mark.row - CALL_CONTEXT_ROWS); r--) { + const t = rowText(r); + context = squash(t) + ' ' + context; + if (CALL_LINE_RE.test(t)) break; + } + for (let skip = 0; skip <= MAX_SKIP && ri - skip >= 0; skip++) { + const words = callWords(ks[ri - skip]); + if (words.length === 0 || words.every((w) => context.includes(w))) { + paired.set(mi, ks[ri - skip].id); + ri -= skip + 1; + break; + } + } + } + } + return paired; +} + +export interface PromptRowInput { + text: string; + /** Whether the pointer cell is drawn on a background of its own (Claude Code's prompt band). */ + shaded: boolean; +} + +const PROMPT_RE = /^\s{0,2}[โฏ>โ€บ]\s+\S/; +const RULE_RE = /^\s*[โ”€โ”-]{8,}\s*$/; + +/** + * Rows holding an earlier prompt the person sent: a pointer, then text, on the prompt band's own + * background. The input box at the bottom is not one โ€” it sits below the last horizontal rule. + */ +export function findPromptRows(rows: readonly PromptRowInput[], firstRow = 0): number[] { + // The live input box sits between two rules near the bottom. Everything from its top rule down is + // the composer, not history. + let end = rows.length; + let bottomRule = -1; + for (let i = rows.length - 1; i >= Math.max(0, rows.length - 40); i--) if (RULE_RE.test(rows[i].text)) { bottomRule = i; break; } + if (bottomRule >= 0) { + end = bottomRule; + for (let i = bottomRule - 1; i >= Math.max(0, bottomRule - 15); i--) if (RULE_RE.test(rows[i].text)) { end = i; break; } + } + const out: number[] = []; + for (let i = 0; i < end; i++) { + const r = rows[i]; + if (r.shaded && PROMPT_RE.test(r.text)) out.push(firstRow + i); + } + return out; +} + +/** The prompt row to jump to from a viewport whose top is `viewportTop`; null means "to the bottom". */ +export function nextPromptRow(prompts: readonly number[], viewportTop: number, dir: -1 | 1): number | null { + if (dir < 0) { + for (let i = prompts.length - 1; i >= 0; i--) if (prompts[i] < viewportTop) return prompts[i]; + return prompts.length ? prompts[0] : null; + } + for (const p of prompts) if (p > viewportTop) return p; + return null; +}