diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fdcdb2..b3d7092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 @@ -114,6 +114,15 @@ jobs: - name: Verify host and two viewers agree on committed session titles if: runner.os == 'Windows' run: node qa/session-sync.mjs + - name: Performance gate (fixture agent; animations, DOM churn, main-process stalls) + if: runner.os == 'Windows' + run: node qa/perf-gate.mjs + - name: Session images, attach and update-all journeys + if: runner.os == 'Windows' + run: | + node qa/images.mjs + node qa/images-remote.mjs + node qa/claude-update.mjs - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() with: @@ -122,3 +131,7 @@ jobs: qa/shots/resilience/ qa/shots/maintenance/ qa/shots/session-sync/ + qa/shots/perf-gate/ + qa/shots/images/ + qa/shots/images-remote/ + qa/shots/claude-update/ diff --git a/docs/quality.md b/docs/quality.md index 2e63394..dcea071 100644 --- a/docs/quality.md +++ b/docs/quality.md @@ -28,7 +28,9 @@ Results and screenshots are written to `qa/shots/maintenance/` and `qa/shots/res ## Manual harnesses -`qa:link` is a supervised two-app harness that may start installed provider CLIs. `qa:close` is a supervised two-app harness for the session close lifecycle (remote, offline, refused and bulk close, adoption focus, restart, Apply update); it starts the installed `claude` CLI in a throwaway repository and sends no prompts. `npm run qa:perf -- 4 --allow-live-agents` is a manual Windows benchmark and explicitly opts into real-agent startup. Performance QA refuses missing terminals, invisible workload, incomplete streaming/repaint activity and missing CPU samples instead of treating low CPU as success. It has not become a deterministic CI performance gate; that conversion remains a separate task. +`qa:link` is a supervised two-app harness that may start installed provider CLIs. `qa:close` is a supervised two-app harness for the session close lifecycle (remote, offline, refused and bulk close, adoption focus, restart, Apply update); it starts the installed `claude` CLI in a throwaway repository and sends no prompts. `npm run qa:perf -- 4 --allow-live-agents` is a manual Windows benchmark and explicitly opts into real-agent startup. Performance QA refuses missing terminals, invisible workload, incomplete streaming/repaint activity and missing CPU samples instead of treating low CPU as success. Its CPU percentages are for a person to read, not for CI. + +The CI performance gate is `qa:perf-gate` (Windows, every push): a fixture agent instead of a real one, and units that do not depend on the runner's speed — running CSS animations and DOM mutation records on an idle deck (both must be ~0), DOM mutations outside the terminals while two tiles repaint at 10fps (a per-chunk sidebar rebuild is ~8,000 against a limit of 400), and the main process's longest event-loop stall while working and while 20,000 lines stream. Each limit was checked by injecting the regression it guards against: an infinite CSS animation, a sidebar render per PTY chunk, and a 700ms synchronous block in the main process all fail it. Results go to `qa/shots/perf-gate/results.json`. `qa:images` (Windows) drives the cockpit image strip, full-size viewer and 📎 attach against an isolated home holding a transcript with a tool screenshot, a pasted image and a written image file; its fixture `claude` echoes what it is sent, so an attached path is read back off the terminal. `qa:update` (Windows) checks "Update all": a fixture `claude` prints Claude Code's failed-update footer, and the harness asserts that `claude update` ran with no session process alive and that every conversation was resumed by id. `qa:images-remote` (Windows) pairs a host and a viewer over real TLS and checks the same from the viewer: thumbnails and the full view cross the link, an attached JPEG lands on the host as PNG with the host path typed, and a non-image named `.png` or an unindexed id is refused over the link. None of these starts a real agent. diff --git a/package.json b/package.json index 280e720..e8e5a34 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "qa:close": "npm run build && node qa/close.mjs", "qa:resilience": "npm run build && node qa/resilience.mjs", "qa:perf": "npm run build && node qa/perf.mjs", + "qa:perf-gate": "npm run build && node qa/perf-gate.mjs", "qa:maintenance": "npm run build && node qa/maintenance.mjs", "check:docs": "node scripts/check-docs.mjs", "qa:session-sync": "npm run build && node qa/session-sync.mjs", diff --git a/qa/perf-gate.mjs b/qa/perf-gate.mjs new file mode 100644 index 0000000..4bf7bf9 --- /dev/null +++ b/qa/perf-gate.mjs @@ -0,0 +1,190 @@ +// Deterministic performance gate (Windows CI): the regressions DevDeck has actually shipped, measured +// in units that do not depend on how fast the runner is. +// +// qa/perf.mjs is the manual benchmark — real agents, CPU percentages, a person reading the numbers. +// CPU percentages are meaningless across CI hardware, and real agents cost money, so this gate asks +// different questions of a fixture agent that prints a known workload: +// +// - IDLE: is anything animating or rewriting the DOM when nothing is happening? Every "the app got +// heavy" report so far was a screen that never stopped drawing (a CSS spinner composited at 60fps, +// a sidebar rebuilt on every PTY chunk). Counted, not timed. +// - WORKING: one tile repainting its whole screen at 10fps the way an agent's TUI does. Does the +// main process keep up (event-loop stall), and does the UI thread stay responsive (long tasks)? +// - STREAM: 20,000 lines into a visible terminal. The "other PC freezes for minutes" class is a main +// process that stops answering; its longest stall is the number that says so. +// +// Limits are set several times above what this machine measures, so a slow runner passes and a real +// regression (an order of magnitude) does not. Never starts a real agent. +import { _electron as electron } from 'playwright'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { closeElectron } from './electron-lifecycle.mjs'; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const out = resolve(process.env.DEVDECK_PERF_OUT || join(root, 'qa', 'shots', 'perf-gate')); +mkdirSync(out, { recursive: true }); +if (process.platform !== 'win32') { console.log('SKIP: Windows native PTY required'); process.exit(0); } + +const LIMITS = { + idleRunningAnimations: 0, // nothing may animate on a quiet deck + idleMutations: 40, // DOM mutation records in 6s of quiet (a rebuild-per-tick loop is thousands) + idleLongTasks: 0, + workingMainStallMs: 250, // longest main-process event-loop stall while a tile repaints at 10fps + workingLongestTaskMs: 250, + workingOutsideMutations: 400, // sidebar/header/footer while two tiles repaint at 10fps for 8s (a rebuild per chunk is ~10k) + streamMainStallMs: 600, // longest main-process stall while 20k lines stream + streamLongestTaskMs: 600, + streamSeconds: 45, // the stream must finish drawing at all +}; + +const sandbox = mkdtempSync(join(tmpdir(), 'devdeck-perf-gate-')); +const repo = join(sandbox, 'perf-workspace'); +const bin = join(sandbox, 'bin'); +const profile = join(sandbox, 'profile'); +const home = join(profile, 'home'); +for (const p of [repo, bin, home]) mkdirSync(p, { recursive: true }); +execFileSync('git', ['init', '-q', '-b', 'qa-perf'], { cwd: repo }); +// The fixture agent: a static screen at rest; "W" repaints the whole screen at 10fps for 8s with a +// Claude spinner in it (so the deck shows the session as working); "S" prints 20,000 lines. +writeFileSync(join(bin, 'agent.js'), ` +const out = (s) => process.stdout.write(s); +const E = '\\x1b'; +const screen = (title) => out(E + '[H' + E + '[2J' + title + '\\r\\n' + Array.from({ length: 30 }, (_, i) => 'line ' + i + ' ' + 'x'.repeat(80)).join('\\r\\n') + '\\r\\n> '); +screen('PERF_FIXTURE_READY'); +if (process.stdin.isTTY) process.stdin.setRawMode(true); +const glyphs = ['\\u00b7', '\\u2722', '\\u2733', '\\u2736', '\\u273b', '\\u273d']; +process.stdin.on('data', (d) => { + const k = d.toString(); + if (k === 'W') { + let i = 0; const end = Date.now() + 8000; + const t = setInterval(() => { + i++; + const rows = Array.from({ length: 34 }, (_, r) => 'row ' + r + ' frame ' + i + ' ' + 'x'.repeat(90)); + out(E + '[H' + rows.join('\\r\\n') + '\\r\\n' + glyphs[i % glyphs.length] + ' Thinking\\u2026 (' + i + 's \\u00b7 esc to interrupt)'); + if (Date.now() > end) { clearInterval(t); screen('WORK_DONE'); } + }, 100); + } else if (k === 'S') { + let n = 0; + const pump = () => { let chunk = ''; for (let j = 0; j < 500 && n < 20000; j++) chunk += 'perf line ' + (++n) + ' ' + 'y'.repeat(60) + '\\r\\n'; out(chunk); if (n < 20000) setImmediate(pump); else out('STREAM_DONE\\r\\n'); }; + pump(); + } +}); +setInterval(() => {}, 1 << 30); +`); +writeFileSync(join(bin, 'claude.cmd'), '@echo off\r\nnode "%~dp0agent.js" %*\r\n'); +const sids = ['8d4e5f6a-1111-4222-8333-944455556666', '9e5f6a7b-1111-4222-8333-944455556666']; +const tdir = join(home, '.claude', 'projects', repo.replace(/[^a-zA-Z0-9]/g, '-')); +mkdirSync(tdir, { recursive: true }); +for (const sid of sids) writeFileSync(join(tdir, sid + '.jsonl'), JSON.stringify({ type: 'user', timestamp: new Date().toISOString(), cwd: repo, sessionId: sid, message: { role: 'user', content: 'hi' } }) + '\n'); +writeFileSync(join(profile, 'state.json'), JSON.stringify({ projects: {}, settings: { + language: 'en', folders: [{ path: repo, kind: 'repo' }], viewMode: 'list', + cockpitSessions: sids.map((sid, i) => ({ tileId: 'tile-perf-' + i, projectPath: repo, name: 'perf-workspace', sessionId: sid, agentId: 'claude', label: 'Perf ' + i, lastActiveMs: Date.now() })), +} })); +const env = { ...process.env, HOME: home, USERPROFILE: home, CLAUDECODE: '', CLAUDE_CODE_SSE_PORT: '', CLAUDE_CODE_ENTRYPOINT: '' }; +const pathKey = Object.keys(env).find((k) => k.toLowerCase() === 'path') || 'PATH'; +env[pathKey] = bin + ';' + env[pathKey]; + +const app = await electron.launch({ args: ['.', '--user-data-dir=' + profile, '--no-sandbox'], cwd: root, env }); +const results = {}; +const failures = []; +const judge = (name, value, limit) => { + results[name] = { value, limit }; + const ok = value <= limit; + console.log(`${ok ? 'PASS' : 'FAIL'} ${name}: ${value} (limit ${limit})`); + if (!ok) failures.push(name); +}; +try { + const win = await app.firstWindow(); + win.setDefaultTimeout(30000); + await win.setViewportSize?.({ width: 1500, height: 950 }).catch(() => {}); + await win.waitForSelector('#shell-session-groups .shell-session', { timeout: 30000 }); + // Both tiles running, one on screen. + const restoreAll = win.getByRole('button', { name: /Restore all/i }); + if (await restoreAll.count()) await restoreAll.first().click(); + await win.waitForFunction(() => document.querySelectorAll('.ck-term').length >= 2, null, { timeout: 30000 }); + await win.locator('#shell-session-groups .shell-session').first().click(); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].some((r) => r.textContent.includes('PERF_FIXTURE_READY'))); + const ids = await win.evaluate(() => window.devdeck.cockpit.liveSessions().then((list) => list.map((s) => s.id))); + assert.ok(ids.length >= 2, 'two sessions running: ' + JSON.stringify(ids)); + // The workload goes to every running session — one is on screen, the other works in the background, + // which is how a deck is actually used. + const sendAll = (key) => win.evaluate(([list, k]) => { for (const i of list) window.devdeck.cockpit.input(i, k); }, [ids, key]); + await win.waitForTimeout(3000); // let restore, meta and git refreshes settle + + // ---- instruments --------------------------------------------------------------------------- + await win.evaluate(() => { + window.__gate = { long: [], mutations: 0, outside: 0 }; + new PerformanceObserver((l) => { for (const e of l.getEntries()) window.__gate.long.push(Math.round(e.duration)); }).observe({ entryTypes: ['longtask'] }); + new MutationObserver((records) => { + window.__gate.mutations += records.length; + // Outside the terminals: the sidebar, header and footer. A terminal redraws by design while its + // agent works; the chrome around it must not rebuild along with every chunk. + for (const r of records) { const t = r.target.nodeType === 1 ? r.target : r.target.parentElement; if (!t?.closest?.('.ck-terms')) window.__gate.outside++; } + }).observe(document.body, { subtree: true, childList: true, attributes: true, characterData: true }); + }); + // Main-process event-loop stall: a 20ms timer's lateness, the longest of the window. + await app.evaluate(() => { + globalThis.__lag = { max: 0, last: Date.now() }; + globalThis.__lagTimer = setInterval(() => { const now = Date.now(); globalThis.__lag.max = Math.max(globalThis.__lag.max, now - globalThis.__lag.last - 20); globalThis.__lag.last = now; }, 20); + }); + const reset = async () => { + await win.evaluate(() => { window.__gate.long = []; window.__gate.mutations = 0; window.__gate.outside = 0; }); + await app.evaluate(() => { globalThis.__lag.max = 0; globalThis.__lag.last = Date.now(); }); + }; + const read = async () => ({ + ...(await win.evaluate(() => ({ long: [...window.__gate.long], mutations: window.__gate.mutations, outside: window.__gate.outside }))), + stall: await app.evaluate(() => globalThis.__lag.max), + }); + const terminalRows = () => win.evaluate(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].map((r) => r.textContent).join('\n')); + + // ---- IDLE ---------------------------------------------------------------------------------- + await reset(); + await win.waitForTimeout(6000); + const idle = await read(); + const running = await win.evaluate(() => document.getAnimations().filter((a) => a.playState === 'running').map((a) => { + const el = a.effect?.target; + return `${a.animationName ?? a.constructor.name} on ${el?.className ?? el?.tagName ?? '?'}`; + })); + if (running.length) console.log(' running animations: ' + running.join(' | ')); + judge('idle running animations', running.length, LIMITS.idleRunningAnimations); + judge('idle DOM mutation records (6s)', idle.mutations, LIMITS.idleMutations); + judge('idle long tasks', idle.long.length, LIMITS.idleLongTasks); + await win.screenshot({ path: join(out, 'idle.png') }); + + // ---- WORKING ------------------------------------------------------------------------------- + await reset(); + await sendAll('W'); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].some((r) => r.textContent.includes('WORK_DONE')), null, { timeout: 30000 }); + const working = await read(); + judge('working: main-process stall ms', working.stall, LIMITS.workingMainStallMs); + judge('working: longest UI task ms', Math.max(0, ...working.long), LIMITS.workingLongestTaskMs); + judge('working: DOM mutation records outside the terminals (8s)', working.outside, LIMITS.workingOutsideMutations); + + // ---- STREAM -------------------------------------------------------------------------------- + await win.waitForTimeout(1500); + await reset(); + const t0 = Date.now(); + await sendAll('S'); + await win.waitForFunction(() => [...document.querySelectorAll('.ck-term.show .xterm-rows')].some((r) => r.textContent.includes('STREAM_DONE')), null, { timeout: LIMITS.streamSeconds * 1000 }); + const streamSeconds = Math.round((Date.now() - t0) / 100) / 10; + // The window includes what follows a stream — scrollback trimming, reflow, the deck catching up — + // so a stall that lands just after the last line is still inside it. + await win.waitForTimeout(3000); + const stream = await read(); + judge('stream: seconds to draw 20k lines', streamSeconds, LIMITS.streamSeconds); + judge('stream: main-process stall ms', stream.stall, LIMITS.streamMainStallMs); + judge('stream: longest UI task ms', Math.max(0, ...stream.long), LIMITS.streamLongestTaskMs); + assert.ok((await terminalRows()).includes('STREAM_DONE')); + await win.screenshot({ path: join(out, 'stream.png') }); +} catch (e) { + console.log('FAIL harness: ' + (e?.message ?? e)); + failures.push('harness'); +} finally { + writeFileSync(join(out, 'results.json'), JSON.stringify({ results, failures }, null, 2)); + await closeElectron(app).catch(() => {}); +} +if (failures.length) { console.log(`\nFAIL: ${failures.join(', ')}`); process.exitCode = 1; } else console.log('\nPASS: performance gate');