From 5dc80f8f79de91a458f2e8552fddabb7cf93e1eb Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:05:49 -0700 Subject: [PATCH 01/10] feat(native-chat): visual directive grammar and host read for a chat's visuals folder A shared grammar for the ::orca-visual{file="..." title="..."} reply line, the per-chat visuals folder location on the owning host, and the agentSession.readVisual runtime method that reads one visual with lexical and canonical containment, a 512 KiB bounded read and UTF-8 refusal. --- .../native-chat-visual-file-read.test.ts | 186 ++++++++++++++++ .../native-chat-visual-file-read.ts | 202 ++++++++++++++++++ .../native-chat/native-chat-visuals-folder.ts | 15 ++ .../structured-agent-session-visual.test.ts | 128 +++++++++++ .../structured-agent-session-visual.ts | 42 ++++ .../methods/structured-agent-session.test.ts | 2 +- .../rpc/methods/structured-agent-session.ts | 2 + .../native-chat-visual-directive.test.ts | 176 +++++++++++++++ src/shared/native-chat-visual-directive.ts | 163 ++++++++++++++ .../agent-session-visual-params.ts | 42 ++++ .../rpc-params-catalog.generated.ts | 2 + 11 files changed, 959 insertions(+), 1 deletion(-) create mode 100644 src/main/native-chat/native-chat-visual-file-read.test.ts create mode 100644 src/main/native-chat/native-chat-visual-file-read.ts create mode 100644 src/main/native-chat/native-chat-visuals-folder.ts create mode 100644 src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts create mode 100644 src/main/runtime/rpc/methods/structured-agent-session-visual.ts create mode 100644 src/shared/native-chat-visual-directive.test.ts create mode 100644 src/shared/native-chat-visual-directive.ts create mode 100644 src/shared/rpc-contract/agent-session-visual-params.ts diff --git a/src/main/native-chat/native-chat-visual-file-read.test.ts b/src/main/native-chat/native-chat-visual-file-read.test.ts new file mode 100644 index 000000000000..901487c688bd --- /dev/null +++ b/src/main/native-chat/native-chat-visual-file-read.test.ts @@ -0,0 +1,186 @@ +import { execFileSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, symlink, writeFile, link } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { NATIVE_CHAT_VISUAL_MAX_BYTES } from '../../shared/native-chat-visual-directive' +import { nativeChatVisualsFolderFor } from './native-chat-visuals-folder' +import { nativeChatVisualRevision, readNativeChatVisualFile } from './native-chat-visual-file-read' + +const posixIt = process.platform === 'win32' ? it.skip : it + +let stateDirectory: string +let folder: string + +beforeEach(async () => { + stateDirectory = await mkdtemp(join(tmpdir(), 'orca-visual-read-')) + folder = nativeChatVisualsFolderFor(stateDirectory, 'session-alpha') + await mkdir(folder, { recursive: true }) +}) + +afterEach(async () => { + await rm(stateDirectory, { recursive: true, force: true }) +}) + +describe('nativeChatVisualsFolderFor', () => { + it('places each chat in its own hashed folder under the state directory', () => { + const alpha = nativeChatVisualsFolderFor('/state', 'session-alpha') + const beta = nativeChatVisualsFolderFor('/state', 'session-beta') + expect(alpha).toMatch(/native-chat-visuals[\\/][0-9a-f]{32}$/) + expect(alpha.startsWith(join('/state', 'native-chat-visuals'))).toBe(true) + expect(alpha).not.toBe(beta) + }) +}) + +describe('readNativeChatVisualFile', () => { + it('reads a UTF-8 visual with its revision and size', async () => { + const html = '

Grüße

' + await writeFile(join(folder, 'chart.html'), html) + const result = await readNativeChatVisualFile(folder, 'chart.html') + expect(result).toEqual({ + ok: true, + html, + revision: nativeChatVisualRevision(Buffer.from(html)), + sizeBytes: Buffer.byteLength(html) + }) + }) + + it('answers unchanged without the bytes when the client already holds the revision', async () => { + await writeFile(join(folder, 'chart.html'), '

a

') + const first = await readNativeChatVisualFile(folder, 'chart.html') + if (!first.ok) { + throw new Error('expected a read') + } + const again = await readNativeChatVisualFile(folder, 'chart.html', first.revision) + expect(again).toEqual({ ok: true, revision: first.revision, sizeBytes: 8, unchanged: true }) + + await writeFile(join(folder, 'chart.html'), '

b

') + const changed = await readNativeChatVisualFile(folder, 'chart.html', first.revision) + expect(changed).toMatchObject({ ok: true, html: '

b

' }) + }) + + it('reports a missing file or a missing folder as not_found', async () => { + expect(await readNativeChatVisualFile(folder, 'nope.html')).toEqual({ + ok: false, + error: 'not_found' + }) + const absent = nativeChatVisualsFolderFor(stateDirectory, 'session-never') + expect(await readNativeChatVisualFile(absent, 'chart.html')).toEqual({ + ok: false, + error: 'not_found' + }) + }) + + it('refuses names that are not a bare visual file name', async () => { + await mkdir(join(stateDirectory, 'other'), { recursive: true }) + await writeFile(join(stateDirectory, 'other', 'secret.html'), 'secret') + for (const name of ['../other/secret.html', '/etc/hosts', 'chart.txt', 'sub/chart.html']) { + expect(await readNativeChatVisualFile(folder, name)).toEqual({ + ok: false, + error: 'outside_folder' + }) + } + }) + + posixIt('refuses a symlink to a sibling chat folder or another workspace', async () => { + const sibling = nativeChatVisualsFolderFor(stateDirectory, 'session-beta') + await mkdir(sibling, { recursive: true }) + await writeFile(join(sibling, 'theirs.html'), '

other chat

') + const worktree = join(stateDirectory, 'worktree') + await mkdir(worktree) + await writeFile(join(worktree, 'index.html'), '

repo

') + + await symlink(join(sibling, 'theirs.html'), join(folder, 'theirs.html')) + await symlink(join(worktree, 'index.html'), join(folder, 'repo.html')) + await symlink(join(folder, 'missing-target.html'), join(folder, 'dangling.html')) + + for (const name of ['theirs.html', 'repo.html', 'dangling.html']) { + expect(await readNativeChatVisualFile(folder, name)).toEqual({ + ok: false, + error: 'outside_folder' + }) + } + }) + + posixIt('refuses a visuals folder replaced by a symlink', async () => { + const elsewhere = join(stateDirectory, 'elsewhere') + await mkdir(elsewhere) + await writeFile(join(elsewhere, 'chart.html'), '

elsewhere

') + await rm(folder, { recursive: true }) + await symlink(elsewhere, folder) + expect(await readNativeChatVisualFile(folder, 'chart.html')).toEqual({ + ok: false, + error: 'outside_folder' + }) + }) + + posixIt('refuses when the shared visuals root is a symlink', async () => { + const root = join(stateDirectory, 'native-chat-visuals') + const moved = join(stateDirectory, 'moved-root') + await rm(root, { recursive: true }) + await mkdir(join(moved, 'x'), { recursive: true }) + await symlink(moved, root) + await mkdir(folder, { recursive: true }) + await writeFile(join(folder, 'chart.html'), '

a

') + expect(await readNativeChatVisualFile(folder, 'chart.html')).toEqual({ + ok: false, + error: 'outside_folder' + }) + }) + + it('refuses a directory named like a visual', async () => { + await mkdir(join(folder, 'dir.html')) + expect(await readNativeChatVisualFile(folder, 'dir.html')).toEqual({ + ok: false, + error: 'not_a_file' + }) + }) + + posixIt('refuses a FIFO without waiting for a writer', async () => { + execFileSync('mkfifo', [join(folder, 'pipe.html')]) + expect(await readNativeChatVisualFile(folder, 'pipe.html')).toEqual({ + ok: false, + error: 'not_a_file' + }) + }) + + it('refuses a file over the byte cap and accepts one exactly at it', async () => { + await writeFile(join(folder, 'max.html'), 'a'.repeat(NATIVE_CHAT_VISUAL_MAX_BYTES)) + expect(await readNativeChatVisualFile(folder, 'max.html')).toMatchObject({ ok: true }) + await writeFile(join(folder, 'big.html'), 'a'.repeat(NATIVE_CHAT_VISUAL_MAX_BYTES + 1)) + expect(await readNativeChatVisualFile(folder, 'big.html')).toEqual({ + ok: false, + error: 'too_large' + }) + }) + + it('refuses binary and invalid UTF-8 content', async () => { + await writeFile(join(folder, 'zero-byte.html'), Buffer.from([0x3c, 0x00, 0x3e])) + await writeFile(join(folder, 'latin1.html'), Buffer.from([0x3c, 0xe9, 0x3e])) + expect(await readNativeChatVisualFile(folder, 'zero-byte.html')).toEqual({ + ok: false, + error: 'not_text' + }) + expect(await readNativeChatVisualFile(folder, 'latin1.html')).toEqual({ + ok: false, + error: 'not_text' + }) + }) + + it('reads a hard link the agent made inside its own folder', async () => { + // A hard link grants nothing a copy would not: the agent could write the same bytes itself. + await writeFile(join(folder, 'a.html'), '

a

') + await link(join(folder, 'a.html'), join(folder, 'b.html')) + expect(await readNativeChatVisualFile(folder, 'b.html')).toMatchObject({ ok: true }) + }) + + it('reads the replacement after a file is swapped for new content', async () => { + await writeFile(join(folder, 'chart.html'), '

old

') + await rm(join(folder, 'chart.html')) + await writeFile(join(folder, 'chart.html'), '

new

') + expect(await readNativeChatVisualFile(folder, 'chart.html')).toMatchObject({ + ok: true, + html: '

new

' + }) + }) +}) diff --git a/src/main/native-chat/native-chat-visual-file-read.ts b/src/main/native-chat/native-chat-visual-file-read.ts new file mode 100644 index 000000000000..e6eaba3d9a32 --- /dev/null +++ b/src/main/native-chat/native-chat-visual-file-read.ts @@ -0,0 +1,202 @@ +// Reading one visual out of a chat's visuals folder. This is the boundary, not the renderer: the +// file must sit directly in the folder both lexically and canonically, be a regular file reached +// without a symlink, and be UTF-8 text within the byte cap. + +import { createHash } from 'node:crypto' +import { constants, type Stats } from 'node:fs' +import { lstat, open, realpath, type FileHandle } from 'node:fs/promises' +import { basename, dirname, join } from 'node:path' +import { + NATIVE_CHAT_VISUAL_MAX_BYTES, + isNativeChatVisualFileName +} from '../../shared/native-chat-visual-directive' +import type { + AgentSessionReadVisualResult, + AgentSessionVisualReadError +} from '../../shared/rpc-contract/agent-session-visual-params' +import { isENOENT } from '../ipc/filesystem-path-containment' +import { readLocalFileBounded } from '../ipc/filesystem/local-regular-file-read' + +// Why O_NOFOLLOW: refuses a final-component symlink at open, so no swap after a check can redirect +// the read. Windows has no such flag; there the lstat and post-open identity checks hold the line. +const VISUAL_OPEN_FLAGS = + constants.O_RDONLY | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0) + +const REVISION_HEX_LENGTH = 32 + +class VisualReadRefusal extends Error { + constructor(readonly refusal: AgentSessionVisualReadError) { + super(refusal) + } +} + +function refuse(refusal: AgentSessionVisualReadError): never { + throw new VisualReadRefusal(refusal) +} + +function errorCode(error: unknown): string | undefined { + return error instanceof Error && 'code' in error && typeof error.code === 'string' + ? error.code + : undefined +} + +function sameFile(left: Stats, right: Stats): boolean { + return left.dev === right.dev && left.ino === right.ino +} + +/** + * The folder and its parent are Orca's own directories below the trusted state directory. Either + * one replaced by a symlink would aim every read somewhere else, so a link there is refused. + */ +async function canonicalFolder(folder: string): Promise { + try { + for (const owned of [dirname(folder), folder]) { + const stats = await lstat(owned) + if (stats.isSymbolicLink()) { + refuse('outside_folder') + } + if (!stats.isDirectory()) { + refuse('not_found') + } + } + return await realpath(folder) + } catch (error) { + if (error instanceof VisualReadRefusal) { + throw error + } + if (isENOENT(error) || errorCode(error) === 'ENOTDIR') { + refuse('not_found') + } + throw error + } +} + +async function openVisual(candidate: string): Promise { + try { + if ((await lstat(candidate)).isSymbolicLink()) { + refuse('outside_folder') + } + return await open(candidate, VISUAL_OPEN_FLAGS) + } catch (error) { + if (error instanceof VisualReadRefusal) { + throw error + } + if (isENOENT(error) || errorCode(error) === 'ENOTDIR') { + refuse('not_found') + } + // ELOOP: the path became a symlink between the lstat and the open. + if (errorCode(error) === 'ELOOP' || errorCode(error) === 'EMLINK') { + refuse('outside_folder') + } + if (errorCode(error) === 'EISDIR') { + refuse('not_a_file') + } + throw error + } +} + +/** After the open: the handle must still be the file directly inside the canonical folder. */ +async function assertStillContained( + candidate: string, + folderReal: string, + file: string, + opened: Stats +): Promise { + let resolved: string + let current: Stats + try { + resolved = await realpath(candidate) + current = await lstat(candidate) + } catch (error) { + if (isENOENT(error)) { + refuse('not_found') + } + throw error + } + // Why compare names case-insensitively: a case-insensitive volume may report the stored case. + if ( + dirname(resolved) !== folderReal || + basename(resolved).toLowerCase() !== file.toLowerCase() || + current.isSymbolicLink() || + !sameFile(current, opened) + ) { + refuse('outside_folder') + } +} + +function decodeVisualText(buffer: Buffer): string { + if (buffer.includes(0)) { + refuse('not_text') + } + try { + return new TextDecoder('utf-8', { fatal: true }).decode(buffer) + } catch { + return refuse('not_text') + } +} + +export function nativeChatVisualRevision(buffer: Buffer): string { + return createHash('sha256').update(buffer).digest('hex').slice(0, REVISION_HEX_LENGTH) +} + +async function readContained( + folder: string, + file: string, + knownRevision: string | undefined +): Promise { + if (!isNativeChatVisualFileName(file)) { + refuse('outside_folder') + } + const folderReal = await canonicalFolder(folder) + const candidate = join(folderReal, file) + if (dirname(candidate) !== folderReal) { + refuse('outside_folder') + } + const handle = await openVisual(candidate) + try { + const stats = await handle.stat() + if (!stats.isFile()) { + refuse('not_a_file') + } + if (stats.size > NATIVE_CHAT_VISUAL_MAX_BYTES) { + refuse('too_large') + } + await assertStillContained(candidate, folderReal, file, stats) + let buffer: Buffer + try { + buffer = await readLocalFileBounded(handle, NATIVE_CHAT_VISUAL_MAX_BYTES, stats.size) + } catch (error) { + if (error instanceof Error && error.message.startsWith('File too large')) { + refuse('too_large') + } + throw error + } + const html = decodeVisualText(buffer) + const revision = nativeChatVisualRevision(buffer) + const sizeBytes = buffer.length + return knownRevision === revision + ? { ok: true, revision, sizeBytes, unchanged: true } + : { ok: true, revision, sizeBytes, html } + } finally { + await handle.close() + } +} + +/** + * The visual `file` from `folder`, or the refusal the host observed. Unexpected filesystem faults + * still throw, so the client reads them as unavailable rather than as a verdict about the file. + */ +export async function readNativeChatVisualFile( + folder: string, + file: string, + knownRevision?: string +): Promise { + try { + return await readContained(folder, file, knownRevision) + } catch (error) { + if (error instanceof VisualReadRefusal) { + return { ok: false, error: error.refusal } + } + throw error + } +} diff --git a/src/main/native-chat/native-chat-visuals-folder.ts b/src/main/native-chat/native-chat-visuals-folder.ts new file mode 100644 index 000000000000..c0736c69bad6 --- /dev/null +++ b/src/main/native-chat/native-chat-visuals-folder.ts @@ -0,0 +1,15 @@ +// Where a structured chat's visuals live on the host that owns the chat: Orca-owned state beside the +// chat journal, never inside the user's workspace. + +import { join } from 'node:path' +import { journalPathSegment } from './agent-session-journal/journal-paths' + +const NATIVE_CHAT_VISUALS_DIR_NAME = 'native-chat-visuals' + +/** + * `/native-chat-visuals/`. Keyed by the session id alone, the + * chat record's primary key, so the folder needs no workspace lookup to find or remove. + */ +export function nativeChatVisualsFolderFor(stateDirectory: string, sessionId: string): string { + return join(stateDirectory, NATIVE_CHAT_VISUALS_DIR_NAME, journalPathSegment(sessionId)) +} diff --git a/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts new file mode 100644 index 000000000000..9e51e89f5bf4 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import type { AgentSessionRecord } from '../../../../shared/agent-session-record' +import { agentSessionRecordFixture } from '../../../../shared/agent-session-record.test-fixture' +import { setStructuredAgentSessionHost } from '../../../native-chat/agent-session-wire/structured-agent-session-registry' +import { nativeChatVisualsFolderFor } from '../../../native-chat/native-chat-visuals-folder' +import { + call, + clearStructuredHostStub, + hostStub, + SESSION, + STRUCTURED_CLIENT +} from './structured-agent-session-rpc.test-fixture' + +let stateDirectory: string +let record: AgentSessionRecord | null + +beforeEach(async () => { + stateDirectory = await mkdtemp(join(tmpdir(), 'orca-visual-rpc-')) + record = { ...agentSessionRecordFixture(), sessionId: SESSION } + setStructuredAgentSessionHost( + Object.assign(hostStub(), { + deps: { + store: { getRecord: (id: string) => (record?.sessionId === id ? record : null) }, + journalDatabase: { stateDirectory } + } + }) + ) +}) + +afterEach(async () => { + clearStructuredHostStub() + await rm(stateDirectory, { recursive: true, force: true }) +}) + +async function writeVisual(name: string, html: string): Promise { + const folder = nativeChatVisualsFolderFor(stateDirectory, SESSION) + await mkdir(folder, { recursive: true }) + await writeFile(join(folder, name), html) +} + +describe('agentSession.readVisual', () => { + it("reads a file from the chat's own folder under the host's state directory", async () => { + await writeVisual('chart.html', '

chart

') + const reply = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + STRUCTURED_CLIENT + ) + expect(reply).toMatchObject({ + ok: true, + result: { ok: true, html: '

chart

', sizeBytes: 12 } + }) + }) + + it('answers unchanged for the revision the client holds', async () => { + await writeVisual('chart.html', '

chart

') + const first = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + STRUCTURED_CLIENT + ) + const revision = first.ok ? (first.result as { revision: string }).revision : '' + const again = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html', knownRevision: revision }, + STRUCTURED_CLIENT + ) + expect(again).toMatchObject({ ok: true, result: { ok: true, unchanged: true, revision } }) + expect(again.ok && 'html' in (again.result as object)).toBe(false) + }) + + it('reports a session this host has no record of', async () => { + record = null + const reply = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + STRUCTURED_CLIENT + ) + expect(reply).toMatchObject({ ok: true, result: { ok: false, error: 'session_not_found' } }) + }) + + it('refuses a chat recorded on another execution host or a WSL distro', async () => { + const base = agentSessionRecordFixture() + record = { + ...base, + sessionId: SESSION, + location: { ...base.location, executionHostId: 'ssh:box' } + } + const ssh = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + STRUCTURED_CLIENT + ) + expect(ssh).toMatchObject({ ok: true, result: { ok: false, error: 'unsupported_location' } }) + + record = { ...base, sessionId: SESSION, location: { ...base.location, wslDistro: 'Ubuntu' } } + const wsl = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + STRUCTURED_CLIENT + ) + expect(wsl).toMatchObject({ ok: true, result: { ok: false, error: 'unsupported_location' } }) + }) + + it.each([ + ['a path', { sessionId: SESSION, file: '../chart.html' }], + ['a non-html file', { sessionId: SESSION, file: 'chart.txt' }], + ['a bad session id', { sessionId: '../x', file: 'chart.html' }], + ['an unknown field', { sessionId: SESSION, file: 'chart.html', path: '/etc' }], + ['a malformed revision', { sessionId: SESSION, file: 'chart.html', knownRevision: 'zz' }] + ])('rejects %s as invalid params', async (_name, params) => { + const reply = await call('agentSession.readVisual', params, STRUCTURED_CLIENT) + expect(reply).toMatchObject({ ok: false, error: { code: 'invalid_argument' } }) + }) + + it('is refused to a client that cannot read structured sessions', async () => { + await writeVisual('chart.html', '

chart

') + const reply = await call( + 'agentSession.readVisual', + { sessionId: SESSION, file: 'chart.html' }, + { clientKind: 'runtime', clientCapabilities: [] } + ) + expect(reply.ok).toBe(false) + }) +}) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-visual.ts b/src/main/runtime/rpc/methods/structured-agent-session-visual.ts new file mode 100644 index 000000000000..b8e2e49f1669 --- /dev/null +++ b/src/main/runtime/rpc/methods/structured-agent-session-visual.ts @@ -0,0 +1,42 @@ +// `agentSession.readVisual` — one HTML visual from a chat's own visuals folder. +// +// Additive: an older host answers `method_not_found` and the client shows the visual as +// unavailable. The host resolves the folder from its own state directory and record; a client +// supplies only the session id and a bare file name. + +import { LOCAL_EXECUTION_HOST_ID } from '../../../../shared/execution-host' +import { + ReadVisualParams, + type AgentSessionReadVisualResult +} from '../../../../shared/rpc-contract/agent-session-visual-params' +import { nativeChatVisualsFolderFor } from '../../../native-chat/native-chat-visuals-folder' +import { readNativeChatVisualFile } from '../../../native-chat/native-chat-visual-file-read' +import { defineMethod } from '../core' +import { requireInstalledStructuredHost } from './structured-agent-session-gate' + +export const STRUCTURED_AGENT_SESSION_VISUAL_METHODS = [ + defineMethod({ + name: 'agentSession.readVisual', + params: ReadVisualParams, + handler: async (params, ctx): Promise => { + const host = await requireInstalledStructuredHost(ctx) + const record = host.deps.store.getRecord(params.sessionId) + if (!record) { + return { ok: false, error: 'session_not_found' } + } + // Structured chats run on their owning runtime's own filesystem; any other location has no + // visuals folder this process can read. + if ( + record.location.executionHostId !== LOCAL_EXECUTION_HOST_ID || + record.location.wslDistro + ) { + return { ok: false, error: 'unsupported_location' } + } + const folder = nativeChatVisualsFolderFor( + host.deps.journalDatabase.stateDirectory, + params.sessionId + ) + return readNativeChatVisualFile(folder, params.file, params.knownRevision) + } + }) +] diff --git a/src/main/runtime/rpc/methods/structured-agent-session.test.ts b/src/main/runtime/rpc/methods/structured-agent-session.test.ts index e33c7800cabb..93fff5d92680 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.test.ts @@ -175,7 +175,7 @@ describe('capability gating', () => { } // Bump deliberately: the whole agentSession.* surface is behind the structured capability, // so an additive method is invisible to old clients and needs no protocol bump. - expect(STRUCTURED_AGENT_SESSION_METHODS).toHaveLength(33) + expect(STRUCTURED_AGENT_SESSION_METHODS).toHaveLength(34) }) it('hides the surface from a declared client that did not advertise it', async () => { diff --git a/src/main/runtime/rpc/methods/structured-agent-session.ts b/src/main/runtime/rpc/methods/structured-agent-session.ts index d2da7b0f92c2..4d9c27cddd21 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session.ts @@ -34,6 +34,7 @@ import { } from './structured-agent-session-create' import { STRUCTURED_AGENT_SESSION_HOLD_METHODS } from './structured-agent-session-hold' import { STRUCTURED_AGENT_SESSION_REVEAL_METHODS } from './structured-agent-session-reveal' +import { STRUCTURED_AGENT_SESSION_VISUAL_METHODS } from './structured-agent-session-visual' import { STRUCTURED_AGENT_SESSION_QUEUED_METHODS } from './structured-agent-session-queued-methods' import { STRUCTURED_AGENT_SESSION_RESTART_RESUME_METHODS } from './structured-agent-session-restart-resume' import { resolveUncommittedStructuredCreate } from './structured-agent-session-precommit-refusal' @@ -316,6 +317,7 @@ export const STRUCTURED_AGENT_SESSION_METHODS = [ }), ...STRUCTURED_AGENT_SESSION_HOLD_METHODS, ...STRUCTURED_AGENT_SESSION_REVEAL_METHODS, + ...STRUCTURED_AGENT_SESSION_VISUAL_METHODS, ...STRUCTURED_AGENT_SESSION_RESTART_RESUME_METHODS, ...STRUCTURED_AGENT_SESSION_STATUS_METHODS, ...STRUCTURED_AGENT_SESSION_TURN_COMPLETION_METHODS, diff --git a/src/shared/native-chat-visual-directive.test.ts b/src/shared/native-chat-visual-directive.test.ts new file mode 100644 index 000000000000..49f09d3346d4 --- /dev/null +++ b/src/shared/native-chat-visual-directive.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from 'vitest' +import { + NATIVE_CHAT_VISUAL_DIRECTIVE_MAX_LINE_LENGTH, + NATIVE_CHAT_VISUAL_FILE_MAX_LENGTH, + NATIVE_CHAT_VISUAL_TITLE_MAX_LENGTH, + isNativeChatVisualFileName, + isPendingNativeChatVisualDirectiveTail, + parseNativeChatVisualDirectiveLine, + withoutPendingNativeChatVisualDirectiveTail +} from './native-chat-visual-directive' + +describe('parseNativeChatVisualDirectiveLine', () => { + it('parses file and title', () => { + expect( + parseNativeChatVisualDirectiveLine( + '::orca-visual{file="usage-3f2a.html" title="Usage by day"}' + ) + ).toEqual({ file: 'usage-3f2a.html', title: 'Usage by day' }) + }) + + it('treats title as optional and trims it', () => { + expect(parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html"}')).toEqual({ + file: 'a.html', + title: null + }) + expect(parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html" title=" "}')).toEqual({ + file: 'a.html', + title: null + }) + }) + + it('accepts attributes in any order, with spaces and tabs around them', () => { + expect( + parseNativeChatVisualDirectiveLine('::orca-visual{ \ttitle="T" file="a.html"\t }') + ).toEqual({ file: 'a.html', title: 'T' }) + }) + + it('accepts up to 3 leading spaces, trailing whitespace, and CRLF or LF endings', () => { + expect(parseNativeChatVisualDirectiveLine(' ::orca-visual{file="a.html"} \t\r\n')).toEqual({ + file: 'a.html', + title: null + }) + expect(parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html"}\n')).not.toBeNull() + expect(parseNativeChatVisualDirectiveLine(' ::orca-visual{file="a.html"}')).toBeNull() + expect(parseNativeChatVisualDirectiveLine('\t::orca-visual{file="a.html"}')).toBeNull() + }) + + it('ignores unknown attributes but refuses a repeated one', () => { + expect(parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html" height="300"}')).toEqual( + { file: 'a.html', title: null } + ) + expect( + parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html" file="b.html"}') + ).toBeNull() + expect( + parseNativeChatVisualDirectiveLine('::orca-visual{file="a.html" title="x" title="y"}') + ).toBeNull() + }) + + it('keeps braces and markdown characters in a title verbatim', () => { + expect( + parseNativeChatVisualDirectiveLine( + '::orca-visual{file="a.html" title="Cost {USD} *by* "}' + ) + ).toEqual({ file: 'a.html', title: 'Cost {USD} *by* ' }) + }) + + it.each([ + ['missing file', '::orca-visual{title="T"}'], + ['empty braces', '::orca-visual{}'], + ['no braces', '::orca-visual'], + ['unclosed', '::orca-visual{file="a.html"'], + ['text after the line', '::orca-visual{file="a.html"} see above'], + ['text before the marker', 'Chart: ::orca-visual{file="a.html"}'], + ['wrong case', '::Orca-Visual{file="a.html"}'], + ['single quotes', "::orca-visual{file='a.html'}"], + ['unquoted value', '::orca-visual{file=a.html}'], + ['escaped quote', '::orca-visual{file="a.html" title="say \\"hi\\""}'], + ['backslash', '::orca-visual{file="a.html" title="a\\b"}'], + ['control character', '::orca-visual{file="a.html" title="a\u0007b"}'], + ['attributes glued together', '::orca-visual{file="a.html"title="T"}'], + ['uppercase key', '::orca-visual{FILE="a.html"}'], + ['embedded newline', '::orca-visual{file="a.html"\ntitle="T"}'], + ['three colons', ':::orca-visual{file="a.html"}'] + ])('refuses %s', (_name, line) => { + expect(parseNativeChatVisualDirectiveLine(line)).toBeNull() + }) + + it('refuses lines and titles past their length caps', () => { + const title = 'x'.repeat(NATIVE_CHAT_VISUAL_TITLE_MAX_LENGTH) + expect( + parseNativeChatVisualDirectiveLine(`::orca-visual{file="a.html" title="${title}"}`) + ).not.toBeNull() + expect( + parseNativeChatVisualDirectiveLine(`::orca-visual{file="a.html" title="${title}x"}`) + ).toBeNull() + const padding = ' '.repeat(NATIVE_CHAT_VISUAL_DIRECTIVE_MAX_LINE_LENGTH) + expect(parseNativeChatVisualDirectiveLine(`::orca-visual{file="a.html"}${padding}`)).toBeNull() + }) +}) + +describe('isNativeChatVisualFileName', () => { + it.each(['a.html', 'usage-chart_3f2a.html', 'A1.b.html', '0.html'])('accepts %s', (name) => { + expect(isNativeChatVisualFileName(name)).toBe(true) + }) + + it.each([ + ['empty', ''], + ['no extension', 'chart'], + ['other extension', 'chart.htm'], + ['uppercase extension', 'chart.HTML'], + ['markdown', 'chart.md'], + ['directory', 'sub/chart.html'], + ['backslash directory', 'sub\\chart.html'], + ['parent', '../chart.html'], + ['double dot inside', 'a..html'], + ['leading dot', '.chart.html'], + ['leading dash', '-chart.html'], + ['absolute posix', '/tmp/chart.html'], + ['drive', 'C:chart.html'], + ['unc', '\\\\server\\share\\chart.html'], + ['alternate stream', 'chart.html:stream'], + ['space', 'my chart.html'], + ['unicode', 'grafik-ä.html'], + ['device name', 'con.html'], + ['device name with dots', 'NUL.tar.html'], + ['com port', 'COM1.html'], + ['superscript com port', 'com¹.html'], + ['too long', `${'a'.repeat(NATIVE_CHAT_VISUAL_FILE_MAX_LENGTH - 4)}.html`] + ])('refuses %s', (_name, name) => { + expect(isNativeChatVisualFileName(name)).toBe(false) + }) + + it('accepts a name exactly at the length cap', () => { + expect( + isNativeChatVisualFileName(`${'a'.repeat(NATIVE_CHAT_VISUAL_FILE_MAX_LENGTH - 5)}.html`) + ).toBe(true) + }) +}) + +describe('streaming tail', () => { + it.each([ + ':', + '::', + '::orca', + '::orca-visual{', + '::orca-visual{file="a.ht', + ' ::orca-visual{file="a.html"}' + ])('holds back %j', (line) => { + expect(isPendingNativeChatVisualDirectiveTail(line)).toBe(true) + }) + + it.each([ + '', + ' ', + 'Here is a chart:', + '::other', + '::orca-visualx', + ' ::orca-visual{', + '- ::orca' + ])('does not hold back %j', (line) => { + expect(isPendingNativeChatVisualDirectiveTail(line)).toBe(false) + }) + + it('drops only the pending final line', () => { + expect(withoutPendingNativeChatVisualDirectiveTail('Intro\n::orca-visual{file="a')).toBe( + 'Intro\n' + ) + expect(withoutPendingNativeChatVisualDirectiveTail('::orca-vis')).toBe('') + expect(withoutPendingNativeChatVisualDirectiveTail('Intro\nMore text')).toBe('Intro\nMore text') + // A finished line (followed by a newline) is no longer the tail. + expect(withoutPendingNativeChatVisualDirectiveTail('::orca-visual{file="a.html"}\n')).toBe( + '::orca-visual{file="a.html"}\n' + ) + }) +}) diff --git a/src/shared/native-chat-visual-directive.ts b/src/shared/native-chat-visual-directive.ts new file mode 100644 index 000000000000..bff46fe0fd69 --- /dev/null +++ b/src/shared/native-chat-visual-directive.ts @@ -0,0 +1,163 @@ +// The one grammar for a native-chat visual: an assistant reply line naming an HTML file in that +// chat's visuals folder. Desktop/web (a markdown syntax extension) and mobile both parse with it. +// +// ::orca-visual{file="usage-chart-3f2a.html" title="Usage by day"} +// +// - The line may carry up to 3 leading spaces and any trailing spaces/tabs; a trailing CR is +// ignored. Nothing else may share the line. +// - Attributes are `key="value"`, separated by spaces/tabs. Values have no escapes: they cannot +// contain `"`, `\`, or control characters. Keys are lowercase ASCII letters. +// - `file` is required. `title` is optional. A repeated key refuses the line. Other keys are +// ignored, so a newer agent's extra attribute still renders on this viewer. +// - The name is case-sensitive. + +export const NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER = '::orca-visual{' + +/** Longest line the grammar considers. Bounds parsing and the streaming-tail check. */ +export const NATIVE_CHAT_VISUAL_DIRECTIVE_MAX_LINE_LENGTH = 512 +export const NATIVE_CHAT_VISUAL_FILE_MAX_LENGTH = 128 +export const NATIVE_CHAT_VISUAL_TITLE_MAX_LENGTH = 120 + +/** + * Directives one reply may mount; later ones show as text. Each visual runs author script in its + * own frame (lazily mounted, at most 512 KiB), so this bounds one reply's live frames and memory + * while leaving room for a reply that compares a few charts. + */ +export const NATIVE_CHAT_VISUAL_MAX_PER_MESSAGE = 8 + +/** Largest visual file, in UTF-8 bytes. The execution host enforces it in a bounded read. */ +export const NATIVE_CHAT_VISUAL_MAX_BYTES = 512 * 1024 + +export type NativeChatVisualDirective = { + file: string + title: string | null +} + +const FILE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.html$/ +// Why: Windows maps these base names to devices whatever the extension. +const WINDOWS_DEVICE_BASE_NAME = /^(?:con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³]|conin\$|conout\$)$/i +const ATTRIBUTE_PATTERN = /^([a-z]+)="([^"]*)"/ +const LINE_SPACE = /^[ \t]*/ + +/** + * A single file name inside the chat's visuals folder: no directories, separators, `..`, drive, + * UNC, device or stream forms, ending in `.html`. + */ +export function isNativeChatVisualFileName(name: string): boolean { + if ( + name.length === 0 || + name.length > NATIVE_CHAT_VISUAL_FILE_MAX_LENGTH || + !FILE_NAME_PATTERN.test(name) || + name.includes('..') + ) { + return false + } + const baseName = name.slice(0, name.indexOf('.')) + return !WINDOWS_DEVICE_BASE_NAME.test(baseName) +} + +function isPlainAttributeValue(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if (code < 0x20 || code === 0x7f || code === 0x5c) { + return false + } + } + return true +} + +function stripLineEnding(line: string): string { + if (line.endsWith('\r\n')) { + return line.slice(0, -2) + } + return line.endsWith('\n') || line.endsWith('\r') ? line.slice(0, -1) : line +} + +function leadingSpaceCount(line: string): number { + let count = 0 + while (line[count] === ' ') { + count += 1 + } + return count +} + +function parseAttributes(body: string): Map | null { + const attributes = new Map() + let rest = body.replace(LINE_SPACE, '') + while (rest.length > 0) { + const match = ATTRIBUTE_PATTERN.exec(rest) + if (!match) { + return null + } + const [whole, key, value] = match + if (attributes.has(key) || !isPlainAttributeValue(value)) { + return null + } + attributes.set(key, value) + rest = rest.slice(whole.length) + const separator = LINE_SPACE.exec(rest)?.[0] ?? '' + if (rest.length > 0 && separator.length === 0) { + return null + } + rest = rest.slice(separator.length) + } + return attributes +} + +/** The directive a whole line spells, or null when the line is anything else. */ +export function parseNativeChatVisualDirectiveLine( + rawLine: string +): NativeChatVisualDirective | null { + const line = stripLineEnding(rawLine) + if (line.length > NATIVE_CHAT_VISUAL_DIRECTIVE_MAX_LINE_LENGTH || /[\r\n]/.test(line)) { + return null + } + const indent = leadingSpaceCount(line) + if (indent > 3 || !line.startsWith(NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER, indent)) { + return null + } + const trimmedEnd = line.replace(/[ \t]+$/, '') + if (!trimmedEnd.endsWith('}')) { + return null + } + const body = trimmedEnd.slice(indent + NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER.length, -1) + const attributes = parseAttributes(body) + const file = attributes?.get('file') + if (!attributes || file === undefined || !isNativeChatVisualFileName(file)) { + return null + } + const title = attributes.get('title')?.trim() ?? '' + if (title.length > NATIVE_CHAT_VISUAL_TITLE_MAX_LENGTH) { + return null + } + return { file, title: title.length > 0 ? title : null } +} + +/** + * Whether the final, still-growing line of a streaming reply may yet become a directive, so the + * viewer holds it back instead of flashing raw syntax. A complete-looking line counts too: the + * agent may still append to it. Only a streaming tail may be held; a finished reply shows it. + */ +export function isPendingNativeChatVisualDirectiveTail(lastLine: string): boolean { + const line = stripLineEnding(lastLine) + if (line.length === 0 || line.length > NATIVE_CHAT_VISUAL_DIRECTIVE_MAX_LINE_LENGTH) { + return false + } + const indent = leadingSpaceCount(line) + if (indent > 3 || indent === line.length) { + return false + } + const rest = line.slice(indent) + if (rest.length <= NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER.length) { + return NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER.startsWith(rest) + } + return rest.startsWith(NATIVE_CHAT_VISUAL_DIRECTIVE_MARKER) && !/[\r\n]/.test(rest) +} + +/** Drops a pending directive tail (see above) from streaming reply text. */ +export function withoutPendingNativeChatVisualDirectiveTail(text: string): string { + const lineStart = text.lastIndexOf('\n') + 1 + return isPendingNativeChatVisualDirectiveTail(text.slice(lineStart)) + ? text.slice(0, lineStart) + : text +} diff --git a/src/shared/rpc-contract/agent-session-visual-params.ts b/src/shared/rpc-contract/agent-session-visual-params.ts new file mode 100644 index 000000000000..bc2e67562615 --- /dev/null +++ b/src/shared/rpc-contract/agent-session-visual-params.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' +import { isNativeChatVisualFileName } from '../native-chat-visual-directive' +import { SessionId } from './structured-agent-session-identifiers' + +// `agentSession.readVisual`: one HTML visual from a structured chat's visuals folder, read on the +// host that owns the chat. The client names only the session and a bare file name; the host +// resolves its own folder, so no client path ever reaches another host. + +/** Content revision: hex digest of the file's bytes. */ +const Revision = z.string().regex(/^[0-9a-f]{16,64}$/) + +export const ReadVisualParams = z + .object({ + sessionId: SessionId, + file: z.string().refine(isNativeChatVisualFileName, 'Invalid visual file name'), + // The revision the client already holds; a match answers `unchanged` without the bytes. + knownRevision: Revision.optional() + }) + .strict() + +export type AgentSessionReadVisualParams = z.infer + +/** + * Why a visual cannot be shown, as positively observed by the owning host. Transport loss and an + * older host are never one of these: the client reads those as unavailable and may retry. + */ +export const AGENT_SESSION_VISUAL_READ_ERRORS = [ + 'session_not_found', + 'unsupported_location', + 'not_found', + 'not_a_file', + 'outside_folder', + 'too_large', + 'not_text' +] as const + +export type AgentSessionVisualReadError = (typeof AGENT_SESSION_VISUAL_READ_ERRORS)[number] + +export type AgentSessionReadVisualResult = + | { ok: true; revision: string; sizeBytes: number; html: string } + | { ok: true; revision: string; sizeBytes: number; unchanged: true } + | { ok: false; error: AgentSessionVisualReadError } diff --git a/src/shared/rpc-contract/rpc-params-catalog.generated.ts b/src/shared/rpc-contract/rpc-params-catalog.generated.ts index b8af9f409373..dedaa3a54c1d 100644 --- a/src/shared/rpc-contract/rpc-params-catalog.generated.ts +++ b/src/shared/rpc-contract/rpc-params-catalog.generated.ts @@ -40,6 +40,7 @@ import { import { PrepareCodexForWslPaneParams } from './agent-hooks-params' import { AgentLaunch, AgentLaunchReplay } from './agent-launch-params' import { CreateAgentSessionParams, EnsureAgentSessionParams } from './agent-session-params' +import { ReadVisualParams } from './agent-session-visual-params' import { AiVaultListSessionsParams, AiVaultPrepareSessionResumeParams, @@ -607,6 +608,7 @@ export const RPC_PARAMS_BY_METHOD = { 'agentSession.queuedMessageDelete': QueuedMessageActionParams, 'agentSession.queuedMessageSend': QueuedMessageActionParams, 'agentSession.queuedMessagesResume': QueuedMessagesResumeParams, + 'agentSession.readVisual': ReadVisualParams, 'agentSession.release': HoldParams, 'agentSession.respondToApproval': RespondParams, 'agentSession.respondToQuestion': RespondToQuestionParams, From 455f5dd4dbca2203f880192c15cb9d4a20281939 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:44:21 -0700 Subject: [PATCH 02/10] feat(native-chat): render chat visuals inline and in the right sidebar Native-chat assistant replies render a ::orca-visual{...} line as the chat's HTML visual in an opaque, scripts-only sandboxed frame: CSP first, the host frame navigation guard registered before content runs, live theme without a reload, fitted height, links opened in the viewer's browser only from a real gesture, lazy mount, and one muted line when the visual cannot be shown. Open in sidebar shows the same frame in the right sidebar, widened while it is open and restored after. --- .../structured-agent-session-visual.test.ts | 5 +- .../host-frame-navigation-guard.test.ts} | 25 +- .../host-frame-navigation-guard.ts} | 34 ++- .../main-window-webview-security.test.ts | 4 +- .../window/main-window-webview-security.ts | 8 +- .../native-chat/NativeChatInlineVisual.tsx | 111 ++++++++ .../native-chat/NativeChatMarkdown.tsx | 26 +- .../NativeChatMarkdown.visual.test.tsx | 127 +++++++++ .../native-chat/NativeChatMessageRow.tsx | 2 + .../NativeChatTranscriptChrome.tsx | 5 +- .../components/native-chat/NativeChatView.tsx | 10 +- .../NativeChatVisualFrame.test.tsx | 176 ++++++++++++ .../native-chat/NativeChatVisualFrame.tsx | 163 +++++++++++ .../native-chat/NativeChatVisualPanel.tsx | 80 ++++++ ...native-chat-visual-height-governor.test.ts | 52 ++++ .../native-chat-visual-height-governor.ts | 62 +++++ .../native-chat-visual-markdown-extension.tsx | 75 ++++++ ...ative-chat-visual-markdown-syntax.test.tsx | 133 +++++++++ .../native-chat-visual-markdown-syntax.ts | 160 +++++++++++ .../native-chat/native-chat-visual-owner.tsx | 53 ++++ .../native-chat-visual-read-client.test.ts | 147 ++++++++++ .../native-chat-visual-read-client.ts | 159 +++++++++++ .../use-native-chat-visual-document.ts | 99 +++++++ .../use-native-chat-visual-theme.ts | 89 +++++++ .../src/components/right-sidebar/index.tsx | 18 +- .../right-sidebar-panel-content.tsx | 19 +- .../right-sidebar/right-sidebar-width.ts | 8 + .../components/sidebar/CommentMarkdown.tsx | 45 +++- src/renderer/src/i18n/locales/en.json | 4 + .../editor/actions/right-sidebar-state.ts | 77 +++++- .../right-sidebar-visual-state.test.ts | 96 +++++++ src/shared/native-chat-visual-shell.test.ts | 161 +++++++++++ src/shared/native-chat-visual-shell.ts | 252 ++++++++++++++++++ 33 files changed, 2448 insertions(+), 37 deletions(-) rename src/main/{plugins/plugin-panel-navigation-guard.test.ts => window/host-frame-navigation-guard.test.ts} (51%) rename src/main/{plugins/plugin-panel-navigation-guard.ts => window/host-frame-navigation-guard.ts} (62%) create mode 100644 src/renderer/src/components/native-chat/NativeChatInlineVisual.tsx create mode 100644 src/renderer/src/components/native-chat/NativeChatMarkdown.visual.test.tsx create mode 100644 src/renderer/src/components/native-chat/NativeChatVisualFrame.test.tsx create mode 100644 src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx create mode 100644 src/renderer/src/components/native-chat/NativeChatVisualPanel.tsx create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-height-governor.test.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-height-governor.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-markdown-extension.tsx create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-markdown-syntax.test.tsx create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-markdown-syntax.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-owner.tsx create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-read-client.test.ts create mode 100644 src/renderer/src/components/native-chat/native-chat-visual-read-client.ts create mode 100644 src/renderer/src/components/native-chat/use-native-chat-visual-document.ts create mode 100644 src/renderer/src/components/native-chat/use-native-chat-visual-theme.ts create mode 100644 src/renderer/src/store/slices/editor/actions/right-sidebar-visual-state.test.ts create mode 100644 src/shared/native-chat-visual-shell.test.ts create mode 100644 src/shared/native-chat-visual-shell.ts diff --git a/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts b/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts index 9e51e89f5bf4..49fec6e7946e 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-visual.test.ts @@ -62,14 +62,15 @@ describe('agentSession.readVisual', () => { { sessionId: SESSION, file: 'chart.html' }, STRUCTURED_CLIENT ) - const revision = first.ok ? (first.result as { revision: string }).revision : '' + const result: unknown = first.ok ? first.result : null + const revision = String(Reflect.get(result ?? {}, 'revision')) const again = await call( 'agentSession.readVisual', { sessionId: SESSION, file: 'chart.html', knownRevision: revision }, STRUCTURED_CLIENT ) expect(again).toMatchObject({ ok: true, result: { ok: true, unchanged: true, revision } }) - expect(again.ok && 'html' in (again.result as object)).toBe(false) + expect(again.ok ? Reflect.has(Object(again.result), 'html') : true).toBe(false) }) it('reports a session this host has no record of', async () => { diff --git a/src/main/plugins/plugin-panel-navigation-guard.test.ts b/src/main/window/host-frame-navigation-guard.test.ts similarity index 51% rename from src/main/plugins/plugin-panel-navigation-guard.test.ts rename to src/main/window/host-frame-navigation-guard.test.ts index 2889e57cf707..1a4aae3d0420 100644 --- a/src/main/plugins/plugin-panel-navigation-guard.test.ts +++ b/src/main/window/host-frame-navigation-guard.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { PLUGIN_PANEL_FRAME_NAME_PREFIX } from '../../shared/plugins/plugin-panel-bridge' -import { PluginPanelNavigationRegistry } from './plugin-panel-navigation-guard' +import { NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX } from '../../shared/native-chat-visual-shell' +import { HostFrameNavigationRegistry, hostFrameKindForName } from './host-frame-navigation-guard' function frame(input: { id: number; name?: string; url?: string }) { let destroyed = false @@ -14,9 +15,9 @@ function frame(input: { id: number; name?: string; url?: string }) { } } -describe('PluginPanelNavigationRegistry', () => { +describe('HostFrameNavigationRegistry', () => { it('blocks only host-marked plugin srcdoc frames', () => { - const registry = new PluginPanelNavigationRegistry() + const registry = new HostFrameNavigationRegistry() const plugin = frame({ id: 1, name: `${PLUGIN_PANEL_FRAME_NAME_PREFIX}demo` }) const notebook = frame({ id: 2 }) registry.register(plugin) @@ -28,7 +29,7 @@ describe('PluginPanelNavigationRegistry', () => { }) it('keeps pre-parse identity after name mutation and prunes destroyed frames', () => { - const registry = new PluginPanelNavigationRegistry() + const registry = new HostFrameNavigationRegistry() const plugin = frame({ id: 1, name: `${PLUGIN_PANEL_FRAME_NAME_PREFIX}demo` }) registry.register(plugin) plugin.name = '' @@ -38,4 +39,20 @@ describe('PluginPanelNavigationRegistry', () => { plugin.destroy() expect(registry.shouldBlock(plugin, plugin, 'https://example.com')).toBe(false) }) + + it('contains chat visual frames the same way, as their own kind', () => { + const registry = new HostFrameNavigationRegistry() + const visual = frame({ id: 3, name: `${NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX}abc` }) + registry.register(visual) + expect(hostFrameKindForName(visual.name)).toBe('chat-visual') + expect(hostFrameKindForName(`${PLUGIN_PANEL_FRAME_NAME_PREFIX}demo`)).toBe('plugin-panel') + expect(hostFrameKindForName('orca-chat-visual')).toBeNull() + + expect(registry.shouldBlock(visual, null, 'about:srcdoc')).toBe(false) + // Its own navigation (location, meta refresh, link) and any child it starts are refused. + expect(registry.shouldBlock(visual, visual, 'https://example.com')).toBe(true) + expect(registry.shouldBlock(visual, null, 'https://example.com')).toBe(true) + const child = frame({ id: 4 }) + expect(registry.shouldBlock(child, visual, 'https://example.com')).toBe(true) + }) }) diff --git a/src/main/plugins/plugin-panel-navigation-guard.ts b/src/main/window/host-frame-navigation-guard.ts similarity index 62% rename from src/main/plugins/plugin-panel-navigation-guard.ts rename to src/main/window/host-frame-navigation-guard.ts index d183eae98881..110b3705c5d4 100644 --- a/src/main/plugins/plugin-panel-navigation-guard.ts +++ b/src/main/window/host-frame-navigation-guard.ts @@ -1,22 +1,40 @@ import type { WebContents, WebFrameMain } from 'electron' +import { NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX } from '../../shared/native-chat-visual-shell' import { PLUGIN_PANEL_FRAME_NAME_PREFIX } from '../../shared/plugins/plugin-panel-bridge' type NavigationFrame = Pick +/** + * Host-built srcdoc frames that hold content Orca did not write. Each kind is only a navigation + * containment class: registering a frame grants it nothing (no plugin identity, no actions). + */ +const HOST_FRAME_KINDS = [ + { kind: 'plugin-panel', namePrefix: PLUGIN_PANEL_FRAME_NAME_PREFIX }, + { kind: 'chat-visual', namePrefix: NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX } +] as const + +export type HostFrameKind = (typeof HOST_FRAME_KINDS)[number]['kind'] + type RegisteredFrame = { frame: NavigationFrame + kind: HostFrameKind initialSrcdocPending: boolean } -/** Records host-marked panel frame identities at browsing-context creation, - * before plugin parsing can mutate window.name. */ -export class PluginPanelNavigationRegistry { +export function hostFrameKindForName(name: string): HostFrameKind | null { + return HOST_FRAME_KINDS.find((entry) => name.startsWith(entry.namePrefix))?.kind ?? null +} + +/** Records host-marked frame identities at browsing-context creation, before their content can + * mutate window.name. */ +export class HostFrameNavigationRegistry { private readonly frames = new Map() register(frame: NavigationFrame): void { this.prune() - if (frame.name.startsWith(PLUGIN_PANEL_FRAME_NAME_PREFIX)) { - this.frames.set(frame.frameTreeNodeId, { frame, initialSrcdocPending: true }) + const kind = hostFrameKindForName(frame.name) + if (kind) { + this.frames.set(frame.frameTreeNodeId, { frame, kind, initialSrcdocPending: true }) } } @@ -52,8 +70,8 @@ export class PluginPanelNavigationRegistry { } } -export function registerPluginPanelNavigationGuard(webContents: WebContents): void { - const registry = new PluginPanelNavigationRegistry() +export function registerHostFrameNavigationGuard(webContents: WebContents): void { + const registry = new HostFrameNavigationRegistry() webContents.on('frame-created', (_event, { frame }) => { if (frame) { registry.register(frame) @@ -62,7 +80,7 @@ export function registerPluginPanelNavigationGuard(webContents: WebContents): vo webContents.on('did-start-navigation', (event) => { if (!event.isMainFrame && event.url === 'about:srcdoc' && event.frame) { // Some Chromium builds populate the frame name only when navigation - // starts; this event still precedes document parsing and plugin code. + // starts; this event still precedes document parsing and frame content. registry.register(event.frame) } }) diff --git a/src/main/window/main-window-webview-security.test.ts b/src/main/window/main-window-webview-security.test.ts index e54dc6a68369..d47a2f8118ef 100644 --- a/src/main/window/main-window-webview-security.test.ts +++ b/src/main/window/main-window-webview-security.test.ts @@ -15,8 +15,8 @@ vi.mock('../browser/browser-manager', () => ({ vi.mock('../browser/browser-session-registry', () => ({ browserSessionRegistry: { isAllowedPartition: mocks.isAllowedPartition } })) -vi.mock('../plugins/plugin-panel-navigation-guard', () => ({ - registerPluginPanelNavigationGuard: mocks.registerPluginGuard +vi.mock('./host-frame-navigation-guard', () => ({ + registerHostFrameNavigationGuard: mocks.registerPluginGuard })) vi.mock('./privileged-window-navigation', () => ({ installPrivilegedWindowNavigationPolicy: mocks.installNavigationPolicy diff --git a/src/main/window/main-window-webview-security.ts b/src/main/window/main-window-webview-security.ts index a662449eb580..7c026d7cf4a3 100644 --- a/src/main/window/main-window-webview-security.ts +++ b/src/main/window/main-window-webview-security.ts @@ -20,7 +20,7 @@ import { revokeAllDocPreviewGrants } from '../browser/doc-preview-grant-registry' import { isDocPreviewSession } from '../browser/doc-preview-protocol' -import { registerPluginPanelNavigationGuard } from '../plugins/plugin-panel-navigation-guard' +import { registerHostFrameNavigationGuard } from './host-frame-navigation-guard' import { installPrivilegedWindowNavigationPolicy } from './privileged-window-navigation' /** @@ -54,9 +54,9 @@ export function installMainWindowWebviewSecurity(mainWindow: BrowserWindow): voi setDocPreviewFailureSink(null) revokeAllDocPreviewGrants() }) - // Why: containment must be listening before any plugin panel frame is created, - // so register it with the window's other navigation policy. - registerPluginPanelNavigationGuard(mainWindow.webContents) + // Why: containment must be listening before any plugin panel or chat visual frame is + // created, so register it with the window's other navigation policy. + registerHostFrameNavigationGuard(mainWindow.webContents) const browserWindowClosePreload = join(__dirname, 'browser-window-close-preload.js') // Why a preview gets a preload at all: it is our own editor surface, not a browsing guest. This diff --git a/src/renderer/src/components/native-chat/NativeChatInlineVisual.tsx b/src/renderer/src/components/native-chat/NativeChatInlineVisual.tsx new file mode 100644 index 000000000000..eda0f0e7f9ea --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatInlineVisual.tsx @@ -0,0 +1,111 @@ +import { useEffect, useRef, useState } from 'react' +import { Maximize2 } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' +import { translate } from '@/i18n/i18n' +import { useAppStore } from '@/store' +import { NATIVE_CHAT_VISUAL_RESERVED_HEIGHT, NativeChatVisualFrame } from './NativeChatVisualFrame' +import { observeTranscriptVisibility } from './NativeChatTranscriptChrome' +import type { NativeChatVisualOwner } from './native-chat-visual-owner' +import { useNativeChatVisualDocument } from './use-native-chat-visual-document' + +export function NativeChatVisualUnavailable(): React.JSX.Element { + return ( +

+ {translate('components.native-chat.visualUnavailable', 'Visualization unavailable')} +

+ ) +} + +/** + * A visual inside an assistant reply. Its frame mounts once the reply scrolls near it and stays + * mounted, so what the reader did in it survives scrolling away and back. + */ +export function NativeChatInlineVisual({ + owner, + messageId, + file, + title +}: { + owner: NativeChatVisualOwner + messageId: string + file: string + title: string | null +}): React.JSX.Element { + const boxRef = useRef(null) + const [near, setNear] = useState(false) + const [retired, setRetired] = useState(false) + const openRightSidebarVisual = useAppStore((state) => state.openRightSidebarVisual) + const state = useNativeChatVisualDocument( + { target: owner.target, sessionId: owner.sessionId, file }, + near + ) + + useEffect(() => { + const element = boxRef.current + if (!element || near) { + return + } + return observeTranscriptVisibility(element, (visible) => { + if (visible) { + setNear(true) + } + }) + }, [near]) + + if (retired || state.status === 'unavailable') { + return + } + const label = title ?? file + const openLabel = translate('components.native-chat.visualOpenInSidebar', 'Open in sidebar') + + return ( +
+ {state.status === 'ready' && near ? ( + <> + setRetired(true)} + /> +
+ + + + + + {openLabel} + + +
+ + ) : ( +
+ )} +
+ ) +} diff --git a/src/renderer/src/components/native-chat/NativeChatMarkdown.tsx b/src/renderer/src/components/native-chat/NativeChatMarkdown.tsx index 3f2e89fb07ea..4033b51fc620 100644 --- a/src/renderer/src/components/native-chat/NativeChatMarkdown.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMarkdown.tsx @@ -1,11 +1,33 @@ import type { ComponentProps } from 'react' import CommentMarkdown from '@/components/sidebar/CommentMarkdown' import { cn } from '@/lib/utils' +import { withoutPendingNativeChatVisualDirectiveTail } from '../../../../shared/native-chat-visual-directive' +import { useNativeChatVisualMarkdownExtension } from './native-chat-visual-markdown-extension' import './native-chat-markdown.css' +type NativeChatMarkdownProps = ComponentProps & { + /** On assistant prose in a structured chat: this message may show visuals. */ + visualMessageId?: string + /** The reply is still arriving, so an unfinished visual line at its end is held back. */ + streaming?: boolean +} + export function NativeChatMarkdown({ className, + visualMessageId, + streaming = false, + content, ...props -}: ComponentProps): React.JSX.Element { - return +}: NativeChatMarkdownProps): React.JSX.Element { + const extension = useNativeChatVisualMarkdownExtension(visualMessageId) + return ( + + ) } diff --git a/src/renderer/src/components/native-chat/NativeChatMarkdown.visual.test.tsx b/src/renderer/src/components/native-chat/NativeChatMarkdown.visual.test.tsx new file mode 100644 index 000000000000..5ffde06b6c5e --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatMarkdown.visual.test.tsx @@ -0,0 +1,127 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' + +import { cleanup, render, waitFor } from '@testing-library/react' +import type { ReactNode } from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const callRuntimeRpc = vi.fn() +vi.mock('@/runtime/runtime-rpc-client', () => ({ + callRuntimeRpc: (...args: unknown[]) => callRuntimeRpc(...args) +})) + +import { NativeChatMarkdown } from './NativeChatMarkdown' +import { + NativeChatVisualOwnerContext, + type NativeChatVisualOwner +} from './native-chat-visual-owner' +import { clearNativeChatVisualCacheForTests } from './native-chat-visual-read-client' +import { TooltipProvider } from '@/components/ui/tooltip' + +const owner: NativeChatVisualOwner = { + target: { kind: 'local' }, + sessionId: 'session-alpha', + tabId: 'tab-1', + worktreeId: 'wt-1' +} +const LINE = '::orca-visual{file="usage.html" title="Usage"}' + +function withOwner(children: ReactNode, value: NativeChatVisualOwner | null = owner): ReactNode { + return ( + + + {children} + + + ) +} + +// The transcript mounts a visual once it scrolls near; here every visual is in view at once. +class InViewObserver { + constructor(private readonly callback: (entries: Partial[]) => void) {} + observe(target: Element): void { + this.callback([{ target, isIntersecting: true }]) + } + unobserve(): void {} + disconnect(): void {} +} + +beforeEach(() => { + vi.stubGlobal('IntersectionObserver', InViewObserver) + clearNativeChatVisualCacheForTests() + callRuntimeRpc.mockReset() + callRuntimeRpc.mockResolvedValue({ ok: true, revision: 'r1', sizeBytes: 9, html: '

v

' }) +}) + +afterEach(() => { + cleanup() + vi.unstubAllGlobals() +}) + +describe('NativeChatMarkdown visuals', () => { + it('holds back an unfinished directive at the end of a streaming reply', () => { + const { container } = render( + withOwner( + + ) + ) + expect(container).toHaveTextContent('Here it is:') + expect(container).not.toHaveTextContent('::orca-visual') + }) + + it('shows unfinished syntax as text once the reply is no longer streaming', () => { + const { container } = render( + withOwner( + + ) + ) + expect(container).toHaveTextContent('::orca-visual{file="usa') + }) + + it('keeps a mounted visual frame while the rest of the reply streams in', async () => { + const { container, rerender } = render( + withOwner( + + ) + ) + await waitFor(() => expect(container.querySelector('iframe')).not.toBeNull()) + const frame = container.querySelector('iframe') + rerender( + withOwner( + + ) + ) + expect(container.querySelector('iframe')).toBe(frame) + expect(container).toHaveTextContent('More text arriving now') + expect(callRuntimeRpc).toHaveBeenCalledTimes(1) + }) + + it('shows one muted line when the host refuses the file', async () => { + callRuntimeRpc.mockResolvedValue({ ok: false, error: 'outside_folder' }) + const { container } = render( + withOwner() + ) + await waitFor(() => expect(container).toHaveTextContent('Visualization unavailable')) + expect(container.querySelector('iframe')).toBeNull() + }) + + it('leaves the line as text where visuals do not apply', () => { + const noMessage = render(withOwner()) + expect(noMessage.container).toHaveTextContent('::orca-visual') + cleanup() + const noOwner = render( + withOwner(, null) + ) + expect(noOwner.container).toHaveTextContent('::orca-visual') + expect(callRuntimeRpc).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatMessageRow.tsx b/src/renderer/src/components/native-chat/NativeChatMessageRow.tsx index 7c2418cf6f79..560c7e4a0e59 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageRow.tsx @@ -305,6 +305,8 @@ export const MessageRow = memo(function MessageRow({ onLinkClick={onLinkClick} allowFileUriLinks={allowFileUriLinks} linkifyFilePaths={onLinkClick !== undefined} + visualMessageId={message.role === 'assistant' ? message.id : undefined} + streaming={message.state === 'running' && activeTurnIsWorking === true} /> ) : null} {tools.length > 0 || subagentGroups.length > 0 || backgroundTasks.length > 0 ? ( diff --git a/src/renderer/src/components/native-chat/NativeChatTranscriptChrome.tsx b/src/renderer/src/components/native-chat/NativeChatTranscriptChrome.tsx index e2f712c742c2..a808ee7e40d8 100644 --- a/src/renderer/src/components/native-chat/NativeChatTranscriptChrome.tsx +++ b/src/renderer/src/components/native-chat/NativeChatTranscriptChrome.tsx @@ -24,7 +24,10 @@ type VisibilityListener = (isVisible: boolean) => void const visibilityListeners = new Map() let visibilityObserver: IntersectionObserver | null = null -function observeTranscriptVisibility(element: Element, listener: VisibilityListener): () => void { +export function observeTranscriptVisibility( + element: Element, + listener: VisibilityListener +): () => void { if (typeof IntersectionObserver === 'undefined') { listener(true) return () => {} diff --git a/src/renderer/src/components/native-chat/NativeChatView.tsx b/src/renderer/src/components/native-chat/NativeChatView.tsx index d17de41a8542..e496ce9ad338 100644 --- a/src/renderer/src/components/native-chat/NativeChatView.tsx +++ b/src/renderer/src/components/native-chat/NativeChatView.tsx @@ -4,6 +4,7 @@ import { NativeChatResolvedView } from './NativeChatResolvedView' import { useNativeChatStatusEntry } from './use-native-chat-status-entry' import type { NativeChatViewProps } from './native-chat-view-types' import { NativeChatPaneFileDropSurface } from './NativeChatPaneFileDropSurface' +import { NativeChatVisualOwnerProvider } from './native-chat-visual-owner' export type { NativeChatViewProps } from './native-chat-view-types' @@ -12,7 +13,14 @@ export default function NativeChatView(props: NativeChatViewProps): React.JSX.El return ( {props.mode === 'structured' ? ( - + + + ) : ( )} diff --git a/src/renderer/src/components/native-chat/NativeChatVisualFrame.test.tsx b/src/renderer/src/components/native-chat/NativeChatVisualFrame.test.tsx new file mode 100644 index 000000000000..15399fbb63de --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatVisualFrame.test.tsx @@ -0,0 +1,176 @@ +// @vitest-environment happy-dom + +import '@testing-library/jest-dom/vitest' + +import { act, cleanup, render } from '@testing-library/react' +import { useRef } from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX, + NATIVE_CHAT_VISUAL_OPEN_LINK_TYPE, + NATIVE_CHAT_VISUAL_SIZE_TYPE, + NATIVE_CHAT_VISUAL_THEME_TYPE +} from '../../../../shared/native-chat-visual-shell' + +const openHttpLink = vi.fn() +vi.mock('@/lib/http-link-routing', () => ({ + openHttpLink: (...args: unknown[]) => openHttpLink(...args) +})) + +import { NativeChatVisualFrame } from './NativeChatVisualFrame' + +const visual = { revision: 'r1', html: '

chart

' } + +function Harness(props: { layout?: 'inline' | 'panel'; onRetired?: () => void }) { + const scope = useRef(null) + return ( +
+ {})} + /> +
+ ) +} + +function frameOf(container: HTMLElement): HTMLIFrameElement { + const frame = container.querySelector('iframe') + if (!frame) { + throw new Error('no frame') + } + return frame +} + +function channelOf(frame: HTMLIFrameElement): string { + return frame.name.slice(NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX.length) +} + +function post(data: unknown, source: MessageEventSource | null): void { + act(() => { + window.dispatchEvent(new MessageEvent('message', { data, source })) + }) +} + +function setUserActivation(isActive: boolean): void { + Object.defineProperty(navigator, 'userActivation', { + configurable: true, + value: { isActive, hasBeenActive: isActive } + }) +} + +beforeEach(() => { + openHttpLink.mockReset() +}) + +afterEach(() => { + cleanup() + document.documentElement.classList.remove('dark') +}) + +describe('NativeChatVisualFrame', () => { + it('runs the visual in a scripts-only sandbox, named for host registration, with its CSP first', () => { + const { container } = render() + const frame = frameOf(container) + expect(frame.getAttribute('sandbox')).toBe('allow-scripts') + expect(frame.name.startsWith(NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX)).toBe(true) + const srcdoc = frame.getAttribute('srcdoc') ?? '' + expect(srcdoc.indexOf('Content-Security-Policy')).toBeLessThan(srcdoc.indexOf('

chart

')) + expect(channelOf(frame)).toMatch(/^[0-9a-f]{32}$/) + }) + + it('fits its height to what the frame reports, ignoring other windows and channels', () => { + const { container } = render() + const frame = frameOf(container) + const box = frame.parentElement + const channel = channelOf(frame) + post({ type: NATIVE_CHAT_VISUAL_SIZE_TYPE, channel, height: 420 }, frame.contentWindow) + expect(box).toHaveStyle({ height: '420px' }) + post({ type: NATIVE_CHAT_VISUAL_SIZE_TYPE, channel, height: 900 }, window) + post({ type: NATIVE_CHAT_VISUAL_SIZE_TYPE, channel: 'other', height: 900 }, frame.contentWindow) + expect(box).toHaveStyle({ height: '420px' }) + }) + + it('restyles in place on a theme change instead of reloading the visual', () => { + const { container } = render() + const frame = frameOf(container) + const contentWindow = frame.contentWindow + if (!contentWindow) { + throw new Error('no content window') + } + const postMessage = vi.spyOn(contentWindow, 'postMessage') + act(() => { + frame.dispatchEvent(new Event('load')) + }) + expect(postMessage).toHaveBeenCalledWith( + expect.objectContaining({ type: NATIVE_CHAT_VISUAL_THEME_TYPE, colorScheme: 'light' }), + '*' + ) + const srcdoc = frame.getAttribute('srcdoc') + act(() => { + document.documentElement.classList.add('dark') + }) + return vi.waitFor(() => { + expect(postMessage).toHaveBeenLastCalledWith( + expect.objectContaining({ type: NATIVE_CHAT_VISUAL_THEME_TYPE, colorScheme: 'dark' }), + '*' + ) + expect(frameOf(container)).toBe(frame) + expect(frame.getAttribute('srcdoc')).toBe(srcdoc) + }) + }) + + it('opens a link only while the frame has focus and the user has just acted', () => { + const { container } = render() + const frame = frameOf(container) + const channel = channelOf(frame) + const link = { type: NATIVE_CHAT_VISUAL_OPEN_LINK_TYPE, channel, url: 'https://example.com/' } + + setUserActivation(true) + post(link, frame.contentWindow) + expect(openHttpLink).not.toHaveBeenCalled() + + frame.tabIndex = 0 + frame.focus() + setUserActivation(false) + post(link, frame.contentWindow) + expect(openHttpLink).not.toHaveBeenCalled() + + setUserActivation(true) + post(link, window) + post({ ...link, url: 'javascript:alert(1)' }, frame.contentWindow) + expect(openHttpLink).not.toHaveBeenCalled() + + post(link, frame.contentWindow) + expect(openHttpLink).toHaveBeenCalledWith('https://example.com/', { forceSystemBrowser: true }) + // A burst of requests opens at most one page. + post(link, frame.contentWindow) + expect(openHttpLink).toHaveBeenCalledTimes(1) + }) + + it('retires itself if the frame loads a second document', () => { + const onRetired = vi.fn() + const { container } = render() + const frame = frameOf(container) + act(() => { + frame.dispatchEvent(new Event('load')) + }) + expect(onRetired).not.toHaveBeenCalled() + act(() => { + frame.dispatchEvent(new Event('load')) + }) + expect(onRetired).toHaveBeenCalledTimes(1) + }) + + it('fills a panel instead of fitting its height', () => { + const { container } = render() + const frame = frameOf(container) + post( + { type: NATIVE_CHAT_VISUAL_SIZE_TYPE, channel: channelOf(frame), height: 420 }, + frame.contentWindow + ) + expect(frame.parentElement?.style.height).toBe('') + }) +}) diff --git a/src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx b/src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx new file mode 100644 index 000000000000..e31951d44a14 --- /dev/null +++ b/src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx @@ -0,0 +1,163 @@ +import { useEffect, useMemo, useRef, useState, type RefObject } from 'react' +import { cn } from '@/lib/utils' +import { openHttpLink } from '@/lib/http-link-routing' +import { + NATIVE_CHAT_VISUAL_FRAME_NAME_PREFIX, + buildNativeChatVisualDocument, + nativeChatVisualThemeMessage, + readNativeChatVisualFrameMessage +} from '../../../../shared/native-chat-visual-shell' +import type { NativeChatVisualDocument } from './native-chat-visual-read-client' +import { createNativeChatVisualHeightGovernor } from './native-chat-visual-height-governor' +import { useNativeChatVisualTheme } from './use-native-chat-visual-theme' + +/** Height reserved before a visual reports its own, so the reply below does not jump far. */ +export const NATIVE_CHAT_VISUAL_RESERVED_HEIGHT = 160 +const LINK_COOLDOWN_MS = 1_000 + +function newChannel(): string { + const bytes = new Uint8Array(16) + crypto.getRandomValues(bytes) + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')).join('') +} + +/** + * One agent-written visual in an opaque-origin frame that may run scripts and nothing else. Inline + * it fits its height to the page; in a panel it fills the panel. A theme change restyles the page in + * place, so interaction state survives it. The frame is never allowed to become another page: the + * main process refuses its navigation, and a second load (a navigation that got through anyway) + * retires it. + */ +export function NativeChatVisualFrame({ + document: visual, + title, + layout, + themeScope, + onRetired +}: { + document: NativeChatVisualDocument + title: string + layout: 'inline' | 'panel' + /** An element already mounted where the frame sits, so the first paint wears that scope's theme. */ + themeScope: RefObject + onRetired: () => void +}): React.JSX.Element { + const frameRef = useRef(null) + const theme = useNativeChatVisualTheme(themeScope) + const themeRef = useRef(theme) + themeRef.current = theme + const [height, setHeight] = useState(NATIVE_CHAT_VISUAL_RESERVED_HEIGHT) + const loadsRef = useRef(0) + + // Built once per revision: the theme at build time paints first, later themes arrive by message. + const built = useMemo(() => { + const channel = newChannel() + return { + channel, + srcDoc: buildNativeChatVisualDocument({ + html: visual.html, + channel, + theme: themeRef.current + }) + } + }, [visual.html]) + + useEffect(() => { + loadsRef.current = 0 + const governor = createNativeChatVisualHeightGovernor() + let deferred: ReturnType | null = null + let latestReported: number | null = null + let lastLinkAt = -Infinity + const applyHeight = (): void => { + deferred = null + if (latestReported === null) { + return + } + const decision = governor.decide(latestReported, performance.now()) + if (decision.kind === 'apply') { + setHeight(decision.height) + } else if (decision.kind === 'defer') { + deferred = setTimeout(applyHeight, decision.retryInMs) + } + } + const onMessage = (event: MessageEvent): void => { + const frame = frameRef.current + if (!frame || event.source !== frame.contentWindow) { + return + } + const message = readNativeChatVisualFrameMessage(event.data, built.channel) + if (!message) { + return + } + if (message.kind === 'size') { + if (layout === 'inline') { + latestReported = message.height + if (!deferred) { + applyHeight() + } + } + return + } + // A link opens only from a real gesture in this very frame: it must hold focus and the user + // must have just acted. This stops opens on load, not a page that waits for the next click. + const now = performance.now() + if ( + window.document.activeElement !== frame || + navigator.userActivation?.isActive !== true || + now - lastLinkAt < LINK_COOLDOWN_MS + ) { + return + } + lastLinkAt = now + openHttpLink(message.url, { forceSystemBrowser: true }) + } + window.addEventListener('message', onMessage) + return () => { + window.removeEventListener('message', onMessage) + if (deferred) { + clearTimeout(deferred) + } + } + }, [built.channel, layout]) + + useEffect(() => { + if (loadsRef.current > 0) { + frameRef.current?.contentWindow?.postMessage( + nativeChatVisualThemeMessage(theme, built.channel), + '*' + ) + } + }, [built.channel, theme]) + + return ( +
+