diff --git a/apps/codex-claw/src/routes/api/send.ts b/apps/codex-claw/src/routes/api/send.ts index 2087030..1b7bf1c 100644 --- a/apps/codex-claw/src/routes/api/send.ts +++ b/apps/codex-claw/src/routes/api/send.ts @@ -1,7 +1,89 @@ +import { Buffer } from 'node:buffer' import { randomUUID } from 'node:crypto' import { createFileRoute } from '@tanstack/react-router' import { json } from '@tanstack/react-start' -import { resolveCodexSession, sendCodexPrompt } from '../../server/codex-cli' +import { + isSupportedCodexImageMimeType, + resolveCodexSession, + sendCodexPrompt, +} from '../../server/codex-cli' + +type ParsedAttachment = { + mimeType: string + content: string + name?: string +} + +type AttachmentParseResult = + | { + ok: true + attachments: Array | undefined + } + | { + ok: false + error: string + } + +function parseAttachments(rawAttachments: unknown): AttachmentParseResult { + if (typeof rawAttachments === 'undefined') { + return { ok: true, attachments: undefined } + } + + if (!Array.isArray(rawAttachments)) { + return { ok: false, error: 'attachments must be an array' } + } + + const attachments: Array = [] + + for (const rawAttachment of rawAttachments) { + if (!rawAttachment || typeof rawAttachment !== 'object') { + return { ok: false, error: 'attachment must be an object' } + } + + const attachment = rawAttachment as Record + const mimeType = + typeof attachment.mimeType === 'string' ? attachment.mimeType.trim() : '' + const rawContent = + typeof attachment.content === 'string' ? attachment.content : '' + const content = normalizeBase64Content(rawContent) + + if (!isSupportedCodexImageMimeType(mimeType)) { + return { + ok: false, + error: + 'Unsupported attachment type. Please use PNG, JPG, GIF, or WebP images.', + } + } + + if (!isValidBase64Content(content)) { + return { + ok: false, + error: 'Attachment image data could not be decoded.', + } + } + + attachments.push({ + mimeType, + content, + name: typeof attachment.name === 'string' ? attachment.name : undefined, + }) + } + + return { ok: true, attachments } +} + +function normalizeBase64Content(content: string) { + const trimmed = content.trim() + const dataUrlMatch = /^data:[^,]+;base64,(?[\s\S]+)$/i.exec(trimmed) + return (dataUrlMatch?.groups?.data ?? trimmed).replace(/\s/g, '') +} + +function isValidBase64Content(content: string) { + if (!content) return false + if (content.length % 4 === 1) return false + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(content)) return false + return Buffer.from(content, 'base64').length > 0 +} export const Route = createFileRoute('/api/send')({ server: { @@ -21,16 +103,14 @@ export const Route = createFileRoute('/api/send')({ const thinking = typeof body.thinking === 'string' ? body.thinking : undefined - const rawAttachments = body.attachments - const attachments = Array.isArray(rawAttachments) - ? rawAttachments.filter( - (a: unknown): a is { mimeType: string; content: string } => - typeof a === 'object' && - a !== null && - typeof (a as Record).mimeType === 'string' && - typeof (a as Record).content === 'string', - ) - : undefined + const parsedAttachments = parseAttachments(body.attachments) + if (!parsedAttachments.ok) { + return json( + { ok: false, error: parsedAttachments.error }, + { status: 400 }, + ) + } + const attachments = parsedAttachments.attachments if (!message.trim() && (!attachments || attachments.length === 0)) { return json( diff --git a/apps/codex-claw/src/screens/chat/chat-screen.tsx b/apps/codex-claw/src/screens/chat/chat-screen.tsx index e5b8a5b..dd61692 100644 --- a/apps/codex-claw/src/screens/chat/chat-screen.tsx +++ b/apps/codex-claw/src/screens/chat/chat-screen.tsx @@ -259,6 +259,7 @@ export function ChatScreen({ const attachmentsPayload = attachments?.map((a) => ({ mimeType: a.file.type, content: a.base64, + name: a.file.name, })) fetch('/api/send', { diff --git a/apps/codex-claw/src/screens/chat/components/message-item.tsx b/apps/codex-claw/src/screens/chat/components/message-item.tsx index ff22c57..3751a0a 100644 --- a/apps/codex-claw/src/screens/chat/components/message-item.tsx +++ b/apps/codex-claw/src/screens/chat/components/message-item.tsx @@ -121,7 +121,7 @@ export function assistantPartRenderOrder( } continue } - if (showToolMessages) { + if (part.type === 'toolCall' && showToolMessages) { order.push('toolCall') } } diff --git a/apps/codex-claw/src/screens/chat/types.ts b/apps/codex-claw/src/screens/chat/types.ts index 082c233..099d973 100644 --- a/apps/codex-claw/src/screens/chat/types.ts +++ b/apps/codex-claw/src/screens/chat/types.ts @@ -27,7 +27,20 @@ export type ThinkingContent = { thinkingSignature?: string } -export type MessageContent = TextContent | ToolCallContent | ThinkingContent +export type ImageContent = { + type: 'image' + source: { + type: 'base64' + media_type: string + data: string + } +} + +export type MessageContent = + | TextContent + | ToolCallContent + | ThinkingContent + | ImageContent export type GatewayMessage = { role?: string diff --git a/apps/codex-claw/src/server/codex-cli.test.ts b/apps/codex-claw/src/server/codex-cli.test.ts new file mode 100644 index 0000000..5482769 --- /dev/null +++ b/apps/codex-claw/src/server/codex-cli.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest' + +import { mergeAssistantText, processCodexJsonLine } from './codex-cli' + +describe('processCodexJsonLine', function () { + it('parses assistant deltas before the final completed item', function () { + const first = processCodexJsonLine( + JSON.stringify({ + type: 'item.updated', + item: { type: 'agent_message', text: 'Hel' }, + }), + ) + const second = processCodexJsonLine( + JSON.stringify({ + type: 'item.updated', + item: { type: 'agent_message', text: 'Hello' }, + }), + ) + const final = processCodexJsonLine( + JSON.stringify({ + type: 'item.completed', + item: { type: 'agent_message', text: 'Hello there' }, + }), + ) + + expect(first).toEqual({ kind: 'assistant-delta', text: 'Hel' }) + expect(second).toEqual({ kind: 'assistant-delta', text: 'Hello' }) + expect(final).toEqual({ kind: 'assistant-final', text: 'Hello there' }) + }) + + it('ignores non-assistant events', function () { + expect( + processCodexJsonLine( + JSON.stringify({ + type: 'item.completed', + item: { type: 'reasoning', text: 'hidden' }, + }), + ), + ).toBeNull() + }) +}) + +describe('mergeAssistantText', function () { + it('keeps cumulative snapshots in order', function () { + const first = mergeAssistantText('', 'Hel', 'assistant-delta') + const second = mergeAssistantText(first, 'Hello', 'assistant-delta') + const final = mergeAssistantText(second, 'Hello there', 'assistant-final') + + expect(first).toBe('Hel') + expect(second).toBe('Hello') + expect(final).toBe('Hello there') + }) + + it('appends token deltas without duplicating repeated chunks', function () { + const first = mergeAssistantText('', 'Hel', 'assistant-delta') + const second = mergeAssistantText(first, 'lo', 'assistant-delta') + const repeated = mergeAssistantText(second, 'lo', 'assistant-delta') + + expect(first).toBe('Hel') + expect(second).toBe('Hello') + expect(repeated).toBe('Hello') + }) +}) diff --git a/apps/codex-claw/src/server/codex-cli.ts b/apps/codex-claw/src/server/codex-cli.ts index f72ec8e..05d5580 100644 --- a/apps/codex-claw/src/server/codex-cli.ts +++ b/apps/codex-claw/src/server/codex-cli.ts @@ -1,10 +1,13 @@ +import { Buffer } from 'node:buffer' import { randomUUID } from 'node:crypto' import { spawn, spawnSync } from 'node:child_process' import { existsSync, mkdirSync, + mkdtempSync, readFileSync, renameSync, + rmSync, writeFileSync, } from 'node:fs' import os from 'node:os' @@ -26,6 +29,14 @@ type MessageContent = arguments?: Record partialJson?: string } + | { + type: 'image' + source: { + type: 'base64' + media_type: string + data: string + } + } type CodexMessage = { id?: string @@ -58,6 +69,7 @@ type SessionStore = { type AttachmentInput = { mimeType: string content: string + name?: string } type SendCodexPromptInput = { @@ -77,9 +89,22 @@ type CodexStreamEvent = { type CodexExecJsonEvent = { type?: string + role?: string + text?: string + delta?: string + message?: unknown + content?: unknown item?: { + id?: string type?: string + role?: string text?: string + delta?: string + command?: string + aggregated_output?: string + exit_code?: number | null + status?: string + content?: unknown name?: string arguments?: Record [key: string]: unknown @@ -95,6 +120,32 @@ type CodexPathsPayload = { storePath: string } +type PreparedAttachmentFiles = { + imagePaths: Array + cleanup: () => void +} + +type ProcessedCodexJsonLine = + | { + kind: 'assistant-delta' + text: string + } + | { + kind: 'assistant-final' + text: string + } + | { + kind: 'message-delta' + message: CodexMessage + } + +const supportedImageMimeTypes = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]) + const listeners = new Map void>>() let storeCache: SessionStore | null = null let stateVersion = 0 @@ -305,25 +356,29 @@ function buildUserPrompt(input: SendCodexPromptInput) { } if (input.attachments && input.attachments.length > 0) { parts.push( - `Attachments were provided, but CodexClaw alpha currently sends text prompts only. Attachment count: ${input.attachments.length}.`, + `Attached image count: ${input.attachments.length}. Review the attached image files when they are relevant to the request.`, ) } return parts.join('\n\n') } -function buildCodexArgs() { +function buildCodexArgs(imagePaths: Array) { const args = ['-s', getCodexSandbox(), '-C', getCodexWorkdir()] args.push('exec') args.push('--ignore-user-config') args.push('--json') args.push('--skip-git-repo-check') + for (const imagePath of imagePaths) { + args.push('--image') + args.push(imagePath) + } args.push('-') return args } -function messageFromAssistantText(text: string): CodexMessage { +function messageFromAssistantText(text: string, id = randomUUID()): CodexMessage { return { - id: randomUUID(), + id, role: 'assistant', timestamp: Date.now(), content: [{ type: 'text', text }], @@ -340,28 +395,312 @@ function errorMessage(text: string): CodexMessage { } } -function processCodexJsonLine(line: string) { +function imageContentFromAttachment(attachment: AttachmentInput): MessageContent { + return { + type: 'image', + source: { + type: 'base64', + media_type: attachment.mimeType, + data: attachment.content, + }, + } +} + +function contentFromUserInput(input: SendCodexPromptInput): Array { + const content: Array = [] + + for (const attachment of input.attachments ?? []) { + if (!isSupportedImageAttachment(attachment)) continue + content.push(imageContentFromAttachment(attachment)) + } + + if (input.message.trim()) { + content.push({ type: 'text', text: input.message }) + } else if (content.length === 0) { + content.push({ type: 'text', text: '' }) + } + + return content +} + +function isSupportedImageAttachment(attachment: AttachmentInput) { + return ( + isSupportedCodexImageMimeType(attachment.mimeType) && + attachment.content.trim().length > 0 + ) +} + +export function isSupportedCodexImageMimeType(mimeType: string) { + return supportedImageMimeTypes.has(mimeType.trim().toLowerCase()) +} + +function attachmentExtension(mimeType: string) { + switch (mimeType.toLowerCase()) { + case 'image/jpeg': + case 'image/jpg': + return '.jpg' + case 'image/png': + return '.png' + case 'image/gif': + return '.gif' + case 'image/webp': + return '.webp' + default: + return '.img' + } +} + +function prepareAttachmentFiles( + runId: string, + attachments: Array | undefined, +): PreparedAttachmentFiles { + const imageAttachments = attachments ?? [] + if (imageAttachments.length === 0) { + return { + imagePaths: [], + cleanup() {}, + } + } + + const attachmentsDir = mkdtempSync( + path.join(os.tmpdir(), `codex-claw-${runId}-`), + ) + const imagePaths: Array = [] + + try { + imageAttachments.forEach((attachment, index) => { + if (!isSupportedImageAttachment(attachment)) { + throw new Error( + 'Unsupported attachment type. Please use PNG, JPG, GIF, or WebP images.', + ) + } + + const imagePath = path.join( + attachmentsDir, + `image-${index + 1}${attachmentExtension(attachment.mimeType)}`, + ) + writeFileSync(imagePath, Buffer.from(attachment.content, 'base64')) + imagePaths.push(imagePath) + }) + } catch (error) { + rmSync(attachmentsDir, { recursive: true, force: true }) + throw error + } + + return { + imagePaths, + cleanup() { + rmSync(attachmentsDir, { recursive: true, force: true }) + }, + } +} + +function extractTextFromContent(value: unknown): string { + if (typeof value === 'string') return value + if (!Array.isArray(value)) return '' + return value + .map((part) => { + if (!part || typeof part !== 'object') return '' + const item = part as Record + if (typeof item.text === 'string') return item.text + if (typeof item.delta === 'string') return item.delta + if (typeof item.content === 'string') return item.content + return '' + }) + .join('') +} + +function extractTextCandidate(value: unknown): string { + if (!value || typeof value !== 'object') return '' + const item = value as Record + const candidates = [ + item.text, + item.delta, + item.output_text, + item.outputText, + item.content, + ] + + for (const candidate of candidates) { + const text = + typeof candidate === 'string' + ? candidate + : extractTextFromContent(candidate) + if (text) return text + } + + const message = item.message + if (message && typeof message === 'object') { + const text = extractTextCandidate(message) + if (text) return text + } + + return '' +} + +function isAssistantJsonEvent(event: CodexExecJsonEvent) { + const type = event.type ?? '' + const itemType = event.item?.type ?? '' + const role = event.item?.role ?? event.role ?? '' + return ( + role === 'assistant' || + itemType === 'agent_message' || + itemType === 'assistant_message' || + type.includes('agent_message') || + type.includes('assistant') || + type.includes('output_text') + ) +} + +function commandExecutionDetails( + item: NonNullable, +) { + const details: Record = {} + if (typeof item.command === 'string' && item.command.trim()) { + details.command = item.command.trim() + } + if (typeof item.status === 'string' && item.status.trim()) { + details.status = item.status.trim() + } + if (typeof item.exit_code === 'number') { + details.exitCode = item.exit_code + } + return details +} + +function commandExecutionStartedMessage( + item: NonNullable, +): CodexMessage { + const id = typeof item.id === 'string' ? item.id.trim() : '' + const command = typeof item.command === 'string' ? item.command.trim() : '' + return { + id: id || undefined, + role: 'assistant', + timestamp: Date.now(), + content: [ + { + type: 'toolCall', + id: id || undefined, + name: 'command_execution', + arguments: command ? { command } : undefined, + }, + ], + } +} + +function commandExecutionCompletedMessage( + item: NonNullable, +): CodexMessage { + const id = typeof item.id === 'string' ? item.id.trim() : '' + const output = String(item.aggregated_output ?? '').trim() + const status = typeof item.status === 'string' ? item.status.trim() : '' + const exitCode = item.exit_code + const isError = + (typeof exitCode === 'number' && exitCode !== 0) || + (status.length > 0 && status !== 'completed') + + return { + id: id ? `${id}:result` : undefined, + role: 'toolResult', + toolCallId: id || undefined, + toolName: 'command_execution', + timestamp: Date.now(), + details: commandExecutionDetails(item), + isError, + content: [ + { + type: 'text', + text: output || status || 'Command finished.', + }, + ], + } +} + +function processCommandExecutionJsonEvent( + event: CodexExecJsonEvent, +): ProcessedCodexJsonLine | null { + if (event.item?.type !== 'command_execution') return null + if (event.type === 'item.started') { + return { + kind: 'message-delta', + message: commandExecutionStartedMessage(event.item), + } + } + if (event.type === 'item.completed') { + return { + kind: 'message-delta', + message: commandExecutionCompletedMessage(event.item), + } + } + return null +} + +export function processCodexJsonLine( + line: string, +): ProcessedCodexJsonLine | null { if (!line.startsWith('{')) return null try { const event = JSON.parse(line) as CodexExecJsonEvent - if (event.type === 'item.completed' && event.item?.type === 'agent_message') { - const text = typeof event.item.text === 'string' ? event.item.text : '' - return text ? messageFromAssistantText(text) : null + const commandEvent = processCommandExecutionJsonEvent(event) + if (commandEvent) return commandEvent + if (!isAssistantJsonEvent(event)) return null + + const text = + extractTextCandidate(event.item) || + extractTextCandidate(event) || + extractTextFromContent(event.content) + + if (!text) return null + + if ( + event.type === 'item.completed' || + event.type === 'response.output_text.done' || + event.type === 'assistant.completed' + ) { + return { kind: 'assistant-final', text } + } + + if ( + event.type?.includes('delta') || + event.type?.includes('updated') || + event.type?.includes('stream') + ) { + return { kind: 'assistant-delta', text } } + + return null } catch { return null } - return null } -function runCodexExec(sessionKey: string, prompt: string, runId: string) { +export function mergeAssistantText( + currentText: string, + incomingText: string, + kind: ProcessedCodexJsonLine['kind'], +) { + if (!incomingText) return currentText + if (kind === 'assistant-final') return incomingText + if (!currentText) return incomingText + if (incomingText.startsWith(currentText)) return incomingText + if (currentText.endsWith(incomingText)) return currentText + return `${currentText}${incomingText}` +} + +function runCodexExec( + sessionKey: string, + prompt: string, + runId: string, + attachments?: Array, +) { const store = readStore() const session = ensureSession(sessionKey) const command = process.platform === 'win32' ? getCodexCommand() : resolveCodexCommand(getCodexCommand()) - const args = buildCodexArgs() + const preparedAttachments = prepareAttachmentFiles(runId, attachments) + const args = buildCodexArgs(preparedAttachments.imagePaths) const child = spawn(command, args, { cwd: getCodexWorkdir(), env: process.env, @@ -371,6 +710,40 @@ function runCodexExec(sessionKey: string, prompt: string, runId: string) { let stdoutBuffer = '' let stderrBuffer = '' let emittedFinal = false + let assistantText = '' + let streamSeq = 0 + + function emitStreamMessage(message: CodexMessage, state: 'delta' | 'final') { + streamSeq += 1 + emit(session.key, { + event: 'chat', + seq: streamSeq, + stateVersion, + payload: { + runId, + sessionKey: session.key, + state, + seq: streamSeq, + message, + }, + }) + } + + function emitAssistantDelta(text: string) { + emitStreamMessage(messageFromAssistantText(text, runId), 'delta') + } + + function emitFinalMessage(message: CodexMessage) { + emitStreamMessage(message, 'final') + } + + function appendFinalText(text: string) { + const message = messageFromAssistantText(text, runId) + appendMessage(session, message) + writeStore(store) + emittedFinal = true + emitFinalMessage(message) + } child.stdout.setEncoding('utf8') child.stdout.on('data', (chunk: string) => { @@ -378,21 +751,24 @@ function runCodexExec(sessionKey: string, prompt: string, runId: string) { const lines = stdoutBuffer.split(/\r?\n/) stdoutBuffer = lines.pop() ?? '' for (const line of lines) { - const message = processCodexJsonLine(line.trim()) - if (!message) continue - appendMessage(session, message) - writeStore(store) - emittedFinal = true - emit(session.key, { - event: 'chat', - stateVersion, - payload: { - runId, - sessionKey: session.key, - state: 'final', - message, - }, - }) + const processed = processCodexJsonLine(line.trim()) + if (!processed) continue + if (processed.kind === 'message-delta') { + appendMessage(session, processed.message) + writeStore(store) + emitStreamMessage(processed.message, 'delta') + continue + } + assistantText = mergeAssistantText( + assistantText, + processed.text, + processed.kind, + ) + if (processed.kind === 'assistant-delta') { + emitAssistantDelta(assistantText) + continue + } + if (!emittedFinal) appendFinalText(assistantText) } }) @@ -404,6 +780,7 @@ function runCodexExec(sessionKey: string, prompt: string, runId: string) { child.stdin.end(prompt) child.on('error', (error) => { + preparedAttachments.cleanup() const message = errorMessage(error.message) appendMessage(session, message) writeStore(store) @@ -420,25 +797,32 @@ function runCodexExec(sessionKey: string, prompt: string, runId: string) { }) child.on('close', (code) => { + preparedAttachments.cleanup() const trailing = processCodexJsonLine(stdoutBuffer.trim()) if (trailing) { - appendMessage(session, trailing) - writeStore(store) - emittedFinal = true - emit(session.key, { - event: 'chat', - stateVersion, - payload: { - runId, - sessionKey: session.key, - state: 'final', - message: trailing, - }, - }) - return + if (trailing.kind === 'message-delta') { + appendMessage(session, trailing.message) + writeStore(store) + emitStreamMessage(trailing.message, 'delta') + } else { + assistantText = mergeAssistantText( + assistantText, + trailing.text, + trailing.kind, + ) + if (!emittedFinal && (trailing.kind === 'assistant-final' || code === 0)) { + appendFinalText(assistantText) + return + } + if (emittedFinal && code === 0) return + } } if (emittedFinal && code === 0) return + if (code === 0 && assistantText) { + appendFinalText(assistantText) + return + } const detail = stderrBuffer.trim() const fallback = @@ -498,7 +882,7 @@ export function sendCodexPrompt(input: SendCodexPromptInput) { clientId: input.idempotencyKey, role: 'user', timestamp: Date.now(), - content: [{ type: 'text', text: input.message }], + content: contentFromUserInput(input), } appendMessage(session, userMessage) writeStore(store) @@ -515,7 +899,7 @@ export function sendCodexPrompt(input: SendCodexPromptInput) { }, }) - runCodexExec(session.key, prompt, runId) + runCodexExec(session.key, prompt, runId, input.attachments) return { runId, sessionKey: session.key } } @@ -581,9 +965,10 @@ export function deleteCodexSession(key: string) { } export function codexCliCheck() { + const command = getCodexCommand() const result = process.platform === 'win32' - ? spawnSync(`${getCodexCommand()} --version`, { + ? spawnSync(`${command} --version`, { cwd: getCodexWorkdir(), env: process.env, stdio: 'pipe',