fix: show live tool result images outside event groups

.changeset/show-tool-images-outside-events.md

@@ -0,0 +1,5 @@
+---
+"@jmfederico/pi-web": patch
+---
+
+Keep tool-result images visible in clearly labeled standard chat cards outside collapsed event groups while retaining technical execution details and final message metadata.

src/client/src/chatGroups.test.ts

@@ -41,6 +41,43 @@ describe("groupChatMessages", () => {
     ]);
   });
 
+  it("keeps image content visible outside collapsed event groups", () => {
+    const image = { type: "image" as const, mimeType: "image/png", data: "QUJD" };
+    const messages: ChatLine[] = [
+      { role: "tool", parts: [{ type: "toolResult", toolName: "read", text: "Read image file [image/png]", isError: false }, image] },
+    ];
+
+    expect(groupChatMessages(messages)).toEqual([
+      {
+        kind: "group",
+        startIndex: 0,
+        endIndex: 0,
+        messages: [{ role: "tool", parts: [{ type: "toolResult", toolName: "read", text: "Read image file [image/png]", isError: false }] }],
+      },
+      { kind: "tool-image", index: 0, message: { role: "tool", parts: [image] }, toolName: "read" },
+    ]);
+  });
+
+  it("preserves image metadata when splitting technical and readable parts", () => {
+    const meta = { timestamp: "2026-07-13T22:00:00.000Z" };
+    const image = { type: "image" as const, mimeType: "image/webp", data: "QUJD" };
+    const message: ChatLine = { role: "tool", parts: [{ type: "toolResult", toolName: "read", text: "ok", isError: false }, image], meta };
+
+    expect(groupChatMessages([message])).toEqual([
+      { kind: "group", startIndex: 0, endIndex: 0, messages: [{ role: "tool", parts: [message.parts[0]], meta }] },
+      { kind: "tool-image", index: 0, message: { role: "tool", parts: [image], meta }, toolName: "read" },
+    ]);
+  });
+
+  it("keeps user images as ordinary messages", () => {
+    const image = { type: "image" as const, mimeType: "image/png", data: "QUJD" };
+    const message: ChatLine = { role: "user", parts: [image] };
+
+    expect(groupChatMessages([message])).toEqual([
+      { kind: "message", index: 0, message },
+    ]);
+  });
+
   it("preserves message metadata when grouping", () => {
     const message: ChatLine = { role: "assistant", parts: [{ type: "thinking", text: "hidden" }, { type: "text", text: "shown" }], meta: { timestamp: "2026-05-09T12:00:00.000Z", model: { provider: "test", id: "model" } } };
 

src/client/src/chatGroups.ts

@@ -2,6 +2,7 @@ import type { ChatLine, ChatPart } from "./components/shared";
 
 export type ChatGroup =
   | { kind: "message"; message: ChatLine; index: number }
+  | { kind: "tool-image"; message: ChatLine; index: number; toolName?: string }
   | { kind: "group"; messages: ChatLine[]; startIndex: number; endIndex: number };
 
 export function groupChatMessages(messages: ChatLine[], indexOffset = 0): ChatGroup[] {
@@ -29,7 +30,13 @@ export function groupChatMessages(messages: ChatLine[], indexOffset = 0): ChatGr
     if (readableParts.length) {
       flushEvents();
       const role = readableParts.every((part) => part.type === "skillRead") ? "skill" : message.role;
-      groups.push({ kind: "message", message: { role, parts: readableParts, ...metadata }, index: absoluteIndex });
+      const readableMessage = { role, parts: readableParts, ...metadata };
+      if (isToolImageMessage(readableMessage)) {
+        const toolName = toolNameFromParts(technicalParts);
+        groups.push({ kind: "tool-image", message: readableMessage, index: absoluteIndex, ...(toolName === undefined ? {} : { toolName }) });
+      } else {
+        groups.push({ kind: "message", message: readableMessage, index: absoluteIndex });
+      }
     }
   });
   flushEvents();
@@ -47,8 +54,19 @@ export function summarizeChatGroup(messages: ChatLine[]): string {
   return `${String(messages.length)} ${messages.length === 1 ? "event" : "events"}${details !== "" ? ` · ${details}` : ""}`;
 }
 
+function isToolImageMessage(message: ChatLine): boolean {
+  return message.role === "tool" && message.parts.length > 0 && message.parts.every((part) => part.type === "image");
+}
+
+function toolNameFromParts(parts: ChatPart[]): string | undefined {
+  for (const part of parts) {
+    if ((part.type === "toolCall" || part.type === "toolExecution" || part.type === "toolResult") && part.toolName !== "") return part.toolName;
+  }
+  return undefined;
+}
+
 function isReadablePart(message: ChatLine, part: ChatPart): boolean {
   if (message.source === "compaction" || message.source === "branch_summary") return false;
-  if (part.type === "skillInvocation" || part.type === "skillRead") return true;
+  if (part.type === "skillInvocation" || part.type === "skillRead" || part.type === "image") return true;
   return part.type === "text" && (message.role === "user" || message.role === "assistant" || message.role === "system" || message.role === "bash");
 }

src/client/src/chatTranscript.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
-import { textMessage } from "./chatMessages";
+import { groupChatMessages } from "./chatGroups";
+import { normalizeMessages, textMessage } from "./chatMessages";
 import { applyTranscriptEvent } from "./chatTranscript";
 import type { ChatLine } from "./components/shared";
 
@@ -169,6 +170,183 @@ describe("applyTranscriptEvent", () => {
     ]);
   });
 
+  it("projects live tool-result images and reconciles final content and metadata", () => {
+    const provisionalImage = { type: "image" as const, mimeType: "image/png", data: "UFJFVklFVw==" };
+    const finalImage = { type: "image" as const, mimeType: "image/png", data: "RklOQUw=" };
+    const finalContent = [{ type: "text", text: "Read image file [image/png]" }, finalImage];
+    const timestamp = "2026-07-13T22:00:00.000Z";
+    let messages: ChatLine[] = [];
+
+    messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "read", toolCallId: "read-image-1", summary: "image.png", args: { path: "image.png" } }) ?? messages;
+    messages = applyTranscriptEvent(messages, {
+      type: "tool.end",
+      toolName: "read",
+      toolCallId: "read-image-1",
+      text: "Read image file [image/png]\n[image]",
+      isError: false,
+      content: [{ type: "text", text: "Read image file [image/png]" }, provisionalImage],
+      details: { source: "tool.end" },
+    }) ?? messages;
+
+    expect(messages[0]?.parts.filter((part) => part.type === "image")).toEqual([provisionalImage]);
+
+    messages = applyTranscriptEvent(messages, {
+      type: "message.end",
+      message: {
+        role: "toolResult",
+        toolCallId: "read-image-1",
+        toolName: "read",
+        content: finalContent,
+        details: { source: "message.end" },
+        isError: false,
+        timestamp,
+      },
+    }) ?? messages;
+    messages = applyTranscriptEvent(messages, { type: "assistant.delta", text: "done" }) ?? messages;
+
+    const finalizedToolLine: ChatLine = {
+      role: "tool",
+      parts: [{
+        type: "toolExecution",
+        toolCallId: "read-image-1",
+        toolName: "read",
+        summary: "image.png",
+        args: { path: "image.png" },
+        status: "success",
+        resultText: "Read image file [image/png]",
+        content: finalContent,
+        details: { source: "message.end" },
+      }, finalImage],
+      meta: { timestamp },
+    };
+    expect(messages).toEqual([finalizedToolLine, textMessage("assistant", "done")]);
+    expect(groupChatMessages(messages)).toEqual([
+      { kind: "group", startIndex: 0, endIndex: 0, messages: [{ ...finalizedToolLine, parts: [finalizedToolLine.parts[0]] }] },
+      { kind: "tool-image", index: 0, message: { ...finalizedToolLine, parts: [finalImage] }, toolName: "read" },
+      { kind: "message", index: 1, message: textMessage("assistant", "done") },
+    ]);
+  });
+
+  it("keeps image-only live tool results visible without inventing text", () => {
+    const image = { type: "image" as const, mimeType: "image/webp", data: "QUJD" };
+    let messages: ChatLine[] = [];
+
+    messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "capture", toolCallId: "capture-1", summary: "screenshot" }) ?? messages;
+    messages = applyTranscriptEvent(messages, { type: "tool.end", toolName: "capture", toolCallId: "capture-1", text: "[image]", isError: false, content: [image] }) ?? messages;
+    messages = applyTranscriptEvent(messages, {
+      type: "message.end",
+      message: { role: "toolResult", toolCallId: "capture-1", toolName: "capture", content: [image], isError: false },
+    }) ?? messages;
+
+    expect(messages).toEqual([{
+      role: "tool",
+      parts: [{
+        type: "toolExecution",
+        toolCallId: "capture-1",
+        toolName: "capture",
+        summary: "screenshot",
+        status: "success",
+        resultText: "",
+        content: [image],
+      }, image],
+    }]);
+    expect(groupChatMessages(messages).map((group) => group.kind)).toEqual(["group", "tool-image"]);
+  });
+
+  it("keeps repeated final tool-result events idempotent", () => {
+    const image = { type: "image" as const, mimeType: "image/png", data: "RklOQUw=" };
+    const finalEvent = {
+      type: "message.end" as const,
+      message: {
+        role: "toolResult",
+        toolCallId: "read-image-repeat",
+        toolName: "read",
+        content: [{ type: "text", text: "Read image file [image/png]" }, image],
+        isError: false,
+        timestamp: "2026-07-13T22:00:00.000Z",
+      },
+    };
+    let messages: ChatLine[] = [];
+
+    messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "read", toolCallId: "read-image-repeat", summary: "image.png" }) ?? messages;
+    messages = applyTranscriptEvent(messages, {
+      type: "tool.end",
+      toolName: "read",
+      toolCallId: "read-image-repeat",
+      text: "Read image file [image/png]\n[image]",
+      isError: false,
+      content: finalEvent.message.content,
+    }) ?? messages;
+    messages = applyTranscriptEvent(messages, finalEvent) ?? messages;
+    messages = applyTranscriptEvent(messages, finalEvent) ?? messages;
+
+    expect(messages).toHaveLength(1);
+    expect(messages[0]?.parts.filter((part) => part.type === "toolExecution")).toHaveLength(1);
+    expect(messages[0]?.parts.filter((part) => part.type === "image")).toEqual([image]);
+    expect(messages[0]?.meta).toEqual({ timestamp: "2026-07-13T22:00:00.000Z" });
+  });
+
+  it("matches hydrated history for technical execution and visible image content", () => {
+    const image = { type: "image" as const, mimeType: "image/png", data: "QUJD" };
+    const timestamp = "2026-07-13T22:00:00.000Z";
+    const finalResult = {
+      role: "toolResult",
+      toolCallId: "read-history-parity",
+      toolName: "read",
+      content: [{ type: "text", text: "Read image file [image/png]" }, image],
+      details: { path: "image.png" },
+      isError: false,
+      timestamp,
+    };
+    const historyGroups = groupChatMessages(normalizeMessages([
+      {
+        role: "assistant",
+        content: [{ type: "toolCall", id: "read-history-parity", name: "read", arguments: { path: "image.png" } }],
+      },
+      finalResult,
+    ]));
+    let liveMessages: ChatLine[] = [];
+
+    liveMessages = applyTranscriptEvent(liveMessages, {
+      type: "tool.start",
+      toolName: "read",
+      toolCallId: "read-history-parity",
+      summary: "image.png",
+      args: { path: "image.png" },
+    }) ?? liveMessages;
+    liveMessages = applyTranscriptEvent(liveMessages, {
+      type: "tool.end",
+      toolName: "read",
+      toolCallId: "read-history-parity",
+      text: "Read image file [image/png]\n[image]",
+      isError: false,
+      content: finalResult.content,
+      details: finalResult.details,
+    }) ?? liveMessages;
+    liveMessages = applyTranscriptEvent(liveMessages, { type: "message.end", message: finalResult }) ?? liveMessages;
+
+    const liveGroups = groupChatMessages(liveMessages);
+    const technicalParts = (groups: ReturnType<typeof groupChatMessages>) => groups.flatMap((group) => group.kind === "group"
+      ? group.messages.flatMap((message) => message.parts.filter((part) => part.type === "toolExecution"))
+      : []);
+    const visibleImages = (groups: ReturnType<typeof groupChatMessages>) => groups.flatMap((group) => group.kind !== "group"
+      ? group.message.parts.filter((part) => part.type === "image")
+      : []);
+    const visibleImageMeta = (groups: ReturnType<typeof groupChatMessages>) => {
+      for (const group of groups) {
+        if (group.kind !== "group" && group.message.parts.some((part) => part.type === "image")) return group.message.meta;
+      }
+      return undefined;
+    };
+
+    expect(historyGroups.map((group) => group.kind)).toEqual(["group", "tool-image"]);
+    expect(liveGroups.map((group) => group.kind)).toEqual(["group", "tool-image"]);
+    expect(technicalParts(liveGroups)).toEqual(technicalParts(historyGroups));
+    expect(visibleImages(liveGroups)).toEqual(visibleImages(historyGroups));
+    expect(visibleImageMeta(historyGroups)).toEqual({ timestamp });
+    expect(visibleImageMeta(liveGroups)).toEqual({ timestamp });
+  });
+
   it("does not merge consecutive streamed skill reads", () => {
     let messages: ChatLine[] = [];
     messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "read", toolCallId: "1", summary: "", args: { path: "/skills/playwright/SKILL.md" } }) ?? messages;