From 4fe2bf0dd03bbb9bb81b78de68aab06dee1d6f24 Mon Sep 17 00:00:00 2001 From: Simon Klee Date: Thu, 30 Jul 2026 23:38:25 +0200 Subject: [PATCH] feat(tui): render session tool images --- packages/tui/src/routes/session/index.tsx | 111 ++++++++++++++++-- .../tui/test/cli/tui/session-images.test.tsx | 104 ++++++++++++++++ 2 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 packages/tui/test/cli/tui/session-images.test.tsx diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index c757a6aa97..7b021caecd 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -107,6 +107,7 @@ const NAVIGATION_SLACK_ID = "session-navigation-slack" const TRANSCRIPT_TAIL_ROWS = 40 const TRANSCRIPT_BACKFILL_CHUNK = 60 const TRANSCRIPT_BACKFILL_DELAY = 120 +const SESSION_IMAGE_LIMIT = 6 const context = createContext<{ width: number @@ -116,6 +117,7 @@ const context = createContext<{ markdownMode: () => "source" | "rendered" groupExploration: () => boolean diffWrapMode: () => "word" | "none" + imageKeys: () => ReadonlySet models: () => ModelInfo[] config: ReturnType["data"] }>() @@ -145,6 +147,7 @@ export function Session() { const promptRef = usePromptRef() const session = createMemo(() => data.session.get(route.sessionID)) const messages = () => data.session.message.list(route.sessionID) + const imageKeys = createMemo(() => sessionImageKeys(messages(), session()?.revert?.messageID)) const location = createMemo(() => session()?.location) const currentLocation = useLocation() @@ -995,6 +998,7 @@ export function Session() { markdownMode, groupExploration, diffWrapMode, + imageKeys, models, config, }} @@ -1384,7 +1388,7 @@ function SessionPartView(props: { partRef: PartRef; message: (messageID: string) /> - + )} @@ -1531,13 +1535,13 @@ function SessionGroupView(props: { if (message?.type !== "assistant") return [] const part = resolvePart(message, ref.partID) if (part?.type !== "tool") return [] - return [part] + return [{ messageID: ref.messageID, part }] }) const grouped = createMemo(() => parts(props.refs)) const pending = createMemo(() => parts(props.pending)) const label = createMemo(() => { - const counts = grouped().reduce>((result, part) => { - const tool = toolDisplay(part.name) + const counts = grouped().reduce>((result, item) => { + const tool = toolDisplay(item.part.name) const name = tool === "grep" || tool === "glob" ? "search" : tool result[name] = (result[name] ?? 0) + 1 return result @@ -1551,7 +1555,11 @@ function SessionGroupView(props: { 0 || pending().length > 0}> {(part) => }} + fallback={ + + {(item) => } + + } > 0}> 0}> - {(part) => } + {(item) => } - {(part) => } + + + {(item) => } + ) @@ -2028,7 +2039,9 @@ function AssistantMessage(props: { message: SessionMessageAssistant; last: boole } + fallback={ + + } > {(summary) => } @@ -2343,6 +2356,88 @@ function ToolPart(props: { part: SessionMessageAssistantTool }) { ) } +function ToolPartWithImages(props: { messageID: string; part: SessionMessageAssistantTool }) { + const ctx = use() + return ( + <> + + + + ) +} + +export function ToolImages(props: { + parts: readonly { messageID: string; part: SessionMessageAssistantTool }[] + visible: ReadonlySet +}) { + const dimensions = useTerminalDimensions() + const images = createMemo(() => + props.parts + .flatMap((item) => inlineToolImages(item.messageID, item.part)) + .filter((image) => props.visible.has(image.key)), + ) + const height = createMemo(() => Math.max(6, Math.min(18, Math.floor((dimensions().width - 6) / 4)))) + + return ( + 0}> + + + {(image) => { + const [failed, setFailed] = createSignal(false) + return ( + + No preview}> + setFailed(true)} + /> + + + ) + }} + + 3}> + +{images().length - 3} more images + + + + ) +} + +export function sessionImageKeys(messages: readonly SessionMessageInfo[], revertBoundary?: string) { + return new Set( + messages + .filter((message) => !revertBoundary || message.id < revertBoundary) + .flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => (part.type === "tool" ? inlineToolImages(message.id, part) : [])) + : [], + ) + .map((image) => image.key) + .slice(-SESSION_IMAGE_LIMIT), + ) +} + +function inlineToolImages(messageID: string, part: SessionMessageAssistantTool) { + return toolDisplayContent(part.state).flatMap((content, index) => + content.type === "file" && content.mime.startsWith("image/") && content.uri.startsWith("data:image/") + ? [{ ...content, key: `${messageID}:${part.id}:${index}` }] + : [], + ) +} + type ToolProps = { input: Record metadata: Record diff --git a/packages/tui/test/cli/tui/session-images.test.tsx b/packages/tui/test/cli/tui/session-images.test.tsx new file mode 100644 index 0000000000..852c243398 --- /dev/null +++ b/packages/tui/test/cli/tui/session-images.test.tsx @@ -0,0 +1,104 @@ +import { afterEach, expect, test } from "bun:test" +import { ImageRenderable } from "@opentui/core" +import { testRender } from "@opentui/solid" +import type { SessionMessageAssistant, SessionMessageAssistantTool } from "@opencode-ai/client" +import { sessionImageKeys, ToolImages } from "../../../src/routes/session" + +const PNG_1X1_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4AWP4z8DwHwAFAAH/e+m+7wAAAABJRU5ErkJggg==" +const image = { type: "file" as const, uri: `data:image/png;base64,${PNG_1X1_BASE64}`, mime: "image/png" } +let setup: Awaited> | undefined + +afterEach(() => { + setup?.renderer.destroy() + setup = undefined +}) + +test("renders bounded inline images from completed tool content", async () => { + setup = await testRender(() => toolImages([image, image, image, image, image, image, image]), { + width: 80, + height: 70, + }) + await setup.renderOnce() + + const first = setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:1") + if (!(first instanceof ImageRenderable)) throw new Error("Tool image did not render") + await first.loadPromise + + expect(first.fit).toBe("fit") + expect(first.height).toBe(18) + expect(setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:2")).toBeInstanceOf( + ImageRenderable, + ) + expect(setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:3")).toBeInstanceOf( + ImageRenderable, + ) + expect(setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:0")).toBeUndefined() + expect(setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:4")).toBeUndefined() + expect(setup.captureCharFrame()).toContain("+3 more images") +}) + +test("does not fetch image tool content from external sources", async () => { + setup = await testRender( + () => + toolImages([ + { type: "file", uri: "https://example.test/image.png", mime: "image/png" }, + { type: "file", uri: "file:///tmp/image.png", mime: "image/png" }, + { type: "file", uri: "data:text/plain;base64,SGVsbG8=", mime: "text/plain" }, + ]), + { width: 80, height: 24 }, + ) + await setup.renderOnce() + + expect(setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:0")).toBeUndefined() +}) + +test("does not reserve image slots for reverted messages", () => { + const visible = assistant("message-1", [tool([image])]) + const reverted = assistant("message-2", [tool([image, image, image, image, image, image])]) + + expect([...sessionImageKeys([visible, reverted], reverted.id)]).toEqual(["message-1:call-1:0"]) +}) + +test("falls back when inline image content is malformed", async () => { + setup = await testRender( + () => toolImages([{ type: "file", uri: "data:image/png;base64,aW52YWxpZA==", mime: "image/png" }]), + { width: 80, height: 24 }, + ) + await setup.renderOnce() + + const preview = setup.renderer.root.findDescendantById("session-tool-image-message-1:call-1:0") + if (!(preview instanceof ImageRenderable)) throw new Error("Tool image did not render") + await preview.loadPromise + + expect(await setup.waitForFrame((frame) => frame.includes("No preview"))).toContain("No preview") +}) + +function toolImages(content: Extract["content"]) { + const part = tool(content) + const message = assistant("message-1", [part]) + return +} + +function assistant(id: string, content: SessionMessageAssistant["content"]): SessionMessageAssistant { + return { + type: "assistant", + id, + agent: "build", + model: { id: "model", providerID: "provider" }, + content, + time: { created: 1 }, + } +} + +function tool( + content: Extract["content"], +): SessionMessageAssistantTool { + return { + type: "tool", + id: "call-1", + name: "image", + state: { status: "completed", input: {}, content }, + time: { created: 0, completed: 1 }, + } +}