From 472a0c998f64f41a980ccbcd12f254c4ee3cfb7c Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 15 Jul 2026 10:51:55 -0500 Subject: [PATCH] feat(core): normalize tool and attachment images at settlement Image resizing previously ran only inside the read tool, so plugin, MCP, and codemode tools could persist unbounded inline base64 images that grow every provider request body (see #36552). - ToolRegistry.settleTool now normalizes image file content generically: a missing resizer keeps the original, an unresizable image is dropped and replaced with an omission note, mirroring V1 processor behavior. - SessionV2.prompt normalizes image attachments at admission, resolving the Location-scoped Image service lazily so text-only prompts do not boot location services. - read keeps its internal normalize call to bound the image persisted in its structured output. --- packages/core/src/session.ts | 49 ++++++++++++++++--- packages/core/src/tool/registry.ts | 45 +++++++++++++++-- packages/core/test/lib/image.ts | 7 +++ packages/core/test/mcp.test.ts | 3 ++ packages/core/test/session-prompt.test.ts | 19 ++++++- .../test/session-runner-tool-registry.test.ts | 48 +++++++++++++++++- packages/core/test/tool-question.test.ts | 3 ++ packages/core/test/tool-skill.test.ts | 3 ++ packages/core/test/tool-webfetch.test.ts | 3 ++ packages/core/test/tool-websearch.test.ts | 3 ++ 10 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 packages/core/test/lib/image.ts diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 34ff7ecfa9..0858058b66 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot" import { SessionRevert } from "./session/revert" import { Session } from "@opencode-ai/schema/session" import { FSUtil } from "./fs-util" +import { Image } from "./image" import { Mime } from "./mime" import type { EventLog } from "@opencode-ai/schema/event-log" import { SkillV2 } from "./skill" @@ -531,9 +532,13 @@ const layer = Layer.effect( // continues from the reverted boundary rather than stale post-boundary history. if (session.revert) yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events)) - const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe( - Effect.provideService(FSUtil.Service, fs), - ) + // Resolved lazily so prompt admission only boots location services when an + // image attachment actually needs the resizer. + const image = Image.Service.pipe(Effect.provide(locations.get(session.location))) + const prompt = yield* resolvePrompt( + { text: input.text, files: input.files, agents: input.agents }, + image, + ).pipe(Effect.provideService(FSUtil.Service, fs)) const messageID = input.id ?? SessionMessage.ID.create() const admittedInput = SessionPending.Message.make({ type: "user", @@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf } } -const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) { +const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* ( + input: PromptInput.Prompt, + image: Effect.Effect, +) { const fs = yield* FSUtil.Service const files = input.files - ? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 }) + ? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 }) : undefined return Prompt.make({ text: input.text, agents: input.agents, files }) }) @@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* ( fs: FSUtil.Interface, input: PromptInput.FileAttachment, + image: Effect.Effect, ) { const resolved = input.uri.startsWith("data:") ? { @@ -900,9 +909,15 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct .join("\n"), ) : resolved.bytes - return FileAttachment.create({ - data: Base64.make(Buffer.from(content).toString("base64")), + const normalized = yield* normalizeImageAttachment( + input, + Base64.make(Buffer.from(content).toString("base64")), mime, + image, + ) + return FileAttachment.create({ + data: normalized.data, + mime: normalized.mime, source: resolved.source, name: input.name ?? resolved.name, description: input.description, @@ -910,6 +925,26 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct }) }) +// V1 parity: bound image attachments at prompt admission so oversized images never +// reach persistence or providers. A missing resizer keeps the original image; an +// image that cannot fit the configured limits fails the prompt. +const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* ( + input: PromptInput.FileAttachment, + data: Base64, + mime: string, + image: Effect.Effect, +) { + if (!mime.startsWith("image/")) return { data, mime } + const service = yield* image + const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri) + const content = { uri: label, content: data, encoding: "base64" as const, mime } + const normalized = yield* service.normalize(label, content).pipe( + Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)), + Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })), + ) + return { data: Base64.make(normalized.content), mime: normalized.mime } +}) + const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) { const url = yield* Effect.try({ try: () => new URL(uri), diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 98120fc2d9..eb041dc414 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry" import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai" import { Context, Effect, Layer, Scope } from "effect" import type { AgentV2 } from "../agent" +import { Image } from "../image" import { PermissionV2 } from "../permission" import { SessionMessage } from "../session/message" import { SessionSchema } from "../session/schema" @@ -57,6 +58,44 @@ const registryLayer = Layer.effect( Effect.gen(function* () { const resources = yield* ToolOutputStore.Service const toolHooks = yield* ToolHooks.Service + const image = yield* Image.Service + + // Generic model-output image bounding: every tool's media content settles through + // here, so individual tools do not add their own resize calls. A missing resizer + // keeps the original image; an image that cannot fit the configured limits is + // dropped and replaced with a note, mirroring V1 settlement behavior. + const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (output: ToolOutput) { + const content = yield* Effect.forEach(output.content, (item) => { + if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item) + const base64 = /^data:[^;,]*;base64,(.*)$/s.exec(item.uri)?.[1] + if (base64 === undefined) return Effect.succeed(item) + const resource = item.name ?? `${item.mime} tool output` + return image + .normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime }) + .pipe( + Effect.map((normalized) => ({ + ...item, + uri: `data:${normalized.mime};base64,${normalized.content}`, + mime: normalized.mime, + })), + Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)), + Effect.catch(() => Effect.succeed(undefined)), + ) + }) + const kept = content.filter((item) => item !== undefined) + const omitted = content.length - kept.length + if (omitted === 0) return { structured: output.structured, content: kept } + return { + structured: output.structured, + content: [ + ...kept, + { + type: "text" as const, + text: `[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the image size limit.]`, + }, + ], + } + }) type Registration = { readonly tool: AnyTool readonly name: string @@ -115,7 +154,7 @@ const registryLayer = Layer.effect( const bounded = yield* resources.bound({ sessionID: input.sessionID, callID: input.call.id, - output: pending.output, + output: yield* normalizeImages(pending.output), }) const result = ToolOutput.toResultValue(bounded.output) settlement = @@ -232,11 +271,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) { export const node = makeLocationNode({ service: Service, layer, - deps: [ToolOutputStore.node, ToolHooks.node], + deps: [ToolOutputStore.node, ToolHooks.node, Image.node], }) export const toolsNode = makeLocationNode({ service: Tools.Service, layer, - deps: [ToolOutputStore.node, ToolHooks.node], + deps: [ToolOutputStore.node, ToolHooks.node, Image.node], }) diff --git a/packages/core/test/lib/image.ts b/packages/core/test/lib/image.ts new file mode 100644 index 0000000000..b899bc9168 --- /dev/null +++ b/packages/core/test/lib/image.ts @@ -0,0 +1,7 @@ +import { Image } from "@opencode-ai/core/image" +import { Effect, Layer } from "effect" + +/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */ +export const imagePassthrough = Layer.mock(Image.Service, { + normalize: (_resource, content) => Effect.succeed(content), +}) diff --git a/packages/core/test/mcp.test.ts b/packages/core/test/mcp.test.ts index 22390b6316..fd6f99f538 100644 --- a/packages/core/test/mcp.test.ts +++ b/packages/core/test/mcp.test.ts @@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { location } from "./fixture/location" import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool" @@ -250,6 +252,7 @@ const it = testEffect( [PermissionV2.node, permissions], [EventV2.node, events], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ]), ) diff --git a/packages/core/test/session-prompt.test.ts b/packages/core/test/session-prompt.test.ts index 7ea2633f62..b86e19444a 100644 --- a/packages/core/test/session-prompt.test.ts +++ b/packages/core/test/session-prompt.test.ts @@ -1,5 +1,5 @@ import { describe, expect } from "bun:test" -import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect" +import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect" import { mkdtemp, rm } from "fs/promises" import { tmpdir } from "os" import path from "path" @@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution" import { SessionPending } from "@opencode-ai/core/session/pending" import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" +import { LocationServiceMap } from "@opencode-ai/core/location-service-map" +import type { LocationServices } from "@opencode-ai/core/location-services" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" const executionCalls: SessionV2.ID[] = [] const interruptCalls: SessionV2.ID[] = [] @@ -49,10 +52,22 @@ const execution = Layer.succeed( awaitIdle: () => Effect.void, }), ) +const locations = Layer.effect( + LocationServiceMap.Service, + LayerMap.make( + () => + // Attachment admission only needs the location-scoped Image service. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion + imagePassthrough as unknown as Layer.Layer, + ), +) const it = testEffect( AppNodeBuilder.build( LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]), - [[SessionExecution.node, execution]], + [ + [SessionExecution.node, execution], + [LocationServiceMap.node, locations], + ], ), ) const sessionID = SessionV2.ID.make("ses_prompt_test") diff --git a/packages/core/test/session-runner-tool-registry.test.ts b/packages/core/test/session-runner-tool-registry.test.ts index ae80449523..be2ca24b07 100644 --- a/packages/core/test/session-runner-tool-registry.test.ts +++ b/packages/core/test/session-runner-tool-registry.test.ts @@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool" import { AgentV2 } from "@opencode-ai/core/agent" import type { PermissionV2 } from "@opencode-ai/core/permission" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { Image } from "@opencode-ai/core/image" import { SessionV2 } from "@opencode-ai/core/session" import { SessionMessage } from "@opencode-ai/core/session/message" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" @@ -28,7 +29,26 @@ const outputStore = Layer.mock(ToolOutputStore.Service, { ) }, }) -const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]]) +const imageStore = Layer.mock(Image.Service, { + normalize: (resource, content) => + resource === "too-large.png" + ? Effect.fail( + new Image.SizeError({ + resource, + width: 9_000, + height: 9_000, + bytes: content.content.length, + maxWidth: 2_000, + maxHeight: 2_000, + maxBytes: 5, + }), + ) + : Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" }), +}) +const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [ + [ToolOutputStore.node, outputStore], + [Image.node, imageStore], +]) const it = testEffect(registryLayer) const identity = { agent: AgentV2.ID.make("build"), @@ -255,6 +275,32 @@ describe("ToolRegistry", () => { }), ) + it.effect("normalizes image tool output at settlement and drops unresizable images", () => + Effect.gen(function* () { + const service = yield* ToolRegistry.Service + yield* service.register({ + snapshot: Tool.make({ + description: "Return images", + input: Schema.Struct({ text: Schema.String }), + output: Schema.Struct({ text: Schema.String }), + execute: ({ text }) => Effect.succeed({ text }), + toModelOutput: ({ output }) => [ + { type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" }, + { type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" }, + { type: "text", text: output.text }, + ], + }), + }, { codemode: false }) + + const settlement = yield* settleTool(service, call("snapshot")) + expect(settlement.output?.content).toEqual([ + { type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" }, + { type: "text", text: "snapshot" }, + { type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" }, + ]) + }), + ) + it.effect("enforces transformed codecs at execution and projection boundaries", () => Effect.gen(function* () { const service = yield* ToolRegistry.Service diff --git a/packages/core/test/tool-question.test.ts b/packages/core/test/tool-question.test.ts index 1c91de9966..f3ef03bf79 100644 --- a/packages/core/test/tool-question.test.ts +++ b/packages/core/test/tool-question.test.ts @@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { QuestionTool } from "@opencode-ai/core/tool/question" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" @@ -84,6 +86,7 @@ const it = testEffect( [PermissionV2.node, permission], [Form.node, form], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ]), ) diff --git a/packages/core/test/tool-skill.test.ts b/packages/core/test/tool-skill.test.ts index 8d81917d8c..8cc3eb25df 100644 --- a/packages/core/test/tool-skill.test.ts +++ b/packages/core/test/tool-skill.test.ts @@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { tmpdir } from "./fixture/tmpdir" +import { Image } from "@opencode-ai/core/image" import { it } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" import { FSUtil } from "@opencode-ai/core/fs-util" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" @@ -90,6 +92,7 @@ describe("SkillTool", () => { [PermissionV2.node, permission], [SkillV2.node, skills], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ], ) diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index 421c09f876..fc38296c9b 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { WebFetchTool } from "@opencode-ai/core/tool/webfetch" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" const webFetchToolNode = makeLocationNode({ @@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) => AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [ [PermissionV2.node, permission], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ...replacements, ]) const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]])) diff --git a/packages/core/test/tool-websearch.test.ts b/packages/core/test/tool-websearch.test.ts index 53984ef69b..8d1cc1e679 100644 --- a/packages/core/test/tool-websearch.test.ts +++ b/packages/core/test/tool-websearch.test.ts @@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { WebSearchTool } from "@opencode-ai/core/tool/websearch" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" const webSearchToolNode = makeLocationNode({ @@ -138,6 +140,7 @@ const it = testEffect( [LayerNodePlatform.httpClient, http], [WebSearchTool.configNode, websearchConfig], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ], ), )