diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index 34ff7ecfa9..0858058b66 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot" import { SessionRevert } from "./session/revert" import { Session } from "@opencode-ai/schema/session" import { FSUtil } from "./fs-util" +import { Image } from "./image" import { Mime } from "./mime" import type { EventLog } from "@opencode-ai/schema/event-log" import { SkillV2 } from "./skill" @@ -531,9 +532,13 @@ const layer = Layer.effect( // continues from the reverted boundary rather than stale post-boundary history. if (session.revert) yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events)) - const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe( - Effect.provideService(FSUtil.Service, fs), - ) + // Resolved lazily so prompt admission only boots location services when an + // image attachment actually needs the resizer. + const image = Image.Service.pipe(Effect.provide(locations.get(session.location))) + const prompt = yield* resolvePrompt( + { text: input.text, files: input.files, agents: input.agents }, + image, + ).pipe(Effect.provideService(FSUtil.Service, fs)) const messageID = input.id ?? SessionMessage.ID.create() const admittedInput = SessionPending.Message.make({ type: "user", @@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf } } -const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) { +const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* ( + input: PromptInput.Prompt, + image: Effect.Effect, +) { const fs = yield* FSUtil.Service const files = input.files - ? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 }) + ? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 }) : undefined return Prompt.make({ text: input.text, agents: input.agents, files }) }) @@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* ( fs: FSUtil.Interface, input: PromptInput.FileAttachment, + image: Effect.Effect, ) { const resolved = input.uri.startsWith("data:") ? { @@ -900,9 +909,15 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct .join("\n"), ) : resolved.bytes - return FileAttachment.create({ - data: Base64.make(Buffer.from(content).toString("base64")), + const normalized = yield* normalizeImageAttachment( + input, + Base64.make(Buffer.from(content).toString("base64")), mime, + image, + ) + return FileAttachment.create({ + data: normalized.data, + mime: normalized.mime, source: resolved.source, name: input.name ?? resolved.name, description: input.description, @@ -910,6 +925,26 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct }) }) +// V1 parity: bound image attachments at prompt admission so oversized images never +// reach persistence or providers. A missing resizer keeps the original image; an +// image that cannot fit the configured limits fails the prompt. +const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* ( + input: PromptInput.FileAttachment, + data: Base64, + mime: string, + image: Effect.Effect, +) { + if (!mime.startsWith("image/")) return { data, mime } + const service = yield* image + const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri) + const content = { uri: label, content: data, encoding: "base64" as const, mime } + const normalized = yield* service.normalize(label, content).pipe( + Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)), + Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })), + ) + return { data: Base64.make(normalized.content), mime: normalized.mime } +}) + const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) { const url = yield* Effect.try({ try: () => new URL(uri), diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 98120fc2d9..eb041dc414 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry" import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai" import { Context, Effect, Layer, Scope } from "effect" import type { AgentV2 } from "../agent" +import { Image } from "../image" import { PermissionV2 } from "../permission" import { SessionMessage } from "../session/message" import { SessionSchema } from "../session/schema" @@ -57,6 +58,44 @@ const registryLayer = Layer.effect( Effect.gen(function* () { const resources = yield* ToolOutputStore.Service const toolHooks = yield* ToolHooks.Service + const image = yield* Image.Service + + // Generic model-output image bounding: every tool's media content settles through + // here, so individual tools do not add their own resize calls. A missing resizer + // keeps the original image; an image that cannot fit the configured limits is + // dropped and replaced with a note, mirroring V1 settlement behavior. + const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (output: ToolOutput) { + const content = yield* Effect.forEach(output.content, (item) => { + if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item) + const base64 = /^data:[^;,]*;base64,(.*)$/s.exec(item.uri)?.[1] + if (base64 === undefined) return Effect.succeed(item) + const resource = item.name ?? `${item.mime} tool output` + return image + .normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime }) + .pipe( + Effect.map((normalized) => ({ + ...item, + uri: `data:${normalized.mime};base64,${normalized.content}`, + mime: normalized.mime, + })), + Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)), + Effect.catch(() => Effect.succeed(undefined)), + ) + }) + const kept = content.filter((item) => item !== undefined) + const omitted = content.length - kept.length + if (omitted === 0) return { structured: output.structured, content: kept } + return { + structured: output.structured, + content: [ + ...kept, + { + type: "text" as const, + text: `[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the image size limit.]`, + }, + ], + } + }) type Registration = { readonly tool: AnyTool readonly name: string @@ -115,7 +154,7 @@ const registryLayer = Layer.effect( const bounded = yield* resources.bound({ sessionID: input.sessionID, callID: input.call.id, - output: pending.output, + output: yield* normalizeImages(pending.output), }) const result = ToolOutput.toResultValue(bounded.output) settlement = @@ -232,11 +271,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) { export const node = makeLocationNode({ service: Service, layer, - deps: [ToolOutputStore.node, ToolHooks.node], + deps: [ToolOutputStore.node, ToolHooks.node, Image.node], }) export const toolsNode = makeLocationNode({ service: Tools.Service, layer, - deps: [ToolOutputStore.node, ToolHooks.node], + deps: [ToolOutputStore.node, ToolHooks.node, Image.node], }) diff --git a/packages/core/test/lib/image.ts b/packages/core/test/lib/image.ts new file mode 100644 index 0000000000..b899bc9168 --- /dev/null +++ b/packages/core/test/lib/image.ts @@ -0,0 +1,7 @@ +import { Image } from "@opencode-ai/core/image" +import { Effect, Layer } from "effect" + +/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */ +export const imagePassthrough = Layer.mock(Image.Service, { + normalize: (_resource, content) => Effect.succeed(content), +}) diff --git a/packages/core/test/mcp.test.ts b/packages/core/test/mcp.test.ts index 22390b6316..fd6f99f538 100644 --- a/packages/core/test/mcp.test.ts +++ b/packages/core/test/mcp.test.ts @@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { location } from "./fixture/location" import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool" @@ -250,6 +252,7 @@ const it = testEffect( [PermissionV2.node, permissions], [EventV2.node, events], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ]), ) diff --git a/packages/core/test/session-prompt.test.ts b/packages/core/test/session-prompt.test.ts index 7ea2633f62..b86e19444a 100644 --- a/packages/core/test/session-prompt.test.ts +++ b/packages/core/test/session-prompt.test.ts @@ -1,5 +1,5 @@ import { describe, expect } from "bun:test" -import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect" +import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect" import { mkdtemp, rm } from "fs/promises" import { tmpdir } from "os" import path from "path" @@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution" import { SessionPending } from "@opencode-ai/core/session/pending" import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql" import { SessionStore } from "@opencode-ai/core/session/store" +import { LocationServiceMap } from "@opencode-ai/core/location-service-map" +import type { LocationServices } from "@opencode-ai/core/location-services" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" const executionCalls: SessionV2.ID[] = [] const interruptCalls: SessionV2.ID[] = [] @@ -49,10 +52,22 @@ const execution = Layer.succeed( awaitIdle: () => Effect.void, }), ) +const locations = Layer.effect( + LocationServiceMap.Service, + LayerMap.make( + () => + // Attachment admission only needs the location-scoped Image service. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion + imagePassthrough as unknown as Layer.Layer, + ), +) const it = testEffect( AppNodeBuilder.build( LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]), - [[SessionExecution.node, execution]], + [ + [SessionExecution.node, execution], + [LocationServiceMap.node, locations], + ], ), ) const sessionID = SessionV2.ID.make("ses_prompt_test") diff --git a/packages/core/test/session-runner-tool-registry.test.ts b/packages/core/test/session-runner-tool-registry.test.ts index ae80449523..be2ca24b07 100644 --- a/packages/core/test/session-runner-tool-registry.test.ts +++ b/packages/core/test/session-runner-tool-registry.test.ts @@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool" import { AgentV2 } from "@opencode-ai/core/agent" import type { PermissionV2 } from "@opencode-ai/core/permission" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { Image } from "@opencode-ai/core/image" import { SessionV2 } from "@opencode-ai/core/session" import { SessionMessage } from "@opencode-ai/core/session/message" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" @@ -28,7 +29,26 @@ const outputStore = Layer.mock(ToolOutputStore.Service, { ) }, }) -const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]]) +const imageStore = Layer.mock(Image.Service, { + normalize: (resource, content) => + resource === "too-large.png" + ? Effect.fail( + new Image.SizeError({ + resource, + width: 9_000, + height: 9_000, + bytes: content.content.length, + maxWidth: 2_000, + maxHeight: 2_000, + maxBytes: 5, + }), + ) + : Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" }), +}) +const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [ + [ToolOutputStore.node, outputStore], + [Image.node, imageStore], +]) const it = testEffect(registryLayer) const identity = { agent: AgentV2.ID.make("build"), @@ -255,6 +275,32 @@ describe("ToolRegistry", () => { }), ) + it.effect("normalizes image tool output at settlement and drops unresizable images", () => + Effect.gen(function* () { + const service = yield* ToolRegistry.Service + yield* service.register({ + snapshot: Tool.make({ + description: "Return images", + input: Schema.Struct({ text: Schema.String }), + output: Schema.Struct({ text: Schema.String }), + execute: ({ text }) => Effect.succeed({ text }), + toModelOutput: ({ output }) => [ + { type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" }, + { type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" }, + { type: "text", text: output.text }, + ], + }), + }, { codemode: false }) + + const settlement = yield* settleTool(service, call("snapshot")) + expect(settlement.output?.content).toEqual([ + { type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" }, + { type: "text", text: "snapshot" }, + { type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" }, + ]) + }), + ) + it.effect("enforces transformed codecs at execution and projection boundaries", () => Effect.gen(function* () { const service = yield* ToolRegistry.Service diff --git a/packages/core/test/tool-question.test.ts b/packages/core/test/tool-question.test.ts index 1c91de9966..f3ef03bf79 100644 --- a/packages/core/test/tool-question.test.ts +++ b/packages/core/test/tool-question.test.ts @@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { QuestionTool } from "@opencode-ai/core/tool/question" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" @@ -84,6 +86,7 @@ const it = testEffect( [PermissionV2.node, permission], [Form.node, form], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ]), ) diff --git a/packages/core/test/tool-skill.test.ts b/packages/core/test/tool-skill.test.ts index 8d81917d8c..8cc3eb25df 100644 --- a/packages/core/test/tool-skill.test.ts +++ b/packages/core/test/tool-skill.test.ts @@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill" import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { tmpdir } from "./fixture/tmpdir" +import { Image } from "@opencode-ai/core/image" import { it } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" import { FSUtil } from "@opencode-ai/core/fs-util" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" @@ -90,6 +92,7 @@ describe("SkillTool", () => { [PermissionV2.node, permission], [SkillV2.node, skills], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ], ) diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index 421c09f876..fc38296c9b 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { WebFetchTool } from "@opencode-ai/core/tool/webfetch" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" const webFetchToolNode = makeLocationNode({ @@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) => AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [ [PermissionV2.node, permission], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ...replacements, ]) const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]])) diff --git a/packages/core/test/tool-websearch.test.ts b/packages/core/test/tool-websearch.test.ts index 53984ef69b..8d1cc1e679 100644 --- a/packages/core/test/tool-websearch.test.ts +++ b/packages/core/test/tool-websearch.test.ts @@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry" import { WebSearchTool } from "@opencode-ai/core/tool/websearch" import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" import { makeLocationNode } from "@opencode-ai/core/effect/app-node" +import { Image } from "@opencode-ai/core/image" import { testEffect } from "./lib/effect" +import { imagePassthrough } from "./lib/image" import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool" const webSearchToolNode = makeLocationNode({ @@ -138,6 +140,7 @@ const it = testEffect( [LayerNodePlatform.httpClient, http], [WebSearchTool.configNode, websearchConfig], [ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig], + [Image.node, imagePassthrough], ], ), )