feat(core): normalize tool and attachment images at settlement
Image resizing previously ran only inside the read tool, so plugin, MCP, and codemode tools could persist unbounded inline base64 images that grow every provider request body (see #36552). - ToolRegistry.settleTool now normalizes image file content generically: a missing resizer keeps the original, an unresizable image is dropped and replaced with an omission note, mirroring V1 processor behavior. - SessionV2.prompt normalizes image attachments at admission, resolving the Location-scoped Image service lazily so text-only prompts do not boot location services. - read keeps its internal normalize call to bound the image persisted in its structured output.
This commit is contained in:
@@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot"
|
||||
import { SessionRevert } from "./session/revert"
|
||||
import { Session } from "@opencode-ai/schema/session"
|
||||
import { FSUtil } from "./fs-util"
|
||||
import { Image } from "./image"
|
||||
import { Mime } from "./mime"
|
||||
import type { EventLog } from "@opencode-ai/schema/event-log"
|
||||
import { SkillV2 } from "./skill"
|
||||
@@ -531,9 +532,13 @@ const layer = Layer.effect(
|
||||
// continues from the reverted boundary rather than stale post-boundary history.
|
||||
if (session.revert)
|
||||
yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events))
|
||||
const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe(
|
||||
Effect.provideService(FSUtil.Service, fs),
|
||||
)
|
||||
// Resolved lazily so prompt admission only boots location services when an
|
||||
// image attachment actually needs the resizer.
|
||||
const image = Image.Service.pipe(Effect.provide(locations.get(session.location)))
|
||||
const prompt = yield* resolvePrompt(
|
||||
{ text: input.text, files: input.files, agents: input.agents },
|
||||
image,
|
||||
).pipe(Effect.provideService(FSUtil.Service, fs))
|
||||
const messageID = input.id ?? SessionMessage.ID.create()
|
||||
const admittedInput = SessionPending.Message.make({
|
||||
type: "user",
|
||||
@@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf
|
||||
}
|
||||
}
|
||||
|
||||
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) {
|
||||
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (
|
||||
input: PromptInput.Prompt,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
const fs = yield* FSUtil.Service
|
||||
const files = input.files
|
||||
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 })
|
||||
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 })
|
||||
: undefined
|
||||
return Prompt.make({ text: input.text, agents: input.agents, files })
|
||||
})
|
||||
@@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024
|
||||
const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* (
|
||||
fs: FSUtil.Interface,
|
||||
input: PromptInput.FileAttachment,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
const resolved = input.uri.startsWith("data:")
|
||||
? {
|
||||
@@ -900,9 +909,15 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
||||
.join("\n"),
|
||||
)
|
||||
: resolved.bytes
|
||||
return FileAttachment.create({
|
||||
data: Base64.make(Buffer.from(content).toString("base64")),
|
||||
const normalized = yield* normalizeImageAttachment(
|
||||
input,
|
||||
Base64.make(Buffer.from(content).toString("base64")),
|
||||
mime,
|
||||
image,
|
||||
)
|
||||
return FileAttachment.create({
|
||||
data: normalized.data,
|
||||
mime: normalized.mime,
|
||||
source: resolved.source,
|
||||
name: input.name ?? resolved.name,
|
||||
description: input.description,
|
||||
@@ -910,6 +925,26 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
||||
})
|
||||
})
|
||||
|
||||
// V1 parity: bound image attachments at prompt admission so oversized images never
|
||||
// reach persistence or providers. A missing resizer keeps the original image; an
|
||||
// image that cannot fit the configured limits fails the prompt.
|
||||
const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* (
|
||||
input: PromptInput.FileAttachment,
|
||||
data: Base64,
|
||||
mime: string,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
if (!mime.startsWith("image/")) return { data, mime }
|
||||
const service = yield* image
|
||||
const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri)
|
||||
const content = { uri: label, content: data, encoding: "base64" as const, mime }
|
||||
const normalized = yield* service.normalize(label, content).pipe(
|
||||
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)),
|
||||
Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })),
|
||||
)
|
||||
return { data: Base64.make(normalized.content), mime: normalized.mime }
|
||||
})
|
||||
|
||||
const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) {
|
||||
const url = yield* Effect.try({
|
||||
try: () => new URL(uri),
|
||||
|
||||
@@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry"
|
||||
import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
|
||||
import { Context, Effect, Layer, Scope } from "effect"
|
||||
import type { AgentV2 } from "../agent"
|
||||
import { Image } from "../image"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { SessionMessage } from "../session/message"
|
||||
import { SessionSchema } from "../session/schema"
|
||||
@@ -57,6 +58,44 @@ const registryLayer = Layer.effect(
|
||||
Effect.gen(function* () {
|
||||
const resources = yield* ToolOutputStore.Service
|
||||
const toolHooks = yield* ToolHooks.Service
|
||||
const image = yield* Image.Service
|
||||
|
||||
// Generic model-output image bounding: every tool's media content settles through
|
||||
// here, so individual tools do not add their own resize calls. A missing resizer
|
||||
// keeps the original image; an image that cannot fit the configured limits is
|
||||
// dropped and replaced with a note, mirroring V1 settlement behavior.
|
||||
const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (output: ToolOutput) {
|
||||
const content = yield* Effect.forEach(output.content, (item) => {
|
||||
if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item)
|
||||
const base64 = /^data:[^;,]*;base64,(.*)$/s.exec(item.uri)?.[1]
|
||||
if (base64 === undefined) return Effect.succeed(item)
|
||||
const resource = item.name ?? `${item.mime} tool output`
|
||||
return image
|
||||
.normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime })
|
||||
.pipe(
|
||||
Effect.map((normalized) => ({
|
||||
...item,
|
||||
uri: `data:${normalized.mime};base64,${normalized.content}`,
|
||||
mime: normalized.mime,
|
||||
})),
|
||||
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)),
|
||||
Effect.catch(() => Effect.succeed(undefined)),
|
||||
)
|
||||
})
|
||||
const kept = content.filter((item) => item !== undefined)
|
||||
const omitted = content.length - kept.length
|
||||
if (omitted === 0) return { structured: output.structured, content: kept }
|
||||
return {
|
||||
structured: output.structured,
|
||||
content: [
|
||||
...kept,
|
||||
{
|
||||
type: "text" as const,
|
||||
text: `[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the image size limit.]`,
|
||||
},
|
||||
],
|
||||
}
|
||||
})
|
||||
type Registration = {
|
||||
readonly tool: AnyTool
|
||||
readonly name: string
|
||||
@@ -115,7 +154,7 @@ const registryLayer = Layer.effect(
|
||||
const bounded = yield* resources.bound({
|
||||
sessionID: input.sessionID,
|
||||
callID: input.call.id,
|
||||
output: pending.output,
|
||||
output: yield* normalizeImages(pending.output),
|
||||
})
|
||||
const result = ToolOutput.toResultValue(bounded.output)
|
||||
settlement =
|
||||
@@ -232,11 +271,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) {
|
||||
export const node = makeLocationNode({
|
||||
service: Service,
|
||||
layer,
|
||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
||||
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||
})
|
||||
|
||||
export const toolsNode = makeLocationNode({
|
||||
service: Tools.Service,
|
||||
layer,
|
||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
||||
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { Effect, Layer } from "effect"
|
||||
|
||||
/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */
|
||||
export const imagePassthrough = Layer.mock(Image.Service, {
|
||||
normalize: (_resource, content) => Effect.succeed(content),
|
||||
})
|
||||
@@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp"
|
||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { location } from "./fixture/location"
|
||||
import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool"
|
||||
|
||||
@@ -250,6 +252,7 @@ const it = testEffect(
|
||||
[PermissionV2.node, permissions],
|
||||
[EventV2.node, events],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect"
|
||||
import { mkdtemp, rm } from "fs/promises"
|
||||
import { tmpdir } from "os"
|
||||
import path from "path"
|
||||
@@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution"
|
||||
import { SessionPending } from "@opencode-ai/core/session/pending"
|
||||
import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql"
|
||||
import { SessionStore } from "@opencode-ai/core/session/store"
|
||||
import { LocationServiceMap } from "@opencode-ai/core/location-service-map"
|
||||
import type { LocationServices } from "@opencode-ai/core/location-services"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
|
||||
const executionCalls: SessionV2.ID[] = []
|
||||
const interruptCalls: SessionV2.ID[] = []
|
||||
@@ -49,10 +52,22 @@ const execution = Layer.succeed(
|
||||
awaitIdle: () => Effect.void,
|
||||
}),
|
||||
)
|
||||
const locations = Layer.effect(
|
||||
LocationServiceMap.Service,
|
||||
LayerMap.make(
|
||||
() =>
|
||||
// Attachment admission only needs the location-scoped Image service.
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
|
||||
imagePassthrough as unknown as Layer.Layer<LocationServices>,
|
||||
),
|
||||
)
|
||||
const it = testEffect(
|
||||
AppNodeBuilder.build(
|
||||
LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]),
|
||||
[[SessionExecution.node, execution]],
|
||||
[
|
||||
[SessionExecution.node, execution],
|
||||
[LocationServiceMap.node, locations],
|
||||
],
|
||||
),
|
||||
)
|
||||
const sessionID = SessionV2.ID.make("ses_prompt_test")
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool"
|
||||
import { AgentV2 } from "@opencode-ai/core/agent"
|
||||
import type { PermissionV2 } from "@opencode-ai/core/permission"
|
||||
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { SessionV2 } from "@opencode-ai/core/session"
|
||||
import { SessionMessage } from "@opencode-ai/core/session/message"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
@@ -28,7 +29,26 @@ const outputStore = Layer.mock(ToolOutputStore.Service, {
|
||||
)
|
||||
},
|
||||
})
|
||||
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]])
|
||||
const imageStore = Layer.mock(Image.Service, {
|
||||
normalize: (resource, content) =>
|
||||
resource === "too-large.png"
|
||||
? Effect.fail(
|
||||
new Image.SizeError({
|
||||
resource,
|
||||
width: 9_000,
|
||||
height: 9_000,
|
||||
bytes: content.content.length,
|
||||
maxWidth: 2_000,
|
||||
maxHeight: 2_000,
|
||||
maxBytes: 5,
|
||||
}),
|
||||
)
|
||||
: Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" }),
|
||||
})
|
||||
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [
|
||||
[ToolOutputStore.node, outputStore],
|
||||
[Image.node, imageStore],
|
||||
])
|
||||
const it = testEffect(registryLayer)
|
||||
const identity = {
|
||||
agent: AgentV2.ID.make("build"),
|
||||
@@ -255,6 +275,32 @@ describe("ToolRegistry", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("normalizes image tool output at settlement and drops unresizable images", () =>
|
||||
Effect.gen(function* () {
|
||||
const service = yield* ToolRegistry.Service
|
||||
yield* service.register({
|
||||
snapshot: Tool.make({
|
||||
description: "Return images",
|
||||
input: Schema.Struct({ text: Schema.String }),
|
||||
output: Schema.Struct({ text: Schema.String }),
|
||||
execute: ({ text }) => Effect.succeed({ text }),
|
||||
toModelOutput: ({ output }) => [
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
|
||||
{ type: "text", text: output.text },
|
||||
],
|
||||
}),
|
||||
}, { codemode: false })
|
||||
|
||||
const settlement = yield* settleTool(service, call("snapshot"))
|
||||
expect(settlement.output?.content).toEqual([
|
||||
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
|
||||
{ type: "text", text: "snapshot" },
|
||||
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("enforces transformed codecs at execution and projection boundaries", () =>
|
||||
Effect.gen(function* () {
|
||||
const service = yield* ToolRegistry.Service
|
||||
|
||||
@@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session"
|
||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { QuestionTool } from "@opencode-ai/core/tool/question"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
@@ -84,6 +86,7 @@ const it = testEffect(
|
||||
[PermissionV2.node, permission],
|
||||
[Form.node, form],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
@@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill"
|
||||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { tmpdir } from "./fixture/tmpdir"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { it } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
@@ -90,6 +92,7 @@ describe("SkillTool", () => {
|
||||
[PermissionV2.node, permission],
|
||||
[SkillV2.node, skills],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
const webFetchToolNode = makeLocationNode({
|
||||
@@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) =>
|
||||
AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [
|
||||
[PermissionV2.node, permission],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
...replacements,
|
||||
])
|
||||
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))
|
||||
|
||||
@@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { WebSearchTool } from "@opencode-ai/core/tool/websearch"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
const webSearchToolNode = makeLocationNode({
|
||||
@@ -138,6 +140,7 @@ const it = testEffect(
|
||||
[LayerNodePlatform.httpClient, http],
|
||||
[WebSearchTool.configNode, websearchConfig],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user