Compare commits

...

7 Commits

Author SHA1 Message Date
Aiden Cline 8e8fef4d35 Merge branch 'dev' into image-handling 2026-05-09 01:19:16 -05:00
Aiden Cline 4850562ff6 load wasm 2026-05-09 01:14:45 -05:00
Aiden Cline a7060c6e64 type errs 2026-05-09 00:44:15 -05:00
Aiden Cline 6785a45776 resize stuff 2026-05-08 23:36:56 -05:00
Aiden Cline c455e9f07e add photon image resizing 2026-05-08 22:40:18 -05:00
Aiden Cline 890fec31fa Merge branch 'dev' into image-handling 2026-05-08 14:46:14 -05:00
Aiden Cline 6918d29e79 wip 2026-05-05 10:28:23 -05:00
11 changed files with 358 additions and 19 deletions
+1
View File
@@ -9,6 +9,7 @@
### General Principles
- Keep things in one function unless composable or reusable
- Do not extract single-use helpers preemptively. Inline the logic at the call site unless the helper is reused, hides a genuinely complex boundary, or has a clear independent name that improves the caller.
- Avoid `try`/`catch` where possible
- Avoid using the `any` type
- Use Bun APIs when possible, like `Bun.file()`
+3
View File
@@ -416,6 +416,7 @@
"@opentui/solid": "catalog:",
"@parcel/watcher": "2.5.1",
"@pierre/diffs": "catalog:",
"@silvia-odwyer/photon-node": "0.3.4",
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
@@ -2045,6 +2046,8 @@
"@sigstore/verify": ["@sigstore/verify@3.1.0", "", { "dependencies": { "@sigstore/bundle": "^4.0.0", "@sigstore/core": "^3.1.0", "@sigstore/protobuf-specs": "^0.5.0" } }, "sha512-mNe0Iigql08YupSOGv197YdHpPPr+EzDZmfCgMc7RPNaZTw5aLN01nBl6CHJOh3BGtnMIj83EeN4butBchc8Ag=="],
"@silvia-odwyer/photon-node": ["@silvia-odwyer/photon-node@0.3.4", "", {}, "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA=="],
"@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="],
"@slack/bolt": ["@slack/bolt@3.22.0", "", { "dependencies": { "@slack/logger": "^4.0.0", "@slack/oauth": "^2.6.3", "@slack/socket-mode": "^1.3.6", "@slack/types": "^2.13.0", "@slack/web-api": "^6.13.0", "@types/express": "^4.16.1", "@types/promise.allsettled": "^1.0.3", "@types/tsscmp": "^1.0.0", "axios": "^1.7.4", "express": "^4.21.0", "path-to-regexp": "^8.1.0", "promise.allsettled": "^1.0.2", "raw-body": "^2.3.3", "tsscmp": "^1.0.6" } }, "sha512-iKDqGPEJDnrVwxSVlFW6OKTkijd7s4qLBeSufoBsTM0reTyfdp/5izIQVkxNfzjHi3o6qjdYbRXkYad5HBsBog=="],
+1
View File
@@ -129,6 +129,7 @@
"@opentui/solid": "catalog:",
"@parcel/watcher": "2.5.1",
"@pierre/diffs": "catalog:",
"@silvia-odwyer/photon-node": "0.3.4",
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
+5
View File
@@ -2,3 +2,8 @@ declare module "*.wav" {
const file: string
export default file
}
declare module "*.wasm" {
const file: string
export default file
}
@@ -0,0 +1,30 @@
export * as ConfigAttachment from "./attachment"
import { Schema } from "effect"
import { zod } from "@/util/effect-zod"
import { PositiveInt, withStatics } from "@/util/schema"
export const Image = Schema.Struct({
auto_resize: Schema.optional(Schema.Boolean).annotate({
description: "Resize images before sending them to the model when they exceed configured limits (default: true)",
}),
max_width: Schema.optional(PositiveInt).annotate({
description: "Maximum image width before resizing or rejecting the attachment (default: 2000)",
}),
max_height: Schema.optional(PositiveInt).annotate({
description: "Maximum image height before resizing or rejecting the attachment (default: 2000)",
}),
max_base64_bytes: Schema.optional(PositiveInt).annotate({
description: "Maximum base64 payload bytes for an image attachment (default: 4718592)",
}),
})
.annotate({ identifier: "ImageAttachmentConfig" })
.pipe(withStatics((s) => ({ zod: zod(s) })))
export type Image = Schema.Schema.Type<typeof Image>
export const Info = Schema.Struct({
image: Schema.optional(Image).annotate({ description: "Image attachment configuration" }),
})
.annotate({ identifier: "AttachmentConfig" })
.pipe(withStatics((s) => ({ zod: zod(s) })))
export type Info = Schema.Schema.Type<typeof Info>
+4
View File
@@ -25,6 +25,7 @@ import { containsPath } from "../project/instance-context"
import { zod } from "@/util/effect-zod"
import { NonNegativeInt, PositiveInt, withStatics, type DeepMutable } from "@/util/schema"
import { ConfigAgent } from "./agent"
import { ConfigAttachment } from "./attachment"
import { ConfigCommand } from "./command"
import { ConfigFormatter } from "./formatter"
import { ConfigLayout } from "./layout"
@@ -241,6 +242,9 @@ export const Info = Schema.Struct({
layout: Schema.optional(ConfigLayout.Layout).annotate({ description: "@deprecated Always uses stretch layout." }),
permission: Schema.optional(ConfigPermission.Info),
tools: Schema.optional(Schema.Record(Schema.String, Schema.Boolean)),
attachment: Schema.optional(ConfigAttachment.Info).annotate({
description: "Attachment processing configuration, including image size limits and resizing behavior",
}),
enterprise: Schema.optional(
Schema.Struct({
url: Schema.optional(Schema.String).annotate({ description: "Enterprise URL" }),
+189
View File
@@ -0,0 +1,189 @@
import { Config } from "@/config/config"
import type { MessageV2 } from "@/session/message-v2"
import * as Log from "@opencode-ai/core/util/log"
import { Context, Effect, Layer, Schema } from "effect"
import fs from "fs"
const MAX_BASE64_BYTES = 4.5 * 1024 * 1024
const MAX_WIDTH = 2000
const MAX_HEIGHT = 2000
const AUTO_RESIZE = true
const JPEG_QUALITIES = [80, 85, 70, 55, 40]
const log = Log.create({ service: "image" })
type Photon = typeof import("@silvia-odwyer/photon-node")
let photonModule: Photon | null = null
let photonPromise: Promise<Photon | null> | null = null
function loadPhoton() {
if (photonModule) return Promise.resolve(photonModule)
if (photonPromise) return photonPromise
photonPromise = (async () => {
const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } })).default
const original = fs.readFileSync
fs.readFileSync = ((file: fs.PathOrFileDescriptor, options?: Parameters<typeof fs.readFileSync>[1]) => {
if (typeof file === "string" && file.endsWith("photon_rs_bg.wasm")) return original(photonWasm, options)
return original(file, options)
}) as typeof fs.readFileSync
try {
photonModule = await import("@silvia-odwyer/photon-node")
return photonModule
} catch {
photonModule = null
return null
} finally {
fs.readFileSync = original
}
})()
return photonPromise
}
export class PhotonUnavailableError extends Schema.TaggedErrorClass<PhotonUnavailableError>()(
"ImagePhotonUnavailableError",
{},
) {
override get message() {
return "Photon image processor is unavailable"
}
}
export class InvalidDataUrlError extends Schema.TaggedErrorClass<InvalidDataUrlError>()("ImageInvalidDataUrlError", {
url: Schema.String,
}) {
override get message() {
return "Image URL must be a base64 data URL"
}
}
export class DecodeError extends Schema.TaggedErrorClass<DecodeError>()("ImageDecodeError", {}) {
override get message() {
return "Image could not be decoded"
}
}
export class SizeError extends Schema.TaggedErrorClass<SizeError>()("ImageSizeError", {
bytes: Schema.Number,
max: Schema.Number,
width: Schema.Number,
height: Schema.Number,
max_width: Schema.Number,
max_height: Schema.Number,
}) {
override get message() {
return `Image ${this.width}x${this.height} with base64 size ${this.bytes} exceeds configured limits and could not be resized below ${this.max_width}x${this.max_height}/${this.max} bytes`
}
}
export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError
export interface Interface {
readonly normalize: (input: MessageV2.FilePart) => Effect.Effect<MessageV2.FilePart, Error>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/Image") {}
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const config = yield* Config.Service
const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) {
const image = (yield* config.get()).attachment?.image
const info = {
autoResize: image?.auto_resize ?? AUTO_RESIZE,
maxWidth: image?.max_width ?? MAX_WIDTH,
maxHeight: image?.max_height ?? MAX_HEIGHT,
maxBase64Bytes: image?.max_base64_bytes ?? MAX_BASE64_BYTES,
}
if (!info.autoResize) return input
if (!input.url.startsWith("data:") || !input.url.includes(";base64,"))
return yield* new InvalidDataUrlError({ url: input.url })
const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length)
const photon = yield* Effect.promise(loadPhoton)
if (!photon) return yield* new PhotonUnavailableError()
const decoded = yield* Effect.sync(() => {
try {
return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64"))
} catch {
return undefined
}
})
if (!decoded) return yield* new DecodeError()
try {
const originalWidth = decoded.get_width()
const originalHeight = decoded.get_height()
if (
originalWidth <= info.maxWidth &&
originalHeight <= info.maxHeight &&
Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes
)
return input
const scale = Math.min(1, info.maxWidth / originalWidth, info.maxHeight / originalHeight)
for (const size of Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
const previous = acc.at(-1) ?? {
width: Math.max(1, Math.round(originalWidth * scale)),
height: Math.max(1, Math.round(originalHeight * scale)),
}
const next = acc.length === 0
? previous
: {
width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
}
return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next]
}, [])) {
const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
const candidate = [
{ data: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
...JPEG_QUALITIES.map((quality) => ({
data: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
mime: "image/jpeg",
})),
]
.map((item) => ({ ...item, bytes: Buffer.byteLength(item.data, "utf8") }))
.filter((item) => item.bytes <= info.maxBase64Bytes)
.sort((a, b) => a.bytes - b.bytes)[0]
resized.free()
if (candidate) {
log.info("using resized image", {
from_mime: input.mime,
to_mime: candidate.mime,
from: `${originalWidth}x${originalHeight}`,
to: `${size.width}x${size.height}`,
})
return {
...input,
mime: candidate.mime,
url: `data:${candidate.mime};base64,${candidate.data}`,
}
}
}
return yield* new SizeError({
bytes: Buffer.byteLength(base64, "utf8"),
max: info.maxBase64Bytes,
width: originalWidth,
height: originalHeight,
max_width: info.maxWidth,
max_height: info.maxHeight,
})
} finally {
decoded.free()
}
})
return Service.of({ normalize })
}),
)
export const defaultLayer = layer.pipe(Layer.provide(Config.defaultLayer))
export * as Image from "./image"
@@ -203,13 +203,15 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
params: { sessionID: SessionID }
payload: typeof InitPayload.Type
}) {
yield* promptSvc.command({
sessionID: ctx.params.sessionID,
messageID: ctx.payload.messageID,
model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
command: Command.Default.INIT,
arguments: "",
})
yield* promptSvc
.command({
sessionID: ctx.params.sessionID,
messageID: ctx.payload.messageID,
model: `${ctx.payload.providerID}/${ctx.payload.modelID}`,
command: Command.Default.INIT,
arguments: "",
})
.pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
return true
})
@@ -297,7 +299,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session",
params: { sessionID: SessionID }
payload: typeof CommandPayload.Type
}) {
return yield* promptSvc.command({ ...ctx.payload, sessionID: ctx.params.sessionID })
return yield* promptSvc
.command({ ...ctx.payload, sessionID: ctx.params.sessionID })
.pipe(Effect.mapError(() => new HttpApiError.BadRequest({})))
})
const shell = Effect.fn("SessionHttpApi.shell")(function* (ctx: {
+20 -4
View File
@@ -1,4 +1,4 @@
import { Cause, Deferred, Effect, Layer, Context, Scope } from "effect"
import { Cause, Deferred, Effect, Exit, Layer, Context, Scope } from "effect"
import * as Stream from "effect/Stream"
import { Agent } from "@/agent/agent"
import { Bus } from "@/bus"
@@ -9,6 +9,7 @@ import { Snapshot } from "@/snapshot"
import * as Session from "./session"
import { LLM } from "./llm"
import { MessageV2 } from "./message-v2"
import { Image } from "@/image/image"
import { isOverflow } from "./overflow"
import { PartID } from "./schema"
import type { SessionID } from "./schema"
@@ -108,6 +109,7 @@ export const layer: Layer.Layer<
const summary = yield* SessionSummary.Service
const scope = yield* Scope.Scope
const status = yield* SessionStatus.Service
const image = yield* Image.Service
const create = Effect.fn("SessionProcessor.create")(function* (input: Input) {
// Pre-capture snapshot before the LLM stream starts. The AI SDK
@@ -183,16 +185,30 @@ export const layer: Layer.Layer<
) {
const match = yield* readToolCall(toolCallID)
if (!match || match.part.state.status !== "running") return
const normalized = output.attachments
? yield* Effect.forEach(output.attachments, (attachment) =>
attachment.mime.startsWith("image/")
? image.normalize(attachment).pipe(Effect.exit)
: Effect.succeed(Exit.succeed(attachment)),
)
: undefined
const omitted = normalized?.filter(Exit.isFailure).length ?? 0
const attachments = normalized
?.filter(Exit.isSuccess)
.map((item) => item.value)
yield* session.updatePart({
...match.part,
state: {
status: "completed",
input: match.part.state.input,
output: output.output,
output:
omitted === 0
? output.output
: `${output.output}\n\n[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the inline image size limit.]`,
metadata: output.metadata,
title: output.title,
time: { start: match.part.state.time.start, end: Date.now() },
attachments: output.attachments,
attachments: attachments?.length ? attachments : undefined,
},
})
yield* settleToolCall(toolCallID)
@@ -746,7 +762,7 @@ export const layer: Layer.Layer<
return Service.of({ create })
}),
)
).pipe(Layer.provide(Image.layer))
export const defaultLayer = Layer.suspend(() =>
layer.pipe(
+15 -7
View File
@@ -43,6 +43,7 @@ import { Shell } from "@/shell/shell"
import { ShellID } from "@/tool/shell/id"
import { AppFileSystem } from "@opencode-ai/core/filesystem"
import { Truncate } from "@/tool/truncate"
import { Image } from "@/image/image"
import { decodeDataUrl } from "@/util/data-url"
import { Process } from "@/util/process"
import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect"
@@ -80,10 +81,10 @@ const elog = EffectLogger.create({ service: "session.prompt" })
export interface Interface {
readonly cancel: (sessionID: SessionID) => Effect.Effect<void>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts>
readonly prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly loop: (input: LoopInput) => Effect.Effect<MessageV2.WithParts>
readonly shell: (input: ShellInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts>
readonly command: (input: CommandInput) => Effect.Effect<MessageV2.WithParts, Image.Error>
readonly resolvePromptParts: (template: string) => Effect.Effect<PromptInput["parts"]>
}
@@ -108,6 +109,7 @@ export const layer = Layer.effect(
const lsp = yield* LSP.Service
const registry = yield* ToolRegistry.Service
const truncate = yield* Truncate.Service
const image = yield* Image.Service
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
const scope = yield* Scope.Scope
const instruction = yield* Instruction.Service
@@ -123,7 +125,7 @@ export const layer = Layer.effect(
return {
cancel: (sessionID: SessionID) => cancel(sessionID),
resolvePromptParts: (template: string) => resolvePromptParts(template),
prompt: (input: PromptInput) => prompt(input),
prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)),
} satisfies TaskPromptOps
})
@@ -1259,7 +1261,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return [{ ...part, messageID: info.id, sessionID: input.sessionID }]
})
const parts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
const resolvedParts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe(
Effect.map((x) => x.flat().map(assign)),
)
@@ -1272,7 +1274,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the
messageID: input.messageID,
variant: input.variant,
},
{ message: info, parts },
{ message: info, parts: resolvedParts },
)
const parts = yield* Effect.forEach(resolvedParts, (part) =>
part.type === "file" && part.mime.startsWith("image/")
? image.normalize(part)
: Effect.succeed(part),
)
const parsed = MessageV2.Info.zod.safeParse(info)
@@ -1368,7 +1376,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
return { info, parts }
}, Effect.scoped)
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts> = Effect.fn("SessionPrompt.prompt")(
const prompt: (input: PromptInput) => Effect.Effect<MessageV2.WithParts, Image.Error> = Effect.fn("SessionPrompt.prompt")(
function* (input: PromptInput) {
const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie)
yield* revert.cleanup(session)
@@ -1766,7 +1774,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
resolvePromptParts,
})
}),
)
).pipe(Layer.provide(Image.layer))
export const defaultLayer = Layer.suspend(() =>
layer.pipe(
@@ -0,0 +1,78 @@
import { describe, expect } from "bun:test"
import { Cause, Effect, Exit, Layer } from "effect"
import { Image } from "@/image/image"
import { MessageID, PartID, SessionID } from "@/session/schema"
import { TestConfig } from "../fixture/config"
import { testEffect } from "../lib/effect"
const it = testEffect(Layer.mergeAll(Image.layer.pipe(Layer.provide(TestConfig.layer()))))
const tiny = testEffect(
Layer.mergeAll(
Image.layer.pipe(
Layer.provide(TestConfig.layer({ get: () => Effect.succeed({ attachment: { image: { max_base64_bytes: 1 } } }) })),
),
),
)
function part(mime: string, data: string) {
return {
id: PartID.ascending(),
messageID: MessageID.ascending(),
sessionID: SessionID.make("test"),
type: "file" as const,
mime,
url: `data:${mime};base64,${data}`,
}
}
describe("Image", () => {
it.effect("normalizes generated png and jpeg attachments", () =>
Effect.gen(function* () {
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
const source = new photon.PhotonImage(
new Uint8Array(Array.from({ length: 64 * 64 * 4 }, (_, index) => (index % 4 === 3 ? 255 : index % 251))),
64,
64,
)
const image = yield* Image.Service
const results = yield* Effect.all([
image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64"))),
image.normalize(part("image/jpeg", Buffer.from(source.get_bytes_jpeg(90)).toString("base64"))),
])
source.free()
expect(results.map((result) => result.url.startsWith(`data:${result.mime};base64,`))).toEqual([true, true])
expect(results.every((result) => result.mime === "image/png" || result.mime === "image/jpeg")).toBe(true)
}),
)
it.effect("accepts webp attachments that are already within limits", () =>
Effect.gen(function* () {
const image = yield* Image.Service
const input = part("image/webp", "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA")
expect(yield* image.normalize(input)).toEqual(input)
}),
)
tiny.effect("fails with a typed size error when no resized candidate fits", () =>
Effect.gen(function* () {
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 4 }, () => 255)), 1, 1)
const image = yield* Image.Service
const exit = yield* image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64"))).pipe(Effect.exit)
source.free()
expect(Exit.isFailure(exit)).toBe(true)
if (Exit.isFailure(exit)) {
const error = Cause.squash(exit.cause)
expect(error).toBeInstanceOf(Image.SizeError)
if (error instanceof Image.SizeError) {
expect(error.width).toBe(1)
expect(error.height).toBe(1)
expect(error.max).toBe(1)
}
}
}),
)
})