Compare commits

...

1 Commits

Author SHA1 Message Date
Aiden Cline 31589330a9 feat(ai): add OpenAI image generation 2026-07-19 05:30:14 +00:00
11 changed files with 565 additions and 18 deletions
+43 -2
View File
@@ -1,6 +1,6 @@
# @opencode-ai/ai
Schema-first LLM core for opencode. One typed request, response, event, and tool language; provider quirks live in adapters, not in calling code.
Schema-first AI primitives for opencode. Provider quirks live in adapters, not in calling code.
```ts
import { Effect } from "effect"
@@ -24,6 +24,45 @@ const program = Effect.gen(function* () {
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
## Image generation
Use `Image.generate` with an image model for direct asset generation:
```ts
import { Image } from "@opencode-ai/ai"
import { OpenAI } from "@opencode-ai/ai/providers"
const program = Effect.gen(function* () {
const response = yield* Image.generate({
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
count: 2,
size: { width: 1024, height: 1024 },
providerOptions: { openai: { quality: "high", outputFormat: "webp" } },
})
return response.images // GeneratedImage[] with owned bytes or a provider URL
})
```
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
```ts
const program = Effect.gen(function* () {
const response = yield* LLM.generate(
LLM.request({
model: OpenAI.configure({ apiKey }).responses("gpt-5"),
prompt: "Design a solarpunk rooftop garden, then show me.",
tools: [OpenAI.imageGeneration({ quality: "high" })],
}),
)
return response.message
})
```
The hosted result is represented as a provider-executed tool call and tool result. Its image is a `file` content item with a data URI, so retaining `response.message` preserves the generated image for continuation.
## Public API
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
@@ -32,6 +71,8 @@ Run `LLMClient.stream(request)` instead of `generate` when you want incremental
- **`Model.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
- **`LLMClient.prepare(request)`** — compile a request through protocol body construction, validation, and HTTP preparation without sending. Useful for inspection and testing.
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
- **`Image.generate({...})`** — generate images through a provider-neutral image request and response model.
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
## Caching
@@ -182,7 +223,7 @@ Adding a new model or deployment is usually 5-15 lines using `Route.make({ proto
## Effect
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for runtime dispatch and import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
## See also
+34
View File
@@ -0,0 +1,34 @@
import { Context, Effect, Layer } from "effect"
import { RequestExecutor } from "./route/executor"
import type { ImageRequest, ImageResponse } from "./image"
import type { LLMError } from "./schema"
export type Execute = RequestExecutor.Interface["execute"]
export interface Interface {
readonly generate: (request: ImageRequest) => Effect.Effect<ImageResponse, LLMError>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
export const generate = (request: ImageRequest): Effect.Effect<ImageResponse, LLMError> =>
Effect.gen(function* () {
const client = yield* Service
return yield* client.generate(request)
}) as Effect.Effect<ImageResponse, LLMError>
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
Service,
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
return Service.of({
generate: (request) => request.model.route.generate(request, executor.execute),
})
}),
)
export const ImageClient = {
Service,
layer,
generate,
} as const
+107
View File
@@ -0,0 +1,107 @@
import { Effect, Schema } from "effect"
import { HttpOptions, LLMError, ModelID, ProviderID, ProviderMetadata, Usage } from "./schema"
import { ImageClient, type Execute as ImageExecute } from "./image-client"
export interface ImageRoute {
readonly id: string
readonly generate: (request: ImageRequest, execute: ImageExecute) => Effect.Effect<ImageResponse, LLMError>
}
export class ImageModel {
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute
readonly defaults?: ImageModelDefaults
constructor(input: ImageModel.Input) {
this.id = input.id
this.provider = input.provider
this.route = input.route
this.defaults = input.defaults
}
static make(input: ImageModel.MakeInput) {
return new ImageModel({
id: ModelID.make(input.id),
provider: ProviderID.make(input.provider),
route: input.route,
defaults: input.defaults,
})
}
}
export namespace ImageModel {
export interface Input {
readonly id: ModelID
readonly provider: ProviderID
readonly route: ImageRoute
readonly defaults?: ImageModelDefaults
}
export interface MakeInput extends Omit<Input, "id" | "provider"> {
readonly id: string | ModelID
readonly provider: string | ProviderID
}
}
export interface ImageModelDefaults {
readonly providerOptions?: Record<string, Record<string, unknown>>
readonly http?: HttpOptions
}
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
expected: "Image.Model",
})
export const ImageSize = Schema.Struct({
width: Schema.Number,
height: Schema.Number,
}).annotate({ identifier: "Image.Size" })
export type ImageSize = Schema.Schema.Type<typeof ImageSize>
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
model: ImageModelSchema,
prompt: Schema.String,
count: Schema.optional(Schema.Number),
size: Schema.optional(ImageSize),
aspectRatio: Schema.optional(Schema.String),
seed: Schema.optional(Schema.Number),
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
http: Schema.optional(HttpOptions),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
export type ImageRequestInput = Omit<ConstructorParameters<typeof ImageRequest>[0], "http"> & {
readonly http?: HttpOptions.Input
}
export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
mediaType: Schema.String,
data: Schema.Union([Schema.String, Schema.Uint8Array]),
providerMetadata: Schema.optional(ProviderMetadata),
}) {}
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
images: Schema.Array(GeneratedImage),
usage: Schema.optional(Usage),
providerMetadata: Schema.optional(ProviderMetadata),
}) {
get image() {
return this.images[0]
}
}
export const request = (input: ImageRequest | ImageRequestInput) => {
if (input instanceof ImageRequest) return input
return new ImageRequest({
...input,
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
})
}
export const generate = (input: ImageRequest | ImageRequestInput) => ImageClient.generate(request(input))
export const Image = {
request,
generate,
} as const
+4
View File
@@ -1,4 +1,5 @@
export { LLMClient } from "./route/client"
export { ImageClient } from "./image-client"
export { Auth } from "./route/auth"
export { Provider } from "./provider"
export { ProviderPackage } from "./provider-package"
@@ -10,6 +11,9 @@ export type {
Service as LLMClientService,
} from "./route/client"
export * from "./schema"
export { GeneratedImage, ImageModel, ImageRequest, ImageResponse, ImageSize } from "./image"
export type { ImageModelDefaults, ImageRequestInput, ImageRoute } from "./image"
export { Image } from "./image"
export { Tool, ToolFailure, toDefinitions } from "./tool"
export { ToolRuntime } from "./tool-runtime"
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime"
+1
View File
@@ -2,6 +2,7 @@ export * as AnthropicMessages from "./anthropic-messages"
export * as BedrockConverse from "./bedrock-converse"
export * as Gemini from "./gemini"
export * as OpenAIChat from "./openai-chat"
export * as OpenAIImages from "./openai-images"
export * as OpenAICompatibleChat from "./openai-compatible-chat"
export * as OpenAICompatibleResponses from "./openai-compatible-responses"
export * as OpenAIResponses from "./openai-responses"
+174
View File
@@ -0,0 +1,174 @@
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
ImageModel,
GeneratedImage,
ImageResponse,
type ImageRequest,
type ImageModelDefaults,
type ImageRoute,
} from "../image"
import { Auth, type Definition as AuthDefinition } from "../route/auth"
import { InvalidProviderOutputReason, LLMError, Usage } from "../schema"
import { ProviderShared } from "./shared"
const ADAPTER = "openai-images"
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/images/generations"
export interface OpenAIImageOptions {
readonly quality?: "auto" | "low" | "medium" | "high"
readonly background?: "auto" | "opaque" | "transparent"
readonly moderation?: "auto" | "low"
readonly outputFormat?: "png" | "jpeg" | "webp"
readonly outputCompression?: number
}
const OpenAIImageBody = Schema.Struct({
model: Schema.String,
prompt: Schema.String,
n: Schema.optional(Schema.Number),
size: Schema.optional(Schema.String),
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
moderation: Schema.optional(Schema.Literals(["auto", "low"])),
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
output_compression: Schema.optional(Schema.Number),
})
export type OpenAIImageBody = Schema.Schema.Type<typeof OpenAIImageBody>
const OpenAIImageResponse = Schema.Struct({
data: Schema.Array(
Schema.Struct({
b64_json: Schema.optional(Schema.String),
url: Schema.optional(Schema.String),
revised_prompt: Schema.optional(Schema.String),
}),
),
output_format: Schema.optional(Schema.String),
usage: Schema.optional(
Schema.Struct({
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}),
),
})
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly defaults?: ImageModelDefaults
}
const providerOptions = (request: ImageRequest): OpenAIImageOptions => ({
...request.model.defaults?.providerOptions?.openai,
...request.providerOptions?.openai,
})
const body = (request: ImageRequest): OpenAIImageBody => {
const options = providerOptions(request)
return {
model: request.model.id,
prompt: request.prompt,
n: request.count,
size: request.size === undefined ? undefined : `${request.size.width}x${request.size.height}`,
quality: options.quality,
background: options.background,
moderation: options.moderation,
output_format: options.outputFormat,
output_compression: options.outputCompression,
}
}
const invalidOutput = (message: string) =>
new LLMError({
module: ADAPTER,
method: "generate",
reason: new InvalidProviderOutputReason({ message, route: ADAPTER }),
})
export const model = (input: ModelInput) => {
const route: ImageRoute = {
id: ADAPTER,
generate: Effect.fn("OpenAIImages.generate")(function* (request: ImageRequest, execute) {
if (request.aspectRatio !== undefined)
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common aspectRatio option")
if (request.seed !== undefined)
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common seed option")
const requestBody = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenAIImageBody))(body(request))
const text = Schema.encodeSync(Schema.fromJsonString(OpenAIImageBody))(requestBody)
const url = `${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`
const http = request.http ?? request.model.defaults?.http
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
const payload = yield* response.json.pipe(
Effect.mapError(() => invalidOutput("Failed to read the OpenAI Images response")),
)
const decoded = yield* Schema.decodeUnknownEffect(OpenAIImageResponse)(payload).pipe(
Effect.mapError(() => invalidOutput("OpenAI Images returned an invalid response")),
)
const format = decoded.output_format ?? providerOptions(request).outputFormat ?? "png"
const images = yield* Effect.forEach(decoded.data, (item, index) => {
if (item.b64_json)
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
Effect.mapError(() => invalidOutput(`OpenAI Images result ${index} contains invalid base64 data`)),
Effect.map(
(data) =>
new GeneratedImage({
mediaType: `image/${format}`,
data,
providerMetadata:
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
}),
),
)
if (item.url)
return Effect.succeed(
new GeneratedImage({
mediaType: `image/${format}`,
data: item.url,
providerMetadata:
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
}),
)
return Effect.fail(invalidOutput(`OpenAI Images result ${index} has neither image data nor a URL`))
})
if (images.length === 0) return yield* invalidOutput("OpenAI Images returned no images")
return new ImageResponse({
images,
usage:
decoded.usage === undefined
? undefined
: new Usage({
inputTokens: decoded.usage.input_tokens,
outputTokens: decoded.usage.output_tokens,
totalTokens: decoded.usage.total_tokens,
providerMetadata: { openai: decoded.usage },
}),
providerMetadata: { openai: { outputFormat: format } },
})
}),
}
return ImageModel.make({ id: input.id, provider: "openai", route, defaults: input.defaults })
}
export const OpenAIImages = {
model,
} as const
+41 -13
View File
@@ -113,11 +113,24 @@ const OpenAIResponsesTool = Schema.Struct({
parameters: JsonObject,
strict: Schema.optional(Schema.Boolean),
})
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTool>
const OpenAIResponsesImageGenerationTool = Schema.Struct({
type: Schema.tag("image_generation"),
action: Schema.optional(Schema.Literals(["auto", "generate", "edit"])),
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
input_fidelity: Schema.optional(Schema.Literals(["low", "high"])),
output_compression: Schema.optional(Schema.Number),
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
partial_images: Schema.optional(Schema.Number),
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
size: Schema.optional(Schema.String),
})
const OpenAIResponsesTools = Schema.Union([OpenAIResponsesTool, OpenAIResponsesImageGenerationTool])
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTools>
const OpenAIResponsesToolChoice = Schema.Union([
Schema.Literals(["auto", "none", "required"]),
Schema.Struct({ type: Schema.tag("function"), name: Schema.String }),
Schema.Struct({ type: Schema.tag("image_generation") }),
])
// Fields shared between the HTTP body and the WebSocket `response.create`
@@ -128,7 +141,7 @@ const OpenAIResponsesCoreFields = {
model: Schema.String,
input: Schema.Array(OpenAIResponsesInputItem),
instructions: Schema.optional(Schema.String),
tools: optionalArray(OpenAIResponsesTool),
tools: optionalArray(OpenAIResponsesTools),
tool_choice: Schema.optional(OpenAIResponsesToolChoice),
store: Schema.optional(Schema.Boolean),
service_tier: Schema.optional(OpenAIOptions.OpenAIServiceTier),
@@ -194,6 +207,7 @@ const OpenAIResponsesStreamItem = Schema.Struct({
outputs: Schema.optional(Schema.Unknown),
server_label: Schema.optional(Schema.String),
output: Schema.optional(Schema.Unknown),
result: Schema.optional(Schema.String),
error: Schema.optional(Schema.Unknown),
encrypted_content: optionalNull(Schema.String),
})
@@ -258,21 +272,30 @@ const invalid = ProviderShared.invalidRequest
// =============================================================================
// Request Lowering
// =============================================================================
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): OpenAIResponsesTool => ({
type: "function",
name: tool.name,
description: tool.description,
parameters: ToolSchemaProjection.openAI(inputSchema),
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
strict: false,
})
const nativeImageTool = (tool: ToolDefinition) => {
const native = tool.native?.openai
return Schema.is(OpenAIResponsesImageGenerationTool)(native) ? native : undefined
}
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): OpenAIResponsesTool =>
nativeImageTool(tool) ?? {
type: "function",
name: tool.name,
description: tool.description,
parameters: ToolSchemaProjection.openAI(inputSchema),
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
strict: false,
}
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolDefinition>) =>
ProviderShared.matchToolChoice("OpenAI Responses", toolChoice, {
auto: () => "auto" as const,
none: () => "none" as const,
required: () => "required" as const,
tool: (name) => ({ type: "function" as const, name }),
tool: (name) =>
tools.some((tool) => tool.name === name && nativeImageTool(tool) !== undefined)
? ({ type: "image_generation" } as const)
: { type: "function" as const, name },
})
const lowerToolCall = (part: ToolCallPart): OpenAIResponsesInputItem => ({
@@ -488,7 +511,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
: request.tools.map((tool) =>
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
),
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined,
stream: true as const,
max_output_tokens: generation?.maxTokens,
temperature: generation?.temperature,
@@ -576,6 +599,11 @@ const isReasoningItem = (
// outputs / sources / status without re-decoding.
const hostedToolResult = (item: OpenAIResponsesStreamItem) => {
const isError = typeof item.error !== "undefined" && item.error !== null
if (item.type === "image_generation_call" && item.result)
return {
type: "content" as const,
value: [{ type: "file" as const, uri: `data:image/png;base64,${item.result}`, mime: "image/png" }],
}
return isError ? { type: "error" as const, value: item.error } : { type: "json" as const, value: item }
}
+51 -1
View File
@@ -1,12 +1,14 @@
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
import type { Route, RouteDefaultsInput } from "../route/client"
import type { ProviderPackage } from "../provider-package"
import { ProviderID, type ModelID } from "../schema"
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema"
import * as OpenAIChat from "../protocols/openai-chat"
import * as OpenAIResponses from "../protocols/openai-responses"
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options"
import { OpenAIImages, type OpenAIImageOptions } from "../protocols/openai-images"
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options"
export type { OpenAIImageOptions } from "../protocols/openai-images"
export const id = ProviderID.make("openai")
@@ -22,6 +24,41 @@ export type Config = RouteDefaultsInput &
readonly providerOptions?: OpenAIProviderOptionsInput
}
export interface ImageConfig {
readonly providerOptions?: OpenAIImageOptions
}
export interface ImageGenerationOptions {
readonly action?: "auto" | "generate" | "edit"
readonly background?: "auto" | "opaque" | "transparent"
readonly inputFidelity?: "low" | "high"
readonly outputCompression?: number
readonly outputFormat?: "png" | "jpeg" | "webp"
readonly partialImages?: number
readonly quality?: "auto" | "low" | "medium" | "high"
readonly size?: string
}
export const imageGeneration = (options: ImageGenerationOptions = {}) =>
ToolDefinition.make({
name: "image_generation",
description: "Generate or edit an image using OpenAI's hosted image generation tool.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
native: {
openai: {
type: "image_generation",
action: options.action,
background: options.background,
input_fidelity: options.inputFidelity,
output_compression: options.outputCompression,
output_format: options.outputFormat,
partial_images: options.partialImages,
quality: options.quality,
size: options.size,
},
},
})
export interface Settings extends ProviderPackage.Settings {
readonly apiKey?: string
readonly baseURL?: string
@@ -55,6 +92,17 @@ export const configure = (input: Config = {}) => {
const responsesWebSocket = (id: string | ModelID) =>
responsesWebSocketRoute.with(withOpenAIOptions(id, modelDefaults, { textVerbosity: true })).model({ id })
const chat = (id: string | ModelID) => chatRoute.with(withOpenAIOptions(id, modelDefaults)).model({ id })
const image = (modelID: string | ModelID, options: ImageConfig = {}) =>
OpenAIImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
defaults: {
providerOptions: options.providerOptions === undefined ? undefined : { openai: { ...options.providerOptions } },
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
},
})
return {
id,
@@ -62,6 +110,7 @@ export const configure = (input: Config = {}) => {
responses,
responsesWebSocket,
chat,
image,
configure,
}
}
@@ -97,3 +146,4 @@ export const chatModel: ProviderPackage.Definition<Settings>["model"] = (modelID
export const responses = provider.responses
export const responsesWebSocket = provider.responsesWebSocket
export const chat = provider.chat
export const image = provider.image
+2 -2
View File
@@ -1,6 +1,6 @@
import { Config, Effect, Redacted } from "effect"
import { Headers } from "effect/unstable/http"
import { AuthenticationReason, InvalidRequestReason, LLMError, type LLMRequest } from "../schema"
import { AuthenticationReason, InvalidRequestReason, LLMError, type HttpOptions } from "../schema"
export class MissingCredentialError extends Error {
readonly _tag = "MissingCredentialError"
@@ -15,7 +15,7 @@ export type AuthError = CredentialError | LLMError
type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redacted>
export interface AuthInput {
readonly request: LLMRequest
readonly request: { readonly http?: HttpOptions }
readonly method: "POST" | "GET"
readonly url: string
readonly body: string
+59
View File
@@ -0,0 +1,59 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Image, ImageClient } from "../src"
import { OpenAI } from "../src/providers"
import { it } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
describe("Image", () => {
it.effect("generates images through the OpenAI Images API", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("gpt-image-2"),
prompt: "A robot tending a rooftop garden",
count: 2,
size: { width: 1024, height: 1024 },
providerOptions: {
openai: { quality: "high", outputFormat: "webp" },
},
})
expect(response.images).toHaveLength(2)
expect(response.image?.mediaType).toBe("image/webp")
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
expect(response.usage?.totalTokens).toBe(12)
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(request.url).toBe("https://api.openai.test/v1/images/generations")
expect(request.headers.get("authorization")).toBe("Bearer test")
expect(JSON.parse(input.text)).toEqual({
model: "gpt-image-2",
prompt: "A robot tending a rooftop garden",
n: 2,
size: "1024x1024",
quality: "high",
output_format: "webp",
})
return input.respond(
JSON.stringify({
data: [{ b64_json: "AQID", revised_prompt: "A precise robot" }, { b64_json: "BAUG" }],
output_format: "webp",
usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 },
}),
{ headers: { "content-type": "application/json" } },
)
}),
),
),
),
),
),
)
})
@@ -58,6 +58,24 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("lowers the hosted OpenAI image generation tool", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
LLM.request({
model,
prompt: "Show me a rooftop garden.",
tools: [OpenAI.imageGeneration({ action: "generate", quality: "high", size: "1024x1024" })],
toolChoice: "image_generation",
}),
)
expect(prepared.body.tools).toEqual([
{ type: "image_generation", action: "generate", quality: "high", size: "1024x1024" },
])
expect(prepared.body.tool_choice).toEqual({ type: "image_generation" })
}),
)
it.effect("lowers semantic service tier options", () =>
Effect.gen(function* () {
const input = LLM.updateRequest(request, { providerOptions: { openai: { serviceTier: "priority" } } })
@@ -1361,6 +1379,37 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("decodes image generation output as image content", () =>
Effect.gen(function* () {
const item = {
type: "image_generation_call",
id: "ig_1",
status: "completed",
result: "AQID",
}
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "response.output_item.done", item },
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
),
),
),
)
expect(response.events.find(LLMEvent.is.toolResult)).toMatchObject({
id: "ig_1",
name: "image_generation",
providerExecuted: true,
result: {
type: "content",
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
},
})
}),
)
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
Effect.gen(function* () {
const item = {