Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 31589330a9 |
+43
-2
@@ -1,6 +1,6 @@
|
||||
# @opencode-ai/ai
|
||||
|
||||
Schema-first LLM core for opencode. One typed request, response, event, and tool language; provider quirks live in adapters, not in calling code.
|
||||
Schema-first AI primitives for opencode. Provider quirks live in adapters, not in calling code.
|
||||
|
||||
```ts
|
||||
import { Effect } from "effect"
|
||||
@@ -24,6 +24,45 @@ const program = Effect.gen(function* () {
|
||||
|
||||
Run `LLMClient.stream(request)` instead of `generate` when you want incremental `LLMEvent`s. The event stream is provider-neutral — same shape across OpenAI Chat, OpenAI Responses, Anthropic Messages, Gemini, Bedrock Converse, and any OpenAI-compatible deployment.
|
||||
|
||||
## Image generation
|
||||
|
||||
Use `Image.generate` with an image model for direct asset generation:
|
||||
|
||||
```ts
|
||||
import { Image } from "@opencode-ai/ai"
|
||||
import { OpenAI } from "@opencode-ai/ai/providers"
|
||||
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY }).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
count: 2,
|
||||
size: { width: 1024, height: 1024 },
|
||||
providerOptions: { openai: { quality: "high", outputFormat: "webp" } },
|
||||
})
|
||||
|
||||
return response.images // GeneratedImage[] with owned bytes or a provider URL
|
||||
})
|
||||
```
|
||||
|
||||
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
|
||||
|
||||
```ts
|
||||
const program = Effect.gen(function* () {
|
||||
const response = yield* LLM.generate(
|
||||
LLM.request({
|
||||
model: OpenAI.configure({ apiKey }).responses("gpt-5"),
|
||||
prompt: "Design a solarpunk rooftop garden, then show me.",
|
||||
tools: [OpenAI.imageGeneration({ quality: "high" })],
|
||||
}),
|
||||
)
|
||||
|
||||
return response.message
|
||||
})
|
||||
```
|
||||
|
||||
The hosted result is represented as a provider-executed tool call and tool result. Its image is a `file` content item with a data URI, so retaining `response.message` preserves the generated image for continuation.
|
||||
|
||||
## Public API
|
||||
|
||||
- **`LLM.request({...})`** — build a provider-neutral `LLMRequest`. Accepts ergonomic inputs (`system: string`, `prompt: string`) that normalize into the canonical Schema classes.
|
||||
@@ -32,6 +71,8 @@ Run `LLMClient.stream(request)` instead of `generate` when you want incremental
|
||||
- **`Model.make(...)` / `ToolCallPart.make(...)` / `ToolResultPart.make(...)` / `ToolDefinition.make(...)`** — model and tool-related constructors from the canonical schema model.
|
||||
- **`LLMClient.prepare(request)`** — compile a request through protocol body construction, validation, and HTTP preparation without sending. Useful for inspection and testing.
|
||||
- **`LLMEvent.is.*`** — typed guards (`is.textDelta`, `is.toolCall`, `is.finish`, …) for filtering streams.
|
||||
- **`Image.generate({...})`** — generate images through a provider-neutral image request and response model.
|
||||
- **`ImageClient`** — Effect service and layer for image execution, parallel to `LLMClient`.
|
||||
|
||||
## Caching
|
||||
|
||||
@@ -182,7 +223,7 @@ Adding a new model or deployment is usually 5-15 lines using `Route.make({ proto
|
||||
|
||||
## Effect
|
||||
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for runtime dispatch and import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
This package is built on Effect. Public methods return `Effect` or `Stream`; provide `LLMClient.layer` for LLM dispatch and `ImageClient.layer` for image dispatch, then import the provider/protocol modules for the routes you use. The example at `example/tutorial.ts` is a runnable walkthrough.
|
||||
|
||||
## See also
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { Context, Effect, Layer } from "effect"
|
||||
import { RequestExecutor } from "./route/executor"
|
||||
import type { ImageRequest, ImageResponse } from "./image"
|
||||
import type { LLMError } from "./schema"
|
||||
|
||||
export type Execute = RequestExecutor.Interface["execute"]
|
||||
|
||||
export interface Interface {
|
||||
readonly generate: (request: ImageRequest) => Effect.Effect<ImageResponse, LLMError>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/ImageClient") {}
|
||||
|
||||
export const generate = (request: ImageRequest): Effect.Effect<ImageResponse, LLMError> =>
|
||||
Effect.gen(function* () {
|
||||
const client = yield* Service
|
||||
return yield* client.generate(request)
|
||||
}) as Effect.Effect<ImageResponse, LLMError>
|
||||
|
||||
export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
return Service.of({
|
||||
generate: (request) => request.model.route.generate(request, executor.execute),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const ImageClient = {
|
||||
Service,
|
||||
layer,
|
||||
generate,
|
||||
} as const
|
||||
@@ -0,0 +1,107 @@
|
||||
import { Effect, Schema } from "effect"
|
||||
import { HttpOptions, LLMError, ModelID, ProviderID, ProviderMetadata, Usage } from "./schema"
|
||||
import { ImageClient, type Execute as ImageExecute } from "./image-client"
|
||||
|
||||
export interface ImageRoute {
|
||||
readonly id: string
|
||||
readonly generate: (request: ImageRequest, execute: ImageExecute) => Effect.Effect<ImageResponse, LLMError>
|
||||
}
|
||||
|
||||
export class ImageModel {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute
|
||||
readonly defaults?: ImageModelDefaults
|
||||
|
||||
constructor(input: ImageModel.Input) {
|
||||
this.id = input.id
|
||||
this.provider = input.provider
|
||||
this.route = input.route
|
||||
this.defaults = input.defaults
|
||||
}
|
||||
|
||||
static make(input: ImageModel.MakeInput) {
|
||||
return new ImageModel({
|
||||
id: ModelID.make(input.id),
|
||||
provider: ProviderID.make(input.provider),
|
||||
route: input.route,
|
||||
defaults: input.defaults,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export namespace ImageModel {
|
||||
export interface Input {
|
||||
readonly id: ModelID
|
||||
readonly provider: ProviderID
|
||||
readonly route: ImageRoute
|
||||
readonly defaults?: ImageModelDefaults
|
||||
}
|
||||
|
||||
export interface MakeInput extends Omit<Input, "id" | "provider"> {
|
||||
readonly id: string | ModelID
|
||||
readonly provider: string | ProviderID
|
||||
}
|
||||
}
|
||||
|
||||
export interface ImageModelDefaults {
|
||||
readonly providerOptions?: Record<string, Record<string, unknown>>
|
||||
readonly http?: HttpOptions
|
||||
}
|
||||
|
||||
export const ImageModelSchema = Schema.declare((value): value is ImageModel => value instanceof ImageModel, {
|
||||
expected: "Image.Model",
|
||||
})
|
||||
|
||||
export const ImageSize = Schema.Struct({
|
||||
width: Schema.Number,
|
||||
height: Schema.Number,
|
||||
}).annotate({ identifier: "Image.Size" })
|
||||
export type ImageSize = Schema.Schema.Type<typeof ImageSize>
|
||||
|
||||
export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
|
||||
model: ImageModelSchema,
|
||||
prompt: Schema.String,
|
||||
count: Schema.optional(Schema.Number),
|
||||
size: Schema.optional(ImageSize),
|
||||
aspectRatio: Schema.optional(Schema.String),
|
||||
seed: Schema.optional(Schema.Number),
|
||||
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
|
||||
http: Schema.optional(HttpOptions),
|
||||
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}) {}
|
||||
|
||||
export type ImageRequestInput = Omit<ConstructorParameters<typeof ImageRequest>[0], "http"> & {
|
||||
readonly http?: HttpOptions.Input
|
||||
}
|
||||
|
||||
export class GeneratedImage extends Schema.Class<GeneratedImage>("Image.Generated")({
|
||||
mediaType: Schema.String,
|
||||
data: Schema.Union([Schema.String, Schema.Uint8Array]),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {}
|
||||
|
||||
export class ImageResponse extends Schema.Class<ImageResponse>("Image.Response")({
|
||||
images: Schema.Array(GeneratedImage),
|
||||
usage: Schema.optional(Usage),
|
||||
providerMetadata: Schema.optional(ProviderMetadata),
|
||||
}) {
|
||||
get image() {
|
||||
return this.images[0]
|
||||
}
|
||||
}
|
||||
|
||||
export const request = (input: ImageRequest | ImageRequestInput) => {
|
||||
if (input instanceof ImageRequest) return input
|
||||
return new ImageRequest({
|
||||
...input,
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
})
|
||||
}
|
||||
|
||||
export const generate = (input: ImageRequest | ImageRequestInput) => ImageClient.generate(request(input))
|
||||
|
||||
export const Image = {
|
||||
request,
|
||||
generate,
|
||||
} as const
|
||||
@@ -1,4 +1,5 @@
|
||||
export { LLMClient } from "./route/client"
|
||||
export { ImageClient } from "./image-client"
|
||||
export { Auth } from "./route/auth"
|
||||
export { Provider } from "./provider"
|
||||
export { ProviderPackage } from "./provider-package"
|
||||
@@ -10,6 +11,9 @@ export type {
|
||||
Service as LLMClientService,
|
||||
} from "./route/client"
|
||||
export * from "./schema"
|
||||
export { GeneratedImage, ImageModel, ImageRequest, ImageResponse, ImageSize } from "./image"
|
||||
export type { ImageModelDefaults, ImageRequestInput, ImageRoute } from "./image"
|
||||
export { Image } from "./image"
|
||||
export { Tool, ToolFailure, toDefinitions } from "./tool"
|
||||
export { ToolRuntime } from "./tool-runtime"
|
||||
export type { DispatchResult as ToolDispatchResult, ToolSettlement } from "./tool-runtime"
|
||||
|
||||
@@ -2,6 +2,7 @@ export * as AnthropicMessages from "./anthropic-messages"
|
||||
export * as BedrockConverse from "./bedrock-converse"
|
||||
export * as Gemini from "./gemini"
|
||||
export * as OpenAIChat from "./openai-chat"
|
||||
export * as OpenAIImages from "./openai-images"
|
||||
export * as OpenAICompatibleChat from "./openai-compatible-chat"
|
||||
export * as OpenAICompatibleResponses from "./openai-compatible-responses"
|
||||
export * as OpenAIResponses from "./openai-responses"
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import { Effect, Encoding, Schema } from "effect"
|
||||
import { Headers, HttpClientRequest } from "effect/unstable/http"
|
||||
import {
|
||||
ImageModel,
|
||||
GeneratedImage,
|
||||
ImageResponse,
|
||||
type ImageRequest,
|
||||
type ImageModelDefaults,
|
||||
type ImageRoute,
|
||||
} from "../image"
|
||||
import { Auth, type Definition as AuthDefinition } from "../route/auth"
|
||||
import { InvalidProviderOutputReason, LLMError, Usage } from "../schema"
|
||||
import { ProviderShared } from "./shared"
|
||||
|
||||
const ADAPTER = "openai-images"
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/images/generations"
|
||||
|
||||
export interface OpenAIImageOptions {
|
||||
readonly quality?: "auto" | "low" | "medium" | "high"
|
||||
readonly background?: "auto" | "opaque" | "transparent"
|
||||
readonly moderation?: "auto" | "low"
|
||||
readonly outputFormat?: "png" | "jpeg" | "webp"
|
||||
readonly outputCompression?: number
|
||||
}
|
||||
|
||||
const OpenAIImageBody = Schema.Struct({
|
||||
model: Schema.String,
|
||||
prompt: Schema.String,
|
||||
n: Schema.optional(Schema.Number),
|
||||
size: Schema.optional(Schema.String),
|
||||
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
|
||||
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
|
||||
moderation: Schema.optional(Schema.Literals(["auto", "low"])),
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
output_compression: Schema.optional(Schema.Number),
|
||||
})
|
||||
export type OpenAIImageBody = Schema.Schema.Type<typeof OpenAIImageBody>
|
||||
|
||||
const OpenAIImageResponse = Schema.Struct({
|
||||
data: Schema.Array(
|
||||
Schema.Struct({
|
||||
b64_json: Schema.optional(Schema.String),
|
||||
url: Schema.optional(Schema.String),
|
||||
revised_prompt: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
output_format: Schema.optional(Schema.String),
|
||||
usage: Schema.optional(
|
||||
Schema.Struct({
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
export interface ModelInput {
|
||||
readonly id: string
|
||||
readonly auth: AuthDefinition
|
||||
readonly baseURL?: string
|
||||
readonly headers?: Record<string, string>
|
||||
readonly defaults?: ImageModelDefaults
|
||||
}
|
||||
|
||||
const providerOptions = (request: ImageRequest): OpenAIImageOptions => ({
|
||||
...request.model.defaults?.providerOptions?.openai,
|
||||
...request.providerOptions?.openai,
|
||||
})
|
||||
|
||||
const body = (request: ImageRequest): OpenAIImageBody => {
|
||||
const options = providerOptions(request)
|
||||
return {
|
||||
model: request.model.id,
|
||||
prompt: request.prompt,
|
||||
n: request.count,
|
||||
size: request.size === undefined ? undefined : `${request.size.width}x${request.size.height}`,
|
||||
quality: options.quality,
|
||||
background: options.background,
|
||||
moderation: options.moderation,
|
||||
output_format: options.outputFormat,
|
||||
output_compression: options.outputCompression,
|
||||
}
|
||||
}
|
||||
|
||||
const invalidOutput = (message: string) =>
|
||||
new LLMError({
|
||||
module: ADAPTER,
|
||||
method: "generate",
|
||||
reason: new InvalidProviderOutputReason({ message, route: ADAPTER }),
|
||||
})
|
||||
|
||||
export const model = (input: ModelInput) => {
|
||||
const route: ImageRoute = {
|
||||
id: ADAPTER,
|
||||
generate: Effect.fn("OpenAIImages.generate")(function* (request: ImageRequest, execute) {
|
||||
if (request.aspectRatio !== undefined)
|
||||
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common aspectRatio option")
|
||||
if (request.seed !== undefined)
|
||||
return yield* ProviderShared.invalidRequest("OpenAI Images does not support the common seed option")
|
||||
|
||||
const requestBody = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenAIImageBody))(body(request))
|
||||
const text = Schema.encodeSync(Schema.fromJsonString(OpenAIImageBody))(requestBody)
|
||||
const url = `${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`
|
||||
const http = request.http ?? request.model.defaults?.http
|
||||
const headers = yield* Auth.toEffect(input.auth)({
|
||||
request,
|
||||
method: "POST",
|
||||
url,
|
||||
body: text,
|
||||
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
|
||||
})
|
||||
const response = yield* execute(
|
||||
HttpClientRequest.post(url).pipe(
|
||||
HttpClientRequest.setHeaders(headers),
|
||||
HttpClientRequest.bodyText(text, "application/json"),
|
||||
),
|
||||
)
|
||||
const payload = yield* response.json.pipe(
|
||||
Effect.mapError(() => invalidOutput("Failed to read the OpenAI Images response")),
|
||||
)
|
||||
const decoded = yield* Schema.decodeUnknownEffect(OpenAIImageResponse)(payload).pipe(
|
||||
Effect.mapError(() => invalidOutput("OpenAI Images returned an invalid response")),
|
||||
)
|
||||
const format = decoded.output_format ?? providerOptions(request).outputFormat ?? "png"
|
||||
const images = yield* Effect.forEach(decoded.data, (item, index) => {
|
||||
if (item.b64_json)
|
||||
return Effect.fromResult(Encoding.decodeBase64(item.b64_json)).pipe(
|
||||
Effect.mapError(() => invalidOutput(`OpenAI Images result ${index} contains invalid base64 data`)),
|
||||
Effect.map(
|
||||
(data) =>
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
),
|
||||
)
|
||||
if (item.url)
|
||||
return Effect.succeed(
|
||||
new GeneratedImage({
|
||||
mediaType: `image/${format}`,
|
||||
data: item.url,
|
||||
providerMetadata:
|
||||
item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } },
|
||||
}),
|
||||
)
|
||||
return Effect.fail(invalidOutput(`OpenAI Images result ${index} has neither image data nor a URL`))
|
||||
})
|
||||
if (images.length === 0) return yield* invalidOutput("OpenAI Images returned no images")
|
||||
return new ImageResponse({
|
||||
images,
|
||||
usage:
|
||||
decoded.usage === undefined
|
||||
? undefined
|
||||
: new Usage({
|
||||
inputTokens: decoded.usage.input_tokens,
|
||||
outputTokens: decoded.usage.output_tokens,
|
||||
totalTokens: decoded.usage.total_tokens,
|
||||
providerMetadata: { openai: decoded.usage },
|
||||
}),
|
||||
providerMetadata: { openai: { outputFormat: format } },
|
||||
})
|
||||
}),
|
||||
}
|
||||
return ImageModel.make({ id: input.id, provider: "openai", route, defaults: input.defaults })
|
||||
}
|
||||
|
||||
export const OpenAIImages = {
|
||||
model,
|
||||
} as const
|
||||
@@ -113,11 +113,24 @@ const OpenAIResponsesTool = Schema.Struct({
|
||||
parameters: JsonObject,
|
||||
strict: Schema.optional(Schema.Boolean),
|
||||
})
|
||||
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTool>
|
||||
const OpenAIResponsesImageGenerationTool = Schema.Struct({
|
||||
type: Schema.tag("image_generation"),
|
||||
action: Schema.optional(Schema.Literals(["auto", "generate", "edit"])),
|
||||
background: Schema.optional(Schema.Literals(["auto", "opaque", "transparent"])),
|
||||
input_fidelity: Schema.optional(Schema.Literals(["low", "high"])),
|
||||
output_compression: Schema.optional(Schema.Number),
|
||||
output_format: Schema.optional(Schema.Literals(["png", "jpeg", "webp"])),
|
||||
partial_images: Schema.optional(Schema.Number),
|
||||
quality: Schema.optional(Schema.Literals(["auto", "low", "medium", "high"])),
|
||||
size: Schema.optional(Schema.String),
|
||||
})
|
||||
const OpenAIResponsesTools = Schema.Union([OpenAIResponsesTool, OpenAIResponsesImageGenerationTool])
|
||||
type OpenAIResponsesTool = Schema.Schema.Type<typeof OpenAIResponsesTools>
|
||||
|
||||
const OpenAIResponsesToolChoice = Schema.Union([
|
||||
Schema.Literals(["auto", "none", "required"]),
|
||||
Schema.Struct({ type: Schema.tag("function"), name: Schema.String }),
|
||||
Schema.Struct({ type: Schema.tag("image_generation") }),
|
||||
])
|
||||
|
||||
// Fields shared between the HTTP body and the WebSocket `response.create`
|
||||
@@ -128,7 +141,7 @@ const OpenAIResponsesCoreFields = {
|
||||
model: Schema.String,
|
||||
input: Schema.Array(OpenAIResponsesInputItem),
|
||||
instructions: Schema.optional(Schema.String),
|
||||
tools: optionalArray(OpenAIResponsesTool),
|
||||
tools: optionalArray(OpenAIResponsesTools),
|
||||
tool_choice: Schema.optional(OpenAIResponsesToolChoice),
|
||||
store: Schema.optional(Schema.Boolean),
|
||||
service_tier: Schema.optional(OpenAIOptions.OpenAIServiceTier),
|
||||
@@ -194,6 +207,7 @@ const OpenAIResponsesStreamItem = Schema.Struct({
|
||||
outputs: Schema.optional(Schema.Unknown),
|
||||
server_label: Schema.optional(Schema.String),
|
||||
output: Schema.optional(Schema.Unknown),
|
||||
result: Schema.optional(Schema.String),
|
||||
error: Schema.optional(Schema.Unknown),
|
||||
encrypted_content: optionalNull(Schema.String),
|
||||
})
|
||||
@@ -258,21 +272,30 @@ const invalid = ProviderShared.invalidRequest
|
||||
// =============================================================================
|
||||
// Request Lowering
|
||||
// =============================================================================
|
||||
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): OpenAIResponsesTool => ({
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: ToolSchemaProjection.openAI(inputSchema),
|
||||
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
|
||||
strict: false,
|
||||
})
|
||||
const nativeImageTool = (tool: ToolDefinition) => {
|
||||
const native = tool.native?.openai
|
||||
return Schema.is(OpenAIResponsesImageGenerationTool)(native) ? native : undefined
|
||||
}
|
||||
|
||||
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
||||
const lowerTool = (tool: ToolDefinition, inputSchema: JsonSchema): OpenAIResponsesTool =>
|
||||
nativeImageTool(tool) ?? {
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: ToolSchemaProjection.openAI(inputSchema),
|
||||
// TODO: Read this from OpenAI-specific tool options so direct LLM callers can opt into strict schemas.
|
||||
strict: false,
|
||||
}
|
||||
|
||||
const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>, tools: ReadonlyArray<ToolDefinition>) =>
|
||||
ProviderShared.matchToolChoice("OpenAI Responses", toolChoice, {
|
||||
auto: () => "auto" as const,
|
||||
none: () => "none" as const,
|
||||
required: () => "required" as const,
|
||||
tool: (name) => ({ type: "function" as const, name }),
|
||||
tool: (name) =>
|
||||
tools.some((tool) => tool.name === name && nativeImageTool(tool) !== undefined)
|
||||
? ({ type: "image_generation" } as const)
|
||||
: { type: "function" as const, name },
|
||||
})
|
||||
|
||||
const lowerToolCall = (part: ToolCallPart): OpenAIResponsesInputItem => ({
|
||||
@@ -488,7 +511,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request:
|
||||
: request.tools.map((tool) =>
|
||||
lowerTool(tool, ToolSchemaProjection.modelCompatibility(tool.inputSchema, toolSchemaCompatibility)),
|
||||
),
|
||||
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined,
|
||||
tool_choice: request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined,
|
||||
stream: true as const,
|
||||
max_output_tokens: generation?.maxTokens,
|
||||
temperature: generation?.temperature,
|
||||
@@ -576,6 +599,11 @@ const isReasoningItem = (
|
||||
// outputs / sources / status without re-decoding.
|
||||
const hostedToolResult = (item: OpenAIResponsesStreamItem) => {
|
||||
const isError = typeof item.error !== "undefined" && item.error !== null
|
||||
if (item.type === "image_generation_call" && item.result)
|
||||
return {
|
||||
type: "content" as const,
|
||||
value: [{ type: "file" as const, uri: `data:image/png;base64,${item.result}`, mime: "image/png" }],
|
||||
}
|
||||
return isError ? { type: "error" as const, value: item.error } : { type: "json" as const, value: item }
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
|
||||
import type { Route, RouteDefaultsInput } from "../route/client"
|
||||
import type { ProviderPackage } from "../provider-package"
|
||||
import { ProviderID, type ModelID } from "../schema"
|
||||
import { HttpOptions, ProviderID, ToolDefinition, type ModelID } from "../schema"
|
||||
import * as OpenAIChat from "../protocols/openai-chat"
|
||||
import * as OpenAIResponses from "../protocols/openai-responses"
|
||||
import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options"
|
||||
import { OpenAIImages, type OpenAIImageOptions } from "../protocols/openai-images"
|
||||
|
||||
export type { OpenAIOptionsInput, OpenAIResponseIncludable } from "./openai-options"
|
||||
export type { OpenAIImageOptions } from "../protocols/openai-images"
|
||||
|
||||
export const id = ProviderID.make("openai")
|
||||
|
||||
@@ -22,6 +24,41 @@ export type Config = RouteDefaultsInput &
|
||||
readonly providerOptions?: OpenAIProviderOptionsInput
|
||||
}
|
||||
|
||||
export interface ImageConfig {
|
||||
readonly providerOptions?: OpenAIImageOptions
|
||||
}
|
||||
|
||||
export interface ImageGenerationOptions {
|
||||
readonly action?: "auto" | "generate" | "edit"
|
||||
readonly background?: "auto" | "opaque" | "transparent"
|
||||
readonly inputFidelity?: "low" | "high"
|
||||
readonly outputCompression?: number
|
||||
readonly outputFormat?: "png" | "jpeg" | "webp"
|
||||
readonly partialImages?: number
|
||||
readonly quality?: "auto" | "low" | "medium" | "high"
|
||||
readonly size?: string
|
||||
}
|
||||
|
||||
export const imageGeneration = (options: ImageGenerationOptions = {}) =>
|
||||
ToolDefinition.make({
|
||||
name: "image_generation",
|
||||
description: "Generate or edit an image using OpenAI's hosted image generation tool.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
native: {
|
||||
openai: {
|
||||
type: "image_generation",
|
||||
action: options.action,
|
||||
background: options.background,
|
||||
input_fidelity: options.inputFidelity,
|
||||
output_compression: options.outputCompression,
|
||||
output_format: options.outputFormat,
|
||||
partial_images: options.partialImages,
|
||||
quality: options.quality,
|
||||
size: options.size,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export interface Settings extends ProviderPackage.Settings {
|
||||
readonly apiKey?: string
|
||||
readonly baseURL?: string
|
||||
@@ -55,6 +92,17 @@ export const configure = (input: Config = {}) => {
|
||||
const responsesWebSocket = (id: string | ModelID) =>
|
||||
responsesWebSocketRoute.with(withOpenAIOptions(id, modelDefaults, { textVerbosity: true })).model({ id })
|
||||
const chat = (id: string | ModelID) => chatRoute.with(withOpenAIOptions(id, modelDefaults)).model({ id })
|
||||
const image = (modelID: string | ModelID, options: ImageConfig = {}) =>
|
||||
OpenAIImages.model({
|
||||
id: modelID,
|
||||
auth: auth(input),
|
||||
baseURL: input.baseURL,
|
||||
headers: input.headers,
|
||||
defaults: {
|
||||
providerOptions: options.providerOptions === undefined ? undefined : { openai: { ...options.providerOptions } },
|
||||
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
id,
|
||||
@@ -62,6 +110,7 @@ export const configure = (input: Config = {}) => {
|
||||
responses,
|
||||
responsesWebSocket,
|
||||
chat,
|
||||
image,
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -97,3 +146,4 @@ export const chatModel: ProviderPackage.Definition<Settings>["model"] = (modelID
|
||||
export const responses = provider.responses
|
||||
export const responsesWebSocket = provider.responsesWebSocket
|
||||
export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Config, Effect, Redacted } from "effect"
|
||||
import { Headers } from "effect/unstable/http"
|
||||
import { AuthenticationReason, InvalidRequestReason, LLMError, type LLMRequest } from "../schema"
|
||||
import { AuthenticationReason, InvalidRequestReason, LLMError, type HttpOptions } from "../schema"
|
||||
|
||||
export class MissingCredentialError extends Error {
|
||||
readonly _tag = "MissingCredentialError"
|
||||
@@ -15,7 +15,7 @@ export type AuthError = CredentialError | LLMError
|
||||
type Secret = string | Redacted.Redacted | Config.Config<string | Redacted.Redacted>
|
||||
|
||||
export interface AuthInput {
|
||||
readonly request: LLMRequest
|
||||
readonly request: { readonly http?: HttpOptions }
|
||||
readonly method: "POST" | "GET"
|
||||
readonly url: string
|
||||
readonly body: string
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientRequest } from "effect/unstable/http"
|
||||
import { Image, ImageClient } from "../src"
|
||||
import { OpenAI } from "../src/providers"
|
||||
import { it } from "./lib/effect"
|
||||
import { dynamicResponse } from "./lib/http"
|
||||
|
||||
describe("Image", () => {
|
||||
it.effect("generates images through the OpenAI Images API", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: OpenAI.configure({ apiKey: "test", baseURL: "https://api.openai.test/v1" }).image("gpt-image-2"),
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
count: 2,
|
||||
size: { width: 1024, height: 1024 },
|
||||
providerOptions: {
|
||||
openai: { quality: "high", outputFormat: "webp" },
|
||||
},
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(2)
|
||||
expect(response.image?.mediaType).toBe("image/webp")
|
||||
expect(response.image?.data).toEqual(Uint8Array.from([1, 2, 3]))
|
||||
expect(response.image?.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } })
|
||||
expect(response.usage?.totalTokens).toBe(12)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
expect(request.url).toBe("https://api.openai.test/v1/images/generations")
|
||||
expect(request.headers.get("authorization")).toBe("Bearer test")
|
||||
expect(JSON.parse(input.text)).toEqual({
|
||||
model: "gpt-image-2",
|
||||
prompt: "A robot tending a rooftop garden",
|
||||
n: 2,
|
||||
size: "1024x1024",
|
||||
quality: "high",
|
||||
output_format: "webp",
|
||||
})
|
||||
return input.respond(
|
||||
JSON.stringify({
|
||||
data: [{ b64_json: "AQID", revised_prompt: "A precise robot" }, { b64_json: "BAUG" }],
|
||||
output_format: "webp",
|
||||
usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 },
|
||||
}),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
@@ -58,6 +58,24 @@ describe("OpenAI Responses route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers the hosted OpenAI image generation tool", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* LLMClient.prepare<OpenAIResponses.OpenAIResponsesBody>(
|
||||
LLM.request({
|
||||
model,
|
||||
prompt: "Show me a rooftop garden.",
|
||||
tools: [OpenAI.imageGeneration({ action: "generate", quality: "high", size: "1024x1024" })],
|
||||
toolChoice: "image_generation",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body.tools).toEqual([
|
||||
{ type: "image_generation", action: "generate", quality: "high", size: "1024x1024" },
|
||||
])
|
||||
expect(prepared.body.tool_choice).toEqual({ type: "image_generation" })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers semantic service tier options", () =>
|
||||
Effect.gen(function* () {
|
||||
const input = LLM.updateRequest(request, { providerOptions: { openai: { serviceTier: "priority" } } })
|
||||
@@ -1361,6 +1379,37 @@ describe("OpenAI Responses route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes image generation output as image content", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
type: "image_generation_call",
|
||||
id: "ig_1",
|
||||
status: "completed",
|
||||
result: "AQID",
|
||||
}
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{ type: "response.output_item.done", item },
|
||||
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.events.find(LLMEvent.is.toolResult)).toMatchObject({
|
||||
id: "ig_1",
|
||||
name: "image_generation",
|
||||
providerExecuted: true,
|
||||
result: {
|
||||
type: "content",
|
||||
value: [{ type: "file", uri: "data:image/png;base64,AQID", mime: "image/png" }],
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("decodes code_interpreter_call as provider-executed events with code input", () =>
|
||||
Effect.gen(function* () {
|
||||
const item = {
|
||||
|
||||
Reference in New Issue
Block a user