From c99e278e28cc53b583dca733a33369f6170ed566 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Wed, 6 May 2026 13:28:49 -0400 Subject: [PATCH] refactor(llm): tighten helper and diagnostic surfaces --- packages/llm/example/tutorial.ts | 42 ++- packages/llm/src/adapter/auth.ts | 17 +- packages/llm/src/adapter/client.ts | 20 +- packages/llm/src/adapter/executor.ts | 142 +++++++++- packages/llm/src/llm.ts | 66 +---- packages/llm/src/protocols/openai-chat.ts | 45 +++- .../llm/src/protocols/openai-responses.ts | 39 ++- packages/llm/src/providers/azure.ts | 34 ++- packages/llm/src/schema.ts | 106 +++++++- packages/llm/test/executor.test.ts | 108 ++++++++ packages/llm/test/lib/llm-client.ts | 18 -- packages/llm/test/llm.test.ts | 8 +- .../anthropic-messages.recorded.test.ts | 3 +- .../test/provider/anthropic-messages.test.ts | 33 ++- .../test/provider/bedrock-converse.test.ts | 41 ++- .../llm/test/provider/gemini.recorded.test.ts | 12 +- packages/llm/test/provider/gemini.test.ts | 251 ++++++++++-------- .../openai-chat-tool-loop.recorded.test.ts | 4 +- .../provider/openai-chat.recorded.test.ts | 3 +- .../llm/test/provider/openai-chat.test.ts | 39 ++- .../openai-compatible-chat.recorded.test.ts | 13 +- .../provider/openai-compatible-chat.test.ts | 13 +- .../openai-responses.recorded.test.ts | 5 +- .../test/provider/openai-responses.test.ts | 33 ++- packages/llm/test/provider/openrouter.test.ts | 5 +- packages/llm/test/recorded-scenarios.ts | 6 +- packages/llm/test/tool-runtime.test.ts | 10 +- packages/opencode/src/provider/llm-bridge.ts | 9 +- .../opencode/test/provider/llm-bridge.test.ts | 2 + .../opencode/test/session/llm-native.test.ts | 2 + 30 files changed, 729 insertions(+), 400 deletions(-) create mode 100644 packages/llm/test/executor.test.ts delete mode 100644 packages/llm/test/lib/llm-client.ts diff --git a/packages/llm/example/tutorial.ts b/packages/llm/example/tutorial.ts index e93545e55e..be66ae39ff 100644 --- a/packages/llm/example/tutorial.ts +++ b/packages/llm/example/tutorial.ts @@ -104,24 +104,26 @@ const tools = { const streamWithTools = Effect.gen(function* () { const runtime = yield* ToolRuntime.Service - return yield* runtime.run({ - request: LLM.request({ - model, - prompt: "Use get_weather for San Francisco, then answer in one sentence.", - generation: { maxTokens: 80, temperature: 0 }, - }), - tools, - maxSteps: 3, - }).pipe( - Stream.tap((event) => - Effect.sync(() => { - if (event.type === "tool-call") console.log("tool call", event.name, event.input) - if (event.type === "tool-result") console.log("tool result", event.name, event.result) - if (event.type === "text-delta") process.stdout.write(event.text) + return yield* runtime + .run({ + request: LLM.request({ + model, + prompt: "Use get_weather for San Francisco, then answer in one sentence.", + generation: { maxTokens: 80, temperature: 0 }, }), - ), - Stream.runDrain, - ) + tools, + maxSteps: 3, + }) + .pipe( + Stream.tap((event) => + Effect.sync(() => { + if (event.type === "tool-call") console.log("tool call", event.name, event.input) + if (event.type === "tool-result") console.log("tool result", event.name, event.result) + if (event.type === "text-delta") process.stdout.write(event.text) + }), + ), + Stream.runDrain, + ) }) // ----------------------------------------------------------------------------- @@ -207,11 +209,7 @@ const program = Effect.gen(function* () { yield* streamWithTools }).pipe( Effect.provide( - Layer.mergeAll( - requestExecutorLayer, - llmClientLayer, - ToolRuntime.layer.pipe(Layer.provide(llmClientLayer)), - ), + Layer.mergeAll(requestExecutorLayer, llmClientLayer, ToolRuntime.layer.pipe(Layer.provide(llmClientLayer))), ), ) diff --git a/packages/llm/src/adapter/auth.ts b/packages/llm/src/adapter/auth.ts index 3c52986f2a..023f5fb2ec 100644 --- a/packages/llm/src/adapter/auth.ts +++ b/packages/llm/src/adapter/auth.ts @@ -11,8 +11,8 @@ import type { LLMError, LLMRequest } from "../schema" * Most adapters use the default `Auth.bearer`, which reads * `request.model.apiKey` and sets `Authorization: Bearer ...`. Providers * that use a different header pick `Auth.apiKeyHeader(name)` (e.g. - * Anthropic's `x-api-key`, Gemini's `x-goog-api-key`) or a provider-aware - * helper such as `Auth.openAI` for Azure OpenAI's static `api-key` header. + * Anthropic's `x-api-key`, Gemini's `x-goog-api-key`, Azure OpenAI's + * `api-key`). * * Adapters that need per-request signing (AWS SigV4, future Vertex IAM, * future Azure AAD) implement `Auth` as a function that hashes the body, @@ -56,19 +56,6 @@ const fromApiKey = */ export const bearer: Auth = fromApiKey((key) => ({ authorization: `Bearer ${key}` })) -/** - * OpenAI-compatible auth with Azure OpenAI's static API-key exception. Azure - * Entra/OAuth callers can still pre-set `authorization` and omit `apiKey`. - */ -export const openAI: Auth = ({ request, headers }) => { - const key = request.model.apiKey - if (!key) return Effect.succeed(headers) - if (request.model.provider === "azure") { - return Effect.succeed(Headers.set(Headers.remove(headers, "authorization"), "api-key", key)) - } - return Effect.succeed(Headers.set(headers, "authorization", `Bearer ${key}`)) -} - /** * Set a custom header to `request.model.apiKey`. No-op when `model.apiKey` * is unset. Used by Anthropic (`x-api-key`) and Gemini (`x-goog-api-key`). diff --git a/packages/llm/src/adapter/client.ts b/packages/llm/src/adapter/client.ts index 0689921fa1..91c9110a20 100644 --- a/packages/llm/src/adapter/client.ts +++ b/packages/llm/src/adapter/client.ts @@ -342,7 +342,7 @@ const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) { } }) -const prepare = Effect.fn("LLMClient.prepare")(function* (request: LLMRequest) { +const prepareWith = Effect.fn("LLMClient.prepare")(function* (request: LLMRequest) { const compiled = yield* compile(request) return new PreparedRequest({ @@ -378,11 +378,24 @@ const generateWith = (stream: Interface["stream"]) => Effect.fn("LLM.generate")( ) }) +export const prepare = (request: LLMRequest) => + prepareWith(request) as Effect.Effect, LLMError> + +export const stream = (request: LLMRequest) => + Stream.unwrap(Effect.gen(function* () { + return (yield* Service).stream(request) + })) + +export const generate = (request: LLMRequest) => + Effect.gen(function* () { + return yield* (yield* Service).generate(request) + }) + export const layer: Layer.Layer = Layer.effect( Service, Effect.gen(function* () { const stream = streamWith(yield* RequestExecutor.Service) - return Service.of({ prepare: prepare as Interface["prepare"], stream, generate: generateWith(stream) }) + return Service.of({ prepare: prepareWith as Interface["prepare"], stream, generate: generateWith(stream) }) }), ) @@ -391,4 +404,7 @@ export const Adapter = { make, model } as const export const LLMClient = { Service, layer, + prepare, + stream, + generate, } as const diff --git a/packages/llm/src/adapter/executor.ts b/packages/llm/src/adapter/executor.ts index c5809e698e..21a0163a76 100644 --- a/packages/llm/src/adapter/executor.ts +++ b/packages/llm/src/adapter/executor.ts @@ -1,12 +1,19 @@ import { Cause, Context, Effect, Layer } from "effect" import { FetchHttpClient, + Headers, HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse, } from "effect/unstable/http" -import { ProviderRequestError, TransportError, type LLMError } from "../schema" +import { + HttpRequestDetails, + HttpResponseDetails, + ProviderRequestError, + TransportError, + type LLMError, +} from "../schema" export interface Interface { readonly execute: ( @@ -16,41 +23,148 @@ export interface Interface { export class Service extends Context.Service()("@opencode/LLM/RequestExecutor") {} -const statusError = (response: HttpClientResponse.HttpClientResponse) => - Effect.gen(function* () { - if (response.status < 400) return response - const body = yield* response.text.pipe(Effect.catch(() => Effect.succeed(undefined))) - return yield* new ProviderRequestError({ - status: response.status, - message: `Provider request failed with HTTP ${response.status}`, - body, - }) +const BODY_LIMIT = 16_384 +const MAX_RETRIES = 2 +const MAX_DELAY_MS = 10_000 +const REDACTED = "" + +const sensitiveName = (name: string) => + /authorization|api[-_]?key|token|secret|credential|signature|x-amz-signature/i.test(name) + +const redactHeaders = (headers: Headers.Headers) => + Object.fromEntries( + Object.entries(headers).map(([name, value]) => [ + name, + sensitiveName(name) ? REDACTED : value, + ]), + ) + +const redactUrl = (value: string) => { + if (!URL.canParse(value)) return REDACTED + const url = new URL(value) + url.searchParams.forEach((_, key) => { + if (sensitiveName(key)) url.searchParams.set(key, REDACTED) + }) + return url.toString() +} + +const normalizedHeaders = (headers: Headers.Headers) => + Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value])) + +const requestId = (headers: Record) => { + return headers["x-request-id"] ?? + headers["request-id"] ?? + headers["x-amzn-requestid"] ?? + headers["x-amz-request-id"] ?? + headers["x-goog-request-id"] ?? + headers["cf-ray"] +} + +const retryableStatus = (status: number) => status === 429 || status === 503 || status === 504 || status === 529 + +const retryAfterMs = (headers: Record) => { + const millis = Number(headers["retry-after-ms"]) + if (Number.isFinite(millis)) return Math.max(0, millis) + + const value = headers["retry-after"] + if (!value) return undefined + + const seconds = Number(value) + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000) + + const date = Date.parse(value) + if (!Number.isNaN(date)) return Math.max(0, date - Date.now()) + return undefined +} + +const requestDetails = (request: HttpClientRequest.HttpClientRequest) => + new HttpRequestDetails({ + method: request.method, + url: redactUrl(request.url), + headers: redactHeaders(request.headers), }) +const responseDetails = (response: HttpClientResponse.HttpClientResponse) => + new HttpResponseDetails({ + status: response.status, + headers: redactHeaders(response.headers), + }) + +const responseBody = (body: string | void) => { + if (body === undefined) return {} + if (body.length <= BODY_LIMIT) return { body } + return { body: body.slice(0, BODY_LIMIT), bodyTruncated: true } +} + +const statusError = (request: HttpClientRequest.HttpClientRequest) => + (response: HttpClientResponse.HttpClientResponse) => + Effect.gen(function* () { + if (response.status < 400) return response + const body = yield* response.text.pipe(Effect.catch(() => Effect.void)) + const headers = normalizedHeaders(response.headers) + const retryable = retryableStatus(response.status) + return yield* new ProviderRequestError({ + status: response.status, + message: `Provider request failed with HTTP ${response.status}`, + ...responseBody(body), + retryable, + retryAfterMs: retryAfterMs(headers), + requestId: requestId(headers), + request: requestDetails(request), + response: responseDetails(response), + }) + }) + const toHttpError = (error: unknown) => { - if (Cause.isTimeoutError(error)) return new TransportError({ message: error.message, reason: "Timeout" }) - if (!HttpClientError.isHttpClientError(error)) return new TransportError({ message: "HTTP transport failed" }) - const url = "request" in error ? error.request.url : undefined + if (Cause.isTimeoutError(error)) { + return new TransportError({ message: error.message, reason: "Timeout", retryable: false }) + } + if (!HttpClientError.isHttpClientError(error)) { + return new TransportError({ message: "HTTP transport failed", retryable: false }) + } + const request = "request" in error ? error.request : undefined + const url = request ? redactUrl(request.url) : undefined if (error.reason._tag === "TransportError") { return new TransportError({ message: error.reason.description ?? "HTTP transport failed", reason: error.reason._tag, url, + retryable: false, + request: request ? requestDetails(request) : undefined, }) } return new TransportError({ message: `HTTP transport failed: ${error.reason._tag}`, reason: error.reason._tag, url, + retryable: false, + request: request ? requestDetails(request) : undefined, }) } +const retryDelay = (error: ProviderRequestError) => Math.min(error.retryAfterMs ?? 500, MAX_DELAY_MS) + +const retryStatusFailures = ( + effect: Effect.Effect, + retries = MAX_RETRIES, +): Effect.Effect => + Effect.catchTag( + effect, + "LLM.ProviderRequestError", + (error): Effect.Effect => { + if (!error.retryable || retries <= 0) return Effect.fail(error) + return Effect.sleep(retryDelay(error)).pipe(Effect.flatMap(() => retryStatusFailures(effect, retries - 1))) + }, + ) + export const layer: Layer.Layer = Layer.effect( Service, Effect.gen(function* () { const http = yield* HttpClient.HttpClient + const executeOnce = (request: HttpClientRequest.HttpClientRequest) => + http.execute(request).pipe(Effect.mapError(toHttpError), Effect.flatMap(statusError(request))) return Service.of({ - execute: (request) => http.execute(request).pipe(Effect.mapError(toHttpError), Effect.flatMap(statusError)), + execute: (request) => retryStatusFailures(executeOnce(request)), }) }), ) diff --git a/packages/llm/src/llm.ts b/packages/llm/src/llm.ts index 0ad4df60f4..bca1b0be74 100644 --- a/packages/llm/src/llm.ts +++ b/packages/llm/src/llm.ts @@ -8,9 +8,7 @@ import { import { GenerationOptions, HttpOptions, - LLMEvent, LLMRequest, - LLMResponse, Message, ToolChoice, ToolDefinition, @@ -26,11 +24,12 @@ export type ModelInput = ModelRefInput export type MessageInput = Message.Input -export type ToolChoiceInput = ToolChoice | ConstructorParameters[0] | ToolDefinition | string -export type ToolChoiceMode = Exclude +export type ToolChoiceInput = ToolChoice.Input +export type ToolChoiceMode = ToolChoice.Mode export type ToolResultInput = Parameters[0] +/** Input accepted by `LLM.request`, normalized into the canonical `LLMRequest` class. */ export type RequestInput = Omit< ConstructorParameters[0], "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" | "providerOptions" @@ -38,11 +37,11 @@ export type RequestInput = Omit< readonly system?: string | SystemPart | ReadonlyArray readonly prompt?: string | ContentPart | ReadonlyArray readonly messages?: ReadonlyArray - readonly tools?: ReadonlyArray[0]> + readonly tools?: ReadonlyArray readonly toolChoice?: ToolChoiceInput - readonly generation?: GenerationOptions | ConstructorParameters[0] + readonly generation?: GenerationOptions.Input readonly providerOptions?: ConstructorParameters[0]["providerOptions"] - readonly http?: HttpOptions | ConstructorParameters[0] + readonly http?: HttpOptions.Input } export const capabilities = modelCapabilities @@ -66,10 +65,7 @@ export const assistant = Message.assistant export const model = modelRef -export const toolDefinition = (input: ToolDefinition | ConstructorParameters[0]) => { - if (input instanceof ToolDefinition) return input - return new ToolDefinition(input) -} +export const toolDefinition = ToolDefinition.make export const toolCall = ToolCallPart.make @@ -77,28 +73,11 @@ export const toolResult = ToolResultPart.make export const toolMessage = Message.tool -export const toolChoiceName = (name: string) => new ToolChoice({ type: "tool", name }) +export const toolChoiceName = ToolChoice.named -const isToolChoiceMode = (value: string): value is ToolChoiceMode => - value === "auto" || value === "none" || value === "required" +export const toolChoice = ToolChoice.make -export const toolChoice = (input: ToolChoiceInput) => { - if (input instanceof ToolChoice) return input - if (input instanceof ToolDefinition) return new ToolChoice({ type: "tool", name: input.name }) - if (typeof input === "string") - return isToolChoiceMode(input) ? new ToolChoice({ type: input }) : toolChoiceName(input) - return new ToolChoice(input) -} - -export const generation = (input: GenerationOptions | ConstructorParameters[0] = {}) => { - if (input instanceof GenerationOptions) return input - return new GenerationOptions(input) -} - -const http = (input: HttpOptions | ConstructorParameters[0] | undefined) => { - if (input === undefined || input instanceof HttpOptions) return input - return new HttpOptions(input) -} +export const generation = GenerationOptions.make export const requestInput = (input: LLMRequest): RequestInput => ({ ...LLMRequest.input(input), @@ -124,32 +103,9 @@ export const request = (input: RequestInput) => { toolChoice: requestToolChoice ? toolChoice(requestToolChoice) : undefined, generation: requestGeneration === undefined ? undefined : generation(requestGeneration), providerOptions: requestProviderOptions, - http: http(requestHttp), + http: requestHttp === undefined ? undefined : HttpOptions.make(requestHttp), }) } export const updateRequest = (input: LLMRequest, patch: Partial) => request({ ...requestInput(input), ...patch }) - -export const outputText = (response: LLMResponse | { readonly events: ReadonlyArray }) => - response.events - .filter(LLMEvent.is.textDelta) - .map((event) => event.text) - .join("") - -export const outputUsage = (response: LLMResponse | { readonly events: ReadonlyArray }) => { - if (response instanceof LLMResponse) return response.usage - return response.events.reduce( - (usage, event) => ("usage" in event && event.usage !== undefined ? event.usage : usage), - undefined, - ) -} - -export const outputToolCalls = (response: LLMResponse | { readonly events: ReadonlyArray }) => - response.events.filter(LLMEvent.is.toolCall) - -export const outputReasoning = (response: LLMResponse | { readonly events: ReadonlyArray }) => - response.events - .filter(LLMEvent.is.reasoningDelta) - .map((event) => event.text) - .join("") diff --git a/packages/llm/src/protocols/openai-chat.ts b/packages/llm/src/protocols/openai-chat.ts index b28f44fe1e..f10ad73007 100644 --- a/packages/llm/src/protocols/openai-chat.ts +++ b/packages/llm/src/protocols/openai-chat.ts @@ -1,7 +1,7 @@ import { Array as Arr, Effect, Schema } from "effect" import { Adapter, type AdapterModelInput } from "../adapter/client" -import { Auth } from "../adapter/auth" -import { Endpoint } from "../adapter/endpoint" +import type { Auth } from "../adapter/auth" +import { Endpoint, type Endpoint as EndpointConfig } from "../adapter/endpoint" import { Framing } from "../adapter/framing" import { capabilities } from "../llm" import { Protocol } from "../adapter/protocol" @@ -19,6 +19,8 @@ import { OpenAIOptions } from "./utils/openai-options" import { ToolStream } from "./utils/tool-stream" const ADAPTER = "openai-chat" +const DEFAULT_BASE_URL = "https://api.openai.com/v1" +const PATH = "/chat/completions" // ============================================================================= // Public Model Input @@ -373,16 +375,35 @@ export const protocol = Protocol.define({ onHalt: finishEvents, }) -export const adapter = Adapter.make({ - id: ADAPTER, - protocol, - // The adapter supplies deployment concerns around the protocol: URL, auth, - // and response framing. Other providers can reuse `protocol` with different - // endpoint/auth choices instead of cloning this whole file. - endpoint: Endpoint.baseURL({ default: "https://api.openai.com/v1", path: "/chat/completions" }), - auth: Auth.openAI, - framing: Framing.sse, -}) +export const endpoint = (input: { + readonly defaultBaseURL?: string | false + readonly required?: string +} = {}) => + Endpoint.baseURL({ + default: input.defaultBaseURL === false ? undefined : input.defaultBaseURL ?? DEFAULT_BASE_URL, + path: PATH, + required: input.required, + }) + +export const makeAdapter = (input: { + readonly id?: string + readonly auth?: Auth + readonly endpoint?: EndpointConfig + readonly defaultBaseURL?: string | false + readonly endpointRequired?: string +} = {}) => + Adapter.make({ + id: input.id ?? ADAPTER, + protocol, + // The adapter supplies deployment concerns around the protocol: URL, auth, + // and response framing. Other providers can reuse `protocol` with different + // endpoint/auth choices instead of cloning this whole file. + endpoint: input.endpoint ?? endpoint({ defaultBaseURL: input.defaultBaseURL, required: input.endpointRequired }), + auth: input.auth, + framing: Framing.sse, + }) + +export const adapter = makeAdapter() // ============================================================================= // Model Helper diff --git a/packages/llm/src/protocols/openai-responses.ts b/packages/llm/src/protocols/openai-responses.ts index b22ac5a1d1..89f7a57b28 100644 --- a/packages/llm/src/protocols/openai-responses.ts +++ b/packages/llm/src/protocols/openai-responses.ts @@ -1,7 +1,7 @@ import { Effect, Schema } from "effect" import { Adapter, type AdapterModelInput } from "../adapter/client" -import { Auth } from "../adapter/auth" -import { Endpoint } from "../adapter/endpoint" +import type { Auth } from "../adapter/auth" +import { Endpoint, type Endpoint as EndpointConfig } from "../adapter/endpoint" import { Framing } from "../adapter/framing" import { capabilities } from "../llm" import { Protocol } from "../adapter/protocol" @@ -19,6 +19,8 @@ import { OpenAIOptions } from "./utils/openai-options" import { ToolStream } from "./utils/tool-stream" const ADAPTER = "openai-responses" +const DEFAULT_BASE_URL = "https://api.openai.com/v1" +const PATH = "/responses" // ============================================================================= // Public Model Input @@ -401,13 +403,32 @@ export const protocol = Protocol.define({ process: processChunk, }) -export const adapter = Adapter.make({ - id: ADAPTER, - protocol, - endpoint: Endpoint.baseURL({ default: "https://api.openai.com/v1", path: "/responses" }), - auth: Auth.openAI, - framing: Framing.sse, -}) +export const endpoint = (input: { + readonly defaultBaseURL?: string | false + readonly required?: string +} = {}) => + Endpoint.baseURL({ + default: input.defaultBaseURL === false ? undefined : input.defaultBaseURL ?? DEFAULT_BASE_URL, + path: PATH, + required: input.required, + }) + +export const makeAdapter = (input: { + readonly id?: string + readonly auth?: Auth + readonly endpoint?: EndpointConfig + readonly defaultBaseURL?: string | false + readonly endpointRequired?: string +} = {}) => + Adapter.make({ + id: input.id ?? ADAPTER, + protocol, + endpoint: input.endpoint ?? endpoint({ defaultBaseURL: input.defaultBaseURL, required: input.endpointRequired }), + auth: input.auth, + framing: Framing.sse, + }) + +export const adapter = makeAdapter() // ============================================================================= // Model Helper diff --git a/packages/llm/src/providers/azure.ts b/packages/llm/src/providers/azure.ts index da11b1127b..762f34d327 100644 --- a/packages/llm/src/providers/azure.ts +++ b/packages/llm/src/providers/azure.ts @@ -1,3 +1,6 @@ +import { Headers } from "effect/unstable/http" +import { Auth } from "../adapter/auth" +import type { Auth as AuthFn } from "../adapter/auth" import { Adapter } from "../adapter/client" import type { ModelInput } from "../llm" import { ProviderID } from "../schema" @@ -6,6 +9,9 @@ import * as OpenAIResponses from "../protocols/openai-responses" import { withOpenAIOptions, type OpenAIProviderOptionsInput } from "./openai-options" export const id = ProviderID.make("azure") +const MISSING_BASE_URL = "Azure OpenAI requires resourceName or baseURL" +const apiKeyAuth = Auth.apiKeyHeader("api-key") +const auth: AuthFn = (input) => apiKeyAuth({ ...input, headers: Headers.remove(input.headers, "authorization") }) export type ModelOptions = Omit & { readonly resourceName?: string @@ -21,7 +27,21 @@ const resourceBaseURL = (resourceName: string | undefined) => { return `https://${resource}.openai.azure.com/openai/v1` } -export const adapters = [OpenAIResponses.adapter, OpenAIChat.adapter] +const responsesAdapter = OpenAIResponses.makeAdapter({ + id: "azure-openai-responses", + auth, + defaultBaseURL: false, + endpointRequired: MISSING_BASE_URL, +}) + +const chatAdapter = OpenAIChat.makeAdapter({ + id: "azure-openai-chat", + auth, + defaultBaseURL: false, + endpointRequired: MISSING_BASE_URL, +}) + +export const adapters = [responsesAdapter, chatAdapter] const mapInput = (input: AzureModelInput) => { const { apiVersion, resourceName, useCompletionUrls, ...rest } = input @@ -35,10 +55,14 @@ const mapInput = (input: AzureModelInput) => { } } -const chatModel = Adapter.model(OpenAIChat.adapter, { provider: id }, { mapInput }) -const responsesModel = Adapter.model(OpenAIResponses.adapter, { provider: id }, { mapInput }) +const chatModel = Adapter.model(chatAdapter, { provider: id }, { mapInput }) +const responsesModel = Adapter.model(responsesAdapter, { provider: id }, { mapInput }) + +export const responses = (modelID: string, options: ModelOptions = {}) => responsesModel({ ...options, id: modelID }) + +export const chat = (modelID: string, options: ModelOptions = {}) => chatModel({ ...options, id: modelID }) export const model = (modelID: string, options: ModelOptions = {}) => { - const create = options.useCompletionUrls === true ? chatModel : responsesModel - return create({ ...options, id: modelID }) + if (options.useCompletionUrls === true) return chat(modelID, options) + return responses(modelID, options) } diff --git a/packages/llm/src/schema.ts b/packages/llm/src/schema.ts index b20e200278..ccb3e26e7f 100644 --- a/packages/llm/src/schema.ts +++ b/packages/llm/src/schema.ts @@ -83,6 +83,13 @@ export class HttpOptions extends Schema.Class("LLM.HttpOptions")({ query: Schema.optional(Schema.Record(Schema.String, Schema.String)), }) {} +export namespace HttpOptions { + export type Input = HttpOptions | ConstructorParameters[0] + + /** Normalize HTTP option input into the canonical `HttpOptions` class. */ + export const make = (input: Input) => input instanceof HttpOptions ? input : new HttpOptions(input) +} + export const mergeHttpOptions = (...items: ReadonlyArray): HttpOptions | undefined => { const body = mergeJsonRecords(...items.map((item) => item?.body)) const headers = mergeStringRecords(...items.map((item) => item?.headers)) @@ -102,6 +109,13 @@ export class GenerationOptions extends Schema.Class("LLM.Gene stop: Schema.optional(Schema.Array(Schema.String)), }) {} +export namespace GenerationOptions { + export type Input = GenerationOptions | ConstructorParameters[0] + + /** Normalize generation option input into the canonical `GenerationOptions` class. */ + export const make = (input: Input = {}) => input instanceof GenerationOptions ? input : new GenerationOptions(input) +} + export type GenerationOptionsFields = { readonly maxTokens?: number readonly temperature?: number @@ -363,11 +377,37 @@ export class ToolDefinition extends Schema.Class("LLM.ToolDefini native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} +export namespace ToolDefinition { + export type Input = ToolDefinition | ConstructorParameters[0] + + /** Normalize tool definition input into the canonical `ToolDefinition` class. */ + export const make = (input: Input) => input instanceof ToolDefinition ? input : new ToolDefinition(input) +} + export class ToolChoice extends Schema.Class("LLM.ToolChoice")({ type: Schema.Literals(["auto", "none", "required", "tool"]), name: Schema.optional(Schema.String), }) {} +export namespace ToolChoice { + export type Mode = Exclude + export type Input = ToolChoice | ConstructorParameters[0] | ToolDefinition | string + + const isMode = (value: string): value is Mode => + value === "auto" || value === "none" || value === "required" + + /** Select a specific named tool. */ + export const named = (value: string) => new ToolChoice({ type: "tool", name: value }) + + /** Normalize ergonomic tool-choice inputs into the canonical `ToolChoice` class. */ + export const make = (input: Input) => { + if (input instanceof ToolChoice) return input + if (input instanceof ToolDefinition) return named(input.name) + if (typeof input === "string") return isMode(input) ? new ToolChoice({ type: input }) : named(input) + return new ToolChoice(input) + } +} + export const ResponseFormat = Schema.Union([ Schema.Struct({ type: Schema.Literal("text") }), Schema.Struct({ type: Schema.Literal("json"), schema: JsonSchema }), @@ -583,29 +623,60 @@ export type PreparedRequestOf = Omit & { readonly payload: Payload } +const responseText = (events: ReadonlyArray) => + events + .filter(LLMEvent.is.textDelta) + .map((event) => event.text) + .join("") + +const responseReasoning = (events: ReadonlyArray) => + events + .filter(LLMEvent.is.reasoningDelta) + .map((event) => event.text) + .join("") + +const responseUsage = (events: ReadonlyArray) => + events.reduce( + (usage, event) => ("usage" in event && event.usage !== undefined ? event.usage : usage), + undefined, + ) + export class LLMResponse extends Schema.Class("LLM.Response")({ events: Schema.Array(LLMEvent), usage: Schema.optional(Usage), }) { + /** Concatenated assistant text assembled from streamed `text-delta` events. */ get text() { - return this.events - .filter(LLMEvent.is.textDelta) - .map((event) => event.text) - .join("") + return responseText(this.events) } + /** Concatenated reasoning text assembled from streamed `reasoning-delta` events. */ get reasoning() { - return this.events - .filter(LLMEvent.is.reasoningDelta) - .map((event) => event.text) - .join("") + return responseReasoning(this.events) } + /** Completed tool calls emitted by the provider. */ get toolCalls() { return this.events.filter(LLMEvent.is.toolCall) } } +export namespace LLMResponse { + export type Output = LLMResponse | { readonly events: ReadonlyArray; readonly usage?: Usage } + + /** Concatenate assistant text from a response or collected event list. */ + export const text = (response: Output) => responseText(response.events) + + /** Return response usage, falling back to the latest usage-bearing event. */ + export const usage = (response: Output) => response.usage ?? responseUsage(response.events) + + /** Return completed tool calls from a response or collected event list. */ + export const toolCalls = (response: Output) => response.events.filter(LLMEvent.is.toolCall) + + /** Concatenate reasoning text from a response or collected event list. */ + export const reasoning = (response: Output) => responseReasoning(response.events) +} + export class InvalidRequestError extends Schema.TaggedErrorClass()("LLM.InvalidRequestError", { message: Schema.String, }) {} @@ -627,10 +698,27 @@ export class ProviderChunkError extends Schema.TaggedErrorClass("LLM.HttpRequestDetails")({ + method: Schema.String, + url: Schema.String, + headers: Schema.Record(Schema.String, Schema.String), +}) {} + +export class HttpResponseDetails extends Schema.Class("LLM.HttpResponseDetails")({ + status: Schema.Number, + headers: Schema.Record(Schema.String, Schema.String), +}) {} + export class ProviderRequestError extends Schema.TaggedErrorClass()("LLM.ProviderRequestError", { status: Schema.Number, message: Schema.String, body: Schema.optional(Schema.String), + bodyTruncated: Schema.optional(Schema.Boolean), + retryable: Schema.Boolean, + retryAfterMs: Schema.optional(Schema.Number), + requestId: Schema.optional(Schema.String), + request: Schema.optional(HttpRequestDetails), + response: Schema.optional(HttpResponseDetails), }) {} export class TransportError extends Schema.TaggedErrorClass()("LLM.TransportError", { @@ -641,6 +729,8 @@ export class TransportError extends Schema.TaggedErrorClass()("L reason: Schema.optional(Schema.String), // Optional URL of the failing request when the transport layer surfaces it. url: Schema.optional(Schema.String), + retryable: Schema.Boolean, + request: Schema.optional(HttpRequestDetails), }) {} /** diff --git a/packages/llm/test/executor.test.ts b/packages/llm/test/executor.test.ts new file mode 100644 index 0000000000..6b905f54a9 --- /dev/null +++ b/packages/llm/test/executor.test.ts @@ -0,0 +1,108 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer, Ref } from "effect" +import { Headers, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { ProviderRequestError } from "../src" +import { RequestExecutor } from "../src/adapter" +import { it } from "./lib/effect" + +const request = HttpClientRequest.post("https://provider.test/v1/chat?api_key=secret&debug=1").pipe( + HttpClientRequest.setHeaders(Headers.fromInput({ authorization: "Bearer secret", "x-safe": "visible" })), +) + +const responsesLayer = (responses: ReadonlyArray) => + RequestExecutor.layer.pipe( + Layer.provide( + Layer.unwrap( + Effect.gen(function* () { + const cursor = yield* Ref.make(0) + return Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.gen(function* () { + const index = yield* Ref.getAndUpdate(cursor, (value) => value + 1) + return HttpClientResponse.fromWeb(request, responses[index] ?? responses[responses.length - 1]) + }), + ), + ) + }), + ), + ), + ) + +describe("RequestExecutor", () => { + it.effect("returns redacted diagnostics for retryable rate limits", () => + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const error = yield* executor.execute(request).pipe(Effect.flip) + + expect(error).toBeInstanceOf(ProviderRequestError) + if (!(error instanceof ProviderRequestError)) throw new Error("expected ProviderRequestError") + expect(error).toMatchObject({ + status: 429, + retryable: true, + retryAfterMs: 0, + requestId: "req_123", + request: { + method: "POST", + url: "https://provider.test/v1/chat?api_key=%3Credacted%3E&debug=1", + headers: { authorization: "", "x-safe": "visible" }, + }, + response: { + status: 429, + headers: { + "retry-after-ms": "0", + "x-request-id": "req_123", + "x-api-key": "", + }, + }, + }) + expect(error.body).toBe("rate limited") + }).pipe( + Effect.provide( + responsesLayer([ + ...Array.from({ length: 3 }, () => new Response("rate limited", { + status: 429, + headers: { "retry-after-ms": "0", "x-request-id": "req_123", "x-api-key": "secret" }, + })), + ]), + ), + ), + ) + + it.effect("retries retryable status responses before returning the stream", () => + Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const response = yield* executor.execute(request) + + expect(response.status).toBe(200) + expect(yield* response.text).toBe("ok") + }).pipe( + Effect.provide( + responsesLayer([ + new Response("busy", { status: 503, headers: { "retry-after-ms": "0" } }), + new Response("ok", { status: 200 }), + ]), + ), + ), + ) + + it.effect("does not retry non-retryable status responses and truncates large bodies", () => { + return Effect.gen(function* () { + const executor = yield* RequestExecutor.Service + const error = yield* executor.execute(request).pipe(Effect.flip) + + expect(error).toBeInstanceOf(ProviderRequestError) + if (!(error instanceof ProviderRequestError)) throw new Error("expected ProviderRequestError") + expect(error.retryable).toBe(false) + expect(error.bodyTruncated).toBe(true) + expect(error.body).toHaveLength(16_384) + }).pipe( + Effect.provide( + responsesLayer([ + new Response("x".repeat(20_000), { status: 401 }), + new Response("should not retry", { status: 200 }), + ]), + ), + ) + }) +}) diff --git a/packages/llm/test/lib/llm-client.ts b/packages/llm/test/lib/llm-client.ts deleted file mode 100644 index de4dbc24b0..0000000000 --- a/packages/llm/test/lib/llm-client.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Effect, Layer, Stream } from "effect" -import { LLMClient, RequestExecutor } from "../../src/adapter" -import type { LLMRequest } from "../../src/schema" - -export const prepare = (request: LLMRequest) => - Effect.gen(function* () { - return yield* (yield* LLMClient.Service).prepare(request) - }).pipe(Effect.provide(LLMClient.layer.pipe(Layer.provide(RequestExecutor.defaultLayer)))) - -export const generate = (request: LLMRequest) => - Effect.gen(function* () { - return yield* (yield* LLMClient.Service).generate(request) - }) - -export const stream = (request: LLMRequest) => - Stream.unwrap(Effect.gen(function* () { - return (yield* LLMClient.Service).stream(request) - })) diff --git a/packages/llm/test/llm.test.ts b/packages/llm/test/llm.test.ts index 74cd9b0b78..ccd755c3f1 100644 --- a/packages/llm/test/llm.test.ts +++ b/packages/llm/test/llm.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test" -import { LLM } from "../src" +import { LLM, LLMResponse } from "../src" import { LLMRequest, Message, ModelRef, ToolChoice, ToolDefinition } from "../src/schema" describe("llm constructors", () => { @@ -119,7 +119,9 @@ describe("llm constructors", () => { ]) }) - test("extracts output text from responses", () => { - expect(LLM.outputText({ events: [{ type: "text-delta", text: "hi" }, { type: "request-finish", reason: "stop" }] })).toBe("hi") + test("extracts output text from response events", () => { + expect(LLMResponse.text({ + events: [{ type: "text-delta", text: "hi" }, { type: "request-finish", reason: "stop" }], + })).toBe("hi") }) }) diff --git a/packages/llm/test/provider/anthropic-messages.recorded.test.ts b/packages/llm/test/provider/anthropic-messages.recorded.test.ts index 624ee78f94..4a56f3f071 100644 --- a/packages/llm/test/provider/anthropic-messages.recorded.test.ts +++ b/packages/llm/test/provider/anthropic-messages.recorded.test.ts @@ -5,7 +5,6 @@ import { LLMClient } from "../../src/adapter" import * as AnthropicMessages from "../../src/protocols/anthropic-messages" import { eventSummary, expectWeatherToolLoop, runWeatherToolLoop, textRequest, weatherToolLoopRequest, weatherToolName, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" -import * as TestLLMClient from "../lib/llm-client" const model = AnthropicMessages.model({ id: "claude-haiku-4-5-20251001", @@ -34,7 +33,7 @@ const recorded = recordedTests({ }) const generate = (request: LLMRequest) => Effect.gen(function* () { - return yield* TestLLMClient.generate(request) + return yield* LLMClient.generate(request) }) const malformedToolOrderRequest = LLM.request({ diff --git a/packages/llm/test/provider/anthropic-messages.test.ts b/packages/llm/test/provider/anthropic-messages.test.ts index 2b79753c85..9861093b87 100644 --- a/packages/llm/test/provider/anthropic-messages.test.ts +++ b/packages/llm/test/provider/anthropic-messages.test.ts @@ -4,7 +4,6 @@ import { CacheHint, LLM, ProviderRequestError } from "../../src" import { LLMClient } from "../../src/adapter" import * as AnthropicMessages from "../../src/protocols/anthropic-messages" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { fixedResponse } from "../lib/http" import { sseEvents } from "../lib/sse" @@ -25,7 +24,7 @@ const request = LLM.request({ describe("Anthropic Messages adapter", () => { it.effect("prepares Anthropic Messages target", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(request) + const prepared = yield* LLMClient.prepare(request) expect(prepared.payload).toEqual({ model: "claude-sonnet-4-5", @@ -40,7 +39,7 @@ describe("Anthropic Messages adapter", () => { it.effect("prepares tool call and tool result messages", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_tool_result", model, @@ -79,12 +78,12 @@ describe("Anthropic Messages adapter", () => { { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } }, { type: "message_stop" }, ) - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputText(response)).toBe("Hello!") - expect(LLM.outputReasoning(response)).toBe("thinking") - expect(LLM.outputUsage(response)).toMatchObject({ + expect(response.text).toBe("Hello!") + expect(response.reasoning).toBe("thinking") + expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, cacheReadInputTokens: 1, @@ -104,14 +103,14 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 0 }, { type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } }, ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) .pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputToolCalls(response)).toEqual([{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } }]) + expect(response.toolCalls).toEqual([{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } }]) expect(response.events).toEqual([ { type: "tool-input-delta", id: "call_1", name: "lookup", text: '{"query"' }, { type: "tool-input-delta", id: "call_1", name: "lookup", text: ':"weather"}' }, @@ -127,7 +126,7 @@ describe("Anthropic Messages adapter", () => { it.effect("emits provider-error events for mid-stream provider errors", () => Effect.gen(function* () { - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe( Effect.provide( fixedResponse(sseEvents({ type: "error", error: { type: "overloaded_error", message: "Overloaded" } })), @@ -140,7 +139,7 @@ describe("Anthropic Messages adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* TestLLMClient.generate(request) + const error = yield* LLMClient.generate(request) .pipe( Effect.provide( fixedResponse('{"type":"error","error":{"type":"invalid_request_error","message":"Bad request"}}', { @@ -179,7 +178,7 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 2 }, { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } }, ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }], }), @@ -202,7 +201,7 @@ describe("Anthropic Messages adapter", () => { result: { type: "json", value: [{ type: "web_search_result", url: "https://example.com", title: "Example" }] }, providerExecuted: true, }) - expect(LLM.outputText(response)).toBe("Found it.") + expect(response.text).toBe("Found it.") expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "stop" }) }), ) @@ -226,7 +225,7 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 1 }, { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }, ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }], }), @@ -246,7 +245,7 @@ describe("Anthropic Messages adapter", () => { it.effect("round-trips provider-executed assistant content into server tool blocks", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_round_trip", model, @@ -297,7 +296,7 @@ describe("Anthropic Messages adapter", () => { it.effect("rejects round-trip for unknown server tool names", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_unknown_server_tool", model, @@ -322,7 +321,7 @@ describe("Anthropic Messages adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, diff --git a/packages/llm/test/provider/bedrock-converse.test.ts b/packages/llm/test/provider/bedrock-converse.test.ts index da06dd32b3..97e0b45ae8 100644 --- a/packages/llm/test/provider/bedrock-converse.test.ts +++ b/packages/llm/test/provider/bedrock-converse.test.ts @@ -6,7 +6,6 @@ import { CacheHint, LLM } from "../../src" import { LLMClient } from "../../src/adapter" import * as BedrockConverse from "../../src/protocols/bedrock-converse" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { fixedResponse } from "../lib/http" import { eventSummary, expectWeatherToolLoop, runWeatherToolLoop, weatherTool, weatherToolLoopRequest, weatherToolName } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" @@ -63,7 +62,7 @@ const baseRequest = LLM.request({ describe("Bedrock Converse adapter", () => { it.effect("prepares Converse target with system, inference config, and messages", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(baseRequest) + const prepared = yield* LLMClient.prepare(baseRequest) expect(prepared.payload).toEqual({ modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0", @@ -76,7 +75,7 @@ describe("Bedrock Converse adapter", () => { it.effect("prepares tool config with toolSpec and toolChoice", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.updateRequest(baseRequest, { tools: [ { @@ -110,7 +109,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers assistant tool-call + tool-result message history", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_history", model, @@ -156,17 +155,17 @@ describe("Bedrock Converse adapter", () => { ["messageStop", { stopReason: "end_turn" }], ["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }], ) - const response = yield* TestLLMClient.generate(baseRequest) + const response = yield* LLMClient.generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) - expect(LLM.outputText(response)).toBe("Hello!") + expect(response.text).toBe("Hello!") const finishes = response.events.filter((event) => event.type === "request-finish") // Bedrock splits the finish across `messageStop` (carries reason) and // `metadata` (carries usage). We consolidate them into a single // terminal `request-finish` event with both. expect(finishes).toHaveLength(1) expect(finishes[0]).toMatchObject({ type: "request-finish", reason: "stop" }) - expect(LLM.outputUsage(response)).toMatchObject({ + expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, totalTokens: 7, @@ -190,14 +189,14 @@ describe("Bedrock Converse adapter", () => { ["contentBlockStop", { contentBlockIndex: 0 }], ["messageStop", { stopReason: "tool_use" }], ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(baseRequest, { tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }], }), ) .pipe(Effect.provide(fixedBytes(body))) - expect(LLM.outputToolCalls(response)).toEqual([ + expect(response.toolCalls).toEqual([ { type: "tool-call", id: "tool_1", name: "lookup", input: { query: "weather" } }, ]) const events = response.events.filter((event) => event.type === "tool-input-delta") @@ -220,10 +219,10 @@ describe("Bedrock Converse adapter", () => { ["contentBlockStop", { contentBlockIndex: 0 }], ["messageStop", { stopReason: "end_turn" }], ) - const response = yield* TestLLMClient.generate(baseRequest) + const response = yield* LLMClient.generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) - expect(LLM.outputReasoning(response)).toBe("Let me think.") + expect(response.reasoning).toBe("Let me think.") }), ) @@ -233,7 +232,7 @@ describe("Bedrock Converse adapter", () => { ["messageStart", { role: "assistant" }], ["throttlingException", { message: "Slow down" }], ) - const response = yield* TestLLMClient.generate(baseRequest) + const response = yield* LLMClient.generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) expect(response.events.find((event) => event.type === "provider-error")).toEqual({ @@ -250,7 +249,7 @@ describe("Bedrock Converse adapter", () => { id: "anthropic.claude-3-5-sonnet-20240620-v1:0", baseURL: "https://bedrock-runtime.test", }) - const error = yield* TestLLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })) + const error = yield* LLMClient.generate(LLM.updateRequest(baseRequest, { model: unsignedModel })) .pipe(Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))), Effect.flip) expect(error.message).toContain("Bedrock Converse requires either model.apiKey") @@ -268,7 +267,7 @@ describe("Bedrock Converse adapter", () => { secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", }, }) - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.updateRequest(baseRequest, { model: signed }), ) @@ -285,7 +284,7 @@ describe("Bedrock Converse adapter", () => { it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () => Effect.gen(function* () { const cache = new CacheHint({ type: "ephemeral" }) - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_cache", model, @@ -317,7 +316,7 @@ describe("Bedrock Converse adapter", () => { it.effect("does not emit cachePoint when no cache hint is set", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(baseRequest) + const prepared = yield* LLMClient.prepare(baseRequest) expect(prepared.payload).toMatchObject({ system: [{ text: "You are concise." }], messages: [{ role: "user", content: [{ text: "Say hello." }] }], @@ -327,7 +326,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers image media into Bedrock image blocks", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_image", model, @@ -363,7 +362,7 @@ describe("Bedrock Converse adapter", () => { it.effect("base64-encodes Uint8Array image bytes", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_image_bytes", model, @@ -389,7 +388,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers document media into Bedrock document blocks with format and name", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_doc", model, @@ -420,7 +419,7 @@ describe("Bedrock Converse adapter", () => { it.effect("rejects unsupported image media types", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_bad_image", model, @@ -435,7 +434,7 @@ describe("Bedrock Converse adapter", () => { it.effect("rejects unsupported document media types", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_bad_doc", model, diff --git a/packages/llm/test/provider/gemini.recorded.test.ts b/packages/llm/test/provider/gemini.recorded.test.ts index a0cbd2801d..4a104630c8 100644 --- a/packages/llm/test/provider/gemini.recorded.test.ts +++ b/packages/llm/test/provider/gemini.recorded.test.ts @@ -1,11 +1,10 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM, type LLMRequest } from "../../src" +import { LLM } from "../../src" import { LLMClient } from "../../src/adapter" import * as Gemini from "../../src/protocols/gemini" import { eventSummary, textRequest, weatherToolName, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" -import * as TestLLMClient from "../lib/llm-client" const model = Gemini.model({ id: "gemini-2.5-flash", @@ -21,15 +20,10 @@ const recorded = recordedTests({ protocol: "gemini", requires: ["GOOGLE_GENERATIVE_AI_API_KEY"], }) -const generate = (request: LLMRequest) => - Effect.gen(function* () { - return yield* TestLLMClient.generate(request) - }) - describe("Gemini recorded", () => { recorded.effect("streams text", () => Effect.gen(function* () { - const response = yield* generate(request) + const response = yield* LLMClient.generate(request) expect(eventSummary(response.events)).toEqual([ { type: "text", value: expect.stringMatching(/^Hello!?$/) }, @@ -40,7 +34,7 @@ describe("Gemini recorded", () => { recorded.effect.with("streams tool call", { tags: ["tool"] }, () => Effect.gen(function* () { - const response = yield* generate(toolRequest) + const response = yield* LLMClient.generate(toolRequest) expect(eventSummary(response.events)).toEqual([ { type: "tool-call", name: weatherToolName, input: { city: "Paris" } }, diff --git a/packages/llm/test/provider/gemini.test.ts b/packages/llm/test/provider/gemini.test.ts index 75d20e9a4c..127477541e 100644 --- a/packages/llm/test/provider/gemini.test.ts +++ b/packages/llm/test/provider/gemini.test.ts @@ -4,7 +4,6 @@ import { LLM, ProviderChunkError } from "../../src" import { LLMClient } from "../../src/adapter" import * as Gemini from "../../src/protocols/gemini" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { fixedResponse } from "../lib/http" import { sseEvents, sseRaw } from "../lib/sse" @@ -25,7 +24,7 @@ const request = LLM.request({ describe("Gemini adapter", () => { it.effect("prepares Gemini target", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(request) + const prepared = yield* LLMClient.prepare(request) expect(prepared.payload).toEqual({ contents: [{ role: "user", parts: [{ text: "Say hello." }] }], @@ -37,15 +36,17 @@ describe("Gemini adapter", () => { it.effect("prepares multimodal user input and tool history", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_tool_result", model, - tools: [{ - name: "lookup", - description: "Lookup data", - inputSchema: { type: "object", properties: { query: { type: "string" } } }, - }], + tools: [ + { + name: "lookup", + description: "Lookup data", + inputSchema: { type: "object", properties: { query: { type: "string" } } }, + }, + ], toolChoice: { type: "tool", name: "lookup" }, messages: [ LLM.user([ @@ -62,10 +63,7 @@ describe("Gemini adapter", () => { contents: [ { role: "user", - parts: [ - { text: "What is in this image?" }, - { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, - ], + parts: [{ text: "What is in this image?" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }], }, { role: "model", @@ -73,16 +71,22 @@ describe("Gemini adapter", () => { }, { role: "user", - parts: [{ functionResponse: { name: "lookup", response: { name: "lookup", content: '{"forecast":"sunny"}' } } }], + parts: [ + { functionResponse: { name: "lookup", response: { name: "lookup", content: '{"forecast":"sunny"}' } } }, + ], + }, + ], + tools: [ + { + functionDeclarations: [ + { + name: "lookup", + description: "Lookup data", + parameters: { type: "object", properties: { query: { type: "string" } } }, + }, + ], }, ], - tools: [{ - functionDeclarations: [{ - name: "lookup", - description: "Lookup data", - parameters: { type: "object", properties: { query: { type: "string" } } }, - }], - }], toolConfig: { functionCallingConfig: { mode: "ANY", allowedFunctionNames: ["lookup"] } }, }) }), @@ -90,7 +94,7 @@ describe("Gemini adapter", () => { it.effect("omits tools when tool choice is none", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_no_tools", model, @@ -108,41 +112,47 @@ describe("Gemini adapter", () => { it.effect("sanitizes integer enums, dangling required, untyped arrays, and scalar object keys", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_schema_patch", model, prompt: "Use the tool.", - tools: [{ - name: "lookup", - description: "Lookup data", - inputSchema: { - type: "object", - required: ["status", "missing"], - properties: { - status: { type: "integer", enum: [1, 2] }, - tags: { type: "array" }, - name: { type: "string", properties: { ignored: { type: "string" } }, required: ["ignored"] }, + tools: [ + { + name: "lookup", + description: "Lookup data", + inputSchema: { + type: "object", + required: ["status", "missing"], + properties: { + status: { type: "integer", enum: [1, 2] }, + tags: { type: "array" }, + name: { type: "string", properties: { ignored: { type: "string" } }, required: ["ignored"] }, + }, }, }, - }], + ], }), ) expect(prepared.payload).toMatchObject({ - tools: [{ - functionDeclarations: [{ - parameters: { - type: "object", - required: ["status"], - properties: { - status: { type: "string", enum: ["1", "2"] }, - tags: { type: "array", items: { type: "string" } }, - name: { type: "string" }, + tools: [ + { + functionDeclarations: [ + { + parameters: { + type: "object", + required: ["status"], + properties: { + status: { type: "string", enum: ["1", "2"] }, + tags: { type: "array", items: { type: "string" } }, + name: { type: "string" }, + }, + }, }, - }, - }], - }], + ], + }, + ], }) }), ) @@ -151,20 +161,26 @@ describe("Gemini adapter", () => { Effect.gen(function* () { const body = sseEvents( { - candidates: [{ - content: { role: "model", parts: [{ text: "thinking", thought: true }] }, - }], + candidates: [ + { + content: { role: "model", parts: [{ text: "thinking", thought: true }] }, + }, + ], }, { - candidates: [{ - content: { role: "model", parts: [{ text: "Hello" }] }, - }], + candidates: [ + { + content: { role: "model", parts: [{ text: "Hello" }] }, + }, + ], }, { - candidates: [{ - content: { role: "model", parts: [{ text: "!" }] }, - finishReason: "STOP", - }], + candidates: [ + { + content: { role: "model", parts: [{ text: "!" }] }, + finishReason: "STOP", + }, + ], }, { usageMetadata: { @@ -176,12 +192,11 @@ describe("Gemini adapter", () => { }, }, ) - const response = yield* TestLLMClient.generate(request) - .pipe(Effect.provide(fixedResponse(body))) + const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputText(response)).toBe("Hello!") - expect(LLM.outputReasoning(response)).toBe("thinking") - expect(LLM.outputUsage(response)).toMatchObject({ + expect(response.text).toBe("Hello!") + expect(response.reasoning).toBe("thinking") + expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, reasoningTokens: 1, @@ -216,32 +231,38 @@ describe("Gemini adapter", () => { it.effect("emits streamed tool calls and maps finish reason", () => Effect.gen(function* () { - const body = sseEvents( - { - candidates: [{ + const body = sseEvents({ + candidates: [ + { content: { role: "model", parts: [{ functionCall: { name: "lookup", args: { query: "weather" } } }], }, finishReason: "STOP", - }], - usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, - }, - ) - const response = yield* TestLLMClient.generate( - LLM.updateRequest(request, { - tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], - }), - ) - .pipe(Effect.provide(fixedResponse(body))) + }, + ], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, + }) + const response = yield* LLMClient.generate( + LLM.updateRequest(request, { + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + }), + ).pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputToolCalls(response)).toEqual([{ type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } }]) + expect(response.toolCalls).toEqual([ + { type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } }, + ]) expect(response.events).toEqual([ { type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } }, { type: "request-finish", reason: "tool-calls", - usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { promptTokenCount: 5, candidatesTokenCount: 1 } }, + usage: { + inputTokens: 5, + outputTokens: 1, + totalTokens: 6, + native: { promptTokenCount: 5, candidatesTokenCount: 1 }, + }, }, ]) }), @@ -249,9 +270,9 @@ describe("Gemini adapter", () => { it.effect("assigns unique ids to multiple streamed tool calls", () => Effect.gen(function* () { - const body = sseEvents( - { - candidates: [{ + const body = sseEvents({ + candidates: [ + { content: { role: "model", parts: [ @@ -260,17 +281,16 @@ describe("Gemini adapter", () => { ], }, finishReason: "STOP", - }], - }, - ) - const response = yield* TestLLMClient.generate( - LLM.updateRequest(request, { - tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], - }), - ) - .pipe(Effect.provide(fixedResponse(body))) + }, + ], + }) + const response = yield* LLMClient.generate( + LLM.updateRequest(request, { + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + }), + ).pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputToolCalls(response)).toEqual([ + expect(response.toolCalls).toEqual([ { type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } }, { type: "tool-call", id: "tool_1", name: "lookup", input: { query: "news" } }, ]) @@ -280,18 +300,18 @@ describe("Gemini adapter", () => { it.effect("maps length and content-filter finish reasons", () => Effect.gen(function* () { - const length = yield* TestLLMClient.generate(request) - .pipe( - Effect.provide( - fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "MAX_TOKENS" }] })), + const length = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "MAX_TOKENS" }] }), ), - ) - const filtered = yield* TestLLMClient.generate(request) - .pipe( - Effect.provide( - fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "SAFETY" }] })), - ), - ) + ), + ) + const filtered = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "SAFETY" }] })), + ), + ) expect(length.events).toEqual([{ type: "request-finish", reason: "length" }]) expect(filtered.events).toEqual([{ type: "request-finish", reason: "content-filter" }]) @@ -300,8 +320,9 @@ describe("Gemini adapter", () => { it.effect("leaves total usage undefined when component counts are missing", () => Effect.gen(function* () { - const response = yield* TestLLMClient.generate(request) - .pipe(Effect.provide(fixedResponse(sseEvents({ usageMetadata: { thoughtsTokenCount: 1 } })))) + const response = yield* LLMClient.generate(request).pipe( + Effect.provide(fixedResponse(sseEvents({ usageMetadata: { thoughtsTokenCount: 1 } }))), + ) expect(response.usage).toMatchObject({ reasoningTokens: 1 }) expect(response.usage?.totalTokens).toBeUndefined() @@ -310,11 +331,10 @@ describe("Gemini adapter", () => { it.effect("fails invalid stream chunks", () => Effect.gen(function* () { - const error = yield* TestLLMClient.generate(request) - .pipe( - Effect.provide(fixedResponse(sseRaw("data: {not json}"))), - Effect.flip, - ) + const error = yield* LLMClient.generate(request).pipe( + Effect.provide(fixedResponse(sseRaw("data: {not json}"))), + Effect.flip, + ) expect(error).toBeInstanceOf(ProviderChunkError) expect(error.message).toContain("Invalid google/gemini stream chunk") @@ -323,16 +343,17 @@ describe("Gemini adapter", () => { it.effect("rejects unsupported assistant media content", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( - LLM.request({ - id: "req_media", - model, - messages: [LLM.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], - }), - ) - .pipe(Effect.flip) + const error = yield* LLMClient.prepare( + LLM.request({ + id: "req_media", + model, + messages: [LLM.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], + }), + ).pipe(Effect.flip) - expect(error.message).toContain("Gemini assistant messages only support text, reasoning, and tool-call content for now") + expect(error.message).toContain( + "Gemini assistant messages only support text, reasoning, and tool-call content for now", + ) }), ) }) diff --git a/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts b/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts index 8833c92c4e..433cc7b789 100644 --- a/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts +++ b/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect, Stream } from "effect" -import { LLM } from "../../src" +import { LLM, LLMResponse } from "../../src" import * as OpenAIChat from "../../src/protocols/openai-chat" import { ToolRuntime } from "../../src/tool-runtime" import { eventSummary, weatherRuntimeTool } from "../recorded-scenarios" @@ -39,7 +39,7 @@ describe("OpenAI Chat tool-loop recorded", () => { yield* TestToolRuntime.runTools({ request, tools: { get_weather: weatherRuntimeTool } }).pipe(Stream.runCollect), ) - expect(LLM.outputText({ events })).toContain("Paris") + expect(LLMResponse.text({ events })).toContain("Paris") expect(eventSummary(events)).toEqual([ { type: "tool-call", name: "get_weather", input: { city: "Paris" } }, { diff --git a/packages/llm/test/provider/openai-chat.recorded.test.ts b/packages/llm/test/provider/openai-chat.recorded.test.ts index 07ab49f585..69a52b2c83 100644 --- a/packages/llm/test/provider/openai-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-chat.recorded.test.ts @@ -5,7 +5,6 @@ import { LLMClient } from "../../src/adapter" import * as OpenAIChat from "../../src/protocols/openai-chat" import { eventSummary, textRequest, weatherToolName, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" -import * as TestLLMClient from "../lib/llm-client" const model = OpenAIChat.model({ id: "gpt-4o-mini", @@ -39,7 +38,7 @@ const recorded = recordedTests({ }) const generate = (request: LLMRequest) => Effect.gen(function* () { - return yield* TestLLMClient.generate(request) + return yield* LLMClient.generate(request) }) describe("OpenAI Chat recorded", () => { diff --git a/packages/llm/test/provider/openai-chat.test.ts b/packages/llm/test/provider/openai-chat.test.ts index 8580fb12a7..05a4ea0aea 100644 --- a/packages/llm/test/provider/openai-chat.test.ts +++ b/packages/llm/test/provider/openai-chat.test.ts @@ -5,8 +5,8 @@ import { LLM, ProviderRequestError } from "../../src" import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" import * as OpenAIChat from "../../src/protocols/openai-chat" +import { LLMClient } from "../../src/adapter" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { dynamicResponse, fixedResponse, truncatedStream } from "../lib/http" import { deltaChunk, usageChunk } from "../lib/openai-chunks" import { sseEvents } from "../lib/sse" @@ -35,7 +35,7 @@ describe("OpenAI Chat adapter", () => { // Pass the OpenAIChat payload type so `prepared.payload` is statically // typed to the adapter's native shape — the assertions below read field // names without `unknown` casts. - const prepared = yield* TestLLMClient.prepare(request) + const prepared = yield* LLMClient.prepare(request) const _typed: { readonly model: string; readonly stream: true } = prepared.payload expect(prepared.payload).toEqual({ @@ -54,7 +54,7 @@ describe("OpenAI Chat adapter", () => { it.effect("maps OpenAI provider options to Chat options", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ model: OpenAI.chat("gpt-4o-mini", { baseURL: "https://api.openai.test/v1/" }), prompt: "think", @@ -68,7 +68,7 @@ describe("OpenAI Chat adapter", () => { ) it.effect("adds native query params to the Chat Completions URL", () => - TestLLMClient.generate(LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, queryParams: { "api-version": "v1" } }) })) + LLMClient.generate(LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, queryParams: { "api-version": "v1" } }) })) .pipe( Effect.provide( dynamicResponse((input) => @@ -85,10 +85,9 @@ describe("OpenAI Chat adapter", () => { ) it.effect("uses Azure api-key header for static OpenAI Chat keys", () => - TestLLMClient.generate( + LLMClient.generate( LLM.updateRequest(request, { - model: Azure.model("gpt-4o-mini", { - useCompletionUrls: true, + model: Azure.chat("gpt-4o-mini", { baseURL: "https://opencode-test.openai.azure.com/openai/v1/", apiKey: "azure-key", headers: { authorization: "Bearer stale" }, @@ -112,7 +111,7 @@ describe("OpenAI Chat adapter", () => { ) it.effect("applies serializable HTTP overlays after payload lowering", () => - TestLLMClient.generate( + LLMClient.generate( LLM.updateRequest(request, { model: OpenAIChat.model({ ...model, apiKey: "fresh-key", headers: { authorization: "Bearer stale" } }), http: { @@ -146,7 +145,7 @@ describe("OpenAI Chat adapter", () => { it.effect("prepares assistant tool-call and tool-result messages", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_tool_result", model, @@ -183,7 +182,7 @@ describe("OpenAI Chat adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, @@ -198,7 +197,7 @@ describe("OpenAI Chat adapter", () => { it.effect("rejects unsupported assistant reasoning content", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_reasoning", model, @@ -225,10 +224,10 @@ describe("OpenAI Chat adapter", () => { completion_tokens_details: { reasoning_tokens: 0 }, }), ) - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputText(response)).toBe("Hello!") + expect(response.text).toBe("Hello!") expect(response.events).toEqual([ { type: "text-delta", text: "Hello" }, { type: "text-delta", text: "!" }, @@ -264,7 +263,7 @@ describe("OpenAI Chat adapter", () => { deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }), deltaChunk({}, "tool_calls"), ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), @@ -289,7 +288,7 @@ describe("OpenAI Chat adapter", () => { }), deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }), ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), @@ -300,14 +299,14 @@ describe("OpenAI Chat adapter", () => { { type: "tool-input-delta", id: "call_1", name: "lookup", text: '{"query"' }, { type: "tool-input-delta", id: "call_1", name: "lookup", text: ':"weather"}' }, ]) - expect(LLM.outputToolCalls(response)).toEqual([]) + expect(response.toolCalls).toEqual([]) }), ) it.effect("fails on malformed stream chunks", () => Effect.gen(function* () { const body = sseEvents(deltaChunk({ content: 123 })) - const error = yield* TestLLMClient.generate(request) + const error = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body)), Effect.flip) expect(error.message).toContain("Invalid openai/openai-chat stream chunk") @@ -319,7 +318,7 @@ describe("OpenAI Chat adapter", () => { const layer = truncatedStream([ `data: ${JSON.stringify(deltaChunk({ role: "assistant", content: "Hello" }))}\n\n`, ]) - const error = yield* TestLLMClient.generate(request) + const error = yield* LLMClient.generate(request) .pipe(Effect.provide(layer), Effect.flip) expect(error.message).toContain("Failed to read openai/openai-chat stream") @@ -328,7 +327,7 @@ describe("OpenAI Chat adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* TestLLMClient.generate(request) + const error = yield* LLMClient.generate(request) .pipe( Effect.provide( fixedResponse('{"error":{"message":"Bad request","type":"invalid_request_error"}}', { @@ -357,7 +356,7 @@ describe("OpenAI Chat adapter", () => { ) const events = Array.from( - yield* TestLLMClient.stream(request).pipe(Stream.take(1), Stream.runCollect, Effect.provide(fixedResponse(body))), + yield* LLMClient.stream(request).pipe(Stream.take(1), Stream.runCollect, Effect.provide(fixedResponse(body))), ) expect(events.map((event) => event.type)).toEqual(["text-delta"]) }), diff --git a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts index 893a52d7b0..5004aff155 100644 --- a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts @@ -7,7 +7,6 @@ import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-cha import * as OpenRouter from "../../src/providers/openrouter" import { expectFinish, expectWeatherToolCall, expectWeatherToolLoop, runWeatherToolLoop, textRequest, weatherToolLoopRequest, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" -import * as TestLLMClient from "../lib/llm-client" const deepseekModel = OpenAICompatible.deepseek.model("deepseek-chat", { apiKey: process.env.DEEPSEEK_API_KEY ?? "fixture", @@ -58,7 +57,7 @@ const xaiToolRequest = weatherToolRequest({ id: "recorded_xai_tool_call", model: const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" }) const generate = (request: LLMRequest) => Effect.gen(function* () { - return yield* TestLLMClient.generate(request) + return yield* LLMClient.generate(request) }) const openrouterToolLoops = [ @@ -87,7 +86,7 @@ describe("OpenAI-compatible Chat recorded", () => { Effect.gen(function* () { const response = yield* generate(deepseekRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expectFinish(response.events, "stop") }), ) @@ -96,7 +95,7 @@ describe("OpenAI-compatible Chat recorded", () => { Effect.gen(function* () { const response = yield* generate(togetherRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expectFinish(response.events, "stop") }), ) @@ -115,7 +114,7 @@ describe("OpenAI-compatible Chat recorded", () => { Effect.gen(function* () { const response = yield* generate(groqRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expectFinish(response.events, "stop") }), ) @@ -144,7 +143,7 @@ describe("OpenAI-compatible Chat recorded", () => { Effect.gen(function* () { const response = yield* generate(openrouterRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expectFinish(response.events, "stop") }), ) @@ -175,7 +174,7 @@ describe("OpenAI-compatible Chat recorded", () => { Effect.gen(function* () { const response = yield* generate(xaiRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expectFinish(response.events, "stop") }), ) diff --git a/packages/llm/test/provider/openai-compatible-chat.test.ts b/packages/llm/test/provider/openai-compatible-chat.test.ts index 276883f8fc..b9a0405d92 100644 --- a/packages/llm/test/provider/openai-compatible-chat.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.test.ts @@ -6,7 +6,6 @@ import { LLMClient } from "../../src/adapter" import * as OpenAICompatible from "../../src/providers/openai-compatible" import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { dynamicResponse } from "../lib/http" import { sseEvents } from "../lib/sse" @@ -53,7 +52,7 @@ const providerFamilies = [ describe("OpenAI-compatible Chat adapter", () => { it.effect("prepares generic Chat target", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], toolChoice: { type: "required" }, @@ -126,7 +125,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("matches AI SDK compatible basic request body fixture", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(request) + const prepared = yield* LLMClient.prepare(request) expect(prepared.payload).toEqual({ model: "deepseek-chat", @@ -144,7 +143,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("matches AI SDK compatible tool request body fixture", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_tool_parity", model, @@ -194,7 +193,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("posts to the configured compatible endpoint and parses text usage", () => Effect.gen(function* () { - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe( Effect.provide( dynamicResponse((input) => @@ -224,8 +223,8 @@ describe("OpenAI-compatible Chat adapter", () => { ), ) - expect(LLM.outputText(response)).toBe("Hello!") - expect(LLM.outputUsage(response)).toMatchObject({ inputTokens: 5, outputTokens: 2, totalTokens: 7 }) + expect(response.text).toBe("Hello!") + expect(response.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, totalTokens: 7 }) expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "stop" }) }), ) diff --git a/packages/llm/test/provider/openai-responses.recorded.test.ts b/packages/llm/test/provider/openai-responses.recorded.test.ts index ded13505a5..54d6aafdf6 100644 --- a/packages/llm/test/provider/openai-responses.recorded.test.ts +++ b/packages/llm/test/provider/openai-responses.recorded.test.ts @@ -5,7 +5,6 @@ import { LLMClient } from "../../src/adapter" import * as OpenAIResponses from "../../src/protocols/openai-responses" import { expectFinish, expectWeatherToolCall, expectWeatherToolLoop, runWeatherToolLoop, weatherTool, weatherToolLoopRequest, weatherToolName } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" -import * as TestLLMClient from "../lib/llm-client" const model = OpenAIResponses.model({ id: "gpt-5.5", @@ -44,7 +43,7 @@ const recorded = recordedTests({ }) const generate = (request: LLMRequest) => Effect.gen(function* () { - return yield* TestLLMClient.generate(request) + return yield* LLMClient.generate(request) }) describe("OpenAI Responses recorded", () => { @@ -52,7 +51,7 @@ describe("OpenAI Responses recorded", () => { Effect.gen(function* () { const response = yield* generate(textRequest) - expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expect(response.text).toMatch(/^Hello!?$/) expect(response.usage?.totalTokens).toBeGreaterThan(0) expectFinish(response.events, "stop") }), diff --git a/packages/llm/test/provider/openai-responses.test.ts b/packages/llm/test/provider/openai-responses.test.ts index d39e9cd120..8da28ea762 100644 --- a/packages/llm/test/provider/openai-responses.test.ts +++ b/packages/llm/test/provider/openai-responses.test.ts @@ -7,7 +7,6 @@ import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" import * as OpenAIResponses from "../../src/protocols/openai-responses" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" import { dynamicResponse, fixedResponse } from "../lib/http" import { sseEvents } from "../lib/sse" @@ -28,7 +27,7 @@ const request = LLM.request({ describe("OpenAI Responses adapter", () => { it.effect("prepares OpenAI Responses target", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare(request) + const prepared = yield* LLMClient.prepare(request) expect(prepared.payload).toEqual({ model: "gpt-4.1-mini", @@ -45,7 +44,7 @@ describe("OpenAI Responses adapter", () => { it.effect("adds native query params to the Responses URL", () => Effect.gen(function* () { - yield* TestLLMClient.generate(LLM.updateRequest(request, { model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }) })) + yield* LLMClient.generate(LLM.updateRequest(request, { model: OpenAIResponses.model({ ...model, queryParams: { "api-version": "v1" } }) })) .pipe( Effect.provide( dynamicResponse((input) => @@ -64,9 +63,9 @@ describe("OpenAI Responses adapter", () => { it.effect("uses Azure api-key header for static OpenAI Responses keys", () => Effect.gen(function* () { - yield* TestLLMClient.generate( + yield* LLMClient.generate( LLM.updateRequest(request, { - model: Azure.model("gpt-4.1-mini", { + model: Azure.responses("gpt-4.1-mini", { baseURL: "https://opencode-test.openai.azure.com/openai/v1/", apiKey: "azure-key", headers: { authorization: "Bearer stale" }, @@ -92,7 +91,7 @@ describe("OpenAI Responses adapter", () => { it.effect("prepares function call and function output input items", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_tool_result", model, @@ -118,7 +117,7 @@ describe("OpenAI Responses adapter", () => { it.effect("maps OpenAI provider options to Responses options", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ model: OpenAI.model("gpt-5.2", { baseURL: "https://api.openai.test/v1/" }), prompt: "think", @@ -143,7 +142,7 @@ describe("OpenAI Responses adapter", () => { it.effect("request OpenAI provider options override model defaults", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ model: OpenAI.model("gpt-4.1-mini", { baseURL: "https://api.openai.test/v1/", @@ -176,10 +175,10 @@ describe("OpenAI Responses adapter", () => { }, }, ) - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body))) - expect(LLM.outputText(response)).toBe("Hello!") + expect(response.text).toBe("Hello!") expect(response.events).toEqual([ { type: "text-delta", id: "msg_1", text: "Hello" }, { type: "text-delta", id: "msg_1", text: "!" }, @@ -226,7 +225,7 @@ describe("OpenAI Responses adapter", () => { }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* TestLLMClient.generate( + const response = yield* LLMClient.generate( LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), @@ -259,7 +258,7 @@ describe("OpenAI Responses adapter", () => { { type: "response.output_item.done", item }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body))) const callsAndResults = response.events.filter((event) => event.type === "tool-call" || event.type === "tool-result") @@ -296,7 +295,7 @@ describe("OpenAI Responses adapter", () => { { type: "response.output_item.done", item }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(body))) const toolCall = response.events.find((event) => event.type === "tool-call") @@ -320,7 +319,7 @@ describe("OpenAI Responses adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* TestLLMClient.prepare( + const error = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, @@ -335,7 +334,7 @@ describe("OpenAI Responses adapter", () => { it.effect("emits provider-error events for mid-stream provider errors", () => Effect.gen(function* () { - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe( Effect.provide( fixedResponse(sseEvents({ type: "error", code: "rate_limit_exceeded", message: "Slow down" })), @@ -348,7 +347,7 @@ describe("OpenAI Responses adapter", () => { it.effect("falls back to error code when no message is present", () => Effect.gen(function* () { - const response = yield* TestLLMClient.generate(request) + const response = yield* LLMClient.generate(request) .pipe(Effect.provide(fixedResponse(sseEvents({ type: "error", code: "internal_error" })))) expect(response.events).toEqual([{ type: "provider-error", message: "internal_error" }]) @@ -357,7 +356,7 @@ describe("OpenAI Responses adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* TestLLMClient.generate(request) + const error = yield* LLMClient.generate(request) .pipe( Effect.provide( fixedResponse('{"error":{"type":"invalid_request_error","message":"Bad request"}}', { diff --git a/packages/llm/test/provider/openrouter.test.ts b/packages/llm/test/provider/openrouter.test.ts index a3e246ee7d..a74a5c6e2c 100644 --- a/packages/llm/test/provider/openrouter.test.ts +++ b/packages/llm/test/provider/openrouter.test.ts @@ -4,7 +4,6 @@ import { LLM } from "../../src" import { LLMClient } from "../../src/adapter" import * as OpenRouter from "../../src/providers/openrouter" import { it } from "../lib/effect" -import * as TestLLMClient from "../lib/llm-client" describe("OpenRouter", () => { it.effect("prepares OpenRouter models through the OpenAI-compatible Chat route", () => @@ -19,7 +18,7 @@ describe("OpenRouter", () => { apiKey: "test-key", }) - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ model, prompt: "Say hello." }), ) @@ -34,7 +33,7 @@ describe("OpenRouter", () => { it.effect("applies OpenRouter payload options from the model helper", () => Effect.gen(function* () { - const prepared = yield* TestLLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ model: OpenRouter.model("anthropic/claude-3.7-sonnet:thinking", { providerOptions: { diff --git a/packages/llm/test/recorded-scenarios.ts b/packages/llm/test/recorded-scenarios.ts index 242b653c69..e31f762816 100644 --- a/packages/llm/test/recorded-scenarios.ts +++ b/packages/llm/test/recorded-scenarios.ts @@ -1,6 +1,6 @@ import { expect } from "bun:test" import { Effect, Schema, Stream } from "effect" -import { LLM, LLMEvent, type LLMRequest, type LLMResponse, type ModelRef } from "../src" +import { LLM, LLMEvent, LLMResponse, type LLMRequest, type ModelRef } from "../src" import { tool } from "../src/tool" import { ToolRuntime } from "../src/tool-runtime" @@ -90,7 +90,7 @@ export const expectFinish = ( ) => expect(events.at(-1)).toMatchObject({ type: "request-finish", reason }) export const expectWeatherToolCall = (response: LLMResponse) => - expect(LLM.outputToolCalls(response)).toMatchObject([ + expect(response.toolCalls).toMatchObject([ { type: "tool-call", id: expect.any(String), name: weatherToolName, input: { city: "Paris" } }, ]) @@ -112,7 +112,7 @@ export const expectWeatherToolLoop = (events: ReadonlyArray) => { result: { type: "json", value: { temperature: 22, condition: "sunny" } }, }) - const output = LLM.outputText({ events }) + const output = LLMResponse.text({ events }) expect(output).toContain("Paris") expect(output.trim().length).toBeGreaterThan(0) } diff --git a/packages/llm/test/tool-runtime.test.ts b/packages/llm/test/tool-runtime.test.ts index dd61fe4034..7f90435764 100644 --- a/packages/llm/test/tool-runtime.test.ts +++ b/packages/llm/test/tool-runtime.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect, Schema, Stream } from "effect" -import { LLM, LLMEvent, LLMRequest } from "../src" +import { LLM, LLMEvent, LLMRequest, LLMResponse } from "../src" import { LLMClient } from "../src/adapter" import * as AnthropicMessages from "../src/protocols/anthropic-messages" import * as OpenAIChat from "../src/protocols/openai-chat" @@ -49,7 +49,7 @@ describe("ToolRuntime", () => { ), ) - expect(LLM.outputText({ events })).toBe("Done.") + expect(LLMResponse.text({ events })).toBe("Done.") }), ) @@ -123,7 +123,7 @@ describe("ToolRuntime", () => { result: { type: "json", value: { temperature: 22, condition: "sunny" } }, }) expect(events.at(-1)?.type).toBe("request-finish") - expect(LLM.outputText({ events })).toBe("It's sunny in Paris.") + expect(LLMResponse.text({ events })).toBe("It's sunny in Paris.") }), ) @@ -205,7 +205,7 @@ describe("ToolRuntime", () => { ) expect(events.map((event) => event.type)).toEqual(["text-delta", "request-finish"]) - expect(LLM.outputText({ events })).toBe("Done.") + expect(LLMResponse.text({ events })).toBe("Done.") }), ) @@ -300,7 +300,7 @@ describe("ToolRuntime", () => { providerExecuted: true, }, ]) - expect(LLM.outputText({ events })).toBe("Done.") + expect(LLMResponse.text({ events })).toBe("Done.") }), ) diff --git a/packages/opencode/src/provider/llm-bridge.ts b/packages/opencode/src/provider/llm-bridge.ts index fb0c486952..2eb4b76f93 100644 --- a/packages/opencode/src/provider/llm-bridge.ts +++ b/packages/opencode/src/provider/llm-bridge.ts @@ -162,13 +162,14 @@ const PROVIDERS: Record = { AmazonBedrock.model(String(input.model.api.id), sharedOptions(input, options, { protocol: "bedrock-converse" })), "@ai-sdk/anthropic": (input, options) => Anthropic.model(String(input.model.api.id), sharedOptions(input, options, { protocol: "anthropic-messages" })), - "@ai-sdk/azure": (input, options) => - Azure.model(String(input.model.api.id), { + "@ai-sdk/azure": (input, options) => { + const create = options.useCompletionUrls === true ? Azure.chat : Azure.responses + return create(String(input.model.api.id), { ...sharedOptions(input, options, { protocol: azureProtocol(options), providerOptions: openAIOptions(options) }), resourceName: stringOption(options, "resourceName"), apiVersion: stringOption(options, "apiVersion"), - useCompletionUrls: options.useCompletionUrls === true, - }), + }) + }, "@ai-sdk/baseten": openAICompatibleModel, "@ai-sdk/cerebras": openAICompatibleModel, "@ai-sdk/deepinfra": openAICompatibleModel, diff --git a/packages/opencode/test/provider/llm-bridge.test.ts b/packages/opencode/test/provider/llm-bridge.test.ts index b59126001a..bb376de9db 100644 --- a/packages/opencode/test/provider/llm-bridge.test.ts +++ b/packages/opencode/test/provider/llm-bridge.test.ts @@ -131,6 +131,7 @@ describe("ProviderLLMBridge", () => { expect(ref).toMatchObject({ provider: "azure", + adapter: "azure-openai-responses", protocol: "openai-responses", baseURL: "https://opencode-test.openai.azure.com/openai/v1", apiKey: "azure-key", @@ -146,6 +147,7 @@ describe("ProviderLLMBridge", () => { expect(ref).toMatchObject({ provider: "azure", + adapter: "azure-openai-chat", protocol: "openai-chat", baseURL: "https://opencode-test.openai.azure.com/openai/v1", queryParams: { "api-version": "v1" }, diff --git a/packages/opencode/test/session/llm-native.test.ts b/packages/opencode/test/session/llm-native.test.ts index 7b2f663159..85d3dae03a 100644 --- a/packages/opencode/test/session/llm-native.test.ts +++ b/packages/opencode/test/session/llm-native.test.ts @@ -799,6 +799,7 @@ describe("LLMNative.request", () => { expect(request.model).toMatchObject({ id: "gpt-5-deployment", provider: "azure", + adapter: "azure-openai-responses", protocol: "openai-responses", baseURL: "https://opencode-test.openai.azure.com/openai/v1", apiKey: "azure-key", @@ -823,6 +824,7 @@ describe("LLMNative.request", () => { expect(request.model).toMatchObject({ id: "gpt-4-1-deployment", provider: "azure", + adapter: "azure-openai-chat", protocol: "openai-chat", baseURL: "https://opencode-test.openai.azure.com/openai/v1", apiKey: "azure-key",