From 3cd13c87c47cc547e57005de4da9a08140c84567 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 26 Apr 2026 21:08:35 -0400 Subject: [PATCH] refactor(llm): standardize native request APIs --- packages/llm/src/adapter.ts | 11 +- packages/llm/src/llm.ts | 18 +++ packages/llm/src/tool-runtime.ts | 23 +--- packages/llm/test/adapter.test.ts | 110 ++++++++++------ packages/llm/test/llm.test.ts | 18 +++ .../anthropic-messages.recorded.test.ts | 4 +- .../test/provider/anthropic-messages.test.ts | 33 +++-- .../test/provider/bedrock-converse.test.ts | 48 ++++--- .../llm/test/provider/gemini.recorded.test.ts | 4 +- packages/llm/test/provider/gemini.test.ts | 32 +++-- .../openai-chat-tool-loop.recorded.test.ts | 4 +- .../provider/openai-chat.recorded.test.ts | 6 +- .../llm/test/provider/openai-chat.test.ts | 30 +++-- .../openai-compatible-chat.recorded.test.ts | 4 +- .../provider/openai-compatible-chat.test.ts | 15 ++- .../test/provider/openai-responses.test.ts | 25 ++-- packages/llm/test/tool-runtime.test.ts | 18 +-- packages/opencode/src/session/llm-native.ts | 70 ++++++++--- .../opencode/test/session/llm-native.test.ts | 118 ++++++++++++++++-- 19 files changed, 388 insertions(+), 203 deletions(-) diff --git a/packages/llm/src/adapter.ts b/packages/llm/src/adapter.ts index 51513b5504..8122042ac4 100644 --- a/packages/llm/src/adapter.ts +++ b/packages/llm/src/adapter.ts @@ -1,5 +1,6 @@ import { Effect, Stream } from "effect" import { HttpClientRequest, type HttpClientResponse } from "effect/unstable/http" +import * as LLM from "./llm" import { RequestExecutor } from "./executor" import type { AnyPatch, Patch, PatchInput, PatchRegistry } from "./patch" import { context, emptyRegistry, plan, registry as makePatchRegistry, target as targetPatch } from "./patch" @@ -97,6 +98,8 @@ export function define(input: AdapterInput): Adapt protocol: input.protocol, patches, get runtime() { + // Runtime registry erases adapter draft/target generics after validation. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion return this as unknown as RuntimeAdapter }, redact: input.redact, @@ -125,7 +128,7 @@ export function compose(input: ComposeInput): Adap }) } -export function client(options: ClientOptions): LLMClient { +const makeClient = (options: ClientOptions): LLMClient => { const registry = normalizeRegistry(options.patches) const adapters = options.adapters.map((adapter) => adapter.runtime) const providerAdapters = adapters @@ -173,13 +176,13 @@ export function client(options: ClientOptions): LLMClient { const patchedRequest = requestBeforeToolPatches.tools.length === 0 ? requestBeforeToolPatches - : { ...requestBeforeToolPatches, tools: requestBeforeToolPatches.tools.map(toolSchemaPlan.apply) } + : LLM.updateRequest(requestBeforeToolPatches, { tools: requestBeforeToolPatches.tools.map(toolSchemaPlan.apply) }) const patchContext = context({ request: patchedRequest }) const draft = yield* adapter.prepare(patchedRequest) const targetPlan = plan({ phase: "target", context: patchContext, - patches: [...adapter.patches, ...(registry.target as ReadonlyArray>)], + patches: [...adapter.patches, ...registry.target], }) const target = yield* adapter.validate(targetPlan.apply(draft)) const targetPatchTrace = [ @@ -241,4 +244,6 @@ export function client(options: ClientOptions): LLMClient { return { prepare, stream, generate } } +export const LLMClient = { make: makeClient } + export * as Adapter from "./adapter" diff --git a/packages/llm/src/llm.ts b/packages/llm/src/llm.ts index e8f80e6276..1b87df4eb9 100644 --- a/packages/llm/src/llm.ts +++ b/packages/llm/src/llm.ts @@ -159,6 +159,21 @@ export const generation = (input: GenerationOptions | ConstructorParameters ({ + id: input.id, + model: input.model, + system: input.system, + messages: input.messages, + tools: input.tools, + toolChoice: input.toolChoice, + generation: input.generation, + reasoning: input.reasoning, + cache: input.cache, + responseFormat: input.responseFormat, + metadata: input.metadata, + native: input.native, +}) + export const request = (input: RequestInput) => { const { system: requestSystem, prompt, messages, tools, toolChoice: requestToolChoice, generation: requestGeneration, ...rest } = input return new LLMRequest({ @@ -171,6 +186,9 @@ export const request = (input: RequestInput) => { }) } +export const updateRequest = (input: LLMRequest, patch: Partial) => + request({ ...requestInput(input), ...patch }) + export const outputText = (response: LLMResponse | { readonly events: ReadonlyArray }) => response.events .filter(LLMEvent.guards["text-delta"]) diff --git a/packages/llm/src/tool-runtime.ts b/packages/llm/src/tool-runtime.ts index 8d1b784e12..d5df4292b3 100644 --- a/packages/llm/src/tool-runtime.ts +++ b/packages/llm/src/tool-runtime.ts @@ -8,7 +8,7 @@ import { type FinishReason, type LLMError, type LLMEvent, - LLMRequest, + type LLMRequest, type ToolCallPart, type ToolResultValue, } from "./schema" @@ -43,21 +43,6 @@ export interface RunOptions { readonly stopWhen?: (state: RuntimeState) => boolean } -const requestInput = (request: LLMRequest): ConstructorParameters[0] => ({ - id: request.id, - model: request.model, - system: request.system, - messages: request.messages, - tools: request.tools, - toolChoice: request.toolChoice, - generation: request.generation, - reasoning: request.reasoning, - cache: request.cache, - responseFormat: request.responseFormat, - metadata: request.metadata, - native: request.native, -}) - /** * Run a model with a typed tool record. The runtime streams the model, on * each `tool-call` event decodes the input against the tool's `parameters` @@ -78,8 +63,7 @@ export const run = ( const concurrency = options.concurrency ?? 10 const tools = options.tools as Tools const runtimeTools = toDefinitions(tools) - const initialRequest = new LLMRequest({ - ...requestInput(options.request), + const initialRequest = LLM.updateRequest(options.request, { tools: [ ...options.request.tools.filter((tool) => !runtimeTools.some((runtimeTool) => runtimeTool.name === tool.name)), ...runtimeTools, @@ -106,8 +90,7 @@ export const run = ( (call) => dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)), { concurrency }, ) - const followUp = new LLMRequest({ - ...requestInput(request), + const followUp = LLM.updateRequest(request, { messages: [ ...request.messages, LLM.assistant(state.assistantContent), diff --git a/packages/llm/test/adapter.test.ts b/packages/llm/test/adapter.test.ts index 0714cb5aa9..eaf47ad061 100644 --- a/packages/llm/test/adapter.test.ts +++ b/packages/llm/test/adapter.test.ts @@ -2,21 +2,53 @@ import { describe, expect } from "bun:test" import { Effect, Schema, Stream } from "effect" import { HttpClientRequest } from "effect/unstable/http" import { LLM } from "../src" -import { Adapter, client } from "../src/adapter" +import { Adapter, LLMClient } from "../src/adapter" import { Patch } from "../src/patch" -import type { LLMRequest } from "../src/schema" +import type { LLMRequest, Message, ModelRef, ToolDefinition } from "../src/schema" import { testEffect } from "./lib/effect" import { dynamicResponse } from "./lib/http" -const mapText = (fn: (text: string) => string) => (request: LLMRequest): LLMRequest => ({ - ...request, - messages: request.messages.map((message) => ({ - ...message, - content: message.content.map((part) => - part.type === "text" ? { ...part, text: fn(part.text) } : part, +const updateMessageContent = (message: Message, content: Message["content"]) => + LLM.message({ + id: message.id, + role: message.role, + content, + metadata: message.metadata, + native: message.native, + }) + +const updateModel = (model: ModelRef, patch: Partial) => + LLM.model({ + id: model.id, + provider: model.provider, + protocol: model.protocol, + baseURL: model.baseURL, + headers: model.headers, + capabilities: model.capabilities, + limits: model.limits, + native: model.native, + ...patch, + }) + +const updateToolDefinition = (tool: ToolDefinition, patch: Partial) => + LLM.toolDefinition({ + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchema, + metadata: tool.metadata, + native: tool.native, + ...patch, + }) + +const mapText = (fn: (text: string) => string) => (request: LLMRequest): LLMRequest => + LLM.updateRequest(request, { + messages: request.messages.map((message) => + updateMessageContent( + message, + message.content.map((part) => (part.type === "text" ? { ...part, text: fn(part.text) } : part)), + ), ), - })), -}) + }) const Json = Schema.fromJsonString(Schema.Unknown) const encodeJson = Schema.encodeSync(Json) @@ -26,9 +58,12 @@ type FakeDraft = { readonly includeUsage?: boolean } -type FakeChunk = - | { readonly type: "text"; readonly text: string } - | { readonly type: "finish"; readonly reason: "stop" } +const FakeChunk = Schema.Union([ + Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }), + Schema.Struct({ type: Schema.Literal("finish"), reason: Schema.Literal("stop") }), +]) +type FakeChunk = Schema.Schema.Type +const FakeChunks = Schema.Array(FakeChunk) const request = LLM.request({ id: "req_1", @@ -68,7 +103,12 @@ const fake = Adapter.define({ ), ), parse: (response) => - Stream.fromEffect(response.json.pipe(Effect.orDie, Effect.map((body) => body as FakeChunk[]))).pipe( + Stream.fromEffect( + response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(FakeChunks)), + Effect.orDie, + ), + ).pipe( Stream.flatMap(Stream.fromIterable), Stream.map(raiseChunk), ), @@ -103,7 +143,7 @@ const it = testEffect(echoLayer) describe("llm adapter", () => { it.effect("prepare applies target patches with trace", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [ fake.withPatches([ fake.patch("include-usage", { @@ -121,7 +161,7 @@ describe("llm adapter", () => { it.effect("stream and generate use the adapter pipeline", () => Effect.gen(function* () { - const llm = client({ adapters: [fake] }) + const llm = LLMClient.make({ adapters: [fake] }) const events = Array.from(yield* llm.stream(request).pipe(Stream.runCollect)) const response = yield* llm.generate(request) @@ -132,11 +172,8 @@ describe("llm adapter", () => { it.effect("selects adapters by request protocol", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [fake, gemini] }).prepare( - LLM.request({ - ...request, - model: LLM.model({ ...request.model, protocol: "gemini" }), - }), + const prepared = yield* LLMClient.make({ adapters: [fake, gemini] }).prepare( + LLM.updateRequest(request, { model: updateModel(request.model, { protocol: "gemini" }) }), ) expect(prepared.adapter).toBe("gemini-fake") @@ -145,7 +182,7 @@ describe("llm adapter", () => { it.effect("prefers provider-specific adapters over protocol fallbacks", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [fake, providerFake] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [fake, providerFake] }).prepare(request) expect(prepared.adapter).toBe("provider-fake") expect(prepared.target).toEqual({ body: "provider:hello" }) @@ -154,12 +191,12 @@ describe("llm adapter", () => { it.effect("request, prompt, and tool-schema patches run before adapter prepare", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [fake], patches: [ Patch.request("test.id", { reason: "rewrite request id", - apply: (request) => ({ ...request, id: "req_patched" }), + apply: (request) => LLM.updateRequest(request, { id: "req_patched" }), }), Patch.prompt("test.message", { reason: "rewrite prompt text", @@ -167,12 +204,11 @@ describe("llm adapter", () => { }), Patch.toolSchema("test.description", { reason: "rewrite tool description", - apply: (tool) => ({ ...tool, description: "patched tool" }), + apply: (tool) => updateToolDefinition(tool, { description: "patched tool" }), }), ], }).prepare( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "original", inputSchema: {} }], }), ) @@ -189,7 +225,7 @@ describe("llm adapter", () => { it.effect("request patches feed into prompt-patch predicates so phases see updated context", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [fake], patches: [ // Earlier phase rewrites the provider, later phase only fires for the @@ -197,10 +233,7 @@ describe("llm adapter", () => { // test fails because the prompt patch's `when` would not match. Patch.request("rewrite-provider", { reason: "swap provider before prompt phase", - apply: (request) => ({ - ...request, - model: LLM.model({ ...request.model, provider: "rewritten" }), - }), + apply: (request) => LLM.updateRequest(request, { model: updateModel(request.model, { provider: "rewritten" }) }), }), Patch.prompt("rewrite-only-when-rewritten", { reason: "rewrite prompt text only after provider swap", @@ -220,7 +253,7 @@ describe("llm adapter", () => { it.effect("patches with the same order sort by id for deterministic application", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [fake], patches: [ Patch.prompt("zeta", { @@ -242,7 +275,7 @@ describe("llm adapter", () => { it.effect("stream patches transform raised events", () => Effect.gen(function* () { - const llm = client({ + const llm = LLMClient.make({ adapters: [fake], patches: [ Patch.stream("test.uppercase", { @@ -262,7 +295,7 @@ describe("llm adapter", () => { Effect.gen(function* () { // Verifies stream patches run on every event, not just the first. const seen: string[] = [] - const llm = client({ + const llm = LLMClient.make({ adapters: [fake], patches: [ Patch.stream("test.tap", { @@ -283,12 +316,9 @@ describe("llm adapter", () => { it.effect("rejects protocol mismatch", () => Effect.gen(function* () { - const error = yield* client({ adapters: [fake] }) + const error = yield* LLMClient.make({ adapters: [fake] }) .prepare( - LLM.request({ - ...request, - model: LLM.model({ ...request.model, protocol: "gemini" }), - }), + LLM.updateRequest(request, { model: updateModel(request.model, { protocol: "gemini" }) }), ) .pipe(Effect.flip) diff --git a/packages/llm/test/llm.test.ts b/packages/llm/test/llm.test.ts index 807f342034..38d82dc2ba 100644 --- a/packages/llm/test/llm.test.ts +++ b/packages/llm/test/llm.test.ts @@ -20,6 +20,24 @@ describe("llm constructors", () => { expect(request.tools).toEqual([]) }) + test("updates requests without spreading schema class instances", () => { + const base = LLM.request({ + id: "req_1", + model: LLM.model({ id: "fake-model", provider: "fake", protocol: "openai-chat" }), + prompt: "Say hello.", + }) + const updated = LLM.updateRequest(base, { + generation: { maxTokens: 20 }, + messages: [...base.messages, LLM.assistant("Hi.")], + }) + + expect(updated).toBeInstanceOf(LLMRequest) + expect(updated.id).toBe("req_1") + expect(updated.model).toEqual(base.model) + expect(updated.generation).toEqual({ maxTokens: 20 }) + expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"]) + }) + test("builds tool choices from names and tools", () => { const tool = LLM.toolDefinition({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }) diff --git a/packages/llm/test/provider/anthropic-messages.recorded.test.ts b/packages/llm/test/provider/anthropic-messages.recorded.test.ts index 5e1a85e209..d3af483402 100644 --- a/packages/llm/test/provider/anthropic-messages.recorded.test.ts +++ b/packages/llm/test/provider/anthropic-messages.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { AnthropicMessages } from "../../src/provider/anthropic-messages" import { recordedTests } from "../recorded-test" @@ -46,7 +46,7 @@ const recorded = recordedTests({ requires: ["ANTHROPIC_API_KEY"], options: { requestHeaders: ["content-type", "anthropic-version"] }, }) -const anthropic = client({ adapters: [AnthropicMessages.adapter] }) +const anthropic = LLMClient.make({ adapters: [AnthropicMessages.adapter] }) describe("Anthropic Messages recorded", () => { recorded.effect("streams text", () => diff --git a/packages/llm/test/provider/anthropic-messages.test.ts b/packages/llm/test/provider/anthropic-messages.test.ts index 28a07abcea..73fb0c98ab 100644 --- a/packages/llm/test/provider/anthropic-messages.test.ts +++ b/packages/llm/test/provider/anthropic-messages.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { CacheHint, LLM, ProviderRequestError } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { AnthropicMessages } from "../../src/provider/anthropic-messages" import { testEffect } from "../lib/effect" import { fixedResponse } from "../lib/http" @@ -26,7 +26,7 @@ const it = testEffect(Layer.empty) describe("Anthropic Messages adapter", () => { it.effect("prepares Anthropic Messages target", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(request) expect(prepared.target).toEqual({ model: "claude-sonnet-4-5", @@ -41,7 +41,7 @@ describe("Anthropic Messages adapter", () => { it.effect("prepares tool call and tool result messages", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare( LLM.request({ id: "req_tool_result", model, @@ -80,7 +80,7 @@ describe("Anthropic Messages adapter", () => { { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } }, { type: "message_stop" }, ) - const response = yield* client({ adapters: [AnthropicMessages.adapter] }) + const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -106,10 +106,9 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 0 }, { type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } }, ) - const response = yield* client({ adapters: [AnthropicMessages.adapter] }) + const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -131,7 +130,7 @@ describe("Anthropic Messages adapter", () => { it.effect("emits provider-error events for mid-stream provider errors", () => Effect.gen(function* () { - const response = yield* client({ adapters: [AnthropicMessages.adapter] }) + const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate(request) .pipe( Effect.provide( @@ -145,7 +144,7 @@ describe("Anthropic Messages adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* client({ adapters: [AnthropicMessages.adapter] }) + const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate(request) .pipe( Effect.provide( @@ -185,10 +184,9 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 2 }, { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } }, ) - const response = yield* client({ adapters: [AnthropicMessages.adapter] }) + const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }], }), ) @@ -234,10 +232,9 @@ describe("Anthropic Messages adapter", () => { { type: "content_block_stop", index: 1 }, { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }, ) - const response = yield* client({ adapters: [AnthropicMessages.adapter] }) + const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }], }), ) @@ -256,7 +253,7 @@ describe("Anthropic Messages adapter", () => { it.effect("round-trips provider-executed assistant content into server tool blocks", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare( LLM.request({ id: "req_round_trip", model, @@ -307,7 +304,7 @@ describe("Anthropic Messages adapter", () => { it.effect("rejects round-trip for unknown server tool names", () => Effect.gen(function* () { - const error = yield* client({ adapters: [AnthropicMessages.adapter] }) + const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .prepare( LLM.request({ id: "req_unknown_server_tool", @@ -333,7 +330,7 @@ describe("Anthropic Messages adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* client({ adapters: [AnthropicMessages.adapter] }) + const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }) .prepare( LLM.request({ id: "req_media", diff --git a/packages/llm/test/provider/bedrock-converse.test.ts b/packages/llm/test/provider/bedrock-converse.test.ts index fb81fa8cc6..653b70732d 100644 --- a/packages/llm/test/provider/bedrock-converse.test.ts +++ b/packages/llm/test/provider/bedrock-converse.test.ts @@ -3,7 +3,7 @@ import { fromUtf8, toUtf8 } from "@smithy/util-utf8" import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { CacheHint, LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { BedrockConverse } from "../../src/provider/bedrock-converse" import { testEffect } from "../lib/effect" import { fixedResponse } from "../lib/http" @@ -63,7 +63,7 @@ const it = testEffect(Layer.empty) describe("Bedrock Converse adapter", () => { it.effect("prepares Converse target with system, inference config, and messages", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest) + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest) expect(prepared.target).toEqual({ modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0", @@ -76,9 +76,8 @@ describe("Bedrock Converse adapter", () => { it.effect("prepares tool config with toolSpec and toolChoice", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( - LLM.request({ - ...baseRequest, + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( + LLM.updateRequest(baseRequest, { tools: [ { name: "lookup", @@ -111,7 +110,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers assistant tool-call + tool-result message history", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( LLM.request({ id: "req_history", model, @@ -157,7 +156,7 @@ describe("Bedrock Converse adapter", () => { ["messageStop", { stopReason: "end_turn" }], ["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }], ) - const response = yield* client({ adapters: [BedrockConverse.adapter] }) + const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) @@ -192,10 +191,9 @@ describe("Bedrock Converse adapter", () => { ["contentBlockStop", { contentBlockIndex: 0 }], ["messageStop", { stopReason: "tool_use" }], ) - const response = yield* client({ adapters: [BedrockConverse.adapter] }) + const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .generate( - LLM.request({ - ...baseRequest, + LLM.updateRequest(baseRequest, { tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }], }), ) @@ -224,7 +222,7 @@ describe("Bedrock Converse adapter", () => { ["contentBlockStop", { contentBlockIndex: 0 }], ["messageStop", { stopReason: "end_turn" }], ) - const response = yield* client({ adapters: [BedrockConverse.adapter] }) + const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) @@ -238,7 +236,7 @@ describe("Bedrock Converse adapter", () => { ["messageStart", { role: "assistant" }], ["throttlingException", { message: "Slow down" }], ) - const response = yield* client({ adapters: [BedrockConverse.adapter] }) + const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .generate(baseRequest) .pipe(Effect.provide(fixedBytes(body))) @@ -256,8 +254,8 @@ describe("Bedrock Converse adapter", () => { id: "anthropic.claude-3-5-sonnet-20240620-v1:0", baseURL: "https://bedrock-runtime.test", }) - const error = yield* client({ adapters: [BedrockConverse.adapter] }) - .generate(LLM.request({ ...baseRequest, model: unsignedModel })) + const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) + .generate(LLM.updateRequest(baseRequest, { model: unsignedModel })) .pipe(Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))), Effect.flip) expect(error.message).toContain("Bedrock Converse requires either a Bearer API key") @@ -275,8 +273,8 @@ describe("Bedrock Converse adapter", () => { secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", }, }) - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( - LLM.request({ ...baseRequest, model: signed }), + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( + LLM.updateRequest(baseRequest, { model: signed }), ) expect(prepared.adapter).toBe("bedrock-converse") @@ -292,7 +290,7 @@ describe("Bedrock Converse adapter", () => { it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () => Effect.gen(function* () { const cache = new CacheHint({ type: "ephemeral" }) - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( LLM.request({ id: "req_cache", model, @@ -324,7 +322,7 @@ describe("Bedrock Converse adapter", () => { it.effect("does not emit cachePoint when no cache hint is set", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest) + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest) expect(prepared.target).toMatchObject({ system: [{ text: "You are concise." }], messages: [{ role: "user", content: [{ text: "Say hello." }] }], @@ -334,7 +332,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers image media into Bedrock image blocks", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( LLM.request({ id: "req_image", model, @@ -370,7 +368,7 @@ describe("Bedrock Converse adapter", () => { it.effect("base64-encodes Uint8Array image bytes", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( LLM.request({ id: "req_image_bytes", model, @@ -396,7 +394,7 @@ describe("Bedrock Converse adapter", () => { it.effect("lowers document media into Bedrock document blocks with format and name", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare( LLM.request({ id: "req_doc", model, @@ -427,7 +425,7 @@ describe("Bedrock Converse adapter", () => { it.effect("rejects unsupported image media types", () => Effect.gen(function* () { - const error = yield* client({ adapters: [BedrockConverse.adapter] }) + const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .prepare( LLM.request({ id: "req_bad_image", @@ -443,7 +441,7 @@ describe("Bedrock Converse adapter", () => { it.effect("rejects unsupported document media types", () => Effect.gen(function* () { - const error = yield* client({ adapters: [BedrockConverse.adapter] }) + const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }) .prepare( LLM.request({ id: "req_bad_doc", @@ -493,7 +491,7 @@ const recorded = recordedTests({ describe("Bedrock Converse recorded", () => { recorded.effect("streams text", () => Effect.gen(function* () { - const llm = client({ adapters: [BedrockConverse.adapter] }) + const llm = LLMClient.make({ adapters: [BedrockConverse.adapter] }) const response = yield* llm.generate( LLM.request({ id: "recorded_bedrock_text", @@ -511,7 +509,7 @@ describe("Bedrock Converse recorded", () => { recorded.effect("streams a tool call", () => Effect.gen(function* () { - const llm = client({ adapters: [BedrockConverse.adapter] }) + const llm = LLMClient.make({ adapters: [BedrockConverse.adapter] }) const response = yield* llm.generate( LLM.request({ id: "recorded_bedrock_tool_call", diff --git a/packages/llm/test/provider/gemini.recorded.test.ts b/packages/llm/test/provider/gemini.recorded.test.ts index c4f21ff99e..4357f0f37b 100644 --- a/packages/llm/test/provider/gemini.recorded.test.ts +++ b/packages/llm/test/provider/gemini.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { Gemini } from "../../src/provider/gemini" import { recordedTests } from "../recorded-test" @@ -42,7 +42,7 @@ const toolRequest = LLM.request({ }) const recorded = recordedTests({ prefix: "gemini", requires: ["GOOGLE_GENERATIVE_AI_API_KEY"] }) -const gemini = client({ adapters: [Gemini.adapter] }) +const gemini = LLMClient.make({ adapters: [Gemini.adapter] }) describe("Gemini recorded", () => { recorded.effect("streams text", () => diff --git a/packages/llm/test/provider/gemini.test.ts b/packages/llm/test/provider/gemini.test.ts index 831caf66c7..08ac264bb8 100644 --- a/packages/llm/test/provider/gemini.test.ts +++ b/packages/llm/test/provider/gemini.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { LLM, ProviderChunkError, ProviderPatch } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { Gemini } from "../../src/provider/gemini" import { testEffect } from "../lib/effect" import { fixedResponse } from "../lib/http" @@ -26,7 +26,7 @@ const it = testEffect(Layer.empty) describe("Gemini adapter", () => { it.effect("prepares Gemini target", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(request) expect(prepared.target).toEqual({ contents: [{ role: "user", parts: [{ text: "Say hello." }] }], @@ -38,7 +38,7 @@ describe("Gemini adapter", () => { it.effect("prepares multimodal user input and tool history", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare( LLM.request({ id: "req_tool_result", model, @@ -91,7 +91,7 @@ describe("Gemini adapter", () => { it.effect("omits tools when tool choice is none", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare( LLM.request({ id: "req_no_tools", model, @@ -109,7 +109,7 @@ describe("Gemini adapter", () => { it.effect("applies Gemini tool-schema patches before preparing the target", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter], patches: [ProviderPatch.sanitizeGeminiToolSchema], }).prepare( @@ -181,7 +181,7 @@ describe("Gemini adapter", () => { }, }, ) - const response = yield* client({ adapters: [Gemini.adapter] }) + const response = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -234,10 +234,9 @@ describe("Gemini adapter", () => { usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, }, ) - const response = yield* client({ adapters: [Gemini.adapter] }) + const response = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -271,10 +270,9 @@ describe("Gemini adapter", () => { }], }, ) - const response = yield* client({ adapters: [Gemini.adapter] }) + const response = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -290,14 +288,14 @@ describe("Gemini adapter", () => { it.effect("maps length and content-filter finish reasons", () => Effect.gen(function* () { - const length = yield* client({ adapters: [Gemini.adapter] }) + const length = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate(request) .pipe( Effect.provide( fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "MAX_TOKENS" }] })), ), ) - const filtered = yield* client({ adapters: [Gemini.adapter] }) + const filtered = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate(request) .pipe( Effect.provide( @@ -312,7 +310,7 @@ describe("Gemini adapter", () => { it.effect("leaves total usage undefined when component counts are missing", () => Effect.gen(function* () { - const response = yield* client({ adapters: [Gemini.adapter] }) + const response = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(sseEvents({ usageMetadata: { thoughtsTokenCount: 1 } })))) @@ -323,7 +321,7 @@ describe("Gemini adapter", () => { it.effect("fails invalid stream chunks", () => Effect.gen(function* () { - const error = yield* client({ adapters: [Gemini.adapter] }) + const error = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .generate(request) .pipe( Effect.provide(fixedResponse(sseRaw("data: {not json}"))), @@ -337,7 +335,7 @@ describe("Gemini adapter", () => { it.effect("rejects unsupported assistant media content", () => Effect.gen(function* () { - const error = yield* client({ adapters: [Gemini.adapter] }) + const error = yield* LLMClient.make({ adapters: [Gemini.adapter] }) .prepare( LLM.request({ id: "req_media", diff --git a/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts b/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts index c748645725..5fc879bbc0 100644 --- a/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts +++ b/packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Schema, Stream } from "effect" import { LLM, LLMEvent } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAIChat } from "../../src/provider/openai-chat" import { tool } from "../../src/tool" import { ToolRuntime } from "../../src/tool-runtime" @@ -39,7 +39,7 @@ const request = LLM.request({ }) const recorded = recordedTests({ prefix: "openai-chat", requires: ["OPENAI_API_KEY"] }) -const openai = client({ adapters: [OpenAIChat.adapter] }) +const openai = LLMClient.make({ adapters: [OpenAIChat.adapter] }) describe("OpenAI Chat tool-loop recorded", () => { recorded.effect("drives a tool loop end-to-end", () => diff --git a/packages/llm/test/provider/openai-chat.recorded.test.ts b/packages/llm/test/provider/openai-chat.recorded.test.ts index 5e5a86c7dd..c55eb5a539 100644 --- a/packages/llm/test/provider/openai-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-chat.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAIChat } from "../../src/provider/openai-chat" import { recordedTests } from "../recorded-test" @@ -58,8 +58,8 @@ const toolResultRequest = LLM.request({ // `length > 0` checks so adapter parsing regressions surface immediately. // Re-record (`RECORD=true`) only when intentionally refreshing a cassette. const recorded = recordedTests({ prefix: "openai-chat", requires: ["OPENAI_API_KEY"] }) -const openai = client({ adapters: [OpenAIChat.adapter] }) -const openaiWithUsage = client({ adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])] }) +const openai = LLMClient.make({ adapters: [OpenAIChat.adapter] }) +const openaiWithUsage = LLMClient.make({ adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])] }) describe("OpenAI Chat recorded", () => { recorded.effect("streams text", () => diff --git a/packages/llm/test/provider/openai-chat.test.ts b/packages/llm/test/provider/openai-chat.test.ts index 614e18727d..f836ca5d8f 100644 --- a/packages/llm/test/provider/openai-chat.test.ts +++ b/packages/llm/test/provider/openai-chat.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer, Schema, Stream } from "effect" import { LLM, ProviderRequestError } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAIChat } from "../../src/provider/openai-chat" import { testEffect } from "../lib/effect" import { fixedResponse, truncatedStream } from "../lib/http" @@ -41,7 +41,7 @@ const usageChunk = (usage: object) => ({ describe("OpenAI Chat adapter", () => { it.effect("prepares OpenAI Chat target", () => Effect.gen(function* () { - const prepared = yield* client({ + const prepared = yield* LLMClient.make({ adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])], }).prepare(request) @@ -62,7 +62,7 @@ describe("OpenAI Chat adapter", () => { it.effect("prepares assistant tool-call and tool-result messages", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAIChat.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }).prepare( LLM.request({ id: "req_tool_result", model, @@ -98,7 +98,7 @@ describe("OpenAI Chat adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* client({ adapters: [OpenAIChat.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .prepare( LLM.request({ id: "req_media", @@ -114,7 +114,7 @@ describe("OpenAI Chat adapter", () => { it.effect("rejects unsupported assistant reasoning content", () => Effect.gen(function* () { - const error = yield* client({ adapters: [OpenAIChat.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .prepare( LLM.request({ id: "req_reasoning", @@ -142,7 +142,7 @@ describe("OpenAI Chat adapter", () => { completion_tokens_details: { reasoning_tokens: 0 }, }), ) - const response = yield* client({ adapters: [OpenAIChat.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -184,10 +184,9 @@ describe("OpenAI Chat adapter", () => { deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }), deltaChunk({}, "tool_calls"), ) - const response = yield* client({ adapters: [OpenAIChat.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -213,10 +212,9 @@ describe("OpenAI Chat adapter", () => { }), deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }), ) - const response = yield* client({ adapters: [OpenAIChat.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -233,7 +231,7 @@ describe("OpenAI Chat adapter", () => { it.effect("fails on malformed stream chunks", () => Effect.gen(function* () { const body = sseEvents(deltaChunk({ content: 123 })) - const error = yield* client({ adapters: [OpenAIChat.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body)), Effect.flip) @@ -246,7 +244,7 @@ describe("OpenAI Chat adapter", () => { const layer = truncatedStream([ `data: ${JSON.stringify(deltaChunk({ role: "assistant", content: "Hello" }))}\n\n`, ]) - const error = yield* client({ adapters: [OpenAIChat.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate(request) .pipe(Effect.provide(layer), Effect.flip) @@ -256,7 +254,7 @@ describe("OpenAI Chat adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* client({ adapters: [OpenAIChat.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }) .generate(request) .pipe( Effect.provide( @@ -276,7 +274,7 @@ describe("OpenAI Chat adapter", () => { it.effect("short-circuits the upstream stream when the consumer takes a prefix", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) // The body has more chunks than we'll consume. If `Stream.take(1)` did // not interrupt the upstream HTTP body the test would hang waiting for // the rest of the stream to drain. diff --git a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts index 296429848c..e00c349346 100644 --- a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" import { LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat" import { recordedTests } from "../recorded-test" @@ -55,7 +55,7 @@ const togetherToolRequest = LLM.request({ }) const recorded = recordedTests({ prefix: "openai-compatible-chat" }) -const llm = client({ adapters: [OpenAICompatibleChat.adapter] }) +const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }) describe("OpenAI-compatible Chat recorded", () => { recorded.effect.with("deepseek streams text", { requires: ["DEEPSEEK_API_KEY"] }, () => diff --git a/packages/llm/test/provider/openai-compatible-chat.test.ts b/packages/llm/test/provider/openai-compatible-chat.test.ts index bd3fb44be4..efaed0e5e9 100644 --- a/packages/llm/test/provider/openai-compatible-chat.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer, Schema } from "effect" import { HttpClientRequest } from "effect/unstable/http" import { LLM } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat" import { testEffect } from "../lib/effect" import { dynamicResponse } from "../lib/http" @@ -53,9 +53,8 @@ const providerFamilies = [ describe("OpenAI-compatible Chat adapter", () => { it.effect("prepares generic Chat target", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare( - LLM.request({ - ...request, + const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare( + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], toolChoice: { type: "required" }, }), @@ -88,7 +87,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("provides model helpers for compatible provider families", () => Effect.gen(function* () { expect( - providerFamilies.map(([provider, makeModel, baseURL]) => { + providerFamilies.map(([provider, makeModel]) => { const model = makeModel({ id: `${provider}-model`, apiKey: "test-key" }) return { id: String(model.id), @@ -126,7 +125,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("matches AI SDK compatible basic request body fixture", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request) expect(prepared.target).toEqual({ model: "deepseek-chat", @@ -143,7 +142,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("matches AI SDK compatible tool request body fixture", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare( LLM.request({ id: "req_tool_parity", model, @@ -192,7 +191,7 @@ describe("OpenAI-compatible Chat adapter", () => { it.effect("posts to the configured compatible endpoint and parses text usage", () => Effect.gen(function* () { - const response = yield* client({ + const response = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter.withPatches([OpenAICompatibleChat.includeUsage])], }) .generate(request) diff --git a/packages/llm/test/provider/openai-responses.test.ts b/packages/llm/test/provider/openai-responses.test.ts index 23f44b578c..e9d4394ee9 100644 --- a/packages/llm/test/provider/openai-responses.test.ts +++ b/packages/llm/test/provider/openai-responses.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { LLM, ProviderRequestError } from "../../src" -import { client } from "../../src/adapter" +import { LLMClient } from "../../src/adapter" import { OpenAIResponses } from "../../src/provider/openai-responses" import { testEffect } from "../lib/effect" import { fixedResponse } from "../lib/http" @@ -26,7 +26,7 @@ const it = testEffect(Layer.empty) describe("OpenAI Responses adapter", () => { it.effect("prepares OpenAI Responses target", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare(request) expect(prepared.target).toEqual({ model: "gpt-4.1-mini", @@ -43,7 +43,7 @@ describe("OpenAI Responses adapter", () => { it.effect("prepares function call and function output input items", () => Effect.gen(function* () { - const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare( + const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare( LLM.request({ id: "req_tool_result", model, @@ -85,7 +85,7 @@ describe("OpenAI Responses adapter", () => { }, }, ) - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -136,10 +136,9 @@ describe("OpenAI Responses adapter", () => { }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate( - LLM.request({ - ...request, + LLM.updateRequest(request, { tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], }), ) @@ -171,7 +170,7 @@ describe("OpenAI Responses adapter", () => { { type: "response.output_item.done", item }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -209,7 +208,7 @@ describe("OpenAI Responses adapter", () => { { type: "response.output_item.done", item }, { type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } }, ) - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(body))) @@ -234,7 +233,7 @@ describe("OpenAI Responses adapter", () => { it.effect("rejects unsupported user media content", () => Effect.gen(function* () { - const error = yield* client({ adapters: [OpenAIResponses.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .prepare( LLM.request({ id: "req_media", @@ -250,7 +249,7 @@ describe("OpenAI Responses adapter", () => { it.effect("emits provider-error events for mid-stream provider errors", () => Effect.gen(function* () { - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe( Effect.provide( @@ -264,7 +263,7 @@ describe("OpenAI Responses adapter", () => { it.effect("falls back to error code when no message is present", () => Effect.gen(function* () { - const response = yield* client({ adapters: [OpenAIResponses.adapter] }) + const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe(Effect.provide(fixedResponse(sseEvents({ type: "error", code: "internal_error" })))) @@ -274,7 +273,7 @@ describe("OpenAI Responses adapter", () => { it.effect("fails HTTP provider errors before stream parsing", () => Effect.gen(function* () { - const error = yield* client({ adapters: [OpenAIResponses.adapter] }) + const error = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }) .generate(request) .pipe( Effect.provide( diff --git a/packages/llm/test/tool-runtime.test.ts b/packages/llm/test/tool-runtime.test.ts index d5de805017..abd7aafcee 100644 --- a/packages/llm/test/tool-runtime.test.ts +++ b/packages/llm/test/tool-runtime.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer, Schema, Stream } from "effect" import { LLM, LLMEvent } from "../src" -import { client, type LLMClient } from "../src/adapter" +import { LLMClient } from "../src/adapter" import { RequestExecutor } from "../src/executor" import { OpenAIChat } from "../src/provider/openai-chat" import { tool, ToolFailure } from "../src/tool" @@ -39,7 +39,7 @@ const get_weather = tool({ describe("ToolRuntime", () => { it.effect("dispatches a tool call, appends results, and resumes streaming", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"Paris"}'), finishChunk("tool_calls")), sseEvents(deltaChunk({ role: "assistant", content: "It's sunny in Paris." }), finishChunk("stop")), @@ -66,7 +66,7 @@ describe("ToolRuntime", () => { it.effect("emits tool-error for unknown tools so the model can self-correct", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents(toolCallChunk("call_1", "missing_tool", "{}"), finishChunk("tool_calls")), sseEvents(deltaChunk({ role: "assistant", content: "Sorry." }), finishChunk("stop")), @@ -93,7 +93,7 @@ describe("ToolRuntime", () => { it.effect("emits tool-error when the LLM input fails the parameters schema", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents(toolCallChunk("call_1", "get_weather", '{"city":42}'), finishChunk("tool_calls")), sseEvents(deltaChunk({ role: "assistant", content: "Done." }), finishChunk("stop")), @@ -114,7 +114,7 @@ describe("ToolRuntime", () => { it.effect("emits tool-error when the handler returns a ToolFailure", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"FAIL"}'), finishChunk("tool_calls")), sseEvents(deltaChunk({ role: "assistant", content: "Sorry." }), finishChunk("stop")), @@ -135,7 +135,7 @@ describe("ToolRuntime", () => { it.effect("stops when the model finishes without requesting more tools", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([sseEvents(deltaChunk({ role: "assistant", content: "Done." }), finishChunk("stop"))]) const events = Array.from( @@ -152,7 +152,7 @@ describe("ToolRuntime", () => { it.effect("respects maxSteps and stops the loop", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) // Every script entry asks for another tool call. With maxSteps: 2 the // runtime should run at most two model rounds and then exit even though // the model still wants to keep going. @@ -172,7 +172,7 @@ describe("ToolRuntime", () => { it.effect("stops when stopWhen returns true after the first step", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"Paris"}'), finishChunk("tool_calls")), sseEvents(deltaChunk({ role: "assistant", content: "Should not run." }), finishChunk("stop")), @@ -254,7 +254,7 @@ describe("ToolRuntime", () => { it.effect("dispatches multiple tool calls in one step concurrently", () => Effect.gen(function* () { - const llm = client({ adapters: [OpenAIChat.adapter] }) + const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] }) const layer = scriptedResponses([ sseEvents( deltaChunk({ diff --git a/packages/opencode/src/session/llm-native.ts b/packages/opencode/src/session/llm-native.ts index b5dc88ada8..7583d0a433 100644 --- a/packages/opencode/src/session/llm-native.ts +++ b/packages/opencode/src/session/llm-native.ts @@ -1,4 +1,4 @@ -import { LLM, type ContentPart, type Message as CoreMessage } from "@opencode-ai/llm" +import { CacheHint, LLM, type ContentPart, type LLMRequest, type Message as CoreMessage } from "@opencode-ai/llm" import { Effect, Schema } from "effect" import { ProviderLLMBridge } from "@/provider/llm-bridge" import * as EffectZod from "@/util/effect-zod" @@ -169,6 +169,42 @@ export const toolDefinition = (input: { readonly model: Provider.Model; readonly }, }) +// Mirrors the AI SDK path's prompt-cache policy, gated by model capability. +const EPHEMERAL_CACHE = new CacheHint({ type: "ephemeral" }) + +const withCacheOnLastText = (content: ReadonlyArray): ReadonlyArray => { + const index = content.findLastIndex((part) => part.type === "text") + if (index === -1) return content + return content.map((part, position) => + position === index && part.type === "text" ? { ...part, cache: EPHEMERAL_CACHE } : part, + ) +} + +const updateMessageContent = (message: CoreMessage, content: ReadonlyArray) => { + if (content === message.content) return message + return LLM.message({ + id: message.id, + role: message.role, + content, + metadata: message.metadata, + native: message.native, + }) +} + +const applyCachePolicy = (request: LLMRequest): LLMRequest => { + if (!request.model.capabilities.cache?.prompt) return request + const system = request.system.map((part, index) => + index < 2 ? { ...part, cache: EPHEMERAL_CACHE } : part, + ) + const lastTwoStart = Math.max(0, request.messages.length - 2) + const messages = request.messages.map((message, index) => + index < lastTwoStart + ? message + : updateMessageContent(message, withCacheOnLastText(message.content)), + ) + return LLM.updateRequest(request, { system, messages }) +} + export const request = Effect.fn("LLMNative.request")(function* (input: RequestInput) { const unsupported = unsupportedPart(input) if (unsupported) { @@ -186,21 +222,23 @@ export const request = Effect.fn("LLMNative.request")(function* (input: RequestI }) } - return LLM.request({ - id: input.id, - model, - system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [], - messages: input.messages.flatMap(messages), - tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [], - toolChoice: input.toolChoice, - generation: input.generation, - metadata: input.metadata, - native: { - opencodeProviderID: input.provider.id, - opencodeModelID: input.model.id, - ...input.native, - }, - }) + return applyCachePolicy( + LLM.request({ + id: input.id, + model, + system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [], + messages: input.messages.flatMap(messages), + tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [], + toolChoice: input.toolChoice, + generation: input.generation, + metadata: input.metadata, + native: { + opencodeProviderID: input.provider.id, + opencodeModelID: input.model.id, + ...input.native, + }, + }), + ) }) export * as LLMNative from "./llm-native" diff --git a/packages/opencode/test/session/llm-native.test.ts b/packages/opencode/test/session/llm-native.test.ts index 3530b6541d..82aab6dc32 100644 --- a/packages/opencode/test/session/llm-native.test.ts +++ b/packages/opencode/test/session/llm-native.test.ts @@ -1,7 +1,5 @@ import { describe, expect } from "bun:test" -import { AnthropicMessages, Gemini, OpenAICompatibleChat } from "@opencode-ai/llm" -import { client } from "@opencode-ai/llm/adapter" -import { OpenAIResponses } from "@opencode-ai/llm/provider/openai-responses" +import { AnthropicMessages, BedrockConverse, Gemini, LLMClient, OpenAICompatibleChat, OpenAIResponses } from "@opencode-ai/llm" import { Cause, Effect, Exit, Layer, Schema } from "effect" import { ModelID, ProviderID } from "../../src/provider/schema" import { LLMNative } from "../../src/session/llm-native" @@ -333,7 +331,7 @@ describe("LLMNative.request", () => { tools: [lookupTool], toolChoice: "lookup", }) - const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare(request) expect(prepared.target).toMatchObject({ model: "gpt-5", @@ -392,7 +390,7 @@ describe("LLMNative.request", () => { tools: [lookupTool], toolChoice: "lookup", }) - const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(request) expect(request.model).toMatchObject({ provider: "anthropic", @@ -461,7 +459,7 @@ describe("LLMNative.request", () => { tools: [lookupTool], toolChoice: "lookup", }) - const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request) expect(request.model).toMatchObject({ provider: "togetherai", @@ -540,7 +538,7 @@ describe("LLMNative.request", () => { tools: [lookupTool], toolChoice: "lookup", }) - const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(request) expect(request.model).toMatchObject({ provider: "google", @@ -577,4 +575,110 @@ describe("LLMNative.request", () => { generationConfig: { maxOutputTokens: 32, temperature: 0 }, }) })) + + // Cache hint policy. The LLM-native path mirrors the AI-SDK applyCaching + // policy from packages/opencode/src/provider/transform.ts: mark the first 2 + // system parts and the last 2 messages as cacheable, gated on the resolved + // model's `capabilities.cache.prompt`. Adapters lower CacheHint to the + // provider-specific marker (cache_control on Anthropic, cachePoint on + // Bedrock); non-cache-capable adapters never see a hint. + + const anthropicModel = () => + model({ + id: ModelID.make("claude-sonnet-4-5"), + providerID: ProviderID.make("anthropic"), + api: { id: "claude-sonnet-4-5", url: "https://api.anthropic.com/v1", npm: "@ai-sdk/anthropic" }, + }) + + const bedrockModel = () => + model({ + id: ModelID.make("us.amazon.nova-micro-v1:0"), + providerID: ProviderID.make("amazon-bedrock"), + api: { + id: "us.amazon.nova-micro-v1:0", + url: "https://bedrock-runtime.us-east-1.amazonaws.com", + npm: "@ai-sdk/amazon-bedrock", + }, + }) + + it.effect("applies cache hints to the first 2 system parts on cache-capable models", () => + Effect.gen(function* () { + const mdl = anthropicModel() + const userID = MessageID.ascending() + const request = yield* LLMNative.request({ + provider: ProviderTest.info({ id: ProviderID.make("anthropic"), key: "anthropic-key" }, mdl), + model: mdl, + system: ["First", "Second", "Third"], + messages: [userMessage(mdl, userID, [textPart(userID, "hello")])], + }) + + expect(request.system).toHaveLength(3) + expect(request.system[0]).toMatchObject({ text: "First", cache: { type: "ephemeral" } }) + expect(request.system[1]).toMatchObject({ text: "Second", cache: { type: "ephemeral" } }) + expect(request.system[2]).toMatchObject({ text: "Third" }) + expect(request.system[2].cache).toBeUndefined() + })) + + it.effect("applies cache hints to the final text part of the last 2 messages on cache-capable models", () => + Effect.gen(function* () { + const mdl = anthropicModel() + const messageIds = [MessageID.ascending(), MessageID.ascending(), MessageID.ascending()] + const request = yield* LLMNative.request({ + provider: ProviderTest.info({ id: ProviderID.make("anthropic"), key: "anthropic-key" }, mdl), + model: mdl, + messages: messageIds.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])), + }) + + expect(request.messages).toHaveLength(3) + // First message: no cache hint. + const first = request.messages[0].content[0] + if (first.type === "text") expect(first.cache).toBeUndefined() + // Last two messages: cache on the (only) text part. + expect(request.messages[1].content[0]).toMatchObject({ type: "text", text: "m1", cache: { type: "ephemeral" } }) + expect(request.messages[2].content[0]).toMatchObject({ type: "text", text: "m2", cache: { type: "ephemeral" } }) + })) + + it.effect("lowers cache hints to Bedrock Converse cachePoint marker blocks end-to-end", () => + Effect.gen(function* () { + const mdl = bedrockModel() + const userID = MessageID.ascending() + const request = yield* LLMNative.request({ + provider: ProviderTest.info({ id: ProviderID.make("amazon-bedrock"), key: "bedrock-bearer" }, mdl), + model: mdl, + system: ["You are concise."], + messages: [userMessage(mdl, userID, [textPart(userID, "hello")])], + }) + const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(request) + + expect(prepared.target).toMatchObject({ + system: [{ text: "You are concise." }, { cachePoint: { type: "default" } }], + messages: [ + { + role: "user", + content: [{ text: "hello" }, { cachePoint: { type: "default" } }], + }, + ], + }) + })) + + it.effect("does not apply cache hints when the model does not support prompt caching", () => + Effect.gen(function* () { + // gpt-5 / openai resolves to openai-responses, which advertises + // capabilities.cache.prompt: false. The bridge must skip the policy. + const mdl = model() + const ids = [MessageID.ascending(), MessageID.ascending()] + const request = yield* LLMNative.request({ + provider: ProviderTest.info({ id: ProviderID.openai, key: "openai-key" }, mdl), + model: mdl, + system: ["A", "B", "C"], + messages: ids.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])), + }) + + for (const part of request.system) expect(part.cache).toBeUndefined() + for (const message of request.messages) { + for (const part of message.content) { + if (part.type === "text") expect(part.cache).toBeUndefined() + } + } + })) })