Mirror_Repos/anomalyco_opencode
2
0
Fork 0
Code Issues Pull Requests Actions 61 Packages Projects Releases Wiki Activity

refactor(llm): standardize native request APIs

Browse Source
This commit is contained in:
Kit Langton
2026-04-26 21:08:35 -04:00
parent 653a830cf6
commit 3cd13c87c4
19 changed files with 388 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/src/adapter.ts
+8 -3
View File
@@ -1,5 +1,6 @@
import { Effect, Stream } from "effect"
import { HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
import * as LLM from "./llm"
import { RequestExecutor } from "./executor"
import type { AnyPatch, Patch, PatchInput, PatchRegistry } from "./patch"
import { context, emptyRegistry, plan, registry as makePatchRegistry, target as targetPatch } from "./patch"
@@ -97,6 +98,8 @@ export function define<Draft, Target>(input: AdapterInput<Draft, Target>): Adapt
protocol: input.protocol,
patches,
get runtime() {
// Runtime registry erases adapter draft/target generics after validation.
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
return this as unknown as RuntimeAdapter
},
redact: input.redact,
@@ -125,7 +128,7 @@ export function compose<Draft, Target>(input: ComposeInput<Draft, Target>): Adap
})
}
export function client(options: ClientOptions): LLMClient {
const makeClient = (options: ClientOptions): LLMClient => {
const registry = normalizeRegistry(options.patches)
const adapters = options.adapters.map((adapter) => adapter.runtime)
const providerAdapters = adapters
@@ -173,13 +176,13 @@ export function client(options: ClientOptions): LLMClient {
const patchedRequest =
requestBeforeToolPatches.tools.length === 0
? requestBeforeToolPatches
: { ...requestBeforeToolPatches, tools: requestBeforeToolPatches.tools.map(toolSchemaPlan.apply) }
: LLM.updateRequest(requestBeforeToolPatches, { tools: requestBeforeToolPatches.tools.map(toolSchemaPlan.apply) })
const patchContext = context({ request: patchedRequest })
const draft = yield* adapter.prepare(patchedRequest)
const targetPlan = plan({
phase: "target",
context: patchContext,
patches: [...adapter.patches, ...(registry.target as ReadonlyArray<Patch<unknown>>)],
patches: [...adapter.patches, ...registry.target],
})
const target = yield* adapter.validate(targetPlan.apply(draft))
const targetPatchTrace = [
@@ -241,4 +244,6 @@ export function client(options: ClientOptions): LLMClient {
return { prepare, stream, generate }
}
export const LLMClient = { make: makeClient }
export * as Adapter from "./adapter"
packages/llm/src/llm.ts
+18
View File
@@ -159,6 +159,21 @@ export const generation = (input: GenerationOptions | ConstructorParameters<type
return new GenerationOptions(input)
}
export const requestInput = (input: LLMRequest): RequestInput => ({
id: input.id,
model: input.model,
system: input.system,
messages: input.messages,
tools: input.tools,
toolChoice: input.toolChoice,
generation: input.generation,
reasoning: input.reasoning,
cache: input.cache,
responseFormat: input.responseFormat,
metadata: input.metadata,
native: input.native,
})
export const request = (input: RequestInput) => {
const { system: requestSystem, prompt, messages, tools, toolChoice: requestToolChoice, generation: requestGeneration, ...rest } = input
return new LLMRequest({
@@ -171,6 +186,9 @@ export const request = (input: RequestInput) => {
})
}
export const updateRequest = (input: LLMRequest, patch: Partial<RequestInput>) =>
request({ ...requestInput(input), ...patch })
export const outputText = (response: LLMResponse | { readonly events: ReadonlyArray<LLMEvent> }) =>
response.events
.filter(LLMEvent.guards["text-delta"])
packages/llm/src/tool-runtime.ts
+3 -20
View File
@@ -8,7 +8,7 @@ import {
type FinishReason,
type LLMError,
type LLMEvent,
LLMRequest,
type LLMRequest,
type ToolCallPart,
type ToolResultValue,
} from "./schema"
@@ -43,21 +43,6 @@ export interface RunOptions<T extends Tools> {
readonly stopWhen?: (state: RuntimeState) => boolean
}
const requestInput = (request: LLMRequest): ConstructorParameters<typeof LLMRequest>[0] => ({
id: request.id,
model: request.model,
system: request.system,
messages: request.messages,
tools: request.tools,
toolChoice: request.toolChoice,
generation: request.generation,
reasoning: request.reasoning,
cache: request.cache,
responseFormat: request.responseFormat,
metadata: request.metadata,
native: request.native,
})
/**
* Run a model with a typed tool record. The runtime streams the model, on
* each `tool-call` event decodes the input against the tool's `parameters`
@@ -78,8 +63,7 @@ export const run = <T extends Tools>(
const concurrency = options.concurrency ?? 10
const tools = options.tools as Tools
const runtimeTools = toDefinitions(tools)
const initialRequest = new LLMRequest({
...requestInput(options.request),
const initialRequest = LLM.updateRequest(options.request, {
tools: [
...options.request.tools.filter((tool) => !runtimeTools.some((runtimeTool) => runtimeTool.name === tool.name)),
...runtimeTools,
@@ -106,8 +90,7 @@ export const run = <T extends Tools>(
(call) => dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)),
{ concurrency },
)
const followUp = new LLMRequest({
...requestInput(request),
const followUp = LLM.updateRequest(request, {
messages: [
...request.messages,
LLM.assistant(state.assistantContent),
packages/llm/test/adapter.test.ts
+70 -40
View File
@@ -2,21 +2,53 @@ import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM } from "../src"
import { Adapter, client } from "../src/adapter"
import { Adapter, LLMClient } from "../src/adapter"
import { Patch } from "../src/patch"
import type { LLMRequest } from "../src/schema"
import type { LLMRequest, Message, ModelRef, ToolDefinition } from "../src/schema"
import { testEffect } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
const mapText = (fn: (text: string) => string) => (request: LLMRequest): LLMRequest => ({
...request,
messages: request.messages.map((message) => ({
...message,
content: message.content.map((part) =>
part.type === "text" ? { ...part, text: fn(part.text) } : part,
const updateMessageContent = (message: Message, content: Message["content"]) =>
LLM.message({
id: message.id,
role: message.role,
content,
metadata: message.metadata,
native: message.native,
})
const updateModel = (model: ModelRef, patch: Partial<LLM.ModelInput>) =>
LLM.model({
id: model.id,
provider: model.provider,
protocol: model.protocol,
baseURL: model.baseURL,
headers: model.headers,
capabilities: model.capabilities,
limits: model.limits,
native: model.native,
...patch,
})
const updateToolDefinition = (tool: ToolDefinition, patch: Partial<ToolDefinition>) =>
LLM.toolDefinition({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema,
metadata: tool.metadata,
native: tool.native,
...patch,
})
const mapText = (fn: (text: string) => string) => (request: LLMRequest): LLMRequest =>
LLM.updateRequest(request, {
messages: request.messages.map((message) =>
updateMessageContent(
message,
message.content.map((part) => (part.type === "text" ? { ...part, text: fn(part.text) } : part)),
),
),
})),
})
})
const Json = Schema.fromJsonString(Schema.Unknown)
const encodeJson = Schema.encodeSync(Json)
@@ -26,9 +58,12 @@ type FakeDraft = {
readonly includeUsage?: boolean
}
type FakeChunk =
| { readonly type: "text"; readonly text: string }
| { readonly type: "finish"; readonly reason: "stop" }
const FakeChunk = Schema.Union([
Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
Schema.Struct({ type: Schema.Literal("finish"), reason: Schema.Literal("stop") }),
])
type FakeChunk = Schema.Schema.Type<typeof FakeChunk>
const FakeChunks = Schema.Array(FakeChunk)
const request = LLM.request({
id: "req_1",
@@ -68,7 +103,12 @@ const fake = Adapter.define<FakeDraft, FakeDraft>({
),
),
parse: (response) =>
Stream.fromEffect(response.json.pipe(Effect.orDie, Effect.map((body) => body as FakeChunk[]))).pipe(
Stream.fromEffect(
response.json.pipe(
Effect.flatMap(Schema.decodeUnknownEffect(FakeChunks)),
Effect.orDie,
),
).pipe(
Stream.flatMap(Stream.fromIterable),
Stream.map(raiseChunk),
),
@@ -103,7 +143,7 @@ const it = testEffect(echoLayer)
describe("llm adapter", () => {
it.effect("prepare applies target patches with trace", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [
fake.withPatches([
fake.patch("include-usage", {
@@ -121,7 +161,7 @@ describe("llm adapter", () => {
it.effect("stream and generate use the adapter pipeline", () =>
Effect.gen(function* () {
const llm = client({ adapters: [fake] })
const llm = LLMClient.make({ adapters: [fake] })
const events = Array.from(yield* llm.stream(request).pipe(Stream.runCollect))
const response = yield* llm.generate(request)
@@ -132,11 +172,8 @@ describe("llm adapter", () => {
it.effect("selects adapters by request protocol", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [fake, gemini] }).prepare(
LLM.request({
...request,
model: LLM.model({ ...request.model, protocol: "gemini" }),
}),
const prepared = yield* LLMClient.make({ adapters: [fake, gemini] }).prepare(
LLM.updateRequest(request, { model: updateModel(request.model, { protocol: "gemini" }) }),
)
expect(prepared.adapter).toBe("gemini-fake")
@@ -145,7 +182,7 @@ describe("llm adapter", () => {
it.effect("prefers provider-specific adapters over protocol fallbacks", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [fake, providerFake] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [fake, providerFake] }).prepare(request)
expect(prepared.adapter).toBe("provider-fake")
expect(prepared.target).toEqual({ body: "provider:hello" })
@@ -154,12 +191,12 @@ describe("llm adapter", () => {
it.effect("request, prompt, and tool-schema patches run before adapter prepare", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [fake],
patches: [
Patch.request("test.id", {
reason: "rewrite request id",
apply: (request) => ({ ...request, id: "req_patched" }),
apply: (request) => LLM.updateRequest(request, { id: "req_patched" }),
}),
Patch.prompt("test.message", {
reason: "rewrite prompt text",
@@ -167,12 +204,11 @@ describe("llm adapter", () => {
}),
Patch.toolSchema("test.description", {
reason: "rewrite tool description",
apply: (tool) => ({ ...tool, description: "patched tool" }),
apply: (tool) => updateToolDefinition(tool, { description: "patched tool" }),
}),
],
}).prepare(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "original", inputSchema: {} }],
}),
)
@@ -189,7 +225,7 @@ describe("llm adapter", () => {
it.effect("request patches feed into prompt-patch predicates so phases see updated context", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [fake],
patches: [
// Earlier phase rewrites the provider, later phase only fires for the
@@ -197,10 +233,7 @@ describe("llm adapter", () => {
// test fails because the prompt patch's `when` would not match.
Patch.request("rewrite-provider", {
reason: "swap provider before prompt phase",
apply: (request) => ({
...request,
model: LLM.model({ ...request.model, provider: "rewritten" }),
}),
apply: (request) => LLM.updateRequest(request, { model: updateModel(request.model, { provider: "rewritten" }) }),
}),
Patch.prompt("rewrite-only-when-rewritten", {
reason: "rewrite prompt text only after provider swap",
@@ -220,7 +253,7 @@ describe("llm adapter", () => {
it.effect("patches with the same order sort by id for deterministic application", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [fake],
patches: [
Patch.prompt("zeta", {
@@ -242,7 +275,7 @@ describe("llm adapter", () => {
it.effect("stream patches transform raised events", () =>
Effect.gen(function* () {
const llm = client({
const llm = LLMClient.make({
adapters: [fake],
patches: [
Patch.stream("test.uppercase", {
@@ -262,7 +295,7 @@ describe("llm adapter", () => {
Effect.gen(function* () {
// Verifies stream patches run on every event, not just the first.
const seen: string[] = []
const llm = client({
const llm = LLMClient.make({
adapters: [fake],
patches: [
Patch.stream("test.tap", {
@@ -283,12 +316,9 @@ describe("llm adapter", () => {
it.effect("rejects protocol mismatch", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [fake] })
const error = yield* LLMClient.make({ adapters: [fake] })
.prepare(
LLM.request({
...request,
model: LLM.model({ ...request.model, protocol: "gemini" }),
}),
LLM.updateRequest(request, { model: updateModel(request.model, { protocol: "gemini" }) }),
)
.pipe(Effect.flip)
packages/llm/test/llm.test.ts
+18
View File
@@ -20,6 +20,24 @@ describe("llm constructors", () => {
expect(request.tools).toEqual([])
})
test("updates requests without spreading schema class instances", () => {
const base = LLM.request({
id: "req_1",
model: LLM.model({ id: "fake-model", provider: "fake", protocol: "openai-chat" }),
prompt: "Say hello.",
})
const updated = LLM.updateRequest(base, {
generation: { maxTokens: 20 },
messages: [...base.messages, LLM.assistant("Hi.")],
})
expect(updated).toBeInstanceOf(LLMRequest)
expect(updated.id).toBe("req_1")
expect(updated.model).toEqual(base.model)
expect(updated.generation).toEqual({ maxTokens: 20 })
expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
})
test("builds tool choices from names and tools", () => {
const tool = LLM.toolDefinition({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })
packages/llm/test/provider/anthropic-messages.recorded.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { AnthropicMessages } from "../../src/provider/anthropic-messages"
import { recordedTests } from "../recorded-test"
@@ -46,7 +46,7 @@ const recorded = recordedTests({
requires: ["ANTHROPIC_API_KEY"],
options: { requestHeaders: ["content-type", "anthropic-version"] },
})
const anthropic = client({ adapters: [AnthropicMessages.adapter] })
const anthropic = LLMClient.make({ adapters: [AnthropicMessages.adapter] })
describe("Anthropic Messages recorded", () => {
recorded.effect("streams text", () =>
packages/llm/test/provider/anthropic-messages.test.ts
+15 -18
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { CacheHint, LLM, ProviderRequestError } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { AnthropicMessages } from "../../src/provider/anthropic-messages"
import { testEffect } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@@ -26,7 +26,7 @@ const it = testEffect(Layer.empty)
describe("Anthropic Messages adapter", () => {
it.effect("prepares Anthropic Messages target", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(request)
expect(prepared.target).toEqual({
model: "claude-sonnet-4-5",
@@ -41,7 +41,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("prepares tool call and tool result messages", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(
LLM.request({
id: "req_tool_result",
model,
@@ -80,7 +80,7 @@ describe("Anthropic Messages adapter", () => {
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } },
{ type: "message_stop" },
)
const response = yield* client({ adapters: [AnthropicMessages.adapter] })
const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -106,10 +106,9 @@ describe("Anthropic Messages adapter", () => {
{ type: "content_block_stop", index: 0 },
{ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: 1 } },
)
const response = yield* client({ adapters: [AnthropicMessages.adapter] })
const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -131,7 +130,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("emits provider-error events for mid-stream provider errors", () =>
Effect.gen(function* () {
const response = yield* client({ adapters: [AnthropicMessages.adapter] })
const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(request)
.pipe(
Effect.provide(
@@ -145,7 +144,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("fails HTTP provider errors before stream parsing", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [AnthropicMessages.adapter] })
const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(request)
.pipe(
Effect.provide(
@@ -185,10 +184,9 @@ describe("Anthropic Messages adapter", () => {
{ type: "content_block_stop", index: 2 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 8 } },
)
const response = yield* client({ adapters: [AnthropicMessages.adapter] })
const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }],
}),
)
@@ -234,10 +232,9 @@ describe("Anthropic Messages adapter", () => {
{ type: "content_block_stop", index: 1 },
{ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } },
)
const response = yield* client({ adapters: [AnthropicMessages.adapter] })
const response = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "web_search", description: "Web search", inputSchema: { type: "object" } }],
}),
)
@@ -256,7 +253,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("round-trips provider-executed assistant content into server tool blocks", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(
LLM.request({
id: "req_round_trip",
model,
@@ -307,7 +304,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("rejects round-trip for unknown server tool names", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [AnthropicMessages.adapter] })
const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.prepare(
LLM.request({
id: "req_unknown_server_tool",
@@ -333,7 +330,7 @@ describe("Anthropic Messages adapter", () => {
it.effect("rejects unsupported user media content", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [AnthropicMessages.adapter] })
const error = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] })
.prepare(
LLM.request({
id: "req_media",
packages/llm/test/provider/bedrock-converse.test.ts
+23 -25
View File
@@ -3,7 +3,7 @@ import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { CacheHint, LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { BedrockConverse } from "../../src/provider/bedrock-converse"
import { testEffect } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@@ -63,7 +63,7 @@ const it = testEffect(Layer.empty)
describe("Bedrock Converse adapter", () => {
it.effect("prepares Converse target with system, inference config, and messages", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest)
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest)
expect(prepared.target).toEqual({
modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0",
@@ -76,9 +76,8 @@ describe("Bedrock Converse adapter", () => {
it.effect("prepares tool config with toolSpec and toolChoice", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
...baseRequest,
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.updateRequest(baseRequest, {
tools: [
{
name: "lookup",
@@ -111,7 +110,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("lowers assistant tool-call + tool-result message history", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
id: "req_history",
model,
@@ -157,7 +156,7 @@ describe("Bedrock Converse adapter", () => {
["messageStop", { stopReason: "end_turn" }],
["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }],
)
const response = yield* client({ adapters: [BedrockConverse.adapter] })
const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.generate(baseRequest)
.pipe(Effect.provide(fixedBytes(body)))
@@ -192,10 +191,9 @@ describe("Bedrock Converse adapter", () => {
["contentBlockStop", { contentBlockIndex: 0 }],
["messageStop", { stopReason: "tool_use" }],
)
const response = yield* client({ adapters: [BedrockConverse.adapter] })
const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.generate(
LLM.request({
...baseRequest,
LLM.updateRequest(baseRequest, {
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }],
}),
)
@@ -224,7 +222,7 @@ describe("Bedrock Converse adapter", () => {
["contentBlockStop", { contentBlockIndex: 0 }],
["messageStop", { stopReason: "end_turn" }],
)
const response = yield* client({ adapters: [BedrockConverse.adapter] })
const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.generate(baseRequest)
.pipe(Effect.provide(fixedBytes(body)))
@@ -238,7 +236,7 @@ describe("Bedrock Converse adapter", () => {
["messageStart", { role: "assistant" }],
["throttlingException", { message: "Slow down" }],
)
const response = yield* client({ adapters: [BedrockConverse.adapter] })
const response = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.generate(baseRequest)
.pipe(Effect.provide(fixedBytes(body)))
@@ -256,8 +254,8 @@ describe("Bedrock Converse adapter", () => {
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
baseURL: "https://bedrock-runtime.test",
})
const error = yield* client({ adapters: [BedrockConverse.adapter] })
.generate(LLM.request({ ...baseRequest, model: unsignedModel }))
const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.generate(LLM.updateRequest(baseRequest, { model: unsignedModel }))
.pipe(Effect.provide(fixedBytes(eventStreamBody(["messageStop", { stopReason: "end_turn" }]))), Effect.flip)
expect(error.message).toContain("Bedrock Converse requires either a Bearer API key")
@@ -275,8 +273,8 @@ describe("Bedrock Converse adapter", () => {
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
},
})
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({ ...baseRequest, model: signed }),
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.updateRequest(baseRequest, { model: signed }),
)
expect(prepared.adapter).toBe("bedrock-converse")
@@ -292,7 +290,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () =>
Effect.gen(function* () {
const cache = new CacheHint({ type: "ephemeral" })
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
id: "req_cache",
model,
@@ -324,7 +322,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("does not emit cachePoint when no cache hint is set", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest)
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest)
expect(prepared.target).toMatchObject({
system: [{ text: "You are concise." }],
messages: [{ role: "user", content: [{ text: "Say hello." }] }],
@@ -334,7 +332,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("lowers image media into Bedrock image blocks", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
id: "req_image",
model,
@@ -370,7 +368,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("base64-encodes Uint8Array image bytes", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
id: "req_image_bytes",
model,
@@ -396,7 +394,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("lowers document media into Bedrock document blocks with format and name", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(
LLM.request({
id: "req_doc",
model,
@@ -427,7 +425,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("rejects unsupported image media types", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [BedrockConverse.adapter] })
const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.prepare(
LLM.request({
id: "req_bad_image",
@@ -443,7 +441,7 @@ describe("Bedrock Converse adapter", () => {
it.effect("rejects unsupported document media types", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [BedrockConverse.adapter] })
const error = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] })
.prepare(
LLM.request({
id: "req_bad_doc",
@@ -493,7 +491,7 @@ const recorded = recordedTests({
describe("Bedrock Converse recorded", () => {
recorded.effect("streams text", () =>
Effect.gen(function* () {
const llm = client({ adapters: [BedrockConverse.adapter] })
const llm = LLMClient.make({ adapters: [BedrockConverse.adapter] })
const response = yield* llm.generate(
LLM.request({
id: "recorded_bedrock_text",
@@ -511,7 +509,7 @@ describe("Bedrock Converse recorded", () => {
recorded.effect("streams a tool call", () =>
Effect.gen(function* () {
const llm = client({ adapters: [BedrockConverse.adapter] })
const llm = LLMClient.make({ adapters: [BedrockConverse.adapter] })
const response = yield* llm.generate(
LLM.request({
id: "recorded_bedrock_tool_call",
packages/llm/test/provider/gemini.recorded.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { Gemini } from "../../src/provider/gemini"
import { recordedTests } from "../recorded-test"
@@ -42,7 +42,7 @@ const toolRequest = LLM.request({
})
const recorded = recordedTests({ prefix: "gemini", requires: ["GOOGLE_GENERATIVE_AI_API_KEY"] })
const gemini = client({ adapters: [Gemini.adapter] })
const gemini = LLMClient.make({ adapters: [Gemini.adapter] })
describe("Gemini recorded", () => {
recorded.effect("streams text", () =>
packages/llm/test/provider/gemini.test.ts
+15 -17
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { LLM, ProviderChunkError, ProviderPatch } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { Gemini } from "../../src/provider/gemini"
import { testEffect } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@@ -26,7 +26,7 @@ const it = testEffect(Layer.empty)
describe("Gemini adapter", () => {
it.effect("prepares Gemini target", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(request)
expect(prepared.target).toEqual({
contents: [{ role: "user", parts: [{ text: "Say hello." }] }],
@@ -38,7 +38,7 @@ describe("Gemini adapter", () => {
it.effect("prepares multimodal user input and tool history", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(
LLM.request({
id: "req_tool_result",
model,
@@ -91,7 +91,7 @@ describe("Gemini adapter", () => {
it.effect("omits tools when tool choice is none", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(
LLM.request({
id: "req_no_tools",
model,
@@ -109,7 +109,7 @@ describe("Gemini adapter", () => {
it.effect("applies Gemini tool-schema patches before preparing the target", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [Gemini.adapter],
patches: [ProviderPatch.sanitizeGeminiToolSchema],
}).prepare(
@@ -181,7 +181,7 @@ describe("Gemini adapter", () => {
},
},
)
const response = yield* client({ adapters: [Gemini.adapter] })
const response = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -234,10 +234,9 @@ describe("Gemini adapter", () => {
usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 },
},
)
const response = yield* client({ adapters: [Gemini.adapter] })
const response = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -271,10 +270,9 @@ describe("Gemini adapter", () => {
}],
},
)
const response = yield* client({ adapters: [Gemini.adapter] })
const response = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -290,14 +288,14 @@ describe("Gemini adapter", () => {
it.effect("maps length and content-filter finish reasons", () =>
Effect.gen(function* () {
const length = yield* client({ adapters: [Gemini.adapter] })
const length = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(request)
.pipe(
Effect.provide(
fixedResponse(sseEvents({ candidates: [{ content: { role: "model", parts: [] }, finishReason: "MAX_TOKENS" }] })),
),
)
const filtered = yield* client({ adapters: [Gemini.adapter] })
const filtered = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(request)
.pipe(
Effect.provide(
@@ -312,7 +310,7 @@ describe("Gemini adapter", () => {
it.effect("leaves total usage undefined when component counts are missing", () =>
Effect.gen(function* () {
const response = yield* client({ adapters: [Gemini.adapter] })
const response = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(sseEvents({ usageMetadata: { thoughtsTokenCount: 1 } }))))
@@ -323,7 +321,7 @@ describe("Gemini adapter", () => {
it.effect("fails invalid stream chunks", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [Gemini.adapter] })
const error = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.generate(request)
.pipe(
Effect.provide(fixedResponse(sseRaw("data: {not json}"))),
@@ -337,7 +335,7 @@ describe("Gemini adapter", () => {
it.effect("rejects unsupported assistant media content", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [Gemini.adapter] })
const error = yield* LLMClient.make({ adapters: [Gemini.adapter] })
.prepare(
LLM.request({
id: "req_media",
packages/llm/test/provider/openai-chat-tool-loop.recorded.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { LLM, LLMEvent } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAIChat } from "../../src/provider/openai-chat"
import { tool } from "../../src/tool"
import { ToolRuntime } from "../../src/tool-runtime"
@@ -39,7 +39,7 @@ const request = LLM.request({
})
const recorded = recordedTests({ prefix: "openai-chat", requires: ["OPENAI_API_KEY"] })
const openai = client({ adapters: [OpenAIChat.adapter] })
const openai = LLMClient.make({ adapters: [OpenAIChat.adapter] })
describe("OpenAI Chat tool-loop recorded", () => {
recorded.effect("drives a tool loop end-to-end", () =>
packages/llm/test/provider/openai-chat.recorded.test.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAIChat } from "../../src/provider/openai-chat"
import { recordedTests } from "../recorded-test"
@@ -58,8 +58,8 @@ const toolResultRequest = LLM.request({
// `length > 0` checks so adapter parsing regressions surface immediately.
// Re-record (`RECORD=true`) only when intentionally refreshing a cassette.
const recorded = recordedTests({ prefix: "openai-chat", requires: ["OPENAI_API_KEY"] })
const openai = client({ adapters: [OpenAIChat.adapter] })
const openaiWithUsage = client({ adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])] })
const openai = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const openaiWithUsage = LLMClient.make({ adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])] })
describe("OpenAI Chat recorded", () => {
recorded.effect("streams text", () =>
packages/llm/test/provider/openai-chat.test.ts
+14 -16
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer, Schema, Stream } from "effect"
import { LLM, ProviderRequestError } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAIChat } from "../../src/provider/openai-chat"
import { testEffect } from "../lib/effect"
import { fixedResponse, truncatedStream } from "../lib/http"
@@ -41,7 +41,7 @@ const usageChunk = (usage: object) => ({
describe("OpenAI Chat adapter", () => {
it.effect("prepares OpenAI Chat target", () =>
Effect.gen(function* () {
const prepared = yield* client({
const prepared = yield* LLMClient.make({
adapters: [OpenAIChat.adapter.withPatches([OpenAIChat.includeUsage])],
}).prepare(request)
@@ -62,7 +62,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("prepares assistant tool-call and tool-result messages", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAIChat.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] }).prepare(
LLM.request({
id: "req_tool_result",
model,
@@ -98,7 +98,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("rejects unsupported user media content", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [OpenAIChat.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.prepare(
LLM.request({
id: "req_media",
@@ -114,7 +114,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("rejects unsupported assistant reasoning content", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [OpenAIChat.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.prepare(
LLM.request({
id: "req_reasoning",
@@ -142,7 +142,7 @@ describe("OpenAI Chat adapter", () => {
completion_tokens_details: { reasoning_tokens: 0 },
}),
)
const response = yield* client({ adapters: [OpenAIChat.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -184,10 +184,9 @@ describe("OpenAI Chat adapter", () => {
deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }),
deltaChunk({}, "tool_calls"),
)
const response = yield* client({ adapters: [OpenAIChat.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -213,10 +212,9 @@ describe("OpenAI Chat adapter", () => {
}),
deltaChunk({ tool_calls: [{ index: 0, function: { arguments: ':"weather"}' } }] }),
)
const response = yield* client({ adapters: [OpenAIChat.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -233,7 +231,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("fails on malformed stream chunks", () =>
Effect.gen(function* () {
const body = sseEvents(deltaChunk({ content: 123 }))
const error = yield* client({ adapters: [OpenAIChat.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)), Effect.flip)
@@ -246,7 +244,7 @@ describe("OpenAI Chat adapter", () => {
const layer = truncatedStream([
`data: ${JSON.stringify(deltaChunk({ role: "assistant", content: "Hello" }))}\n\n`,
])
const error = yield* client({ adapters: [OpenAIChat.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(request)
.pipe(Effect.provide(layer), Effect.flip)
@@ -256,7 +254,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("fails HTTP provider errors before stream parsing", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [OpenAIChat.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIChat.adapter] })
.generate(request)
.pipe(
Effect.provide(
@@ -276,7 +274,7 @@ describe("OpenAI Chat adapter", () => {
it.effect("short-circuits the upstream stream when the consumer takes a prefix", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
// The body has more chunks than we'll consume. If `Stream.take(1)` did
// not interrupt the upstream HTTP body the test would hang waiting for
// the rest of the stream to drain.
packages/llm/test/provider/openai-compatible-chat.recorded.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat"
import { recordedTests } from "../recorded-test"
@@ -55,7 +55,7 @@ const togetherToolRequest = LLM.request({
})
const recorded = recordedTests({ prefix: "openai-compatible-chat" })
const llm = client({ adapters: [OpenAICompatibleChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] })
describe("OpenAI-compatible Chat recorded", () => {
recorded.effect.with("deepseek streams text", { requires: ["DEEPSEEK_API_KEY"] }, () =>
packages/llm/test/provider/openai-compatible-chat.test.ts
+7 -8
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Layer, Schema } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat"
import { testEffect } from "../lib/effect"
import { dynamicResponse } from "../lib/http"
@@ -53,9 +53,8 @@ const providerFamilies = [
describe("OpenAI-compatible Chat adapter", () => {
it.effect("prepares generic Chat target", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(
LLM.request({
...request,
const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
toolChoice: { type: "required" },
}),
@@ -88,7 +87,7 @@ describe("OpenAI-compatible Chat adapter", () => {
it.effect("provides model helpers for compatible provider families", () =>
Effect.gen(function* () {
expect(
providerFamilies.map(([provider, makeModel, baseURL]) => {
providerFamilies.map(([provider, makeModel]) => {
const model = makeModel({ id: `${provider}-model`, apiKey: "test-key" })
return {
id: String(model.id),
@@ -126,7 +125,7 @@ describe("OpenAI-compatible Chat adapter", () => {
it.effect("matches AI SDK compatible basic request body fixture", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request)
expect(prepared.target).toEqual({
model: "deepseek-chat",
@@ -143,7 +142,7 @@ describe("OpenAI-compatible Chat adapter", () => {
it.effect("matches AI SDK compatible tool request body fixture", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(
LLM.request({
id: "req_tool_parity",
model,
@@ -192,7 +191,7 @@ describe("OpenAI-compatible Chat adapter", () => {
it.effect("posts to the configured compatible endpoint and parses text usage", () =>
Effect.gen(function* () {
const response = yield* client({
const response = yield* LLMClient.make({
adapters: [OpenAICompatibleChat.adapter.withPatches([OpenAICompatibleChat.includeUsage])],
})
.generate(request)
packages/llm/test/provider/openai-responses.test.ts
+12 -13
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { LLM, ProviderRequestError } from "../../src"
import { client } from "../../src/adapter"
import { LLMClient } from "../../src/adapter"
import { OpenAIResponses } from "../../src/provider/openai-responses"
import { testEffect } from "../lib/effect"
import { fixedResponse } from "../lib/http"
@@ -26,7 +26,7 @@ const it = testEffect(Layer.empty)
describe("OpenAI Responses adapter", () => {
it.effect("prepares OpenAI Responses target", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare(request)
expect(prepared.target).toEqual({
model: "gpt-4.1-mini",
@@ -43,7 +43,7 @@ describe("OpenAI Responses adapter", () => {
it.effect("prepares function call and function output input items", () =>
Effect.gen(function* () {
const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare(
const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare(
LLM.request({
id: "req_tool_result",
model,
@@ -85,7 +85,7 @@ describe("OpenAI Responses adapter", () => {
},
},
)
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -136,10 +136,9 @@ describe("OpenAI Responses adapter", () => {
},
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
)
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(
LLM.request({
...request,
LLM.updateRequest(request, {
tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }],
}),
)
@@ -171,7 +170,7 @@ describe("OpenAI Responses adapter", () => {
{ type: "response.output_item.done", item },
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
)
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -209,7 +208,7 @@ describe("OpenAI Responses adapter", () => {
{ type: "response.output_item.done", item },
{ type: "response.completed", response: { usage: { input_tokens: 5, output_tokens: 1 } } },
)
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(body)))
@@ -234,7 +233,7 @@ describe("OpenAI Responses adapter", () => {
it.effect("rejects unsupported user media content", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [OpenAIResponses.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.prepare(
LLM.request({
id: "req_media",
@@ -250,7 +249,7 @@ describe("OpenAI Responses adapter", () => {
it.effect("emits provider-error events for mid-stream provider errors", () =>
Effect.gen(function* () {
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(
Effect.provide(
@@ -264,7 +263,7 @@ describe("OpenAI Responses adapter", () => {
it.effect("falls back to error code when no message is present", () =>
Effect.gen(function* () {
const response = yield* client({ adapters: [OpenAIResponses.adapter] })
const response = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(Effect.provide(fixedResponse(sseEvents({ type: "error", code: "internal_error" }))))
@@ -274,7 +273,7 @@ describe("OpenAI Responses adapter", () => {
it.effect("fails HTTP provider errors before stream parsing", () =>
Effect.gen(function* () {
const error = yield* client({ adapters: [OpenAIResponses.adapter] })
const error = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] })
.generate(request)
.pipe(
Effect.provide(
packages/llm/test/tool-runtime.test.ts
+9 -9
View File
@@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Layer, Schema, Stream } from "effect"
import { LLM, LLMEvent } from "../src"
import { client, type LLMClient } from "../src/adapter"
import { LLMClient } from "../src/adapter"
import { RequestExecutor } from "../src/executor"
import { OpenAIChat } from "../src/provider/openai-chat"
import { tool, ToolFailure } from "../src/tool"
@@ -39,7 +39,7 @@ const get_weather = tool({
describe("ToolRuntime", () => {
it.effect("dispatches a tool call, appends results, and resumes streaming", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"Paris"}'), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "It's sunny in Paris." }), finishChunk("stop")),
@@ -66,7 +66,7 @@ describe("ToolRuntime", () => {
it.effect("emits tool-error for unknown tools so the model can self-correct", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "missing_tool", "{}"), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "Sorry." }), finishChunk("stop")),
@@ -93,7 +93,7 @@ describe("ToolRuntime", () => {
it.effect("emits tool-error when the LLM input fails the parameters schema", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "get_weather", '{"city":42}'), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "Done." }), finishChunk("stop")),
@@ -114,7 +114,7 @@ describe("ToolRuntime", () => {
it.effect("emits tool-error when the handler returns a ToolFailure", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"FAIL"}'), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "Sorry." }), finishChunk("stop")),
@@ -135,7 +135,7 @@ describe("ToolRuntime", () => {
it.effect("stops when the model finishes without requesting more tools", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([sseEvents(deltaChunk({ role: "assistant", content: "Done." }), finishChunk("stop"))])
const events = Array.from(
@@ -152,7 +152,7 @@ describe("ToolRuntime", () => {
it.effect("respects maxSteps and stops the loop", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
// Every script entry asks for another tool call. With maxSteps: 2 the
// runtime should run at most two model rounds and then exit even though
// the model still wants to keep going.
@@ -172,7 +172,7 @@ describe("ToolRuntime", () => {
it.effect("stops when stopWhen returns true after the first step", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(toolCallChunk("call_1", "get_weather", '{"city":"Paris"}'), finishChunk("tool_calls")),
sseEvents(deltaChunk({ role: "assistant", content: "Should not run." }), finishChunk("stop")),
@@ -254,7 +254,7 @@ describe("ToolRuntime", () => {
it.effect("dispatches multiple tool calls in one step concurrently", () =>
Effect.gen(function* () {
const llm = client({ adapters: [OpenAIChat.adapter] })
const llm = LLMClient.make({ adapters: [OpenAIChat.adapter] })
const layer = scriptedResponses([
sseEvents(
deltaChunk({
packages/opencode/src/session/llm-native.ts
+54 -16
View File
@@ -1,4 +1,4 @@
import { LLM, type ContentPart, type Message as CoreMessage } from "@opencode-ai/llm"
import { CacheHint, LLM, type ContentPart, type LLMRequest, type Message as CoreMessage } from "@opencode-ai/llm"
import { Effect, Schema } from "effect"
import { ProviderLLMBridge } from "@/provider/llm-bridge"
import * as EffectZod from "@/util/effect-zod"
@@ -169,6 +169,42 @@ export const toolDefinition = (input: { readonly model: Provider.Model; readonly
},
})
// Mirrors the AI SDK path's prompt-cache policy, gated by model capability.
const EPHEMERAL_CACHE = new CacheHint({ type: "ephemeral" })
const withCacheOnLastText = (content: ReadonlyArray<ContentPart>): ReadonlyArray<ContentPart> => {
const index = content.findLastIndex((part) => part.type === "text")
if (index === -1) return content
return content.map((part, position) =>
position === index && part.type === "text" ? { ...part, cache: EPHEMERAL_CACHE } : part,
)
}
const updateMessageContent = (message: CoreMessage, content: ReadonlyArray<ContentPart>) => {
if (content === message.content) return message
return LLM.message({
id: message.id,
role: message.role,
content,
metadata: message.metadata,
native: message.native,
})
}
const applyCachePolicy = (request: LLMRequest): LLMRequest => {
if (!request.model.capabilities.cache?.prompt) return request
const system = request.system.map((part, index) =>
index < 2 ? { ...part, cache: EPHEMERAL_CACHE } : part,
)
const lastTwoStart = Math.max(0, request.messages.length - 2)
const messages = request.messages.map((message, index) =>
index < lastTwoStart
? message
: updateMessageContent(message, withCacheOnLastText(message.content)),
)
return LLM.updateRequest(request, { system, messages })
}
export const request = Effect.fn("LLMNative.request")(function* (input: RequestInput) {
const unsupported = unsupportedPart(input)
if (unsupported) {
@@ -186,21 +222,23 @@ export const request = Effect.fn("LLMNative.request")(function* (input: RequestI
})
}
return LLM.request({
id: input.id,
model,
system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [],
messages: input.messages.flatMap(messages),
tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [],
toolChoice: input.toolChoice,
generation: input.generation,
metadata: input.metadata,
native: {
opencodeProviderID: input.provider.id,
opencodeModelID: input.model.id,
...input.native,
},
})
return applyCachePolicy(
LLM.request({
id: input.id,
model,
system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [],
messages: input.messages.flatMap(messages),
tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [],
toolChoice: input.toolChoice,
generation: input.generation,
metadata: input.metadata,
native: {
opencodeProviderID: input.provider.id,
opencodeModelID: input.model.id,
...input.native,
},
}),
)
})
export * as LLMNative from "./llm-native"
packages/opencode/test/session/llm-native.test.ts
+111 -7
View File
@@ -1,7 +1,5 @@
import { describe, expect } from "bun:test"
import { AnthropicMessages, Gemini, OpenAICompatibleChat } from "@opencode-ai/llm"
import { client } from "@opencode-ai/llm/adapter"
import { OpenAIResponses } from "@opencode-ai/llm/provider/openai-responses"
import { AnthropicMessages, BedrockConverse, Gemini, LLMClient, OpenAICompatibleChat, OpenAIResponses } from "@opencode-ai/llm"
import { Cause, Effect, Exit, Layer, Schema } from "effect"
import { ModelID, ProviderID } from "../../src/provider/schema"
import { LLMNative } from "../../src/session/llm-native"
@@ -333,7 +331,7 @@ describe("LLMNative.request", () => {
tools: [lookupTool],
toolChoice: "lookup",
})
const prepared = yield* client({ adapters: [OpenAIResponses.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [OpenAIResponses.adapter] }).prepare(request)
expect(prepared.target).toMatchObject({
model: "gpt-5",
@@ -392,7 +390,7 @@ describe("LLMNative.request", () => {
tools: [lookupTool],
toolChoice: "lookup",
})
const prepared = yield* client({ adapters: [AnthropicMessages.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [AnthropicMessages.adapter] }).prepare(request)
expect(request.model).toMatchObject({
provider: "anthropic",
@@ -461,7 +459,7 @@ describe("LLMNative.request", () => {
tools: [lookupTool],
toolChoice: "lookup",
})
const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }).prepare(request)
expect(request.model).toMatchObject({
provider: "togetherai",
@@ -540,7 +538,7 @@ describe("LLMNative.request", () => {
tools: [lookupTool],
toolChoice: "lookup",
})
const prepared = yield* client({ adapters: [Gemini.adapter] }).prepare(request)
const prepared = yield* LLMClient.make({ adapters: [Gemini.adapter] }).prepare(request)
expect(request.model).toMatchObject({
provider: "google",
@@ -577,4 +575,110 @@ describe("LLMNative.request", () => {
generationConfig: { maxOutputTokens: 32, temperature: 0 },
})
}))
// Cache hint policy. The LLM-native path mirrors the AI-SDK applyCaching
// policy from packages/opencode/src/provider/transform.ts: mark the first 2
// system parts and the last 2 messages as cacheable, gated on the resolved
// model's `capabilities.cache.prompt`. Adapters lower CacheHint to the
// provider-specific marker (cache_control on Anthropic, cachePoint on
// Bedrock); non-cache-capable adapters never see a hint.
const anthropicModel = () =>
model({
id: ModelID.make("claude-sonnet-4-5"),
providerID: ProviderID.make("anthropic"),
api: { id: "claude-sonnet-4-5", url: "https://api.anthropic.com/v1", npm: "@ai-sdk/anthropic" },
})
const bedrockModel = () =>
model({
id: ModelID.make("us.amazon.nova-micro-v1:0"),
providerID: ProviderID.make("amazon-bedrock"),
api: {
id: "us.amazon.nova-micro-v1:0",
url: "https://bedrock-runtime.us-east-1.amazonaws.com",
npm: "@ai-sdk/amazon-bedrock",
},
})
it.effect("applies cache hints to the first 2 system parts on cache-capable models", () =>
Effect.gen(function* () {
const mdl = anthropicModel()
const userID = MessageID.ascending()
const request = yield* LLMNative.request({
provider: ProviderTest.info({ id: ProviderID.make("anthropic"), key: "anthropic-key" }, mdl),
model: mdl,
system: ["First", "Second", "Third"],
messages: [userMessage(mdl, userID, [textPart(userID, "hello")])],
})
expect(request.system).toHaveLength(3)
expect(request.system[0]).toMatchObject({ text: "First", cache: { type: "ephemeral" } })
expect(request.system[1]).toMatchObject({ text: "Second", cache: { type: "ephemeral" } })
expect(request.system[2]).toMatchObject({ text: "Third" })
expect(request.system[2].cache).toBeUndefined()
}))
it.effect("applies cache hints to the final text part of the last 2 messages on cache-capable models", () =>
Effect.gen(function* () {
const mdl = anthropicModel()
const messageIds = [MessageID.ascending(), MessageID.ascending(), MessageID.ascending()]
const request = yield* LLMNative.request({
provider: ProviderTest.info({ id: ProviderID.make("anthropic"), key: "anthropic-key" }, mdl),
model: mdl,
messages: messageIds.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])),
})
expect(request.messages).toHaveLength(3)
// First message: no cache hint.
const first = request.messages[0].content[0]
if (first.type === "text") expect(first.cache).toBeUndefined()
// Last two messages: cache on the (only) text part.
expect(request.messages[1].content[0]).toMatchObject({ type: "text", text: "m1", cache: { type: "ephemeral" } })
expect(request.messages[2].content[0]).toMatchObject({ type: "text", text: "m2", cache: { type: "ephemeral" } })
}))
it.effect("lowers cache hints to Bedrock Converse cachePoint marker blocks end-to-end", () =>
Effect.gen(function* () {
const mdl = bedrockModel()
const userID = MessageID.ascending()
const request = yield* LLMNative.request({
provider: ProviderTest.info({ id: ProviderID.make("amazon-bedrock"), key: "bedrock-bearer" }, mdl),
model: mdl,
system: ["You are concise."],
messages: [userMessage(mdl, userID, [textPart(userID, "hello")])],
})
const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(request)
expect(prepared.target).toMatchObject({
system: [{ text: "You are concise." }, { cachePoint: { type: "default" } }],
messages: [
{
role: "user",
content: [{ text: "hello" }, { cachePoint: { type: "default" } }],
},
],
})
}))
it.effect("does not apply cache hints when the model does not support prompt caching", () =>
Effect.gen(function* () {
// gpt-5 / openai resolves to openai-responses, which advertises
// capabilities.cache.prompt: false. The bridge must skip the policy.
const mdl = model()
const ids = [MessageID.ascending(), MessageID.ascending()]
const request = yield* LLMNative.request({
provider: ProviderTest.info({ id: ProviderID.openai, key: "openai-key" }, mdl),
model: mdl,
system: ["A", "B", "C"],
messages: ids.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])),
})
for (const part of request.system) expect(part.cache).toBeUndefined()
for (const message of request.messages) {
for (const part of message.content) {
if (part.type === "text") expect(part.cache).toBeUndefined()
}
}
}))
})