From e8d703108d3abb520ba246bc4bb8bd2eb91af6b0 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 5 May 2026 16:53:27 -0400 Subject: [PATCH] refactor(llm): simplify provider protocol wiring --- packages/llm/src/protocol.ts | 14 ++++- .../llm/src/provider/anthropic-messages.ts | 25 ++++++-- packages/llm/src/provider/bedrock-converse.ts | 28 +++++++-- packages/llm/src/provider/gemini.ts | 23 +++++++- packages/llm/src/provider/openai-chat.ts | 42 ++++++++------ packages/llm/src/provider/openai-responses.ts | 25 ++++++-- packages/llm/src/provider/openrouter.ts | 53 +++++++++++++++-- .../openai-compatible-chat.recorded.test.ts | 12 ++-- packages/llm/test/provider/openrouter.test.ts | 57 +++++++++++++++++++ 9 files changed, 230 insertions(+), 49 deletions(-) create mode 100644 packages/llm/test/provider/openrouter.test.ts diff --git a/packages/llm/src/protocol.ts b/packages/llm/src/protocol.ts index 01742be923..4ad58a4336 100644 --- a/packages/llm/src/protocol.ts +++ b/packages/llm/src/protocol.ts @@ -55,9 +55,17 @@ export interface Protocol { } /** - * Construct a `Protocol` from its parts. Currently a typed identity, but kept - * as the public constructor so future cross-cutting concerns (tracing spans, - * instrumentation) can be added in one place. + * Construct a `Protocol` from the four protocol-local pieces: + * + * - `payload` infers the provider-native request body shape. + * - `chunk` infers the framed response item and decoded chunk shape. + * - `initial`, `process`, and `onHalt` infer the parser state shape. + * - `prepare` ties the common `LLMRequest` to the provider payload. + * + * Provider implementations should usually call `Protocol.define({ ... })` + * without explicit type arguments; the schemas and parser functions are the + * source of truth. The constructor remains as the public seam for future + * cross-cutting concerns such as tracing or instrumentation. */ export const define = ( input: Protocol, diff --git a/packages/llm/src/provider/anthropic-messages.ts b/packages/llm/src/provider/anthropic-messages.ts index cfae8d01ce..349d4399d8 100644 --- a/packages/llm/src/provider/anthropic-messages.ts +++ b/packages/llm/src/provider/anthropic-messages.ts @@ -19,8 +19,14 @@ import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./share const ADAPTER = "anthropic-messages" +// ============================================================================= +// Public Model Input +// ============================================================================= export type AnthropicMessagesModelInput = AdapterModelInput +// ============================================================================= +// Request Payload Schema +// ============================================================================= const AnthropicCacheControl = Schema.Struct({ type: Schema.Literal("ephemeral") }) const AnthropicTextBlock = Schema.Struct({ @@ -188,6 +194,9 @@ interface ParserState { const invalid = ProviderShared.invalidRequest +// ============================================================================= +// Request Lowering +// ============================================================================= const cacheControl = (cache: CacheHint | undefined) => cache?.type === "ephemeral" ? { type: "ephemeral" as const } : undefined const lowerTool = (tool: ToolDefinition): AnthropicTool => ({ @@ -320,6 +329,9 @@ const prepare = Effect.fn("AnthropicMessages.prepare")(function* (request: LLMRe } }) +// ============================================================================= +// Stream Parsing +// ============================================================================= const mapFinishReason = (reason: string | null | undefined): FinishReason => { if (reason === "end_turn" || reason === "stop_sequence" || reason === "pause_turn") return "stop" if (reason === "max_tokens") return "length" @@ -474,18 +486,16 @@ const processChunk = (state: ParserState, chunk: AnthropicChunk) => return [state, []] as const }) +// ============================================================================= +// Protocol And Anthropic Adapter +// ============================================================================= /** * The Anthropic Messages protocol — request lowering, payload schema, and the * streaming-chunk state machine. Used by native * Anthropic Cloud and (once registered) Vertex Anthropic / Bedrock-hosted * Anthropic passthrough. */ -export const protocol = Protocol.define< - AnthropicMessagesPayload, - string, - AnthropicChunk, - ParserState ->({ +export const protocol = Protocol.define({ id: ADAPTER, payload: AnthropicMessagesPayload, prepare, @@ -503,6 +513,9 @@ export const adapter = Adapter.make({ headers: () => ({ "anthropic-version": "2023-06-01" }), }) +// ============================================================================= +// Model Helper +// ============================================================================= export const model = Adapter.model(adapter, { provider: "anthropic", capabilities: capabilities({ diff --git a/packages/llm/src/provider/bedrock-converse.ts b/packages/llm/src/provider/bedrock-converse.ts index cb72ba9b88..42352f3f8a 100644 --- a/packages/llm/src/provider/bedrock-converse.ts +++ b/packages/llm/src/provider/bedrock-converse.ts @@ -21,6 +21,9 @@ import { JsonObject, optionalArray, ProviderShared } from "./shared" const ADAPTER = "bedrock-converse" +// ============================================================================= +// Public Model Input +// ============================================================================= /** * AWS credentials for SigV4 signing. Bedrock also supports Bearer API key auth * via `model.apiKey`, which bypasses SigV4 signing. STS-vended credentials @@ -48,6 +51,9 @@ export type BedrockConverseModelInput = AdapterModelInput & { readonly headers?: Record } +// ============================================================================= +// Request Payload Schema +// ============================================================================= const BedrockTextBlock = Schema.Struct({ text: Schema.String, }) @@ -265,6 +271,9 @@ type BedrockChunk = Schema.Schema.Type const invalid = ProviderShared.invalidRequest +// ============================================================================= +// Request Lowering +// ============================================================================= const region = (request: LLMRequest) => { const fromNative = request.model.native?.aws_region if (typeof fromNative === "string" && fromNative !== "") return fromNative @@ -477,6 +486,9 @@ const prepare = Effect.fn("BedrockConverse.prepare")(function* (request: LLMRequ } }) +// ============================================================================= +// Auth +// ============================================================================= // Credentials live on `model.native.aws_credentials` so the OpenCode bridge // can resolve them via `@aws-sdk/credential-providers` and stuff them in // without exposing the auth machinery to the rest of the LLM core. Schema @@ -544,6 +556,9 @@ const auth: Auth = (input) => { }) } +// ============================================================================= +// Stream Parsing +// ============================================================================= const mapFinishReason = (reason: string): FinishReason => { if (reason === "end_turn" || reason === "stop_sequence") return "stop" if (reason === "max_tokens") return "length" @@ -682,16 +697,14 @@ const onHalt = (state: ParserState): ReadonlyArray => ? [{ type: "request-finish", reason: mapFinishReason(state.pendingStopReason) }] : [] +// ============================================================================= +// Protocol And Bedrock Adapter +// ============================================================================= /** * The Bedrock Converse protocol — request lowering, payload schema, and the * streaming-chunk state machine. */ -export const protocol = Protocol.define< - BedrockConversePayload, - object, - BedrockChunk, - ParserState ->({ +export const protocol = Protocol.define({ id: ADAPTER, payload: BedrockConversePayload, prepare, @@ -715,6 +728,9 @@ export const adapter = Adapter.make({ framing, }) +// ============================================================================= +// Model Helper +// ============================================================================= export const defaultCapabilities = capabilities({ output: { reasoning: true }, tools: { calls: true, streamingInput: true }, diff --git a/packages/llm/src/provider/gemini.ts b/packages/llm/src/provider/gemini.ts index 9d5e6d5eef..2bd3936eb0 100644 --- a/packages/llm/src/provider/gemini.ts +++ b/packages/llm/src/provider/gemini.ts @@ -20,8 +20,14 @@ import { JsonObject, optionalArray, ProviderShared } from "./shared" const ADAPTER = "gemini" +// ============================================================================= +// Public Model Input +// ============================================================================= export type GeminiModelInput = AdapterModelInput +// ============================================================================= +// Request Payload Schema +// ============================================================================= const GeminiTextPart = Schema.Struct({ text: Schema.String, thought: Schema.optional(Schema.Boolean), @@ -140,6 +146,9 @@ const mediaData = ProviderShared.mediaBytes const isRecord = ProviderShared.isRecord +// ============================================================================= +// Tool Schema Conversion +// ============================================================================= // Tool-schema conversion has two distinct concerns: // // 1. Sanitize — fix common authoring mistakes Gemini rejects: integer/number @@ -253,6 +262,9 @@ const projectToolSchemaNode = (schema: unknown): Record | undef const convertToolSchema = (schema: unknown) => projectToolSchemaNode(sanitizeToolSchemaNode(schema)) +// ============================================================================= +// Request Lowering +// ============================================================================= const lowerTool = (tool: ToolDefinition) => ({ name: tool.name, description: tool.description, @@ -367,6 +379,9 @@ const prepare = Effect.fn("Gemini.prepare")(function* (request: LLMRequest) { } }) +// ============================================================================= +// Stream Parsing +// ============================================================================= const mapUsage = (usage: GeminiUsage | undefined) => { if (!usage) return undefined return new Usage({ @@ -436,12 +451,15 @@ const processChunk = (state: ParserState, chunk: GeminiChunk) => { }, events] as const) } +// ============================================================================= +// Protocol And Gemini Adapter +// ============================================================================= /** * The Gemini protocol — request lowering, payload schema, and the streaming- * chunk state machine. Used by Google AI Studio Gemini and * (once registered) Vertex Gemini. */ -export const protocol = Protocol.define({ +export const protocol = Protocol.define({ id: ADAPTER, payload: GeminiPayload, prepare, @@ -463,6 +481,9 @@ export const adapter = Adapter.make({ framing: Framing.sse, }) +// ============================================================================= +// Model Helper +// ============================================================================= export const model = Adapter.model(adapter, { provider: "google", capabilities: capabilities({ diff --git a/packages/llm/src/provider/openai-chat.ts b/packages/llm/src/provider/openai-chat.ts index c6ead61357..4f9809052c 100644 --- a/packages/llm/src/provider/openai-chat.ts +++ b/packages/llm/src/provider/openai-chat.ts @@ -79,6 +79,9 @@ const OpenAIChatPayloadFields = { tool_choice: Schema.optional(OpenAIChatToolChoice), stream: Schema.Literal(true), stream_options: Schema.optional(Schema.Struct({ include_usage: Schema.Boolean })), + usage: Schema.optional(JsonObject), + reasoning: Schema.optional(JsonObject), + prompt_cache_key: Schema.optional(Schema.String), max_tokens: Schema.optional(Schema.Number), temperature: Schema.optional(Schema.Number), top_p: Schema.optional(Schema.Number), @@ -293,37 +296,44 @@ const pushToolDelta = (tools: Record, de } }) +const applyToolDeltas = Effect.fn("OpenAIChat.applyToolDeltas")(function* ( + stateTools: Record, + toolDeltas: ReadonlyArray, +) { + const tools = toolDeltas.length === 0 ? stateTools : { ...stateTools } + const events: LLMEvent[] = [] + for (const tool of toolDeltas) { + const current = yield* pushToolDelta(tools, tool) + tools[tool.index] = current + if (tool.function?.arguments) { + events.push({ type: "tool-input-delta", id: current.id, name: current.name, text: tool.function.arguments }) + } + } + return { tools, events } +}) + const finalizeToolCalls = (tools: Record) => Effect.forEach(Object.values(tools), (tool) => ProviderShared.parsedToolCall(ADAPTER, tool)) const processChunk = (state: ParserState, chunk: OpenAIChatChunk) => Effect.gen(function* () { - const events: LLMEvent[] = [] const usage = mapUsage(chunk.usage) ?? state.usage const choice = chunk.choices[0] const finishReason = choice?.finish_reason ? mapFinishReason(choice.finish_reason) : state.finishReason const delta = choice?.delta - const toolDeltas = delta?.tool_calls ?? [] - const tools = toolDeltas.length === 0 ? state.tools : { ...state.tools } - - if (delta?.content) events.push({ type: "text-delta", text: delta.content }) - - for (const tool of toolDeltas) { - const current = yield* pushToolDelta(tools, tool) - tools[tool.index] = current - if (tool.function?.arguments) { - events.push({ type: "tool-input-delta", id: current.id, name: current.name, text: tool.function.arguments }) - } - } + const toolDeltas = yield* applyToolDeltas(state.tools, delta?.tool_calls ?? []) // Finalize accumulated tool inputs eagerly when finish_reason arrives so // JSON parse failures fail the stream at the boundary rather than at halt. const toolCalls = - finishReason !== undefined && state.finishReason === undefined && Object.keys(tools).length > 0 - ? yield* finalizeToolCalls(tools) + finishReason !== undefined && state.finishReason === undefined && Object.keys(toolDeltas.tools).length > 0 + ? yield* finalizeToolCalls(toolDeltas.tools) : state.toolCalls - return [{ tools, toolCalls, usage, finishReason }, events] as const + return [ + { tools: toolDeltas.tools, toolCalls, usage, finishReason }, + [...(delta?.content ? ([{ type: "text-delta", text: delta.content }] satisfies LLMEvent[]) : []), ...toolDeltas.events], + ] as const }) const finishEvents = (state: ParserState): ReadonlyArray => { diff --git a/packages/llm/src/provider/openai-responses.ts b/packages/llm/src/provider/openai-responses.ts index 383b7332a6..0fddfa5ca8 100644 --- a/packages/llm/src/provider/openai-responses.ts +++ b/packages/llm/src/provider/openai-responses.ts @@ -18,8 +18,14 @@ import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./share const ADAPTER = "openai-responses" +// ============================================================================= +// Public Model Input +// ============================================================================= export type OpenAIResponsesModelInput = AdapterModelInput +// ============================================================================= +// Request Payload Schema +// ============================================================================= const OpenAIResponsesInputText = Schema.Struct({ type: Schema.Literal("input_text"), text: Schema.String, @@ -130,6 +136,9 @@ interface ParserState { const invalid = ProviderShared.invalidRequest +// ============================================================================= +// Request Lowering +// ============================================================================= const lowerTool = (tool: ToolDefinition): OpenAIResponsesTool => ({ type: "function", name: tool.name, @@ -209,6 +218,9 @@ const prepare = Effect.fn("OpenAIResponses.prepare")(function* (request: LLMRequ } }) +// ============================================================================= +// Stream Parsing +// ============================================================================= const mapUsage = (usage: OpenAIResponsesUsage | null | undefined) => { if (!usage) return undefined return new Usage({ @@ -350,17 +362,15 @@ const processChunk = (state: ParserState, chunk: OpenAIResponsesChunk) => return [state, []] as const }) +// ============================================================================= +// Protocol And OpenAI Adapter +// ============================================================================= /** * The OpenAI Responses protocol — request lowering, payload schema, and the * streaming-chunk state machine. Used by native OpenAI and * (once registered) Azure OpenAI Responses. */ -export const protocol = Protocol.define< - OpenAIResponsesPayload, - string, - OpenAIResponsesChunk, - ParserState ->({ +export const protocol = Protocol.define({ id: ADAPTER, payload: OpenAIResponsesPayload, prepare, @@ -377,6 +387,9 @@ export const adapter = Adapter.make({ framing: Framing.sse, }) +// ============================================================================= +// Model Helper +// ============================================================================= export const model = Adapter.model(adapter, { provider: "openai", capabilities: capabilities({ tools: { calls: true, streamingInput: true } }), diff --git a/packages/llm/src/provider/openrouter.ts b/packages/llm/src/provider/openrouter.ts index b2f291051e..62954adcbc 100644 --- a/packages/llm/src/provider/openrouter.ts +++ b/packages/llm/src/provider/openrouter.ts @@ -1,14 +1,59 @@ -import { OpenAICompatibleChat, type ProviderFamilyModelInput } from "./openai-compatible-chat" +import { Adapter, type AdapterModelInput } from "../adapter" +import { capabilities } from "../llm" +import { payload as payloadPatch } from "../patch" +import { OpenAICompatibleChat } from "./openai-compatible-chat" import { OpenAICompatibleProfiles } from "./openai-compatible-profile" +import type { OpenAIChatPayload } from "./openai-chat" +import { isRecord } from "./shared" export const profile = OpenAICompatibleProfiles.profiles.openrouter -export type ModelOptions = Omit +export interface OpenRouterOptions { + readonly usage?: boolean | Record + readonly reasoning?: Record + readonly promptCacheKey?: string +} -export const adapters = [OpenAICompatibleChat.adapter] +export type ModelOptions = Omit & OpenRouterOptions + +const nativeOptions = (options: ModelOptions) => { + const openrouter = { + ...(isRecord(options.native?.openrouter) ? options.native.openrouter : {}), + ...(options.usage === undefined ? {} : { usage: options.usage === true ? { include: true } : options.usage }), + ...(options.reasoning === undefined ? {} : { reasoning: options.reasoning }), + ...(options.promptCacheKey === undefined ? {} : { promptCacheKey: options.promptCacheKey }), + } + if (Object.keys(openrouter).length === 0) return options.native + return { ...options.native, openrouter } +} + +export const applyOptions = payloadPatch("openrouter.options", { + reason: "apply OpenRouter provider options to the Chat payload", + when: (context) => context.model.provider === profile.provider && isRecord(context.model.native?.openrouter), + apply: (payload, context) => { + const openrouter = isRecord(context.model.native?.openrouter) ? context.model.native.openrouter : undefined + if (!openrouter) return payload + return { + ...payload, + ...(openrouter.usage === true ? { usage: { include: true } } : isRecord(openrouter.usage) ? { usage: openrouter.usage } : {}), + ...(isRecord(openrouter.reasoning) ? { reasoning: openrouter.reasoning } : {}), + ...(typeof openrouter.promptCacheKey === "string" ? { prompt_cache_key: openrouter.promptCacheKey } : {}), + } + }, +}) + +export const adapter = OpenAICompatibleChat.adapter.withPatches([applyOptions]) + +export const adapters = [adapter] + +const modelRef = Adapter.model(adapter, { + provider: profile.provider, + baseURL: profile.baseURL, + capabilities: capabilities({ tools: { calls: true, streamingInput: true } }), +}) export const model = (id: string, options: ModelOptions = {}) => - OpenAICompatibleChat.profileModel(profile, { ...options, id }) + modelRef({ ...options, id, native: nativeOptions(options) }) export const chat = model diff --git a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts index e077762923..c79f3ca409 100644 --- a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts @@ -3,6 +3,7 @@ import { Effect } from "effect" import { LLM } from "../../src" import { LLMClient } from "../../src/adapter" import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat" +import { OpenRouter } from "../../src/provider/openrouter" import { expectFinish, expectWeatherToolCall, expectWeatherToolLoop, runWeatherToolLoop, textRequest, weatherToolLoopRequest, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" @@ -29,21 +30,18 @@ const groqModel = OpenAICompatibleChat.groq({ const groqRequest = textRequest({ id: "recorded_groq_text", model: groqModel }) const groqToolRequest = weatherToolRequest({ id: "recorded_groq_tool_call", model: groqModel }) -const openrouterModel = OpenAICompatibleChat.openrouter({ - id: "openai/gpt-4o-mini", +const openrouterModel = OpenRouter.model("openai/gpt-4o-mini", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", }) const openrouterRequest = textRequest({ id: "recorded_openrouter_text", model: openrouterModel }) const openrouterToolRequest = weatherToolRequest({ id: "recorded_openrouter_tool_call", model: openrouterModel }) -const openrouterGpt55Model = OpenAICompatibleChat.openrouter({ - id: "openai/gpt-5.5", +const openrouterGpt55Model = OpenRouter.model("openai/gpt-5.5", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", }) -const openrouterOpus47Model = OpenAICompatibleChat.openrouter({ - id: "anthropic/claude-opus-4.7", +const openrouterOpus47Model = OpenRouter.model("anthropic/claude-opus-4.7", { apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", }) @@ -61,7 +59,7 @@ const xaiRequest = textRequest({ id: "recorded_xai_text", model: xaiModel }) const xaiToolRequest = weatherToolRequest({ id: "recorded_xai_tool_call", model: xaiModel }) const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" }) -const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }) +const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter, ...OpenRouter.adapters] }) const openrouterToolLoops = [ { diff --git a/packages/llm/test/provider/openrouter.test.ts b/packages/llm/test/provider/openrouter.test.ts new file mode 100644 index 0000000000..d5875dcddb --- /dev/null +++ b/packages/llm/test/provider/openrouter.test.ts @@ -0,0 +1,57 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import { LLM } from "../../src" +import { LLMClient } from "../../src/adapter" +import { OpenRouter } from "../../src/provider/openrouter" +import { testEffect } from "../lib/effect" + +const it = testEffect(Layer.empty) + +describe("OpenRouter", () => { + it.effect("prepares OpenRouter models through the OpenAI-compatible Chat route", () => + Effect.gen(function* () { + const model = OpenRouter.model("openai/gpt-4o-mini", { apiKey: "test-key" }) + + expect(model).toMatchObject({ + id: "openai/gpt-4o-mini", + provider: "openrouter", + protocol: "openai-compatible-chat", + baseURL: "https://openrouter.ai/api/v1", + apiKey: "test-key", + }) + + const prepared = yield* LLMClient.make({ adapters: OpenRouter.adapters }).prepare( + LLM.request({ model, prompt: "Say hello." }), + ) + + expect(prepared.adapter).toBe("openai-compatible-chat") + expect(prepared.payload).toMatchObject({ + model: "openai/gpt-4o-mini", + messages: [{ role: "user", content: "Say hello." }], + stream: true, + }) + }), + ) + + it.effect("applies OpenRouter payload options from the model helper", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.make({ adapters: OpenRouter.adapters }).prepare( + LLM.request({ + model: OpenRouter.model("anthropic/claude-3.7-sonnet:thinking", { + usage: true, + reasoning: { effort: "high" }, + promptCacheKey: "session_123", + }), + prompt: "Think briefly.", + }), + ) + + expect(prepared.payload).toMatchObject({ + usage: { include: true }, + reasoning: { effort: "high" }, + prompt_cache_key: "session_123", + }) + expect(prepared.patchTrace.map((item) => item.id)).toContain("payload.openrouter.options") + }), + ) +})