diff --git a/packages/llm/AGENTS.md b/packages/llm/AGENTS.md index 5cc074e719..cf8ceffbf4 100644 --- a/packages/llm/AGENTS.md +++ b/packages/llm/AGENTS.md @@ -107,9 +107,13 @@ Do not blanket re-record an entire test file when adding one cassette. `RECORD=t ### Provider Coverage -- [ ] Add OpenAI-compatible Chat adapter support for non-OpenAI providers that still use `/chat/completions`. +- [x] Add a generic OpenAI-compatible Chat adapter for non-OpenAI providers that expose `/chat/completions`; use `../ai/packages/openai-compatible` as the behavior reference. +- [ ] Keep OpenAI Responses as a separate first-class protocol for providers that actually implement `/responses`; do not treat generic OpenAI-compatible providers as Responses-capable by default. +- [ ] Cover OpenAI-compatible provider families that can share the generic adapter first: DeepSeek, TogetherAI, Cerebras, Baseten, Fireworks, DeepInfra, and similar providers. +- [ ] Decide which providers need thin dedicated wrappers over OpenAI-compatible Chat because they have custom parsing/options: Mistral, Groq, xAI, Perplexity, and Cohere. - [ ] Add Bedrock Converse support or a clear compatibility layer before moving Amazon Bedrock traffic onto `packages/llm`. -- [ ] Decide whether Vertex Gemini and Vertex Anthropic are target patches over existing adapters or separate adapters with their own auth/URL handling. +- [ ] Decide Vertex shape after Bedrock/OpenAI-compatible are stable: Vertex Gemini as Gemini target/http patch vs adapter, and Vertex Anthropic as Anthropic target/http patch vs adapter. +- [ ] Add Gateway/OpenRouter-style routing support only after the generic OpenAI-compatible adapter and provider option patch model are stable. ### OpenCode Parity Patches @@ -118,13 +122,15 @@ Do not blanket re-record an entire test file when adding one cassette. `RECORD=t - [ ] Port DeepSeek reasoning handling and interleaved reasoning field mapping. - [ ] Add unsupported attachment fallback patches keyed by model capabilities. - [ ] Add cache hint patches for Anthropic, OpenRouter, Bedrock, OpenAI-compatible, Copilot, and Alibaba-style providers. -- [ ] Add provider option namespacing patches for Gateway, OpenRouter, Azure, and other provider-specific option bags. +- [ ] Add provider option namespacing patches for Gateway, OpenRouter, Azure, OpenAI-compatible wrappers, and other provider-specific option bags. - [ ] Add model-specific reasoning option patches for providers that need effort, summary, or native reasoning fields. +- [ ] Add provider-specific metadata extraction patches only where OpenCode needs returned reasoning, citations, usage details, or provider-native fields. ### OpenCode Bridge -- [ ] Build a `Provider.Model` -> `LLM.ModelRef` bridge for OpenCode, including protocol selection, base URLs, headers, limits, capabilities, and native provider metadata. +- [ ] Build a `Provider.Model` -> `LLM.ModelRef` bridge for OpenCode, including protocol selection, base URLs, headers, limits, capabilities, native provider metadata, and OpenAI-compatible provider family detection. - [ ] Build a `session.llm` -> `LLM.request(...)` bridge for system prompts, message history, tools, tool choice, generation options, reasoning variants, cache hints, and attachments. +- [ ] Keep auth and deployment concerns in the OpenCode bridge where possible: Bedrock credentials/region/profile, Vertex project/location/token, Azure deployment/API version, and Gateway/OpenRouter routing headers. - [ ] Keep initial OpenCode integration behind a local flag/path until request payload parity and stream event parity are proven against the existing `session/llm.test.ts` cases. ### Test And Recording Gaps @@ -133,3 +139,4 @@ Do not blanket re-record an entire test file when adding one cassette. `RECORD=t - [x] Cover provider-error and HTTP-status sad paths with deterministic fixtures across adapters (Anthropic mid-stream + 4xx; OpenAI Responses mid-stream + 4xx; OpenAI Chat 4xx). Live recordings of provider errors are still TODO when stable cassettes can be captured. - [ ] Improve cassette ergonomics if more providers need custom matching, redaction, or multi-interaction flows. - [ ] Mirror OpenCode request-body parity tests through the new LLM path for OpenAI Responses, Anthropic Messages, Gemini, OpenAI-compatible Chat, and Bedrock once supported. +- [ ] Add adapter parity fixtures against `../ai` behavior for generic OpenAI-compatible Chat before adding provider-specific wrappers. diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index ea10cc7b17..a69fba1464 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -9,4 +9,5 @@ export * as Schema from "./schema" export { AnthropicMessages } from "./provider/anthropic-messages" export { Gemini } from "./provider/gemini" export { OpenAIChat } from "./provider/openai-chat" +export { OpenAICompatibleChat } from "./provider/openai-compatible-chat" export { OpenAIResponses } from "./provider/openai-responses" diff --git a/packages/llm/src/provider/openai-compatible-chat.ts b/packages/llm/src/provider/openai-compatible-chat.ts new file mode 100644 index 0000000000..ecaefc0f7e --- /dev/null +++ b/packages/llm/src/provider/openai-compatible-chat.ts @@ -0,0 +1,88 @@ +import { Effect, Stream } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { Adapter } from "../adapter" +import { capabilities, model as llmModel, type ModelInput } from "../llm" +import { InvalidRequestError, ProviderChunkError, type LLMError, type LLMRequest } from "../schema" +import { OpenAIChat, type OpenAIChatTarget } from "./openai-chat" +import { ProviderShared } from "./shared" + +const ADAPTER = "openai-compatible-chat" + +export type OpenAICompatibleChatModelInput = Omit & { + readonly baseURL: string + readonly apiKey?: string + readonly headers?: Record + readonly queryParams?: Record +} + +const invalid = (message: string) => new InvalidRequestError({ message }) + +const isStringRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) && Object.values(value).every((item) => typeof item === "string") + +const queryParams = (request: LLMRequest) => { + const value = request.model.native?.queryParams + if (!isStringRecord(value)) return undefined + return value +} + +const completionUrl = (request: LLMRequest) => { + if (!request.model.baseURL) return undefined + const url = new URL(`${request.model.baseURL.replace(/\/+$/, "")}/chat/completions`) + for (const [key, value] of Object.entries(queryParams(request) ?? {})) url.searchParams.set(key, value) + return url.toString() +} + +const toHttp = (target: OpenAIChatTarget, request: LLMRequest) => + Effect.gen(function* () { + const url = completionUrl(request) + if (!url) return yield* invalid("OpenAI-compatible Chat requires a baseURL") + + return HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders({ + ...request.model.headers, + "content-type": "application/json", + }), + HttpClientRequest.bodyText(ProviderShared.encodeJson(target), "application/json"), + ) + }) + +const mapParseError = (error: LLMError) => { + if (!(error instanceof ProviderChunkError)) return error + return new ProviderChunkError({ + adapter: ADAPTER, + message: error.message.replace("OpenAI Chat", "OpenAI-compatible Chat"), + raw: error.raw, + }) +} + +export const adapter = Adapter.define({ + id: ADAPTER, + protocol: "openai-compatible-chat", + redact: OpenAIChat.adapter.redact, + prepare: OpenAIChat.adapter.prepare, + validate: OpenAIChat.adapter.validate, + toHttp: (target, context) => toHttp(target, context.request), + parse: (response) => OpenAIChat.adapter.parse(response).pipe(Stream.mapError(mapParseError)), +}) + +export const model = (input: OpenAICompatibleChatModelInput) => { + const { apiKey, headers, queryParams, native, ...rest } = input + return llmModel({ + ...rest, + protocol: "openai-compatible-chat", + headers: apiKey ? { authorization: `Bearer ${apiKey}`, ...headers } : headers, + native: queryParams ? { ...native, queryParams } : native, + capabilities: input.capabilities ?? capabilities({ tools: { calls: true, streamingInput: true } }), + }) +} + +export const includeUsage = adapter.patch("include-usage", { + reason: "request final usage chunk from OpenAI-compatible Chat streaming responses", + apply: (target) => ({ + ...target, + stream_options: { ...target.stream_options, include_usage: true }, + }), +}) + +export * as OpenAICompatibleChat from "./openai-compatible-chat" diff --git a/packages/llm/src/schema.ts b/packages/llm/src/schema.ts index 000e8a19ad..73e4574590 100644 --- a/packages/llm/src/schema.ts +++ b/packages/llm/src/schema.ts @@ -2,6 +2,7 @@ import { Schema } from "effect" export const Protocol = Schema.Literals([ "openai-chat", + "openai-compatible-chat", "openai-responses", "anthropic-messages", "gemini", diff --git a/packages/llm/test/provider/openai-compatible-chat.test.ts b/packages/llm/test/provider/openai-compatible-chat.test.ts new file mode 100644 index 0000000000..56b220d00c --- /dev/null +++ b/packages/llm/test/provider/openai-compatible-chat.test.ts @@ -0,0 +1,120 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer, Schema } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { LLM } from "../../src" +import { client } from "../../src/adapter" +import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat" +import { testEffect } from "../lib/effect" +import { dynamicResponse } from "../lib/http" +import { sseEvents } from "../lib/sse" + +const Json = Schema.fromJsonString(Schema.Unknown) +const decodeJson = Schema.decodeUnknownSync(Json) + +const model = OpenAICompatibleChat.model({ + id: "deepseek-chat", + provider: "deepseek", + baseURL: "https://api.deepseek.test/v1/", + apiKey: "test-key", + queryParams: { "api-version": "2026-01-01" }, +}) + +const request = LLM.request({ + id: "req_1", + model, + system: "You are concise.", + prompt: "Say hello.", + generation: { maxTokens: 20, temperature: 0 }, +}) + +const it = testEffect(Layer.empty) + +const deltaChunk = (delta: object, finishReason: string | null = null) => ({ + id: "chatcmpl_fixture", + choices: [{ delta, finish_reason: finishReason }], + usage: null, +}) + +const usageChunk = (usage: object) => ({ + id: "chatcmpl_fixture", + choices: [], + usage, +}) + +describe("OpenAI-compatible Chat adapter", () => { + it.effect("prepares generic Chat target", () => + Effect.gen(function* () { + const prepared = yield* client({ adapters: [OpenAICompatibleChat.adapter] }).prepare( + LLM.request({ + ...request, + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + toolChoice: { type: "required" }, + }), + ) + + expect(prepared.adapter).toBe("openai-compatible-chat") + expect(prepared.model).toMatchObject({ + id: "deepseek-chat", + provider: "deepseek", + protocol: "openai-compatible-chat", + baseURL: "https://api.deepseek.test/v1/", + headers: { authorization: "Bearer test-key" }, + native: { queryParams: { "api-version": "2026-01-01" } }, + }) + expect(prepared.target).toEqual({ + model: "deepseek-chat", + messages: [ + { role: "system", content: "You are concise." }, + { role: "user", content: "Say hello." }, + ], + tools: [{ type: "function", function: { name: "lookup", description: "Lookup data", parameters: { type: "object" } } }], + tool_choice: "required", + stream: true, + max_tokens: 20, + temperature: 0, + }) + }), + ) + + it.effect("posts to the configured compatible endpoint and parses text usage", () => + Effect.gen(function* () { + const response = yield* client({ + adapters: [OpenAICompatibleChat.adapter.withPatches([OpenAICompatibleChat.includeUsage])], + }) + .generate(request) + .pipe( + Effect.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + expect(web.url).toBe("https://api.deepseek.test/v1/chat/completions?api-version=2026-01-01") + expect(web.headers.get("authorization")).toBe("Bearer test-key") + expect(decodeJson(input.text)).toMatchObject({ + model: "deepseek-chat", + stream: true, + stream_options: { include_usage: true }, + messages: [ + { role: "system", content: "You are concise." }, + { role: "user", content: "Say hello." }, + ], + }) + return new Response( + sseEvents( + deltaChunk({ role: "assistant", content: "Hello" }), + deltaChunk({ content: "!" }), + deltaChunk({}, "stop"), + usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }), + ), + { headers: { "content-type": "text/event-stream" } }, + ) + }), + ), + ), + ) + + expect(LLM.outputText(response)).toBe("Hello!") + expect(LLM.outputUsage(response)).toMatchObject({ inputTokens: 5, outputTokens: 2, totalTokens: 7 }) + expect(response.events.at(-1)).toMatchObject({ type: "request-finish", reason: "stop" }) + }), + ) +})