diff --git a/packages/llm/AGENTS.md b/packages/llm/AGENTS.md index cf8ceffbf4..c2713273e3 100644 --- a/packages/llm/AGENTS.md +++ b/packages/llm/AGENTS.md @@ -109,7 +109,7 @@ Do not blanket re-record an entire test file when adding one cassette. `RECORD=t - [x] Add a generic OpenAI-compatible Chat adapter for non-OpenAI providers that expose `/chat/completions`; use `../ai/packages/openai-compatible` as the behavior reference. - [ ] Keep OpenAI Responses as a separate first-class protocol for providers that actually implement `/responses`; do not treat generic OpenAI-compatible providers as Responses-capable by default. -- [ ] Cover OpenAI-compatible provider families that can share the generic adapter first: DeepSeek, TogetherAI, Cerebras, Baseten, Fireworks, DeepInfra, and similar providers. +- [x] Cover OpenAI-compatible provider families that can share the generic adapter first: DeepSeek, TogetherAI, Cerebras, Baseten, Fireworks, DeepInfra, and similar providers. - [ ] Decide which providers need thin dedicated wrappers over OpenAI-compatible Chat because they have custom parsing/options: Mistral, Groq, xAI, Perplexity, and Cohere. - [ ] Add Bedrock Converse support or a clear compatibility layer before moving Amazon Bedrock traffic onto `packages/llm`. - [ ] Decide Vertex shape after Bedrock/OpenAI-compatible are stable: Vertex Gemini as Gemini target/http patch vs adapter, and Vertex Anthropic as Anthropic target/http patch vs adapter. diff --git a/packages/llm/src/provider/openai-compatible-chat.ts b/packages/llm/src/provider/openai-compatible-chat.ts index ecaefc0f7e..f4700cdd66 100644 --- a/packages/llm/src/provider/openai-compatible-chat.ts +++ b/packages/llm/src/provider/openai-compatible-chat.ts @@ -15,6 +15,24 @@ export type OpenAICompatibleChatModelInput = Omit } +export type ProviderFamilyModelInput = Omit & { + readonly baseURL?: string +} + +interface ProviderFamily { + readonly provider: string + readonly baseURL: string +} + +const families = { + baseten: { provider: "baseten", baseURL: "https://inference.baseten.co/v1" }, + cerebras: { provider: "cerebras", baseURL: "https://api.cerebras.ai/v1" }, + deepinfra: { provider: "deepinfra", baseURL: "https://api.deepinfra.com/v1/openai" }, + deepseek: { provider: "deepseek", baseURL: "https://api.deepseek.com/v1" }, + fireworks: { provider: "fireworks", baseURL: "https://api.fireworks.ai/inference/v1" }, + togetherai: { provider: "togetherai", baseURL: "https://api.together.xyz/v1" }, +} as const satisfies Record + const invalid = (message: string) => new InvalidRequestError({ message }) const isStringRecord = (value: unknown): value is Record => @@ -77,6 +95,26 @@ export const model = (input: OpenAICompatibleChatModelInput) => { }) } +const familyModel = (family: ProviderFamily, input: ProviderFamilyModelInput) => + model({ + ...input, + provider: family.provider, + baseURL: input.baseURL ?? family.baseURL, + native: { ...input.native, openaiCompatibleProvider: family.provider }, + }) + +export const baseten = (input: ProviderFamilyModelInput) => familyModel(families.baseten, input) + +export const cerebras = (input: ProviderFamilyModelInput) => familyModel(families.cerebras, input) + +export const deepinfra = (input: ProviderFamilyModelInput) => familyModel(families.deepinfra, input) + +export const deepseek = (input: ProviderFamilyModelInput) => familyModel(families.deepseek, input) + +export const fireworks = (input: ProviderFamilyModelInput) => familyModel(families.fireworks, input) + +export const togetherai = (input: ProviderFamilyModelInput) => familyModel(families.togetherai, input) + export const includeUsage = adapter.patch("include-usage", { reason: "request final usage chunk from OpenAI-compatible Chat streaming responses", apply: (target) => ({ diff --git a/packages/llm/test/provider/openai-compatible-chat.test.ts b/packages/llm/test/provider/openai-compatible-chat.test.ts index 56b220d00c..37c813f943 100644 --- a/packages/llm/test/provider/openai-compatible-chat.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.test.ts @@ -41,6 +41,15 @@ const usageChunk = (usage: object) => ({ usage, }) +const providerFamilies = [ + ["baseten", OpenAICompatibleChat.baseten, "https://inference.baseten.co/v1"], + ["cerebras", OpenAICompatibleChat.cerebras, "https://api.cerebras.ai/v1"], + ["deepinfra", OpenAICompatibleChat.deepinfra, "https://api.deepinfra.com/v1/openai"], + ["deepseek", OpenAICompatibleChat.deepseek, "https://api.deepseek.com/v1"], + ["fireworks", OpenAICompatibleChat.fireworks, "https://api.fireworks.ai/inference/v1"], + ["togetherai", OpenAICompatibleChat.togetherai, "https://api.together.xyz/v1"], +] as const + describe("OpenAI-compatible Chat adapter", () => { it.effect("prepares generic Chat target", () => Effect.gen(function* () { @@ -76,6 +85,45 @@ describe("OpenAI-compatible Chat adapter", () => { }), ) + it.effect("provides model helpers for compatible provider families", () => + Effect.gen(function* () { + expect( + providerFamilies.map(([provider, makeModel, baseURL]) => { + const model = makeModel({ id: `${provider}-model`, apiKey: "test-key" }) + return { + id: model.id, + provider: model.provider, + protocol: model.protocol, + baseURL: model.baseURL, + headers: model.headers, + native: model.native, + } + }), + ).toEqual( + providerFamilies.map(([provider, _, baseURL]) => ({ + id: `${provider}-model`, + provider, + protocol: "openai-compatible-chat", + baseURL, + headers: { authorization: "Bearer test-key" }, + native: { openaiCompatibleProvider: provider }, + })), + ) + + const custom = OpenAICompatibleChat.deepseek({ + id: "deepseek-chat", + apiKey: "test-key", + baseURL: "https://custom.deepseek.test/v1", + }) + expect(custom).toMatchObject({ + provider: "deepseek", + protocol: "openai-compatible-chat", + baseURL: "https://custom.deepseek.test/v1", + native: { openaiCompatibleProvider: "deepseek" }, + }) + }), + ) + it.effect("posts to the configured compatible endpoint and parses text usage", () => Effect.gen(function* () { const response = yield* client({