From 9e3868fed7faead157e0d7576034ffd6aa7f690d Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 3 May 2026 14:07:33 -0400 Subject: [PATCH] test(llm): add openrouter recorded coverage --- .../src/provider/openai-compatible-chat.ts | 10 ++++++ .../openrouter-streams-text.json | 31 ++++++++++++++++++ .../openrouter-streams-tool-call.json | 32 +++++++++++++++++++ .../openai-compatible-chat.recorded.test.ts | 27 ++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json create mode 100644 packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json diff --git a/packages/llm/src/provider/openai-compatible-chat.ts b/packages/llm/src/provider/openai-compatible-chat.ts index e4a6362ac1..4c7189ea32 100644 --- a/packages/llm/src/provider/openai-compatible-chat.ts +++ b/packages/llm/src/provider/openai-compatible-chat.ts @@ -61,8 +61,18 @@ export const deepseek = (input: ProviderFamilyModelInput) => familyModel(familie export const fireworks = (input: ProviderFamilyModelInput) => familyModel(families.fireworks, input) +export const groq = (input: ProviderFamilyModelInput) => familyModel(families.groq, input) + +export const mistral = (input: ProviderFamilyModelInput) => familyModel(families.mistral, input) + +export const openrouter = (input: ProviderFamilyModelInput) => familyModel(families.openrouter, input) + +export const perplexity = (input: ProviderFamilyModelInput) => familyModel(families.perplexity, input) + export const togetherai = (input: ProviderFamilyModelInput) => familyModel(families.togetherai, input) +export const venice = (input: ProviderFamilyModelInput) => familyModel(families.venice, input) + export const includeUsage = adapter.patch("include-usage", { reason: "request final usage chunk from OpenAI-compatible Chat streaming responses", apply: (target) => ({ diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json new file mode 100644 index 0000000000..138b19a0d4 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/openrouter-streams-text", + "recordedAt": "2026-05-03T18:06:03.649Z", + "tags": [ + "prefix:openai-compatible-chat", + "protocol:openai-compatible-chat", + "provider:openrouter" + ] + }, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":\"Reply with exactly: Hello!\"}],\"stream\":true,\"max_tokens\":20,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":21,\"completion_tokens\":3,\"total_tokens\":24,\"cost\":0.00000495,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00000495,\"upstream_inference_prompt_cost\":0.00000315,\"upstream_inference_completions_cost\":0.0000018},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json new file mode 100644 index 0000000000..e8fada77f4 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/openrouter-streams-tool-call", + "recordedAt": "2026-05-03T18:06:04.205Z", + "tags": [ + "prefix:openai-compatible-chat", + "protocol:openai-compatible-chat", + "provider:openrouter", + "tool" + ] + }, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Call tools exactly as requested.\"},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}},\"stream\":true,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_xbVlNaHfU9J19mE70TdORhwX\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"stop\"}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":67,\"completion_tokens\":5,\"total_tokens\":72,\"cost\":0.00001305,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00001305,\"upstream_inference_prompt_cost\":0.00001005,\"upstream_inference_completions_cost\":0.000003},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts index 5921fe156f..bd1c6139fd 100644 --- a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts @@ -21,6 +21,14 @@ const togetherModel = OpenAICompatibleChat.togetherai({ const togetherRequest = textRequest({ id: "recorded_togetherai_text", model: togetherModel }) const togetherToolRequest = weatherToolRequest({ id: "recorded_togetherai_tool_call", model: togetherModel }) +const openrouterModel = OpenAICompatibleChat.openrouter({ + id: "openai/gpt-4o-mini", + apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", +}) + +const openrouterRequest = textRequest({ id: "recorded_openrouter_text", model: openrouterModel }) +const openrouterToolRequest = weatherToolRequest({ id: "recorded_openrouter_tool_call", model: openrouterModel }) + const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" }) const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }) @@ -52,4 +60,23 @@ describe("OpenAI-compatible Chat recorded", () => { expectFinish(response.events, "tool-calls") }), ) + + recorded.effect.with("openrouter streams text", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"] }, () => + Effect.gen(function* () { + const response = yield* llm.generate(openrouterRequest) + + expect(LLM.outputText(response)).toMatch(/^Hello!?$/) + expectFinish(response.events, "stop") + }), + ) + + recorded.effect.with("openrouter streams tool call", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool"] }, () => + Effect.gen(function* () { + const response = yield* llm.generate(openrouterToolRequest) + + expect(response.events.some((event) => event.type === "tool-input-delta")).toBe(true) + expectWeatherToolCall(response) + expectFinish(response.events, "tool-calls") + }), + ) })