Mirror_Repos/anomalyco_opencode
2
0
Fork 0
Code Issues Pull Requests Actions 63 Packages Projects Releases Wiki Activity

test(llm): add openrouter recorded coverage

Browse Source
This commit is contained in:
Kit Langton
2026-05-03 14:07:33 -04:00
parent a1c1d0766f
commit 9e3868fed7
4 changed files with 100 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/llm/src/provider/openai-compatible-chat.ts
+10
View File
@@ -61,8 +61,18 @@ export const deepseek = (input: ProviderFamilyModelInput) => familyModel(familie
export const fireworks = (input: ProviderFamilyModelInput) => familyModel(families.fireworks, input)
export const groq = (input: ProviderFamilyModelInput) => familyModel(families.groq, input)
export const mistral = (input: ProviderFamilyModelInput) => familyModel(families.mistral, input)
export const openrouter = (input: ProviderFamilyModelInput) => familyModel(families.openrouter, input)
export const perplexity = (input: ProviderFamilyModelInput) => familyModel(families.perplexity, input)
export const togetherai = (input: ProviderFamilyModelInput) => familyModel(families.togetherai, input)
export const venice = (input: ProviderFamilyModelInput) => familyModel(families.venice, input)
export const includeUsage = adapter.patch("include-usage", {
reason: "request final usage chunk from OpenAI-compatible Chat streaming responses",
apply: (target) => ({
packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-text.json
Vendored
+31
View File
@@ -0,0 +1,31 @@
{
"version": 1,
"metadata": {
"name": "openai-compatible-chat/openrouter-streams-text",
"recordedAt": "2026-05-03T18:06:03.649Z",
"tags": [
"prefix:openai-compatible-chat",
"protocol:openai-compatible-chat",
"provider:openrouter"
]
},
"interactions": [
{
"request": {
"method": "POST",
"url": "https://openrouter.ai/api/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"You are concise.\"},{\"role\":\"user\",\"content\":\"Reply with exactly: Hello!\"}],\"stream\":true,\"max_tokens\":20,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}]}\n\ndata: {\"id\":\"gen-1777831563-gkUknIabxEwXqNocnRG3\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"Azure\",\"system_fingerprint\":\"fp_eb37e061ec\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":21,\"completion_tokens\":3,\"total_tokens\":24,\"cost\":0.00000495,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00000495,\"upstream_inference_prompt_cost\":0.00000315,\"upstream_inference_completions_cost\":0.0000018},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
}
}
]
}
packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-streams-tool-call.json
Vendored
+32
View File
@@ -0,0 +1,32 @@
{
"version": 1,
"metadata": {
"name": "openai-compatible-chat/openrouter-streams-tool-call",
"recordedAt": "2026-05-03T18:06:04.205Z",
"tags": [
"prefix:openai-compatible-chat",
"protocol:openai-compatible-chat",
"provider:openrouter",
"tool"
]
},
"interactions": [
{
"request": {
"method": "POST",
"url": "https://openrouter.ai/api/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Call tools exactly as requested.\"},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}},\"stream\":true,\"max_tokens\":80,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_xbVlNaHfU9J19mE70TdORhwX\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"stop\"}]}\n\ndata: {\"id\":\"gen-1777831563-zmGngY6IapHbeA0TiubD\",\"object\":\"chat.completion.chunk\",\"created\":1777831563,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_9075db19fa\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":67,\"completion_tokens\":5,\"total_tokens\":72,\"cost\":0.00001305,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00001305,\"upstream_inference_prompt_cost\":0.00001005,\"upstream_inference_completions_cost\":0.000003},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
}
}
]
}
packages/llm/test/provider/openai-compatible-chat.recorded.test.ts
+27
View File
@@ -21,6 +21,14 @@ const togetherModel = OpenAICompatibleChat.togetherai({
const togetherRequest = textRequest({ id: "recorded_togetherai_text", model: togetherModel })
const togetherToolRequest = weatherToolRequest({ id: "recorded_togetherai_tool_call", model: togetherModel })
const openrouterModel = OpenAICompatibleChat.openrouter({
id: "openai/gpt-4o-mini",
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
})
const openrouterRequest = textRequest({ id: "recorded_openrouter_text", model: openrouterModel })
const openrouterToolRequest = weatherToolRequest({ id: "recorded_openrouter_tool_call", model: openrouterModel })
const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" })
const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] })
@@ -52,4 +60,23 @@ describe("OpenAI-compatible Chat recorded", () => {
expectFinish(response.events, "tool-calls")
}),
)
recorded.effect.with("openrouter streams text", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"] }, () =>
Effect.gen(function* () {
const response = yield* llm.generate(openrouterRequest)
expect(LLM.outputText(response)).toMatch(/^Hello!?$/)
expectFinish(response.events, "stop")
}),
)
recorded.effect.with("openrouter streams tool call", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool"] }, () =>
Effect.gen(function* () {
const response = yield* llm.generate(openrouterToolRequest)
expect(response.events.some((event) => event.type === "tool-input-delta")).toBe(true)
expectWeatherToolCall(response)
expectFinish(response.events, "tool-calls")
}),
)
})