test(llm): add openrouter golden tool loops
This commit is contained in:
Kit Langton
@@ -261,8 +261,8 @@ AI SDK-style mocked tests are still the right tool for exhaustive parser weirdne
|
||||
Reference examples:
|
||||
|
||||
- `test/provider/openai-chat-tool-loop.recorded.test.ts` is the current recorded multi-interaction tool-loop scaffold.
|
||||
- `test/provider/openai-compatible-chat.recorded.test.ts` shows provider-matrix cassettes for generic OpenAI-compatible providers, including OpenRouter text/tool recordings.
|
||||
- When a first-class golden loop reaches the full standard above, add its relative path here and prefer copying that structure for new providers.
|
||||
- `test/provider/openai-compatible-chat.recorded.test.ts` shows provider-matrix cassettes for generic OpenAI-compatible providers, including OpenRouter text/tool recordings and OpenRouter golden loops for baseline and flagship routes.
|
||||
- Prefer copying the OpenRouter golden-loop structure in `test/provider/openai-compatible-chat.recorded.test.ts` when adding new provider/protocol golden cassettes.
|
||||
|
||||
## TODO
|
||||
|
||||
@@ -349,3 +349,4 @@ Reference examples:
|
||||
- [x] Bedrock Converse basic text and tool-call cassettes (recorded against `us.amazon.nova-micro-v1:0` in us-east-1). Cache-hint cassettes still TODO.
|
||||
- [ ] Vertex Gemini and Vertex Anthropic basic/tool cassettes after the Vertex adapter/patch shape is decided.
|
||||
- [ ] Gateway/OpenRouter routing-header cassettes after routing support lands.
|
||||
- [x] OpenRouter OpenAI-compatible Chat golden tool-loop cassettes for `openai/gpt-4o-mini`, `openai/gpt-5.5`, and `anthropic/claude-opus-4.7`.
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"name": "openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop",
|
||||
"recordedAt": "2026-05-03T18:43:52.616Z",
|
||||
"tags": [
|
||||
"prefix:openai-compatible-chat",
|
||||
"protocol:openai-compatible-chat",
|
||||
"provider:openrouter",
|
||||
"tool",
|
||||
"tool-loop",
|
||||
"golden",
|
||||
"flagship"
|
||||
]
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-opus-4.7\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Paris\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}],\"usage\":{\"prompt_tokens\":802,\"completion_tokens\":66,\"total_tokens\":868,\"cost\":0.00566,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00566,\"upstream_inference_prompt_cost\":0.00401,\"upstream_inference_completions_cost\":0.00165},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic/claude-opus-4.7\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris is currently s\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"unny with a\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" temperature of 22°C\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":899,\"completion_tokens\":24,\"total_tokens\":923,\"cost\":0.005095,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.005095,\"upstream_inference_prompt_cost\":0.004495,\"upstream_inference_completions_cost\":0.0006},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+51
File diff suppressed because one or more lines are too long
+52
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"name": "openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop",
|
||||
"recordedAt": "2026-05-03T18:43:50.860Z",
|
||||
"tags": [
|
||||
"prefix:openai-compatible-chat",
|
||||
"protocol:openai-compatible-chat",
|
||||
"provider:openrouter",
|
||||
"tool",
|
||||
"tool-loop",
|
||||
"golden",
|
||||
"flagship"
|
||||
]
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"openai/gpt-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"completed\"}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"completed\"}],\"usage\":{\"prompt_tokens\":69,\"completion_tokens\":18,\"total_tokens\":87,\"cost\":0.000885,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.000885,\"upstream_inference_prompt_cost\":0.000345,\"upstream_inference_completions_cost\":0.00054},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://openrouter.ai/api/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"openai/gpt-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sunny\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"22\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"completed\"}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"completed\"}],\"usage\":{\"prompt_tokens\":108,\"completion_tokens\":12,\"total_tokens\":120,\"cost\":0.0009,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0009,\"upstream_inference_prompt_cost\":0.00054,\"upstream_inference_completions_cost\":0.00036},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM } from "../../src"
|
||||
import { Effect, Stream } from "effect"
|
||||
import { LLM, LLMEvent, type ModelRef } from "../../src"
|
||||
import { LLMClient } from "../../src/adapter"
|
||||
import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat"
|
||||
import { expectFinish, expectWeatherToolCall, textRequest, weatherToolRequest } from "../recorded-scenarios"
|
||||
import { ToolRuntime } from "../../src/tool-runtime"
|
||||
import { expectFinish, expectWeatherToolCall, textRequest, weatherRuntimeTool, weatherToolRequest } from "../recorded-scenarios"
|
||||
import { recordedTests } from "../recorded-test"
|
||||
|
||||
const deepseekModel = OpenAICompatibleChat.deepseek({
|
||||
@@ -29,9 +30,42 @@ const openrouterModel = OpenAICompatibleChat.openrouter({
|
||||
const openrouterRequest = textRequest({ id: "recorded_openrouter_text", model: openrouterModel })
|
||||
const openrouterToolRequest = weatherToolRequest({ id: "recorded_openrouter_tool_call", model: openrouterModel })
|
||||
|
||||
const openrouterGpt55Model = OpenAICompatibleChat.openrouter({
|
||||
id: "openai/gpt-5.5",
|
||||
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
|
||||
})
|
||||
|
||||
const openrouterOpus47Model = OpenAICompatibleChat.openrouter({
|
||||
id: "anthropic/claude-opus-4.7",
|
||||
apiKey: process.env.OPENROUTER_API_KEY ?? "fixture",
|
||||
})
|
||||
|
||||
const openrouterToolLoopRequest = (input: { readonly id: string; readonly model: ModelRef }) =>
|
||||
LLM.request({
|
||||
id: input.id,
|
||||
model: input.model,
|
||||
system: "Use the get_weather tool exactly once, then answer in one short sentence.",
|
||||
prompt: "What is the weather in Paris?",
|
||||
generation: { maxTokens: 200 },
|
||||
})
|
||||
|
||||
const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" })
|
||||
const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] })
|
||||
|
||||
const expectWeatherToolLoop = (events: ReadonlyArray<LLMEvent>) => {
|
||||
const finishes = events.filter(LLMEvent.is.requestFinish)
|
||||
expect(finishes).toHaveLength(2)
|
||||
expect(finishes[0]?.reason).toBe("tool-calls")
|
||||
expect(finishes.at(-1)?.reason).toBe("stop")
|
||||
|
||||
expect(events.find(LLMEvent.is.toolResult)).toMatchObject({
|
||||
type: "tool-result",
|
||||
name: "get_weather",
|
||||
result: { type: "json", value: { temperature: 22, condition: "sunny" } },
|
||||
})
|
||||
expect(LLM.outputText({ events })).toContain("Paris")
|
||||
}
|
||||
|
||||
describe("OpenAI-compatible Chat recorded", () => {
|
||||
recorded.effect.with("deepseek streams text", { provider: "deepseek", requires: ["DEEPSEEK_API_KEY"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
@@ -79,4 +113,37 @@ describe("OpenAI-compatible Chat recorded", () => {
|
||||
expectFinish(response.events, "tool-calls")
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with("openrouter gpt-4o-mini drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
expectWeatherToolLoop(Array.from(
|
||||
yield* ToolRuntime.run(llm, {
|
||||
request: openrouterToolLoopRequest({ id: "recorded_openrouter_gpt_4o_mini_tool_loop", model: openrouterModel }),
|
||||
tools: { get_weather: weatherRuntimeTool },
|
||||
}).pipe(Stream.runCollect),
|
||||
))
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with("openrouter gpt-5.5 drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden", "flagship"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
expectWeatherToolLoop(Array.from(
|
||||
yield* ToolRuntime.run(llm, {
|
||||
request: openrouterToolLoopRequest({ id: "recorded_openrouter_gpt_5_5_tool_loop", model: openrouterGpt55Model }),
|
||||
tools: { get_weather: weatherRuntimeTool },
|
||||
}).pipe(Stream.runCollect),
|
||||
))
|
||||
}),
|
||||
)
|
||||
|
||||
recorded.effect.with("openrouter claude opus 4.7 drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden", "flagship"] }, () =>
|
||||
Effect.gen(function* () {
|
||||
expectWeatherToolLoop(Array.from(
|
||||
yield* ToolRuntime.run(llm, {
|
||||
request: openrouterToolLoopRequest({ id: "recorded_openrouter_claude_opus_4_7_tool_loop", model: openrouterOpus47Model }),
|
||||
tools: { get_weather: weatherRuntimeTool },
|
||||
}).pipe(Stream.runCollect),
|
||||
))
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user