diff --git a/packages/llm/AGENTS.md b/packages/llm/AGENTS.md index 26631473e8..99a906906d 100644 --- a/packages/llm/AGENTS.md +++ b/packages/llm/AGENTS.md @@ -261,8 +261,8 @@ AI SDK-style mocked tests are still the right tool for exhaustive parser weirdne Reference examples: - `test/provider/openai-chat-tool-loop.recorded.test.ts` is the current recorded multi-interaction tool-loop scaffold. -- `test/provider/openai-compatible-chat.recorded.test.ts` shows provider-matrix cassettes for generic OpenAI-compatible providers, including OpenRouter text/tool recordings. -- When a first-class golden loop reaches the full standard above, add its relative path here and prefer copying that structure for new providers. +- `test/provider/openai-compatible-chat.recorded.test.ts` shows provider-matrix cassettes for generic OpenAI-compatible providers, including OpenRouter text/tool recordings and OpenRouter golden loops for baseline and flagship routes. +- Prefer copying the OpenRouter golden-loop structure in `test/provider/openai-compatible-chat.recorded.test.ts` when adding new provider/protocol golden cassettes. ## TODO @@ -349,3 +349,4 @@ Reference examples: - [x] Bedrock Converse basic text and tool-call cassettes (recorded against `us.amazon.nova-micro-v1:0` in us-east-1). Cache-hint cassettes still TODO. - [ ] Vertex Gemini and Vertex Anthropic basic/tool cassettes after the Vertex adapter/patch shape is decided. - [ ] Gateway/OpenRouter routing-header cassettes after routing support lands. +- [x] OpenRouter OpenAI-compatible Chat golden tool-loop cassettes for `openai/gpt-4o-mini`, `openai/gpt-5.5`, and `anthropic/claude-opus-4.7`. diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json new file mode 100644 index 0000000000..bf9319cc94 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop.json @@ -0,0 +1,52 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/openrouter-claude-opus-4-7-drives-a-tool-loop", + "recordedAt": "2026-05-03T18:43:52.616Z", + "tags": [ + "prefix:openai-compatible-chat", + "protocol:openai-compatible-chat", + "provider:openrouter", + "tool", + "tool-loop", + "golden", + "flagship" + ] + }, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-opus-4.7\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Paris\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}]}\n\ndata: {\"id\":\"gen-1777833830-vBk2NZQAlSaGxN6kK3Lw\",\"object\":\"chat.completion.chunk\",\"created\":1777833830,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}],\"usage\":{\"prompt_tokens\":802,\"completion_tokens\":66,\"total_tokens\":868,\"cost\":0.00566,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.00566,\"upstream_inference_prompt_cost\":0.00401,\"upstream_inference_completions_cost\":0.00165},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + }, + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-opus-4.7\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_bdrk_01H8Ttw4bjFojszuNqgRb3oG\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris is currently s\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"unny with a\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" temperature of 22°C\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1777833831-M9dJroEHjSf7j7Eo3tsb\",\"object\":\"chat.completion.chunk\",\"created\":1777833831,\"model\":\"anthropic/claude-4.7-opus-20260416\",\"provider\":\"Amazon Bedrock\",\"service_tier\":\"standard\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":899,\"completion_tokens\":24,\"total_tokens\":923,\"cost\":0.005095,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.005095,\"upstream_inference_prompt_cost\":0.004495,\"upstream_inference_completions_cost\":0.0006},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json new file mode 100644 index 0000000000..9831a328bd --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/openrouter-gpt-4o-mini-drives-a-tool-loop", + "recordedAt": "2026-05-03T18:43:48.240Z", + "tags": [ + "prefix:openai-compatible-chat", + "protocol:openai-compatible-chat", + "provider:openrouter", + "tool", + "tool-loop", + "golden" + ] + }, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_HLcMEFJpsngj7SJ7KfIwSPXI\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_calls\"}]}\n\ndata: {\"id\":\"gen-1777833826-j2hVvfMKVHat3HVnACwK\",\"object\":\"chat.completion.chunk\",\"created\":1777833826,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":66,\"completion_tokens\":14,\"total_tokens\":80,\"cost\":0.0000183,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0000183,\"upstream_inference_prompt_cost\":0.0000099,\"upstream_inference_completions_cost\":0.0000084},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + }, + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_HLcMEFJpsngj7SJ7KfIwSPXI\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_HLcMEFJpsngj7SJ7KfIwSPXI\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"The\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" weather\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sunny\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" with\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" temperature\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"22\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}]}\n\ndata: {\"id\":\"gen-1777833827-6amwFnEcZ60BSi1gJubt\",\"object\":\"chat.completion.chunk\",\"created\":1777833827,\"model\":\"openai/gpt-4o-mini\",\"provider\":\"OpenAI\",\"system_fingerprint\":\"fp_ff247d5857\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":98,\"completion_tokens\":15,\"total_tokens\":113,\"cost\":0.0000237,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0000237,\"upstream_inference_prompt_cost\":0.0000147,\"upstream_inference_completions_cost\":0.000009},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json new file mode 100644 index 0000000000..dfb4328fa2 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop.json @@ -0,0 +1,52 @@ +{ + "version": 1, + "metadata": { + "name": "openai-compatible-chat/openrouter-gpt-5-5-drives-a-tool-loop", + "recordedAt": "2026-05-03T18:43:50.860Z", + "tags": [ + "prefix:openai-compatible-chat", + "protocol:openai-compatible-chat", + "provider:openrouter", + "tool", + "tool-loop", + "golden", + "flagship" + ] + }, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"completed\"}]}\n\ndata: {\"id\":\"gen-1777833828-iz6abT1gmyb8TG7USkhf\",\"object\":\"chat.completion.chunk\",\"created\":1777833828,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"completed\"}],\"usage\":{\"prompt_tokens\":69,\"completion_tokens\":18,\"total_tokens\":87,\"cost\":0.000885,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.000885,\"upstream_inference_prompt_cost\":0.000345,\"upstream_inference_completions_cost\":0.00054},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + }, + { + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"openai/gpt-5.5\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_07tB2cTah7dsGpTh8rISbG8D\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":200}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sunny\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"22\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"completed\"}]}\n\ndata: {\"id\":\"gen-1777833829-0hoA0SolxNrNyVmYPrq9\",\"object\":\"chat.completion.chunk\",\"created\":1777833829,\"model\":\"openai/gpt-5.5-20260423\",\"provider\":\"OpenAI\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"completed\"}],\"usage\":{\"prompt_tokens\":108,\"completion_tokens\":12,\"total_tokens\":120,\"cost\":0.0009,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.0009,\"upstream_inference_prompt_cost\":0.00054,\"upstream_inference_completions_cost\":0.00036},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts index bd1c6139fd..9fa6d7f23d 100644 --- a/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts +++ b/packages/llm/test/provider/openai-compatible-chat.recorded.test.ts @@ -1,9 +1,10 @@ import { describe, expect } from "bun:test" -import { Effect } from "effect" -import { LLM } from "../../src" +import { Effect, Stream } from "effect" +import { LLM, LLMEvent, type ModelRef } from "../../src" import { LLMClient } from "../../src/adapter" import { OpenAICompatibleChat } from "../../src/provider/openai-compatible-chat" -import { expectFinish, expectWeatherToolCall, textRequest, weatherToolRequest } from "../recorded-scenarios" +import { ToolRuntime } from "../../src/tool-runtime" +import { expectFinish, expectWeatherToolCall, textRequest, weatherRuntimeTool, weatherToolRequest } from "../recorded-scenarios" import { recordedTests } from "../recorded-test" const deepseekModel = OpenAICompatibleChat.deepseek({ @@ -29,9 +30,42 @@ const openrouterModel = OpenAICompatibleChat.openrouter({ const openrouterRequest = textRequest({ id: "recorded_openrouter_text", model: openrouterModel }) const openrouterToolRequest = weatherToolRequest({ id: "recorded_openrouter_tool_call", model: openrouterModel }) +const openrouterGpt55Model = OpenAICompatibleChat.openrouter({ + id: "openai/gpt-5.5", + apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", +}) + +const openrouterOpus47Model = OpenAICompatibleChat.openrouter({ + id: "anthropic/claude-opus-4.7", + apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", +}) + +const openrouterToolLoopRequest = (input: { readonly id: string; readonly model: ModelRef }) => + LLM.request({ + id: input.id, + model: input.model, + system: "Use the get_weather tool exactly once, then answer in one short sentence.", + prompt: "What is the weather in Paris?", + generation: { maxTokens: 200 }, + }) + const recorded = recordedTests({ prefix: "openai-compatible-chat", protocol: "openai-compatible-chat" }) const llm = LLMClient.make({ adapters: [OpenAICompatibleChat.adapter] }) +const expectWeatherToolLoop = (events: ReadonlyArray) => { + const finishes = events.filter(LLMEvent.is.requestFinish) + expect(finishes).toHaveLength(2) + expect(finishes[0]?.reason).toBe("tool-calls") + expect(finishes.at(-1)?.reason).toBe("stop") + + expect(events.find(LLMEvent.is.toolResult)).toMatchObject({ + type: "tool-result", + name: "get_weather", + result: { type: "json", value: { temperature: 22, condition: "sunny" } }, + }) + expect(LLM.outputText({ events })).toContain("Paris") +} + describe("OpenAI-compatible Chat recorded", () => { recorded.effect.with("deepseek streams text", { provider: "deepseek", requires: ["DEEPSEEK_API_KEY"] }, () => Effect.gen(function* () { @@ -79,4 +113,37 @@ describe("OpenAI-compatible Chat recorded", () => { expectFinish(response.events, "tool-calls") }), ) + + recorded.effect.with("openrouter gpt-4o-mini drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden"] }, () => + Effect.gen(function* () { + expectWeatherToolLoop(Array.from( + yield* ToolRuntime.run(llm, { + request: openrouterToolLoopRequest({ id: "recorded_openrouter_gpt_4o_mini_tool_loop", model: openrouterModel }), + tools: { get_weather: weatherRuntimeTool }, + }).pipe(Stream.runCollect), + )) + }), + ) + + recorded.effect.with("openrouter gpt-5.5 drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden", "flagship"] }, () => + Effect.gen(function* () { + expectWeatherToolLoop(Array.from( + yield* ToolRuntime.run(llm, { + request: openrouterToolLoopRequest({ id: "recorded_openrouter_gpt_5_5_tool_loop", model: openrouterGpt55Model }), + tools: { get_weather: weatherRuntimeTool }, + }).pipe(Stream.runCollect), + )) + }), + ) + + recorded.effect.with("openrouter claude opus 4.7 drives a tool loop", { provider: "openrouter", requires: ["OPENROUTER_API_KEY"], tags: ["tool", "tool-loop", "golden", "flagship"] }, () => + Effect.gen(function* () { + expectWeatherToolLoop(Array.from( + yield* ToolRuntime.run(llm, { + request: openrouterToolLoopRequest({ id: "recorded_openrouter_claude_opus_4_7_tool_loop", model: openrouterOpus47Model }), + tools: { get_weather: weatherRuntimeTool }, + }).pipe(Stream.runCollect), + )) + }), + ) })