From b5ca62d1ea70d1fbe5a91f74a20049bd15fe2d3e Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 26 Apr 2026 12:06:34 -0400 Subject: [PATCH] test(llm): record OpenAI Chat tool-loop cassette Captures both model rounds of the typed ToolRuntime tool loop into a single multi-interaction cassette: round 1 carries the user prompt and returns a get_weather tool call; round 2 carries the assistant tool call plus tool result and returns a final answer. Verifies the multi-interaction cassette infrastructure end-to-end against a real provider. --- .../drives-a-tool-loop-end-to-end.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 packages/llm/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json diff --git a/packages/llm/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json b/packages/llm/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json new file mode 100644 index 0000000000..46f2114219 --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-chat/drives-a-tool-loop-end-to-end.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "interactions": [ + { + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"index\":0,\"id\":\"call_rd6eAGNs5DfA2jqg9to4Qwxz\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}],\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"d\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"EUb5Gwu8Tap\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"city\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Ybq9qovqKs\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\":\\\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"5mciJoX1T\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"AVUYhGEj3\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"AscuuECkg2P\"}\n\ndata: {\"id\":\"chatcmpl-DYwHYYmB5VYyBXbltE2hZpEd76wNh\",\"object\":\"chat.completion.chunk\",\"created\":1777219560,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"obfuscation\":\"D4JV5kBot8Vv\"}\n\ndata: [DONE]\n\n" + } + }, + { + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool, then answer in one short sentence.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_rd6eAGNs5DfA2jqg9to4Qwxz\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_rd6eAGNs5DfA2jqg9to4Qwxz\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"max_tokens\":80,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "data: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\",\"refusal\":null},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"uZFVhA\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"The\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"gnErV\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" weather\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" in\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"gYLjq\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" Paris\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"NH\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"AfXOl\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" sunny\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"ry\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" with\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"Y27\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" a\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"qTcUSW\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" temperature\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"vHIqlgLoa8JP\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" of\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"a3gN7\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"smqNnpb\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"22\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"VctHB6\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"°C\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"j3LkAV\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"obfuscation\":\"CeO6m2R\"}\n\ndata: {\"id\":\"chatcmpl-DYwHaJIvOecX0LXw5kFqM5OcX7ram\",\"object\":\"chat.completion.chunk\",\"created\":1777219562,\"model\":\"gpt-4o-mini-2024-07-18\",\"service_tier\":\"default\",\"system_fingerprint\":\"fp_c42fec8f39\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"obfuscation\":\"QZ\"}\n\ndata: [DONE]\n\n" + } + } + ] +}