From bdf01ee676b0b13b110e1cec0f0e734da3e22117 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 22 Jul 2026 23:44:28 -0500 Subject: [PATCH] test(llm): record response message phases --- ...-trips-commentary-into-a-final-answer.json | 52 ++++++++++++++ .../openai-responses-phase.recorded.test.ts | 71 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 packages/llm/test/fixtures/recordings/openai-responses-phase/round-trips-commentary-into-a-final-answer.json create mode 100644 packages/llm/test/provider/openai-responses-phase.recorded.test.ts diff --git a/packages/llm/test/fixtures/recordings/openai-responses-phase/round-trips-commentary-into-a-final-answer.json b/packages/llm/test/fixtures/recordings/openai-responses-phase/round-trips-commentary-into-a-final-answer.json new file mode 100644 index 0000000000..1b1d0eb49f --- /dev/null +++ b/packages/llm/test/fixtures/recordings/openai-responses-phase/round-trips-commentary-into-a-final-answer.json @@ -0,0 +1,52 @@ +{ + "version": 1, + "metadata": { + "name": "openai-responses-phase/round-trips-commentary-into-a-final-answer", + "recordedAt": "2026-07-23T04:43:51.014Z", + "tags": [ + "prefix:openai-responses-phase", + "provider:openai", + "protocol:openai-responses", + "phase", + "tool" + ] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.3-codex\",\"input\":[{\"role\":\"system\",\"content\":\"Before calling get_weather, briefly tell the user you are checking. Then call get_weather exactly once. Do not provide the final answer until its result is available.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"store\":false,\"include\":[\"reasoning.encrypted_content\"],\"reasoning\":{\"effort\":\"medium\",\"summary\":\"auto\"},\"max_output_tokens\":100,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0ecac6d08046be6d016a619c04c6288197a706dd55dda0568b\",\"object\":\"response\",\"created_at\":1784781828,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0ecac6d08046be6d016a619c04c6288197a706dd55dda0568b\",\"object\":\"response\",\"created_at\":1784781828,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqYZwFMExWpbwN1BbxV6i_pwTUtW3NghP_TA8Nnn3nAEUkx5_qsuIDxCWfrhbRyX5FXiD2ExkDduAIRq8xFugsg9Tu7Zh2h0NFM2besi67npXyc1k9u-2wFsfHH1tkFh1bCvyLJ3rwlK2IbW98XqgiQD9nU-gCYRtKRx_TeaLgrG0-uBGOJtDklNjqNUjre1SpsAANGy6e4OrU5I4_jzNMv7CNTCl1_pTWi4HztxFb8-KBV2gM7rKHQYdzYPi5PCwmndgagb_s95zUtVrVEQqOxGb--iDrEASbL2jiIuqTSxzNz9e0GraTn0MELU55xEZN9zLgg8E-K4_0lYNwdK3VmnIiwTjCk-8sRoP6nJ4YiZJJT5qyc3FMgcoFHE2qQZSyuMDyqsyfxLTGewv4At2FpLQGRIp7wcTXeT9PVIk368HmLZehgPMwJgMhRQ9WdD1BuTczH0UuwFhb4yRhVBfgI9hotuRLtTBVp8n3RKQoRQESNnPs003_MEx3xLzJlSc87WMaDIRigX_PY1CYohAEJDgjlR455IM_aADgvfxgjR-NMMjaqbf_Nno2bnaB8EAHQO5bF1UR9CzjtOroGFEv15vsYI0gbBaXEFweKBBK0vgz7mz7zhMbcpua3P8_1ev8wPnweEmNvpoxYQBx1mWk8JRi_cs-DoHlI6jbZPk5gqENtH9w8P9pdbfhcG0X1WKw9dWzeJpUewFIqrTmm5guHPnNL6sDXxSMctdIfQeO3cDiO0Jxxk5yPdNn09jgngkgnV9Yd8x5kGwUrOuzpLoWX3XBaZx36eSK8e32cMNbhuz1W_6hf-4jae5UyBzcLnEaoIp3gB1mbtwABw0LoRVY-A1mHQZ4IFCEASjg9tpBx2he7a1f0pZadUXpNvlQRIghh3MIzbd5r-2VZ6QhVShg11gvlg==\",\"summary\":[]},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.reasoning_summary_part.added\ndata: {\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"output_index\":0,\"part\":{\"type\":\"summary_text\",\"text\":\"\"},\"sequence_number\":3,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"**Planning user notification before tool call**\",\"item_id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"obfuscation\":\"W\",\"output_index\":0,\"sequence_number\":4,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.done\ndata: {\"type\":\"response.reasoning_summary_text.done\",\"item_id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"output_index\":0,\"sequence_number\":5,\"summary_index\":0,\"text\":\"**Planning user notification before tool call**\"}\n\nevent: response.reasoning_summary_part.done\ndata: {\"type\":\"response.reasoning_summary_part.done\",\"item_id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"output_index\":0,\"part\":{\"type\":\"summary_text\",\"text\":\"**Planning user notification before tool call**\"},\"sequence_number\":6,\"summary_index\":0}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqYZwGyVLGBn6tGMJmqmb50TN4SnPR7-rSDHo_v3Mxlmq3xwDIvV8TsBIhbc-i7yg4QMEfgyrmE2DYFN_XQ87a6YSUUzsCzQ7Jvr27vJzXgjSO6caVOrkdvYlLc6wvOtsyAoLv_HfX_NkO0ru8AV6ytkXmOwhhmjBbywY9vOHc6C8PME41EI9TQNY2TK_botCsR4Ki2rNt0-OLp5_w8TwfMFZvYextuDjQjRz1A3PxVaC3Jx9VfSqQ32ewFSbw532nU66j08VFuMkTY2u9NRknxlLtZdm08BgMSSC3bwX2DhqmUDYozLAyVpnE7q2HZGsS-lodMEjcM3VNJb0UFIV3ht9AURwxbTYFqPlAlGNFTXpDsiHh82SBp8fJhYi_wXXYRjWxcN_6TtDRn2ZGi9VXWVvX3GofqFNLJ3579Z0cNTgPfmCFWc5wg0S7MwdvyW2ZKMsiiUvwbsmF_hVjWlhCAJQ40Cjv6xXIU9lg63NeqC2WrSYOsZ9-SX9lMlSIw_q2UfEiypSfCrhQA1-aRpcSumAmaXmJlGXxklwVqBj7jEQ7ClKY54bvPjuDF-GZB8DQPA6Tk_grPt114XKpHue-tFQsnd4c7c0JxzUX33LpPvO368uYhgjJF08i7OCwFXg0PEINAcSa6ywEY-bCf2zUh8XJaGC7yZjX-bhE12huv47IiTwVH_w_i69uJO9ONeNHfTlZSBuFs4j52YkHbSAMkZKDeqLbtMLkBhVReX7hrxu3EBIIveo-MViK_qWXmt2hHq-haQwfdrA-r96N1i3JEQk03ev6GdB6tPE1r4v8CBxy6x93irVjBVzMCSDOQxLvds_YjXb5A9gNgkh2AOZVow6Da4B2WY-eink0iuSIWRFubRYbzi39dNWQFU_HLquiA1q99zDGRH_7gm-9Ah20xuVos4Zh3mW30uWFcMebdFzmCEU_f0_WG1PY2VJhPQWIYBiAeIA3O03hhDgWyL4fUH98KBYG6wI8kGRalSmaIP6qqNIDaw5pv0qAg_zQS6k9Bxmo2pknZJbd1SeKKjG9CEOTVemzgji0ObqxG-DDk8sWyCblPdZauDEtFETGSQBCKfdT\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"**Planning user notification before tool call**\"}]},\"output_index\":0,\"sequence_number\":7}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"commentary\",\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":8}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Sure\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"1qkMKJgRzeLJ\",\"output_index\":1,\"sequence_number\":10}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" —\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"WElRgYPLDGFgDO\",\"output_index\":1,\"sequence_number\":11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" I\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"6RIIsBZkwuZayL\",\"output_index\":1,\"sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"’ll\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"KbDforPQUeilB\",\"output_index\":1,\"sequence_number\":13}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" check\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"W32Z7dFfzb\",\"output_index\":1,\"sequence_number\":14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" the\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"rw8aKVDflJC9\",\"output_index\":1,\"sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" current\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"wvmzjAUq\",\"output_index\":1,\"sequence_number\":16}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" weather\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"w1LMlm21\",\"output_index\":1,\"sequence_number\":17}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" in\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"AWcXoWhpnHjwV\",\"output_index\":1,\"sequence_number\":18}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Paris\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"5GJ2dnt33I\",\"output_index\":1,\"sequence_number\":19}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" now\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"pQHRQOCWFqoJ\",\"output_index\":1,\"sequence_number\":20}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"obfuscation\":\"Cm6iBNQZH9Vzcpr\",\"output_index\":1,\"sequence_number\":21}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":22,\"text\":\"Sure — I’ll check the current weather in Paris now.\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Sure — I’ll check the current weather in Paris now.\"},\"sequence_number\":23}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Sure — I’ll check the current weather in Paris now.\"}],\"phase\":\"commentary\",\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":24}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_GIHs8XkFO327HvlfRrkglsF6\",\"name\":\"get_weather\"},\"output_index\":2,\"sequence_number\":25}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"obfuscation\":\"zEI8XPutbX0ZY0\",\"output_index\":2,\"sequence_number\":26}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"city\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"obfuscation\":\"cZbXPfhlSc5k\",\"output_index\":2,\"sequence_number\":27}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"obfuscation\":\"du8JdCcwvNVCt\",\"output_index\":2,\"sequence_number\":28}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Paris\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"obfuscation\":\"P9UghLQ34Ks\",\"output_index\":2,\"sequence_number\":29}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"obfuscation\":\"0mvJbz6R9NogPP\",\"output_index\":2,\"sequence_number\":30}\n\nevent: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"item_id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"output_index\":2,\"sequence_number\":31}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_GIHs8XkFO327HvlfRrkglsF6\",\"name\":\"get_weather\"},\"output_index\":2,\"sequence_number\":32}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0ecac6d08046be6d016a619c04c6288197a706dd55dda0568b\",\"object\":\"response\",\"created_at\":1784781828,\"status\":\"completed\",\"background\":false,\"completed_at\":1784781830,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[{\"id\":\"rs_0ecac6d08046be6d016a619c054d888197b68aaf98efa7ffc6\",\"type\":\"reasoning\",\"content\":[],\"encrypted_content\":\"gAAAAABqYZwGUg2GcHy-O_C1x7OU7oj_7h-CDjy8qfmCESeOoExvr3itpKLi0J5fE_89Wp76_Ml3wM_h1F4f1STvZd6qhaznG806FaqopEuKCpUJJGjWsisSmNSCJAMCKlPghfV9AF92mkH_wZmBNRm29kfiYcA6g6NK9mx1Dl-Z26Y1C0-GhvmNcl1T_RSXl3F7Xok4PkptId-cxCqVGP-JlIK7FWmaAyX0BuTQVSQ_sPnI4y-dJcsUhBnj1__lmVE-tBYnhfEl30-l__hoWqC2CxvrvToUZV0eqLwgEsAwYB__FNZ-QRTFghPSxTQfxYQPrAQGSiSFBvJJUTAmtQc9mhXoUTnJd-cH0ueaGuFbG4u1NF9WeVFBFFRUWbwO6zelE7BYjd5lpk9ALW_ZKhYNhq1TPhrx_lhAdBY51KcjFLLgoAS41X8YTC2B92lW8FtYdrHoNINfIUic4pdo94aL-04RfWCybOfDOxNMJW6k-9k62VtBoeiRh12NqG0Jn6jfMiWj61td7GAZj0aJoZ-_MpA1azkrUiN2M-rBcfPyUrqj8DzcFytY6Le3ijIcgxrZfdNbsjTtaCTRHa7dAdKJ345YB54so5BS_3IfKa4wOpRqWNT-Z8y1QXJ0VZsd2G9iPWy7Pk7wG42fGprJV-CNTaPutBqD6Zoe-ft6M6P1SfJ-67z0Erjll2FJ_s46kmIqOod7EMB5GScmg2OWur3IMLCqPEeJ2fjnJtTOIHhSm-N3RzSEpsd1lbQH8NkdINuqZYJwf3JDf7bYbSpc-4949VmrTj_eSGscEEHDc-3ALr-zWsIcKFzyZ0yRIQkMIpy1oLEtORTH1TxR34XBDtDOVA32RepQzm9Z60nvW6JDQZsg6Cq030f-vvkEiykuMwjCI0dijYDwHm7EAYlzE_RJXlysckJc-JDGXjnwpBDXVuFlopGbw3BbZR6ERfTQaySgyEFgEfczp2X_rRQvIo_w_qJRM4yMTG2C0k3cgUuY8yqgycxDnfob3EMV3y4epYMcUBA-3m6mAH9r5aSMXlu4xEBvW4EkLKbO5TviDVs5fVVvU4mPS-zK_XDZ9I25-f4rbtz2x3gr\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"**Planning user notification before tool call**\"}]},{\"id\":\"msg_0ecac6d08046be6d016a619c061b2c81978a89e6d2cf73bec8\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Sure — I’ll check the current weather in Paris now.\"}],\"phase\":\"commentary\",\"role\":\"assistant\"},{\"id\":\"fc_0ecac6d08046be6d016a619c0629088197be5d8ee7f7ad8b75\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\",\"call_id\":\"call_GIHs8XkFO327HvlfRrkglsF6\",\"name\":\"get_weather\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":86,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":66,\"output_tokens_details\":{\"reasoning_tokens\":27},\"total_tokens\":152},\"user\":null,\"metadata\":{}},\"sequence_number\":33}\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-5.3-codex\",\"input\":[{\"role\":\"system\",\"content\":\"Before calling get_weather, briefly tell the user you are checking. Then call get_weather exactly once. After its result, answer exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"What is the weather in Paris?\"}]},{\"type\":\"reasoning\",\"summary\":[{\"type\":\"summary_text\",\"text\":\"**Planning user notification before tool call**\"}],\"encrypted_content\":\"gAAAAABqYZwGyVLGBn6tGMJmqmb50TN4SnPR7-rSDHo_v3Mxlmq3xwDIvV8TsBIhbc-i7yg4QMEfgyrmE2DYFN_XQ87a6YSUUzsCzQ7Jvr27vJzXgjSO6caVOrkdvYlLc6wvOtsyAoLv_HfX_NkO0ru8AV6ytkXmOwhhmjBbywY9vOHc6C8PME41EI9TQNY2TK_botCsR4Ki2rNt0-OLp5_w8TwfMFZvYextuDjQjRz1A3PxVaC3Jx9VfSqQ32ewFSbw532nU66j08VFuMkTY2u9NRknxlLtZdm08BgMSSC3bwX2DhqmUDYozLAyVpnE7q2HZGsS-lodMEjcM3VNJb0UFIV3ht9AURwxbTYFqPlAlGNFTXpDsiHh82SBp8fJhYi_wXXYRjWxcN_6TtDRn2ZGi9VXWVvX3GofqFNLJ3579Z0cNTgPfmCFWc5wg0S7MwdvyW2ZKMsiiUvwbsmF_hVjWlhCAJQ40Cjv6xXIU9lg63NeqC2WrSYOsZ9-SX9lMlSIw_q2UfEiypSfCrhQA1-aRpcSumAmaXmJlGXxklwVqBj7jEQ7ClKY54bvPjuDF-GZB8DQPA6Tk_grPt114XKpHue-tFQsnd4c7c0JxzUX33LpPvO368uYhgjJF08i7OCwFXg0PEINAcSa6ywEY-bCf2zUh8XJaGC7yZjX-bhE12huv47IiTwVH_w_i69uJO9ONeNHfTlZSBuFs4j52YkHbSAMkZKDeqLbtMLkBhVReX7hrxu3EBIIveo-MViK_qWXmt2hHq-haQwfdrA-r96N1i3JEQk03ev6GdB6tPE1r4v8CBxy6x93irVjBVzMCSDOQxLvds_YjXb5A9gNgkh2AOZVow6Da4B2WY-eink0iuSIWRFubRYbzi39dNWQFU_HLquiA1q99zDGRH_7gm-9Ah20xuVos4Zh3mW30uWFcMebdFzmCEU_f0_WG1PY2VJhPQWIYBiAeIA3O03hhDgWyL4fUH98KBYG6wI8kGRalSmaIP6qqNIDaw5pv0qAg_zQS6k9Bxmo2pknZJbd1SeKKjG9CEOTVemzgji0ObqxG-DDk8sWyCblPdZauDEtFETGSQBCKfdT\"},{\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"Sure — I’ll check the current weather in Paris now.\"}],\"phase\":\"commentary\"},{\"type\":\"function_call\",\"call_id\":\"call_GIHs8XkFO327HvlfRrkglsF6\",\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"},{\"type\":\"function_call_output\",\"call_id\":\"call_GIHs8XkFO327HvlfRrkglsF6\",\"output\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"store\":false,\"include\":[\"reasoning.encrypted_content\"],\"reasoning\":{\"effort\":\"medium\",\"summary\":\"auto\"},\"max_output_tokens\":100,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_080fac734aba50ee016a619c066b1081978f0b37911e9c7435\",\"object\":\"response\",\"created_at\":1784781830,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_080fac734aba50ee016a619c066b1081978f0b37911e9c7435\",\"object\":\"response\",\"created_at\":1784781830,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Paris\",\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"logprobs\":[],\"obfuscation\":\"BaUsfkiRh9B\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" is\",\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"logprobs\":[],\"obfuscation\":\"8ZrEQyY0MIAC1\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" sunny\",\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"logprobs\":[],\"obfuscation\":\"F4dhP79Qkm\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"logprobs\":[],\"obfuscation\":\"sJVdef8T7wBDA0d\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":8,\"text\":\"Paris is sunny.\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"},\"sequence_number\":9}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":10}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_080fac734aba50ee016a619c066b1081978f0b37911e9c7435\",\"object\":\"response\",\"created_at\":1784781830,\"status\":\"completed\",\"background\":false,\"completed_at\":1784781831,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":100,\"max_tool_calls\":null,\"model\":\"gpt-5.3-codex\",\"moderation\":null,\"output\":[{\"id\":\"msg_080fac734aba50ee016a619c06e38081978d0404c7b35ed057\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"Paris is sunny.\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"medium\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Get current weather for a city.\",\"name\":\"get_weather\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":172,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":8,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":180},\"user\":null,\"metadata\":{}},\"sequence_number\":11}\n\n" + } + } + ] +} diff --git a/packages/llm/test/provider/openai-responses-phase.recorded.test.ts b/packages/llm/test/provider/openai-responses-phase.recorded.test.ts new file mode 100644 index 0000000000..cd7d5612d6 --- /dev/null +++ b/packages/llm/test/provider/openai-responses-phase.recorded.test.ts @@ -0,0 +1,71 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, Message } from "../../src" +import * as OpenAI from "../../src/providers/openai" +import { LLMClient } from "../../src/route" +import { weatherTool } from "../recorded-scenarios" +import { recordedTests } from "../recorded-test" + +const model = OpenAI.configure({ + apiKey: process.env.OPENAI_API_KEY ?? "fixture", +}).responses("gpt-5.3-codex") + +const recorded = recordedTests({ + prefix: "openai-responses-phase", + provider: "openai", + protocol: "openai-responses", + requires: ["OPENAI_API_KEY"], +}) + +describe("OpenAI Responses phase recorded", () => { + recorded.effect.with("round-trips commentary into a final answer", { tags: ["phase", "tool"] }, () => + Effect.gen(function* () { + const user = Message.user("What is the weather in Paris?") + const first = yield* LLMClient.generate( + LLM.request({ + model, + system: + "Before calling get_weather, briefly tell the user you are checking. Then call get_weather exactly once. Do not provide the final answer until its result is available.", + messages: [user], + tools: [weatherTool], + generation: { maxTokens: 100 }, + }), + ) + const call = first.toolCalls[0] + if (!call) throw new Error("OpenAI Responses did not return the expected weather tool call") + + expect(call).toMatchObject({ name: "get_weather", input: { city: "Paris" } }) + expect( + first.message.content.some( + (part) => part.type === "text" && part.providerMetadata?.openai?.phase === "commentary", + ), + ).toBeTrue() + + const second = yield* LLMClient.generate( + LLM.request({ + model, + system: + "Before calling get_weather, briefly tell the user you are checking. Then call get_weather exactly once. After its result, answer exactly: Paris is sunny.", + messages: [ + user, + first.message, + Message.tool({ + id: call.id, + name: call.name, + result: { temperature: 22, condition: "sunny" }, + }), + ], + tools: [weatherTool], + generation: { maxTokens: 100 }, + }), + ) + + expect(second.text.trim()).toBe("Paris is sunny.") + expect( + second.message.content.some( + (part) => part.type === "text" && part.providerMetadata?.openai?.phase === "final_answer", + ), + ).toBeTrue() + }), + ) +})