From e82f68fafbad2e86737153b72a2935b77138ebb2 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:58:16 +0000 Subject: [PATCH] fix(ai): parse compatible reasoning deltas --- packages/ai/src/protocols/openai-chat.ts | 187 +++++++++++++++++- .../openrouter-reasoning-details.json | 54 +++++ .../vercel-ai-gateway-reasoning-details.json | 54 +++++ .../openai-chat-reasoning.recorded.test.ts | 110 +++++++++++ packages/ai/test/provider/openai-chat.test.ts | 187 +++++++++++++++++- packages/core/src/models-dev.ts | 3 +- packages/core/src/v1/config/provider.ts | 6 +- packages/core/test/config/provider-v1.test.ts | 11 ++ 8 files changed, 599 insertions(+), 13 deletions(-) create mode 100644 packages/ai/test/fixtures/recordings/openrouter-reasoning-details.json create mode 100644 packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-details.json create mode 100644 packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts create mode 100644 packages/core/test/config/provider-v1.test.ts diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index c74ab7fd64..eb07708f75 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -56,6 +56,8 @@ const OpenAIChatAssistantToolCall = Schema.Struct({ }) type OpenAIChatAssistantToolCall = Schema.Schema.Type +type OpenAIChatReasoningDetail = Schema.Schema.Type + const OpenAIChatUserContent = Schema.Union([ Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }), Schema.Struct({ @@ -75,6 +77,9 @@ const OpenAIChatMessage = Schema.Union([ content: Schema.NullOr(Schema.String), tool_calls: optionalArray(OpenAIChatAssistantToolCall), reasoning_content: Schema.optional(Schema.String), + reasoning: Schema.optional(Schema.String), + reasoning_text: Schema.optional(Schema.String), + reasoning_details: optionalArray(JsonObject), }), Schema.Struct({ role: Schema.Literal("tool"), tool_call_id: Schema.String, content: Schema.String }), ]).pipe(Schema.toTaggedUnion("role")) @@ -145,6 +150,9 @@ type OpenAIChatToolCallDelta = Schema.Schema.Type + readonly reasoningField?: NonNullable>["field"] } // ============================================================================= @@ -208,6 +218,27 @@ const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart const openAICompatibleReasoningContent = (native: unknown) => isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined +const reasoningState = (part: ReasoningPart | ToolCallPart) => { + const state = part.providerMetadata?.openai + return isRecord(state) ? state : undefined +} + +const reasoningField = (part: ReasoningPart) => { + const field = reasoningState(part)?.reasoningField + if ( + field === "reasoning" || + field === "reasoning_content" || + field === "reasoning_text" || + field === "reasoning_details" + ) + return field +} + +const reasoningDetails = (part: ReasoningPart | ToolCallPart) => { + const details = reasoningState(part)?.reasoningDetails + return Array.isArray(details) ? details.filter(isRecord) : [] +} + const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) { const content: Array> = [] for (const part of message.content) { @@ -248,14 +279,24 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func continue } } + const text = reasoning.map((part) => part.text).join("") + const field = reasoning.map(reasoningField).find((item) => item !== undefined) ?? "reasoning_content" + const details = message.content.flatMap((part) => + part.type === "reasoning" || part.type === "tool-call" ? reasoningDetails(part) : [], + ) return { role: "assistant" as const, content: content.length === 0 ? null : ProviderShared.joinText(content), tool_calls: toolCalls.length === 0 ? undefined : toolCalls, reasoning_content: - reasoning.length > 0 - ? reasoning.map((part) => part.text).join("") - : openAICompatibleReasoningContent(message.native?.openaiCompatible), + reasoning.length === 0 + ? openAICompatibleReasoningContent(message.native?.openaiCompatible) + : field === "reasoning_content" + ? text + : undefined, + reasoning: reasoning.length > 0 && field === "reasoning" ? text : undefined, + reasoning_text: reasoning.length > 0 && field === "reasoning_text" ? text : undefined, + reasoning_details: details.length > 0 ? details : undefined, } }) @@ -400,6 +441,97 @@ const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => { }) } +const reasoningDelta = (delta: Schema.Schema.Type | null | undefined) => { + if (delta?.reasoning_content) return { field: "reasoning_content", text: delta.reasoning_content } as const + if (delta?.reasoning) return { field: "reasoning", text: delta.reasoning } as const + if (delta?.reasoning_text) return { field: "reasoning_text", text: delta.reasoning_text } as const + const text = delta?.reasoning_details + ?.flatMap((detail) => { + if (detail.type === "reasoning.text" && typeof detail.text === "string" && detail.text) return [detail.text] + if (detail.type === "reasoning.summary" && typeof detail.summary === "string" && detail.summary) + return [detail.summary] + return [] + }) + .join("") + return text ? ({ field: "reasoning_details", text } as const) : undefined +} + +const reasoningMetadata = ( + field: NonNullable>["field"], + details: ReadonlyArray, +) => ({ + openai: { + reasoningField: field, + ...(details.length > 0 ? { reasoningDetails: details } : {}), + }, +}) + +const withEncryptedReasoningDetails = ( + events: ReadonlyArray, + details: ReadonlyArray, +) => { + const encrypted = details.filter( + (detail) => detail.type === "reasoning.encrypted" && typeof detail.data === "string" && detail.data, + ) + let attached = false + return events.map((event) => { + if (event.type !== "tool-call" || attached || encrypted.length === 0) return event + attached = true + const current = event.providerMetadata?.openai + return LLMEvent.toolCall({ + ...event, + providerMetadata: { + ...event.providerMetadata, + openai: { ...(isRecord(current) ? current : {}), reasoningDetails: encrypted }, + }, + }) + }) +} + +const mergeReasoningDetails = ( + current: ReadonlyArray, + incoming: ReadonlyArray, +) => { + const result = [...current] + for (const detail of incoming) { + let index = result.findIndex((item) => { + if (item.type !== detail.type) return false + if (typeof item.id === "string" && typeof detail.id === "string") return item.id === detail.id + return typeof detail.index === "number" && item.index === detail.index + }) + if (index === -1 && typeof detail.id !== "string" && typeof detail.index !== "number") { + const last = result.length - 1 + if (result[last]?.type === detail.type) index = last + } + if (index === -1) { + result.push(detail) + continue + } + const previous = result[index]! + result[index] = { + ...previous, + ...detail, + ...(typeof detail.signature === "string" && detail.signature + ? { signature: detail.signature } + : typeof previous.signature === "string" && previous.signature + ? { signature: previous.signature } + : {}), + ...(typeof previous.format === "string" && previous.format + ? { format: previous.format } + : typeof detail.format === "string" && detail.format + ? { format: detail.format } + : {}), + ...(typeof detail.text === "string" + ? { text: `${typeof previous.text === "string" ? previous.text : ""}${detail.text}` } + : {}), + ...(typeof detail.summary === "string" + ? { summary: `${typeof previous.summary === "string" ? previous.summary : ""}${detail.summary}` } + : {}), + } + } + return result +} + const step = (state: ParserState, event: OpenAIChatEvent) => Effect.gen(function* () { const events: LLMEvent[] = [] @@ -408,19 +540,46 @@ const step = (state: ParserState, event: OpenAIChatEvent) => const finishReason = choice?.finish_reason ? mapFinishReason(choice.finish_reason) : state.finishReason const delta = choice?.delta const toolDeltas = delta?.tool_calls ?? [] + const reasoningDetails = mergeReasoningDetails(state.reasoningDetails, delta?.reasoning_details ?? []) let tools = state.tools let lifecycle = state.lifecycle - if (delta?.reasoning_content) - lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", delta.reasoning_content) + const reasoning = reasoningDelta(delta) + const reasoningField = state.reasoningField ?? reasoning?.field + const currentReasoningMetadata = reasoningField + ? reasoningMetadata( + reasoningField, + reasoningDetails.filter((detail) => detail.type !== "reasoning.encrypted"), + ) + : undefined + const completeReasoningMetadata = reasoningField ? reasoningMetadata(reasoningField, reasoningDetails) : undefined + if (reasoning) { + lifecycle = Lifecycle.reasoningStart(lifecycle, events, "reasoning-0", currentReasoningMetadata) + events.push( + LLMEvent.reasoningDelta({ + id: "reasoning-0", + text: reasoning.text, + providerMetadata: currentReasoningMetadata, + }), + ) + } if (delta?.content) { - lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0") + lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0", completeReasoningMetadata) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content) } - if (toolDeltas.length) lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0") + if (toolDeltas.length) + lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0", currentReasoningMetadata) + + if (finishReason !== undefined) + lifecycle = Lifecycle.reasoningEnd( + lifecycle, + events, + "reasoning-0", + toolDeltas.length > 0 || Object.keys(tools).length > 0 ? currentReasoningMetadata : completeReasoningMetadata, + ) for (const tool of toolDeltas) { const result = ToolStream.appendOrStart( @@ -446,10 +605,14 @@ const step = (state: ParserState, event: OpenAIChatEvent) => return [ { tools: finished?.tools ?? tools, - toolCallEvents: finished?.events ?? state.toolCallEvents, + toolCallEvents: finished + ? withEncryptedReasoningDetails(finished.events, reasoningDetails) + : state.toolCallEvents, usage, finishReason, lifecycle, + reasoningDetails, + reasoningField, }, events, ] as const @@ -482,7 +645,13 @@ export const protocol = Protocol.make({ }, stream: { event: Protocol.jsonEvent(OpenAIChatEvent), - initial: () => ({ tools: ToolStream.empty(), toolCallEvents: [], lifecycle: Lifecycle.initial() }), + initial: () => ({ + tools: ToolStream.empty(), + toolCallEvents: [], + lifecycle: Lifecycle.initial(), + reasoningDetails: [], + reasoningField: undefined, + }), step, onHalt: finishEvents, }, diff --git a/packages/ai/test/fixtures/recordings/openrouter-reasoning-details.json b/packages/ai/test/fixtures/recordings/openrouter-reasoning-details.json new file mode 100644 index 0000000000..849f711dee --- /dev/null +++ b/packages/ai/test/fixtures/recordings/openrouter-reasoning-details.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "metadata": { + "model": "anthropic/claude-sonnet-4.6", + "tags": [ + "prefix:openai-compatible-chat", + "provider:openrouter", + "protocol:openai-chat", + "reasoning", + "reasoning-details", + "continuation" + ], + "name": "openrouter-reasoning-details", + "recordedAt": "2026-07-18T02:55:47.590Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Think through the arithmetic, then reply with only the final integer.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"temperature\":0,\"reasoning\":{\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"173\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\\n\\n34,600 + 3,287 = 37,887\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"signature\":\"EtgCCosBCA8YAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMBcWxRZfFAcF6/dgHGgwBznuEFqA7dO2/kEYiME/xUOaFAvea+RRlv1lAeIhPhWOBogF1r/Yp+OiIn3DmdBYYI2G0sqp3CBOPCaI6wSp6dPgxy94TkcgupFFukOgXpS7X4sfbVxII8fqxpqZ5lyv7/FWRZOgAMM96VHwRGEO7udkbGBLiYGe0AOE4ngXDxf5IAKhbTExErglpaQ66eSjzvHWwGW3rMqrrWeenpi1hci6Q4SHG20RCRPSEHiq+R8Mfv82GDUCpECgYAQ==\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"37887\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":null},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1784343343-KU2JBkRB0HtmE1D9E6s8\",\"object\":\"chat.completion.chunk\",\"created\":1784343343,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":61,\"completion_tokens\":80,\"total_tokens\":141,\"cost\":0.001383,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.001383,\"upstream_inference_prompt_cost\":0.000183,\"upstream_inference_completions_cost\":0.0012},\"completion_tokens_details\":{\"reasoning_tokens\":29,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Call the requested tool exactly once.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"},{\"role\":\"assistant\",\"content\":\"37887\",\"reasoning\":\"173 × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173 × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"format\":\"anthropic-claude-v1\",\"index\":0,\"signature\":\"EtgCCosBCA8YAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMBcWxRZfFAcF6/dgHGgwBznuEFqA7dO2/kEYiME/xUOaFAvea+RRlv1lAeIhPhWOBogF1r/Yp+OiIn3DmdBYYI2G0sqp3CBOPCaI6wSp6dPgxy94TkcgupFFukOgXpS7X4sfbVxII8fqxpqZ5lyv7/FWRZOgAMM96VHwRGEO7udkbGBLiYGe0AOE4ngXDxf5IAKhbTExErglpaQ66eSjzvHWwGW3rMqrrWeenpi1hci6Q4SHG20RCRPSEHiq+R8Mfv82GDUCpECgYAQ==\"}]},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get the weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"temperature\":0,\"reasoning\":{\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"The user wants me to call get\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants me to call get\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"_weather with city \\\"Paris\\\".\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"_weather with city \\\"Paris\\\".\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"signature\":\"EqYCCosBCA8YAipA5h8QqVInA5bNQkP9UtLyzVgxwWJboJoQi4UtAKsuv+VzDrJ3MR3WfAIl/O/9VQGObKrZHqTD8dDjSFmYuzhUhjIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMAQNk7fZQo1GcFMkxGgxOEyaTlL2JrG+QZrEiMDUJnYAGn6q8VqUeKwMd7mcP4kiJEBcOzR8cZwS6AhbTeof/+PzLIwMgSMIxO0J9oSpIqRfNYdSIZAzWtyXulpYHqBOXZhhyDC8ZNGeHVjPqLMR0WV1ldtBFV+XWVguS+4TXaRWfzX9jedg43jlQZQzZYPUwAYpXksHzGAE=\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure!\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"toolu_01HD1WhXiZU2WX2gcC7mLX4Y\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \\\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":null},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}]}\n\ndata: {\"id\":\"gen-1784343345-PMJCRVF8BOEoQ9QZePNc\",\"object\":\"chat.completion.chunk\",\"created\":1784343345,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}],\"usage\":{\"prompt_tokens\":712,\"completion_tokens\":78,\"total_tokens\":790,\"cost\":0.003306,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.003306,\"upstream_inference_prompt_cost\":0.002136,\"upstream_inference_completions_cost\":0.00117},\"completion_tokens_details\":{\"reasoning_tokens\":14,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-details.json b/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-details.json new file mode 100644 index 0000000000..74e2dd812c --- /dev/null +++ b/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-details.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "metadata": { + "model": "anthropic/claude-sonnet-4.6", + "tags": [ + "prefix:openai-compatible-chat", + "provider:vercel-ai-gateway", + "protocol:openai-chat", + "reasoning", + "reasoning-details", + "continuation" + ], + "name": "vercel-ai-gateway-reasoning-details", + "recordedAt": "2026-07-18T02:55:57.821Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://ai-gateway.vercel.sh/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Think through the arithmetic, then reply with only the final integer.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"temperature\":0,\"reasoning\":{\"enabled\":true,\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"173\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\" - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\",\"signature\":\"EtgCCosBCA8YAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDNiOTNhNWRkLTczMDItNDgzZi1hZWFlLTM2MjA3NTU0OGFlMRIMYncPCRdN0FNVEmtEGgzbu1yKgT1lZ+IoZigiMI0JPBEMzyJ5yismjEEej858hptJ3dB/UcVr9SJFFAt8csIglueLjzLRpj9k+xlMVSp6E8+95YtdRk8XCJ88n/TsA++cxZi3QFz25ncsrsmJn31cuhUotGgsICK9RCCJxLB8fWLFDkpwjlMk0MgLQA4SnQVf2mZlP86m9VDo/FFRmgag2caBI8UAHx3Ov/lqzUP/q66raMYhSQKMyknvm4EbllFMq3Wu/h2VprwYAQ==\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"37887\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_bv1ej3v2vu\"}\n\ndata: {\"id\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\",\"object\":\"chat.completion.chunk\",\"created\":1784343354,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":61,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":80,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":73}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-4.6\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"vertexAnthropic\",\"bedrock\"],\"planningReasoning\":\"System credentials planned for: anthropic, vertexAnthropic, bedrock. Total execution order: anthropic(system) → vertexAnthropic(system) → bedrock(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1784343354086,\"endTime\":1784343355892,\"providerRequestId\":\"req_011Cd8mdSSnUQKCAzPU2T27V\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cd8mdSqqj7WD1D2CVqJnY\"}]}],\"totalProviderAttemptCount\":1},\"cost\":\"0.001383\",\"marketCost\":\"0.001383\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.001383\",\"inferenceCost\":\"0.001383\",\"inputInferenceCost\":\"0.000183\",\"outputInferenceCost\":\"0.0012\",\"generationId\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\"}}},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":61,\"completion_tokens\":80,\"total_tokens\":141,\"cost\":0.001383,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":26,\"reasoning_tokens_estimated\":true,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.001383,\"gateway_cost\":0.001383},\"system_fingerprint\":\"fp_bv1ej3v2vu\",\"generationId\":\"gen_01KXSJDPNTZVKESHJZ3SF9W0WA\"}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://ai-gateway.vercel.sh/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Call the requested tool exactly once.\"},{\"role\":\"user\",\"content\":\"What is 173 multiplied by 219?\"},{\"role\":\"assistant\",\"content\":\"37887\",\"reasoning\":\"173 × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"173 × 219\\n\\n173 × 200 = 34,600\\n173 × 19 = 173 × 20 - 173 = 3,460 - 173 = 3,287\\n\\n34,600 + 3,287 = 37,887\",\"signature\":\"EtgCCosBCA8YAipA0W4viH3kgBs43Cl5ewwVBPXTQElvzfbA2TLF4iSbKy9ZZDCSDjjAlF3Bs4ELEnP3vrrTuTioC6OB380lXQdyIDIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDNiOTNhNWRkLTczMDItNDgzZi1hZWFlLTM2MjA3NTU0OGFlMRIMYncPCRdN0FNVEmtEGgzbu1yKgT1lZ+IoZigiMI0JPBEMzyJ5yismjEEej858hptJ3dB/UcVr9SJFFAt8csIglueLjzLRpj9k+xlMVSp6E8+95YtdRk8XCJ88n/TsA++cxZi3QFz25ncsrsmJn31cuhUotGgsICK9RCCJxLB8fWLFDkpwjlMk0MgLQA4SnQVf2mZlP86m9VDo/FFRmgag2caBI8UAHx3Ov/lqzUP/q66raMYhSQKMyknvm4EbllFMq3Wu/h2VprwYAQ==\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},{\"role\":\"user\",\"content\":\"Call get_weather with city exactly Paris.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get the weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"temperature\":0,\"reasoning\":{\"enabled\":true,\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"The user wants me\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants me\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\" to call get_weather with city \\\"Paris\\\".\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" to call get_weather with city \\\"Paris\\\".\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\",\"signature\":\"EqYCCosBCA8YAipA5h8QqVInA5bNQkP9UtLyzVgxwWJboJoQi4UtAKsuv+VzDrJ3MR3WfAIl/O/9VQGObKrZHqTD8dDjSFmYuzhUhjIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDNiOTNhNWRkLTczMDItNDgzZi1hZWFlLTM2MjA3NTU0OGFlMRIMh8hFzgKdyOol0A6QGgwyRXOI6j+59ATJwzMiMPbs+tQEmNb/j+yEsf5/tcDN30lQ9M54Elq0DwlhTyczc7xb9VihjT9JF8fS22YYxCpIDgX+fWeh7CLHjuff+tmW4s7XR4Abf5ALVmr759UqBLir29bhegJ8cu7BuuW62Wy8waYNKv3itdG01HXDXb+qBBIKuaQ/xyY2GAE=\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure!\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"toolu_014DfykHMJ5idh57fu7q61nt\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \\\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_mar74xh8to\"}\n\ndata: {\"id\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\",\"object\":\"chat.completion.chunk\",\"created\":1784343356,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":712,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":78,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":20}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-4.6\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"vertexAnthropic\",\"bedrock\"],\"planningReasoning\":\"System credentials planned for: anthropic, vertexAnthropic, bedrock. Total execution order: anthropic(system) → vertexAnthropic(system) → bedrock(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1784343355974,\"endTime\":1784343357792,\"providerRequestId\":\"req_011Cd8mdaftBLTHGSr9c1epX\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cd8mdbdvmKXsgtJk5y3At\"}]}],\"totalProviderAttemptCount\":1},\"cost\":\"0.003306\",\"marketCost\":\"0.003306\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.003306\",\"inferenceCost\":\"0.003306\",\"inputInferenceCost\":\"0.002136\",\"outputInferenceCost\":\"0.00117\",\"generationId\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\"}}},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":712,\"completion_tokens\":78,\"total_tokens\":790,\"cost\":0.003306,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":14,\"reasoning_tokens_estimated\":true,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.003306,\"gateway_cost\":0.003306},\"system_fingerprint\":\"fp_mar74xh8to\",\"generationId\":\"gen_01KXSJDRH74ATB3NZSY7V5Q40W\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts new file mode 100644 index 0000000000..1138983a75 --- /dev/null +++ b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts @@ -0,0 +1,110 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { LLM, LLMEvent, Message, ToolDefinition } from "../../src" +import * as OpenAICompatible from "../../src/providers/openai-compatible" +import * as OpenRouter from "../../src/providers/openrouter" +import { LLMClient } from "../../src/route" +import { recordedTests } from "../recorded-test" + +const weather = ToolDefinition.make({ + name: "get_weather", + description: "Get the weather for a city.", + inputSchema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + additionalProperties: false, + }, +}) + +const openRouter = OpenRouter.configure({ + apiKey: process.env.OPENROUTER_API_KEY ?? "fixture", + providerOptions: { openrouter: { reasoning: { max_tokens: 1024 } } }, +}).model("anthropic/claude-sonnet-4.6") + +const vercel = OpenAICompatible.configure({ + provider: "vercel-ai-gateway", + baseURL: "https://ai-gateway.vercel.sh/v1", + apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture", + http: { body: { reasoning: { enabled: true, max_tokens: 1024 } } }, +}).model("anthropic/claude-sonnet-4.6") + +const cases = [ + { + name: "OpenRouter", + model: openRouter, + requires: ["OPENROUTER_API_KEY"], + cassette: "openrouter-reasoning-details", + }, + { + name: "Vercel AI Gateway", + model: vercel, + requires: ["AI_GATEWAY_API_KEY"], + cassette: "vercel-ai-gateway-reasoning-details", + }, +] as const + +for (const item of cases) { + const recorded = recordedTests({ + prefix: "openai-compatible-chat", + provider: item.model.provider, + protocol: "openai-chat", + requires: item.requires, + tags: ["reasoning", "reasoning-details", "continuation"], + metadata: { model: item.model.id }, + }) + + describe(`${item.name} reasoning details recorded`, () => { + recorded.effect.with( + "streams and preserves reasoning details", + { cassette: item.cassette }, + () => + Effect.gen(function* () { + const response = yield* LLMClient.generate( + LLM.request({ + model: item.model, + system: "Think through the arithmetic, then reply with only the final integer.", + prompt: "What is 173 multiplied by 219?", + generation: { maxTokens: 1536, temperature: 0 }, + }), + ) + expect(response.text.replaceAll(",", "").trim()).toBe("37887") + expect(response.reasoning.length).toBeGreaterThan(0) + expect(response.events.some(LLMEvent.is.reasoningDelta)).toBe(true) + const reasoning = response.message.content.find((part) => part.type === "reasoning") + expect(reasoning?.providerMetadata?.openai?.reasoningField).toBe("reasoning") + const details = reasoning?.providerMetadata?.openai?.reasoningDetails + expect(Array.isArray(details)).toBe(true) + expect( + Array.isArray(details) && + details.some( + (detail) => + typeof detail === "object" && + detail !== null && + "type" in detail && + detail.type === "reasoning.text" && + "signature" in detail && + typeof detail.signature === "string" && + detail.signature.length > 0, + ), + ).toBe(true) + + const tool = yield* LLMClient.generate( + LLM.request({ + model: item.model, + system: "Call the requested tool exactly once.", + messages: [ + Message.user("What is 173 multiplied by 219?"), + response.message, + Message.user("Call get_weather with city exactly Paris."), + ], + tools: [weather], + generation: { maxTokens: 1536, temperature: 0 }, + }), + ) + expect(tool.toolCalls).toMatchObject([{ name: "get_weather", input: { city: "Paris" } }]) + }), + 30_000, + ) + }) +} diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index a08a22b037..523e3d421c 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -92,6 +92,73 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("replays provider reasoning fields and structured details", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "thinking", + providerMetadata: { + openai: { + reasoningField: "reasoning_text", + reasoningDetails: [{ type: "reasoning.text", text: "thinking", id: "reasoning-1" }], + }, + }, + }, + ToolCallPart.make({ + id: "call_1", + name: "lookup", + input: { query: "weather" }, + providerMetadata: { + openai: { + reasoningDetails: [ + { + type: "reasoning.encrypted", + id: "call_1", + data: "opaque", + format: "unknown", + provider_field: "preserved", + }, + ], + }, + }, + }), + ]), + ], + }), + ) + + expect(prepared.body.messages).toEqual([ + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "lookup", arguments: '{"query":"weather"}' }, + }, + ], + reasoning_text: "thinking", + reasoning_details: [ + { type: "reasoning.text", text: "thinking", id: "reasoning-1" }, + { + type: "reasoning.encrypted", + id: "call_1", + data: "opaque", + format: "unknown", + provider_field: "preserved", + }, + ], + }, + ]) + }), + ) + it.effect("maps OpenAI provider options to Chat options", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( @@ -540,22 +607,72 @@ describe("OpenAI Chat route", () => { }), ) - it.effect("parses OpenAI-compatible reasoning content deltas", () => + it.effect("parses OpenAI-compatible reasoning deltas", () => Effect.gen(function* () { const body = sseEvents( { choices: [{ delta: { reasoning_content: "thinking" } }] }, + { choices: [{ delta: { reasoning: " more" } }] }, + { choices: [{ delta: { reasoning_text: " deeply" } }] }, + { + choices: [ + { + delta: { + reasoning_details: [ + { type: "reasoning.text", text: " about", index: 0 }, + { type: "reasoning.summary", summary: " this", index: 1 }, + { type: "reasoning.encrypted", data: "opaque" }, + ], + }, + }, + ], + }, + { + choices: [ + { + delta: { + reasoning_details: [ + { + type: "reasoning.text", + text: "", + signature: "signature", + format: "anthropic-claude-v1", + index: 0, + }, + ], + }, + }, + ], + }, { choices: [{ delta: { content: "Hello" } }] }, { choices: [{ delta: {}, finish_reason: "stop" }] }, ) const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) - expect(response.reasoning).toBe("thinking") + expect(response.reasoning).toBe("thinking more deeply about this") expect(response.text).toBe("Hello") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toMatchObject({ + openai: { + reasoningDetails: [ + { + type: "reasoning.text", + text: " about", + signature: "signature", + format: "anthropic-claude-v1", + index: 0, + }, + { type: "reasoning.summary", summary: " this", index: 1 }, + { type: "reasoning.encrypted", data: "opaque" }, + ], + }, + }) expect(response.events).toMatchObject([ { type: "step-start", index: 0 }, { type: "reasoning-start", id: "reasoning-0" }, { type: "reasoning-delta", id: "reasoning-0", text: "thinking" }, + { type: "reasoning-delta", id: "reasoning-0", text: " more" }, + { type: "reasoning-delta", id: "reasoning-0", text: " deeply" }, + { type: "reasoning-delta", id: "reasoning-0", text: " about this" }, { type: "reasoning-end", id: "reasoning-0" }, { type: "text-start", id: "text-0" }, { type: "text-delta", id: "text-0", text: "Hello" }, @@ -566,6 +683,72 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("preserves encrypted reasoning details on the first tool call", () => + Effect.gen(function* () { + const body = sseEvents( + { + choices: [ + { + delta: { + reasoning_details: [{ type: "reasoning.encrypted", data: "opaque", format: "unknown" }], + }, + }, + ], + }, + deltaChunk({ + role: "assistant", + tool_calls: [{ index: 0, id: "call_1", function: { name: "lookup", arguments: "{}" } }], + }), + deltaChunk({}, "tool_calls"), + ) + const response = yield* LLMClient.generate( + LLM.updateRequest(request, { + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + }), + ).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.find(LLMEvent.is.toolCall)).toMatchObject({ + providerMetadata: { + openai: { + reasoningDetails: [{ type: "reasoning.encrypted", data: "opaque", format: "unknown" }], + }, + }, + }) + }), + ) + + it.effect("merges identity-less reasoning detail signatures", () => + Effect.gen(function* () { + const body = sseEvents( + { choices: [{ delta: { reasoning_details: [{ type: "reasoning.text", text: "think" }] } }] }, + { choices: [{ delta: { reasoning_details: [{ type: "reasoning.text", text: "ing" }] } }] }, + { + choices: [ + { + delta: { + reasoning_details: [ + { type: "reasoning.text", text: "", signature: "signature", format: "anthropic-claude-v1" }, + ], + }, + }, + ], + }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ) + const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) + + expect(response.reasoning).toBe("thinking") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toMatchObject({ + openai: { + reasoningDetails: [ + { type: "reasoning.text", text: "thinking", signature: "signature", format: "anthropic-claude-v1" }, + ], + }, + }) + }), + ) + it.effect("assembles streamed tool call input", () => Effect.gen(function* () { const body = sseEvents( diff --git a/packages/core/src/models-dev.ts b/packages/core/src/models-dev.ts index 0caf414cce..640b5b1085 100644 --- a/packages/core/src/models-dev.ts +++ b/packages/core/src/models-dev.ts @@ -35,6 +35,7 @@ type ReasoningOption = | { readonly type: "budget_tokens"; readonly min?: number; readonly max?: number } type Modality = "text" | "audio" | "image" | "video" | "pdf" +type InterleavedField = "reasoning" | "reasoning_content" | "reasoning_text" | "reasoning_details" | (string & {}) type SourceModel = { readonly id: string @@ -46,7 +47,7 @@ type SourceModel = { readonly reasoning_options?: readonly ReasoningOption[] readonly temperature?: boolean readonly tool_call: boolean - readonly interleaved?: true | { readonly field: "reasoning" | "reasoning_content" | "reasoning_details" } + readonly interleaved?: true | { readonly field: InterleavedField } readonly cost?: Cost readonly limit: { readonly context: number; readonly input?: number; readonly output: number } readonly modalities?: { readonly input: readonly Modality[]; readonly output: readonly Modality[] } diff --git a/packages/core/src/v1/config/provider.ts b/packages/core/src/v1/config/provider.ts index d54a3f08f9..b199cbffa3 100644 --- a/packages/core/src/v1/config/provider.ts +++ b/packages/core/src/v1/config/provider.ts @@ -4,6 +4,10 @@ import { Schema } from "effect" import { PositiveInt } from "../../schema" export const ModelStatus = Schema.Literals(["alpha", "beta", "deprecated", "active"]) +const InterleavedField = Schema.Union([ + Schema.Literals(["reasoning", "reasoning_content", "reasoning_text", "reasoning_details"]), + Schema.String, +]) export const Model = Schema.Struct({ id: Schema.optional(Schema.String), @@ -18,7 +22,7 @@ export const Model = Schema.Struct({ Schema.Union([ Schema.Literal(true), Schema.Struct({ - field: Schema.Literals(["reasoning", "reasoning_content", "reasoning_details"]), + field: InterleavedField, }), ]), ), diff --git a/packages/core/test/config/provider-v1.test.ts b/packages/core/test/config/provider-v1.test.ts new file mode 100644 index 0000000000..902c9ab4ef --- /dev/null +++ b/packages/core/test/config/provider-v1.test.ts @@ -0,0 +1,11 @@ +import { expect, test } from "bun:test" +import { ConfigProviderV1 } from "@opencode-ai/core/v1/config/provider" +import { Schema } from "effect" + +const decode = Schema.decodeUnknownSync(ConfigProviderV1.Model) + +test("accepts known and custom interleaved reasoning fields", () => { + const fields = ["reasoning", "reasoning_content", "reasoning_text", "reasoning_details", "vendor_reasoning"] + + for (const field of fields) expect(decode({ interleaved: { field } }).interleaved).toEqual({ field }) +})