diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index 68ab2e3949..44f173aec4 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -77,6 +77,7 @@ const OpenAIChatMessage = Schema.Union([ reasoning_content: Schema.optional(Schema.String), reasoning: Schema.optional(Schema.String), reasoning_text: Schema.optional(Schema.String), + reasoning_details: optionalArray(Schema.Unknown), }), Schema.Struct({ role: Schema.Literal("tool"), tool_call_id: Schema.String, content: Schema.String }), ]).pipe(Schema.toTaggedUnion("role")) @@ -149,6 +150,7 @@ const OpenAIChatDelta = Schema.Struct({ reasoning_content: optionalNull(Schema.String), reasoning: optionalNull(Schema.String), reasoning_text: optionalNull(Schema.String), + reasoning_details: optionalNull(Schema.Array(Schema.Unknown)), tool_calls: optionalNull(Schema.Array(OpenAIChatToolCallDelta)), }) @@ -163,6 +165,10 @@ export const OpenAIChatEvent = Schema.Struct({ }) export type OpenAIChatEvent = Schema.Schema.Type type OpenAIChatRequestMessage = LLMRequest["messages"][number] +interface ReasoningDetails { + readonly value: ReadonlyArray + readonly previous?: ReasoningDetails +} export interface ParserState { readonly tools: ToolStream.State @@ -171,6 +177,10 @@ export interface ParserState { readonly finishReason?: FinishReason readonly lifecycle: Lifecycle.State readonly reasoningField?: "reasoning" | "reasoning_content" | "reasoning_text" + readonly reasoningDetails?: ReasoningDetails + readonly reasoningDetailsObserved: boolean + readonly pendingReasoningDetailText: string + readonly reasoningEmitted: boolean } // ============================================================================= @@ -216,7 +226,21 @@ const openAICompatibleReasoningContent = (native: unknown) => const reasoningField = (part: ReasoningPart) => { const field = part.providerMetadata?.openai?.reasoningField if (field === "reasoning" || field === "reasoning_content" || field === "reasoning_text") return field - return "reasoning_content" +} + +const reasoningDetails = (parts: ReadonlyArray, native: unknown) => { + const observed = parts.flatMap((part) => { + const details = part.providerMetadata?.openai?.reasoningDetails + return Array.isArray(details) ? details : [] + }) + if (parts.some((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails))) return observed + if (isRecord(native) && Array.isArray(native.reasoning_details)) return native.reasoning_details +} + +const accumulatedReasoningDetails = (details: ReasoningDetails | undefined) => { + const chunks: Array> = [] + for (let current = details; current; current = current.previous) chunks.push(current.value) + return chunks.reverse().flat() } const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) { @@ -260,19 +284,22 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func } } const text = reasoning.map((part) => part.text).join("") - const field = reasoning[0] ? reasoningField(reasoning[0]) : "reasoning_content" + const details = reasoningDetails(reasoning, message.native?.openaiCompatible) + const observedField = reasoning.map(reasoningField).find((value) => value !== undefined) + const nativeReasoning = openAICompatibleReasoningContent(message.native?.openaiCompatible) + const fullyStructured = reasoning.every((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails)) + const field = + reasoning.length > 0 + ? (observedField ?? (nativeReasoning !== undefined || !fullyStructured ? "reasoning_content" : undefined)) + : undefined return { role: "assistant" as const, content: content.length === 0 ? null : ProviderShared.joinText(content), tool_calls: toolCalls.length === 0 ? undefined : toolCalls, - reasoning_content: - reasoning.length === 0 - ? openAICompatibleReasoningContent(message.native?.openaiCompatible) - : field === "reasoning_content" - ? text - : undefined, + reasoning_content: reasoning.length === 0 ? nativeReasoning : field === "reasoning_content" ? text : undefined, reasoning: reasoning.length > 0 && field === "reasoning" ? text : undefined, reasoning_text: reasoning.length > 0 && field === "reasoning_text" ? text : undefined, + reasoning_details: details, } }) @@ -418,11 +445,29 @@ const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => { } const reasoningDelta = (delta: Schema.Schema.Type | null | undefined) => { - if (delta?.reasoning_content) return { field: "reasoning_content", text: delta.reasoning_content } as const - if (delta?.reasoning) return { field: "reasoning", text: delta.reasoning } as const - if (delta?.reasoning_text) return { field: "reasoning_text", text: delta.reasoning_text } as const + if (typeof delta?.reasoning_content === "string") + return { field: "reasoning_content", text: delta.reasoning_content } as const + if (typeof delta?.reasoning === "string") return { field: "reasoning", text: delta.reasoning } as const + if (typeof delta?.reasoning_text === "string") return { field: "reasoning_text", text: delta.reasoning_text } as const } +const detailText = (details: ReadonlyArray) => + details + .flatMap((detail) => { + if (!isRecord(detail)) return [] + if (detail.type === "reasoning.text" && typeof detail.text === "string") return [detail.text] + if (detail.type === "reasoning.summary" && typeof detail.summary === "string") return [detail.summary] + return [] + }) + .join("") + +const reasoningMetadata = (field: ParserState["reasoningField"], details?: ReadonlyArray) => ({ + openai: { + ...(field ? { reasoningField: field } : {}), + ...(details ? { reasoningDetails: details } : {}), + }, +}) + const step = (state: ParserState, event: OpenAIChatEvent) => Effect.gen(function* () { const events: LLMEvent[] = [] @@ -437,17 +482,25 @@ const step = (state: ParserState, event: OpenAIChatEvent) => const reasoning = reasoningDelta(delta) const reasoningField = state.reasoningField ?? reasoning?.field - if (reasoning) - lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", reasoning.text, { - openai: { reasoningField: reasoningField ?? reasoning.field }, - }) + const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined + const reasoningDetails = + detailDelta === undefined ? state.reasoningDetails : { value: detailDelta, previous: state.reasoningDetails } + const reasoningDetailsObserved = state.reasoningDetailsObserved || detailDelta !== undefined + const deltaMetadata = reasoningMetadata(reasoningField) + // Details are an opaque replay channel. Buffer their display fallback until halt + // because a scalar reasoning channel may still arrive and remains authoritative. + const pendingReasoningDetailText = + state.pendingReasoningDetailText + (reasoningField === undefined ? detailText(detailDelta ?? []) : "") + if (reasoning) lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", reasoning.text, deltaMetadata) + else if ( + reasoningField === undefined && + !state.reasoningEmitted && + reasoningDetailsObserved && + (Boolean(delta?.content) || toolDeltas.length > 0) + ) + lifecycle = Lifecycle.reasoningStart(lifecycle, events, "reasoning-0", deltaMetadata) - if (delta?.content) { - lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0") - lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content) - } - - if (toolDeltas.length) lifecycle = Lifecycle.reasoningEnd(lifecycle, events, "reasoning-0") + if (delta?.content) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content) for (const tool of toolDeltas) { const result = ToolStream.appendOrStart( @@ -478,6 +531,15 @@ const step = (state: ParserState, event: OpenAIChatEvent) => finishReason, lifecycle, reasoningField, + reasoningDetails, + reasoningDetailsObserved, + pendingReasoningDetailText: reasoning ? "" : pendingReasoningDetailText, + reasoningEmitted: + state.reasoningEmitted || + reasoning !== undefined || + (reasoningField === undefined && + reasoningDetailsObserved && + (Boolean(delta?.content) || toolDeltas.length > 0)), }, events, ] as const @@ -487,7 +549,23 @@ const finishEvents = (state: ParserState): ReadonlyArray => { const events: LLMEvent[] = [] const hasToolCalls = state.toolCallEvents.length > 0 const reason = state.finishReason === "stop" && hasToolCalls ? "tool-calls" : state.finishReason - const lifecycle = state.toolCallEvents.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle + const metadata = reasoningMetadata( + state.reasoningField, + state.reasoningDetailsObserved ? accumulatedReasoningDetails(state.reasoningDetails) : undefined, + ) + const flushed = state.pendingReasoningDetailText + ? Lifecycle.reasoningDelta( + state.lifecycle, + events, + "reasoning-0", + state.pendingReasoningDetailText, + reasoningMetadata(state.reasoningField), + ) + : state.reasoningDetailsObserved && !state.reasoningEmitted + ? Lifecycle.reasoningStart(state.lifecycle, events, "reasoning-0", reasoningMetadata(state.reasoningField)) + : state.lifecycle + const ended = Lifecycle.reasoningEnd(flushed, events, "reasoning-0", metadata) + const lifecycle = state.toolCallEvents.length ? Lifecycle.stepStart(ended, events) : ended events.push(...state.toolCallEvents) if (reason) Lifecycle.finish(lifecycle, events, { reason, usage: state.usage }) return events @@ -515,6 +593,10 @@ export const protocol = Protocol.make({ toolCallEvents: [], lifecycle: Lifecycle.initial(), reasoningField: undefined, + reasoningDetails: undefined, + reasoningDetailsObserved: false, + pendingReasoningDetailText: "", + reasoningEmitted: false, }), step, onHalt: finishEvents, diff --git a/packages/ai/src/protocols/utils/lifecycle.ts b/packages/ai/src/protocols/utils/lifecycle.ts index eb6c95dfbd..6d0189df9a 100644 --- a/packages/ai/src/protocols/utils/lifecycle.ts +++ b/packages/ai/src/protocols/utils/lifecycle.ts @@ -44,7 +44,7 @@ export const reasoningDelta = ( providerMetadata?: ProviderMetadata, ): State => { const started = reasoningStart(state, events, id, providerMetadata) - events.push(LLMEvent.reasoningDelta({ id, text })) + events.push(LLMEvent.reasoningDelta({ id, text, providerMetadata })) return started } diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index 914d7c0a0b..27e00fa2a5 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -41,13 +41,39 @@ export const protocol = Protocol.make({ schema: OpenRouterBody, from: (request) => OpenAIChat.protocol.body.from(request).pipe( - Effect.map( - (body) => - ({ - ...body, - ...bodyOptions(request.providerOptions?.openrouter), - }) as OpenRouterBody, - ), + Effect.map((body) => { + const sourceAssistants = request.messages.filter((message) => message.role === "assistant") + let assistantIndex = 0 + const messages = body.messages.map((message) => { + if (message.role !== "assistant") return message + const source = sourceAssistants[assistantIndex++] + if (!Array.isArray(message.reasoning_details)) return message + const detailParts = + source?.content.filter( + (part) => part.type === "reasoning" && Array.isArray(part.providerMetadata?.openai?.reasoningDetails), + ).length ?? 0 + if (detailParts > 1) + return { + ...message, + reasoning_details: source?.content.flatMap((part) => { + if (part.type !== "reasoning") return [] + const details = part.providerMetadata?.openai?.reasoningDetails + return Array.isArray(details) ? mergeReasoningTextDetails(details) : [] + }), + } + // OpenRouter streams one logical signed block as text fragments followed by its signature. + // Replaying those wire fragments separately is rejected as an invalid thinking signature. + return { + ...message, + reasoning_details: mergeReasoningTextDetails(message.reasoning_details), + } + }) + return { + ...body, + messages, + ...bodyOptions(request.providerOptions?.openrouter), + } as OpenRouterBody + }), ), }, stream: OpenAIChat.protocol.stream, @@ -66,6 +92,76 @@ const bodyOptions = (input: unknown) => { } } +const reasoningTextKeys = new Set(["type", "id", "index", "format", "text", "signature"]) + +const mergeReasoningTextDetails = (details: ReadonlyArray) => + details.reduce((merged, detail) => { + const previous = merged.at(-1) + const previousID = isRecord(previous) && typeof previous.id === "string" && previous.id ? previous.id : undefined + const detailID = isRecord(detail) && typeof detail.id === "string" && detail.id ? detail.id : undefined + const sameID = previousID !== undefined && detailID !== undefined && previousID === detailID + const sameIndex = + isRecord(previous) && + isRecord(detail) && + typeof previous.index === "number" && + previous.index === detail.index && + typeof previous.format === "string" && + previous.format === detail.format + const conflictingIdentity = + isRecord(previous) && + isRecord(detail) && + ((previousID !== undefined && detailID !== undefined && previousID !== detailID) || + (!sameID && previous.index !== undefined && detail.index !== undefined && previous.index !== detail.index) || + (!sameID && previous.format !== undefined && detail.format !== undefined && previous.format !== detail.format)) + const conflictingExtra = + isRecord(previous) && + isRecord(detail) && + Object.keys(previous).some( + (key) => + !reasoningTextKeys.has(key) && + previous[key] !== undefined && + detail[key] !== undefined && + !sameDetailValue(previous[key], detail[key]), + ) + if ( + !isRecord(previous) || + !isRecord(detail) || + previous.type !== "reasoning.text" || + detail.type !== "reasoning.text" || + (!sameID && !sameIndex) || + conflictingIdentity || + conflictingExtra || + (typeof previous.signature === "string" && previous.signature.length > 0) || + (previous.signature && detail.signature && previous.signature !== detail.signature) + ) { + merged.push(detail) + return merged + } + const definedDetail = Object.fromEntries(Object.entries(detail).filter((entry) => entry[1] !== undefined)) + merged[merged.length - 1] = { + ...previous, + ...definedDetail, + text: `${typeof previous.text === "string" ? previous.text : ""}${typeof detail.text === "string" ? detail.text : ""}`, + ...(previousID ? { id: previousID } : {}), + ...(previous.index !== undefined ? { index: previous.index } : {}), + ...(previous.signature ? { signature: previous.signature } : {}), + ...(previous.format !== undefined ? { format: previous.format } : {}), + } + return merged + }, []) + +const sameDetailValue = (left: unknown, right: unknown): boolean => { + if (Object.is(left, right)) return true + if (Array.isArray(left) && Array.isArray(right)) + return left.length === right.length && left.every((value, index) => sameDetailValue(value, right[index])) + if (!isRecord(left) || !isRecord(right)) return false + const keys = Object.keys(left) + return ( + keys.length === Object.keys(right).length && + keys.every((key) => Object.hasOwn(right, key) && sameDetailValue(left[key], right[key])) + ) +} + export const route = Route.make({ id: ADAPTER, provider: profile.provider, diff --git a/packages/ai/test/fixtures/recordings/openrouter-reasoning-tool-loop.json b/packages/ai/test/fixtures/recordings/openrouter-reasoning-tool-loop.json new file mode 100644 index 0000000000..343f4d8160 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/openrouter-reasoning-tool-loop.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "metadata": { + "model": "anthropic/claude-sonnet-4.6", + "tags": [ + "prefix:openai-compatible-chat", + "provider:openrouter", + "protocol:openai-chat", + "reasoning", + "continuation", + "tool", + "tool-loop" + ], + "name": "openrouter-reasoning-tool-loop", + "recordedAt": "2026-07-18T18:02:15.859Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\"The user wants\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":\" to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" to know the weather in Paris. I'll use the get_weather tool.\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMNFb5O6pb4nX0HojdGgyL5h+CAIpsxpdM1QgiMGm/i3ST6F5mAhxB+Uez0Cm95ra9yvQkrzHaA/AmWoXpdmPlczSn1S1RDk2IqeA57Spbf7JT44jygtLQt6yZmGzoTBHn3VkwaNZsuuAtbdo4B5QJXooa/AoKKs54QZ2kfS640vsv5flQVCg7CoQCFuLKjIeLMO7MnxVyuskXJr1DgesTa7I0ScF53U9JGhgB\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure! Let me check the weather in Paris for you right now!\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \\\"Paris\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"reasoning\":null},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}]}\n\ndata: {\"id\":\"gen-1784397732-OaggEWvVIDtBtcPF4n0O\",\"object\":\"chat.completion.chunk\",\"created\":1784397732,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"tool_calls\",\"native_finish_reason\":\"tool_use\"}],\"usage\":{\"prompt_tokens\":623,\"completion_tokens\":95,\"total_tokens\":718,\"cost\":0.003294,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.003294,\"upstream_inference_prompt_cost\":0.001869,\"upstream_inference_completions_cost\":0.001425},\"completion_tokens_details\":{\"reasoning_tokens\":19,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"Sure! Let me check the weather in Paris for you right now!\",\"tool_calls\":[{\"id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"format\":\"anthropic-claude-v1\",\"index\":0,\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDRjMGYwNDZmLTI1ZmQtNDVmYi1iZmIzLWEwOGE4ZTI0OWNhNxIMNFb5O6pb4nX0HojdGgyL5h+CAIpsxpdM1QgiMGm/i3ST6F5mAhxB+Uez0Cm95ra9yvQkrzHaA/AmWoXpdmPlczSn1S1RDk2IqeA57Spbf7JT44jygtLQt6yZmGzoTBHn3VkwaNZsuuAtbdo4B5QJXooa/AoKKs54QZ2kfS640vsv5flQVCg7CoQCFuLKjIeLMO7MnxVyuskXJr1DgesTa7I0ScF53U9JGhgB\"}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_01PaChhcyw3yu2P2bDS2bgAA\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": ": OPENROUTER PROCESSING\n\n: OPENROUTER PROCESSING\n\ndata: {\"id\":\"gen-1784397734-HY78YkUzZ8xIXuimN9bb\",\"object\":\"chat.completion.chunk\",\"created\":1784397734,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397734-HY78YkUzZ8xIXuimN9bb\",\"object\":\"chat.completion.chunk\",\"created\":1784397734,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is sunny.\",\"role\":\"assistant\"},\"finish_reason\":null,\"native_finish_reason\":null}]}\n\ndata: {\"id\":\"gen-1784397734-HY78YkUzZ8xIXuimN9bb\",\"object\":\"chat.completion.chunk\",\"created\":1784397734,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}]}\n\ndata: {\"id\":\"gen-1784397734-HY78YkUzZ8xIXuimN9bb\",\"object\":\"chat.completion.chunk\",\"created\":1784397734,\"model\":\"anthropic/claude-sonnet-4.6\",\"provider\":\"Anthropic\",\"service_tier\":\"default\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":\"stop\",\"native_finish_reason\":\"end_turn\"}],\"usage\":{\"prompt_tokens\":739,\"completion_tokens\":7,\"total_tokens\":746,\"cost\":0.002322,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":0.002322,\"upstream_inference_prompt_cost\":0.002217,\"upstream_inference_completions_cost\":0.000105},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0,\"audio_tokens\":0}}}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-tool-loop.json b/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-tool-loop.json new file mode 100644 index 0000000000..7c5c6cc2b0 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/vercel-ai-gateway-reasoning-tool-loop.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "metadata": { + "model": "anthropic/claude-sonnet-4.6", + "tags": [ + "prefix:openai-compatible-chat", + "provider:vercel-ai-gateway", + "protocol:openai-chat", + "reasoning", + "continuation", + "tool", + "tool-loop" + ], + "name": "vercel-ai-gateway-reasoning-tool-loop", + "recordedAt": "2026-07-18T18:01:07.272Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://ai-gateway.vercel.sh/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"enabled\":true,\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"The user wants\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\" to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\" to know the weather in Paris. I'll use the get_weather tool.\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"\",\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDNiOTNhNWRkLTczMDItNDgzZi1hZWFlLTM2MjA3NTU0OGFlMRIMCQuLq3s9p+BrTNS7GgwAuD583tUZH6OqCmsiMMrNsOyZCCwLT02gcGQ89jpE7btmhkC3O77ozFdZdwjafHtjzUnpp/EggS7QcBSJ4ypbSAZXdpX7yxMsstHIDWHhQ1OvqHrPFR7a7/RTBzbKND3zGrO8y9bf8Q/xf5CAJgpUBPXMxS261RY2ZHcw73PS6XHBqlfpv6UDXKsl+42NXj+8TKvtU/lllOt+ERgB\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Sure! Let me check the weather in Paris for you!\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"toolu_01Ws1rPHU4pg5Hq2zQtvkXkW\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\": \\\"Paris\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"}\"}}]},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_czfkniribx\"}\n\ndata: {\"id\":\"gen_01KXV670KSD04747Z6KG26YHYB\",\"object\":\"chat.completion.chunk\",\"created\":1784397664,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":623,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":93,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":24}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-4.6\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"vertexAnthropic\",\"bedrock\"],\"planningReasoning\":\"System credentials planned for: anthropic, vertexAnthropic, bedrock. Total execution order: anthropic(system) → vertexAnthropic(system) → bedrock(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1784397662437,\"endTime\":1784397665110,\"providerRequestId\":\"req_011Cd9xekHeLEvTpLGGAnTTD\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cd9xemFC7GVY8DQdn5cPS\"}]}],\"totalProviderAttemptCount\":1},\"cost\":\"0.003264\",\"marketCost\":\"0.003264\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.003264\",\"inferenceCost\":\"0.003264\",\"inputInferenceCost\":\"0.001869\",\"outputInferenceCost\":\"0.001395\",\"generationId\":\"gen_01KXV670KSD04747Z6KG26YHYB\"}}},\"logprobs\":null,\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":623,\"completion_tokens\":93,\"total_tokens\":716,\"cost\":0.003264,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":19,\"reasoning_tokens_estimated\":true,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.003264,\"gateway_cost\":0.003264},\"system_fingerprint\":\"fp_czfkniribx\",\"generationId\":\"gen_01KXV670KSD04747Z6KG26YHYB\"}\n\ndata: [DONE]\n\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://ai-gateway.vercel.sh/v1/chat/completions", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"anthropic/claude-sonnet-4.6\",\"messages\":[{\"role\":\"system\",\"content\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\"},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":\"Sure! Let me check the weather in Paris for you!\",\"tool_calls\":[{\"id\":\"toolu_01Ws1rPHU4pg5Hq2zQtvkXkW\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}],\"reasoning\":\"The user wants to know the weather in Paris. I'll use the get_weather tool.\",\"reasoning_details\":[{\"type\":\"reasoning.text\",\"text\":\"The user wants\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0},{\"type\":\"reasoning.text\",\"text\":\" to know the weather in Paris. I'll use the get_weather tool.\",\"signature\":\"\",\"format\":\"anthropic-claude-v1\",\"index\":0},{\"type\":\"reasoning.text\",\"text\":\"\",\"signature\":\"ErkCCosBCA8YAipAjKnRKpxkZ4eHrMPJ63IWEOYPSzb+XSHyG+vLK+2ks2O9T4N9M37Xn2kausQSH1rfsrdmKxgUlBg6yUFRgMVR7DIRY2xhdWRlLXNvbm5ldC00LTY4AEIIdGhpbmtpbmdaJDNiOTNhNWRkLTczMDItNDgzZi1hZWFlLTM2MjA3NTU0OGFlMRIMCQuLq3s9p+BrTNS7GgwAuD583tUZH6OqCmsiMMrNsOyZCCwLT02gcGQ89jpE7btmhkC3O77ozFdZdwjafHtjzUnpp/EggS7QcBSJ4ypbSAZXdpX7yxMsstHIDWHhQ1OvqHrPFR7a7/RTBzbKND3zGrO8y9bf8Q/xf5CAJgpUBPXMxS261RY2ZHcw73PS6XHBqlfpv6UDXKsl+42NXj+8TKvtU/lllOt+ERgB\",\"format\":\"anthropic-claude-v1\",\"index\":0}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_01Ws1rPHU4pg5Hq2zQtvkXkW\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"max_tokens\":1536,\"reasoning\":{\"enabled\":true,\"max_tokens\":1024}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"id\":\"gen_01KXV674WN74Y43YHA1MDABBHT\",\"object\":\"chat.completion.chunk\",\"created\":1784397666,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_8knxiehgay\"}\n\ndata: {\"id\":\"gen_01KXV674WN74Y43YHA1MDABBHT\",\"object\":\"chat.completion.chunk\",\"created\":1784397666,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Paris\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_8knxiehgay\"}\n\ndata: {\"id\":\"gen_01KXV674WN74Y43YHA1MDABBHT\",\"object\":\"chat.completion.chunk\",\"created\":1784397666,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" is sunny.\"},\"logprobs\":null,\"finish_reason\":null}],\"system_fingerprint\":\"fp_8knxiehgay\"}\n\ndata: {\"id\":\"gen_01KXV674WN74Y43YHA1MDABBHT\",\"object\":\"chat.completion.chunk\",\"created\":1784397666,\"model\":\"anthropic/claude-sonnet-4.6\",\"choices\":[{\"index\":0,\"delta\":{\"provider_metadata\":{\"anthropic\":{\"usage\":{\"input_tokens\":737,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":7,\"service_tier\":\"standard\",\"inference_geo\":\"global\",\"output_tokens_details\":{\"thinking_tokens\":0}},\"cacheCreationInputTokens\":0,\"stopSequence\":null,\"iterations\":null,\"container\":null,\"contextManagement\":null},\"gateway\":{\"routing\":{\"originalModelId\":\"anthropic/claude-sonnet-4.6\",\"resolvedProvider\":\"anthropic\",\"fallbacksAvailable\":[\"vertexAnthropic\",\"bedrock\"],\"planningReasoning\":\"System credentials planned for: anthropic, vertexAnthropic, bedrock. Total execution order: anthropic(system) → vertexAnthropic(system) → bedrock(system)\",\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"finalProvider\":\"anthropic\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"anthropic/claude-sonnet-4.6\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"anthropic\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1784397665203,\"endTime\":1784397667273,\"providerRequestId\":\"req_011Cd9xewv8h6XzESKbznUXu\",\"statusCode\":200,\"providerResponseId\":\"msg_011Cd9xexPfEBake1vtRVooY\"}]}],\"totalProviderAttemptCount\":1},\"cost\":\"0.002316\",\"marketCost\":\"0.002316\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.002316\",\"inferenceCost\":\"0.002316\",\"inputInferenceCost\":\"0.002211\",\"outputInferenceCost\":\"0.000105\",\"generationId\":\"gen_01KXV674WN74Y43YHA1MDABBHT\"}}},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":737,\"completion_tokens\":7,\"total_tokens\":744,\"cost\":0.002316,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0,\"upstream_inference_completions_cost\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0},\"cache_creation_input_tokens\":0,\"market_cost\":0.002316,\"gateway_cost\":0.002316},\"system_fingerprint\":\"fp_8knxiehgay\",\"generationId\":\"gen_01KXV674WN74Y43YHA1MDABBHT\"}\n\ndata: [DONE]\n\n" + } + } + ] +} diff --git a/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts index 32ef7820b8..5b4e830c6d 100644 --- a/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts +++ b/packages/ai/test/provider/openai-chat-reasoning.recorded.test.ts @@ -1,10 +1,12 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM, LLMEvent } from "../../src" +import { LLM, LLMEvent, LLMResponse } from "../../src" +import { OpenAIChat } from "../../src/protocols/openai-chat" import * as OpenAICompatible from "../../src/providers/openai-compatible" import * as OpenRouter from "../../src/providers/openrouter" import { LLMClient } from "../../src/route" import { recordedTests } from "../recorded-test" +import { expectWeatherToolLoop, goldenWeatherToolLoopRequest, runWeatherToolLoop } from "../recorded-scenarios" const cases = [ { @@ -15,6 +17,7 @@ const cases = [ }).model("anthropic/claude-sonnet-4.6"), requires: ["OPENROUTER_API_KEY"], cassette: "openrouter-reasoning", + structured: true, }, { name: "Vercel AI Gateway", @@ -26,6 +29,7 @@ const cases = [ }).model("anthropic/claude-sonnet-4.6"), requires: ["AI_GATEWAY_API_KEY"], cassette: "vercel-ai-gateway-reasoning", + structured: true, }, ] as const @@ -57,11 +61,85 @@ for (const item of cases) { expect(response.text.replaceAll(",", "").trim()).toBe("37887") expect(response.reasoning.length).toBeGreaterThan(0) expect(response.events.some(LLMEvent.is.reasoningDelta)).toBe(true) - expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ - openai: { reasoningField: "reasoning" }, - }) + const metadata = response.message.content.find((part) => part.type === "reasoning")?.providerMetadata + expect(metadata?.openai?.reasoningField).toBe(item.structured ? "reasoning" : "reasoning_content") + expect(Array.isArray(metadata?.openai?.reasoningDetails)).toBe(item.structured) + if (!item.structured) return + const details = metadata?.openai?.reasoningDetails + if (!Array.isArray(details)) return + expect( + details.some( + (detail) => + typeof detail === "object" && + detail !== null && + "signature" in detail && + typeof detail.signature === "string" && + detail.signature.length > 0, + ), + ).toBe(true) + + const replay = yield* LLMClient.prepare( + LLM.request({ model: item.model, messages: [response.message] }), + ) + expect(replay.body.messages).toMatchObject([ + { role: "assistant", content: response.text, reasoning: response.reasoning }, + ]) + const replayDetails = + replay.body.messages[0]?.role === "assistant" ? replay.body.messages[0].reasoning_details : undefined + expect(Array.isArray(replayDetails)).toBe(true) + if (!Array.isArray(replayDetails)) return + if (item.name === "Vercel AI Gateway") expect(replayDetails).toEqual(details) + if (item.name === "OpenRouter") { + expect(replayDetails).toHaveLength(1) + expect(replayDetails).not.toEqual(details) + expect(replayDetails[0]).toMatchObject({ + type: "reasoning.text", + text: response.reasoning, + signature: expect.any(String), + }) + } }), 30_000, ) + + recorded.effect.with( + "continues signed reasoning through a tool loop", + { cassette: `${item.cassette}-tool-loop`, tags: ["continuation", "tool", "tool-loop"] }, + () => + Effect.gen(function* () { + const events = yield* runWeatherToolLoop( + goldenWeatherToolLoopRequest({ + id: `${item.cassette}-tool-loop`, + model: item.model, + maxTokens: 1536, + temperature: false, + }), + ) + + expectWeatherToolLoop(events) + expect( + LLMResponse.text({ + events: events.slice(events.findIndex(LLMEvent.is.stepFinish) + 1), + }).trim(), + ).toMatch(/^Paris is sunny\.?$/) + const details = events + .filter(LLMEvent.is.reasoningEnd) + .map((event) => event.providerMetadata?.openai?.reasoningDetails) + .find(Array.isArray) + expect(Array.isArray(details)).toBe(item.structured) + if (!item.structured || !Array.isArray(details)) return + expect( + details.some( + (detail) => + typeof detail === "object" && + detail !== null && + "signature" in detail && + typeof detail.signature === "string" && + detail.signature.length > 0, + ), + ).toBe(true) + }), + 60_000, + ) }) } diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index e2efa1717d..cf22d3b95d 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -570,6 +570,404 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("preserves and replays reasoning details alongside scalar reasoning", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "thinking", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.encrypted", data: "opaque", format: "anthropic-claude-v1", index: 1 }, + ] + const response = yield* LLMClient.generate( + LLM.updateRequest(request, { + tools: [{ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } }], + }), + ).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning: "thinking", reasoning_details: [details[0]] } }] }, + { choices: [{ delta: { reasoning_details: [details[1]] } }] }, + { + choices: [ + { + delta: { + tool_calls: [ + { index: 0, id: "call_1", function: { name: "lookup", arguments: '{"query":"weather"}' } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("thinking") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + + const replay = yield* LLMClient.prepare( + LLM.request({ model, messages: [response.message] }), + ) + expect(replay.body.messages).toEqual([ + { + role: "assistant", + content: null, + reasoning: "thinking", + reasoning_details: details, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "lookup", arguments: '{"query":"weather"}' }, + }, + ], + }, + ]) + }), + ) + + it.effect("uses reasoning details as display fallback without inventing a scalar replay field", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.summary", summary: "thinking", format: "openai-responses-v1", index: 0 }, + { type: "reasoning.encrypted", data: "opaque", format: "openai-responses-v1", index: 1 }, + ] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: [details[0]] } }] }, + { choices: [{ delta: { reasoning_details: [details[1]] } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("thinking") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningDetails: details }, + }) + + const replay = yield* LLMClient.prepare( + LLM.request({ model, messages: [response.message] }), + ) + expect(replay.body.messages).toEqual([{ role: "assistant", content: "Hello", reasoning_details: details }]) + }), + ) + + it.effect("prefers scalar reasoning when reasoning details arrive first", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.text", text: "thinking", format: "anthropic-claude-v1", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: { reasoning: "thinking" } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("thinking") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + }), + ) + + it.effect("uses scalar reasoning independently from details-first text", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.text", text: "think", format: "unknown", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: { reasoning: "ing" } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("ing") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + }), + ) + + it.effect("keeps scalar reasoning authoritative when later details contain text", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.text", text: "detail-only", format: "unknown", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning: "scalar" } }] }, + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("scalar") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + }), + ) + + it.effect("keeps late scalar reasoning authoritative after content starts", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.text", text: "detail", format: "unknown", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: { reasoning: "scalar" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("scalar") + expect(response.events.filter(LLMEvent.is.reasoningStart)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toHaveLength(1) + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + }), + ) + + it.effect("treats an explicitly empty scalar reasoning field as authoritative", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.text", text: "detail", format: "unknown", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: { reasoning: "" } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + }), + ) + + it.effect("preserves an explicitly empty reasoning details array", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: [] } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("") + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningDetails: [] }, + }) + + const replay = yield* LLMClient.prepare( + LLM.request({ model, messages: [response.message] }), + ) + expect(replay.body.messages).toEqual([{ role: "assistant", content: "Hello", reasoning_details: [] }]) + }), + ) + + it.effect("attaches signature-only details that arrive after content", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "thinking", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed", format: "anthropic-claude-v1", index: 0 }, + ] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning: "thinking", reasoning_details: [details[0]] } }] }, + { choices: [{ delta: { content: "Hello" } }] }, + { choices: [{ delta: { reasoning_details: [details[1]] } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("thinking") + expect(response.message.content.filter((part) => part.type === "reasoning")).toHaveLength(1) + expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + expect(response.events.filter(LLMEvent.is.reasoningStart)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.reasoningDelta)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.reasoningEnd).at(-1)?.providerMetadata).toEqual({ + openai: { reasoningField: "reasoning", reasoningDetails: details }, + }) + + const replay = yield* LLMClient.prepare( + LLM.request({ model, messages: [response.message] }), + ) + expect(replay.body.messages).toEqual([ + { role: "assistant", content: "Hello", reasoning: "thinking", reasoning_details: details }, + ]) + }), + ) + + it.effect("preserves metadata-only reasoning when the stream ends", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.encrypted", data: "opaque", format: "openai-responses-v1", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.message.content).toEqual([ + { type: "reasoning", text: "", providerMetadata: { openai: { reasoningDetails: details } } }, + ]) + expect(response.events.filter(LLMEvent.is.reasoningStart)).toHaveLength(1) + expect(response.events.filter(LLMEvent.is.reasoningEnd)).toHaveLength(1) + + const replay = yield* LLMClient.prepare( + LLM.request({ model, messages: [response.message] }), + ) + expect(replay.body.messages).toEqual([{ role: "assistant", content: null, reasoning_details: details }]) + }), + ) + + it.effect("flushes details-only display reasoning when the stream ends", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.summary", summary: "summary", format: "openai-responses-v1", index: 0 }] + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { choices: [{ delta: { reasoning_details: details } }] }, + { choices: [{ delta: {}, finish_reason: "stop" }] }, + ), + ), + ), + ) + + expect(response.reasoning).toBe("summary") + expect(response.message.content).toEqual([ + { type: "reasoning", text: "summary", providerMetadata: { openai: { reasoningDetails: details } } }, + ]) + }), + ) + + it.effect("replays details from multiple reasoning parts in order", () => + Effect.gen(function* () { + const first = { type: "reasoning.text", text: "first", signature: "signed-0", index: 0 } + const second = { type: "reasoning.text", text: "second", signature: "signed-1", index: 1 } + const replay = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "first", + providerMetadata: { openai: { reasoningDetails: [first] } }, + }, + { + type: "reasoning", + text: "second", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: [second] } }, + }, + ]), + ], + }), + ) + + expect(replay.body.messages).toEqual([ + { role: "assistant", content: null, reasoning: "firstsecond", reasoning_details: [first, second] }, + ]) + }), + ) + + it.effect("retains scalar replay for mixed structured reasoning parts", () => + Effect.gen(function* () { + const detail = { type: "reasoning.encrypted", data: "opaque", index: 0 } + const replay = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "A", + providerMetadata: { openai: { reasoningDetails: [detail] } }, + }, + { type: "reasoning", text: "B" }, + ]), + ], + }), + ) + + expect(replay.body.messages).toEqual([ + { role: "assistant", content: null, reasoning_content: "AB", reasoning_details: [detail] }, + ]) + }), + ) + + it.effect("replays native scalar reasoning alongside native details", () => + Effect.gen(function* () { + const details = [{ type: "reasoning.encrypted", data: "opaque", index: 0 }] + const replay = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [ + Message.make({ + role: "assistant", + content: [{ type: "reasoning", text: "thinking" }], + native: { openaiCompatible: { reasoning_content: "thinking", reasoning_details: details } }, + }), + ], + }), + ) + + expect(replay.body.messages).toEqual([ + { role: "assistant", content: null, reasoning_content: "thinking", reasoning_details: details }, + ]) + }), + ) + it.effect("assembles streamed tool call input", () => Effect.gen(function* () { const body = sseEvents( diff --git a/packages/ai/test/provider/openrouter.test.ts b/packages/ai/test/provider/openrouter.test.ts index 86d1317b3e..1dedadaffd 100644 --- a/packages/ai/test/provider/openrouter.test.ts +++ b/packages/ai/test/provider/openrouter.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM } from "../../src" +import { LLM, Message } from "../../src" import { LLMClient } from "../../src/route" import * as OpenRouter from "../../src/providers/openrouter" import { it } from "../lib/effect" @@ -53,4 +53,452 @@ describe("OpenRouter", () => { }) }), ) + + it.effect("merges streamed reasoning text fragments before replay", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "Think", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", text: "ing", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.encrypted", data: "opaque", format: "openai-responses-v1", index: 1 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "Thinking", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages).toEqual([ + { + role: "assistant", + content: null, + reasoning: "Thinking", + reasoning_details: [ + { + type: "reasoning.text", + text: "Thinking", + signature: "signed", + format: "anthropic-claude-v1", + index: 0, + }, + { type: "reasoning.encrypted", data: "opaque", format: "openai-responses-v1", index: 1 }, + ], + }, + ]) + }), + ) + + it.effect("keeps independently signed reasoning blocks separate", () => + Effect.gen(function* () { + const details = [ + { + type: "reasoning.text", + text: "First", + signature: "signed-0", + format: "anthropic-claude-v1", + index: 0, + }, + { + type: "reasoning.text", + text: "Second", + signature: "signed-1", + format: "anthropic-claude-v1", + index: 1, + }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "FirstSecond", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ reasoning_details: details }) + }), + ) + + it.effect("preserves unsigned and repeated reasoning details across history", () => + Effect.gen(function* () { + const encrypted = { type: "reasoning.encrypted", id: "reasoning-1", data: "opaque", index: 0 } + const unsigned = { type: "reasoning.text", text: "Unsigned", format: "anthropic-claude-v1", index: 0 } + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "Unsigned", + providerMetadata: { + openai: { + reasoningField: "reasoning", + reasoningDetails: [unsigned], + }, + }, + }, + ]), + Message.assistant([ + { + type: "reasoning", + text: "Encrypted", + providerMetadata: { openai: { reasoningDetails: [encrypted] } }, + }, + ]), + Message.assistant([ + { + type: "reasoning", + text: "Encrypted", + providerMetadata: { openai: { reasoningDetails: [encrypted] } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages).toMatchObject([ + { reasoning: "Unsigned", reasoning_details: [unsigned] }, + { reasoning_details: [encrypted] }, + { reasoning_details: [encrypted] }, + ]) + }), + ) + + it.effect("keeps identical independently signed blocks and detail types", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "same", signature: "signed-0", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", text: "same", signature: "signed-1", format: "anthropic-claude-v1", index: 1 }, + { type: "reasoning.summary", summary: "same", format: "openai-responses-v1", index: 2 }, + { type: "reasoning.encrypted", data: "same", format: "openai-responses-v1", index: 3 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "samesame", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ reasoning_details: details }) + }), + ) + + it.effect("does not merge adjacent text details without a shared identity", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", id: "first", text: "A", format: "openai-responses-v1" }, + { type: "reasoning.text", text: "B", format: "openai-responses-v1" }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("openai/gpt-5"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "AB", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ reasoning_details: details }) + }), + ) + + it.effect("treats a signature as the boundary when block indices restart", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "A", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed-a", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", text: "B", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed-b", format: "anthropic-claude-v1", index: 0 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "AB", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [ + { type: "reasoning.text", text: "A", signature: "signed-a", index: 0 }, + { type: "reasoning.text", text: "B", signature: "signed-b", index: 0 }, + ], + }) + }), + ) + + it.effect("does not merge details across canonical reasoning parts", () => + Effect.gen(function* () { + const first = [ + { type: "reasoning.text", text: "A", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed-a", format: "anthropic-claude-v1", index: 0 }, + ] + const second = [ + { type: "reasoning.text", text: "B", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", signature: "signed-b", format: "anthropic-claude-v1", index: 0 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "A", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: first } }, + }, + { + type: "reasoning", + text: "B", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: second } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [ + { ...first[0], signature: "signed-a" }, + { ...second[0], signature: "signed-b" }, + ], + }) + }), + ) + + it.effect("treats empty detail ids as absent during fragment merging", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", id: "", text: "A", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", id: "block", signature: "signed", format: "anthropic-claude-v1", index: 0 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "A", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [{ type: "reasoning.text", id: "block", text: "A", signature: "signed", index: 0 }], + }) + }), + ) + + it.effect("uses a matching non-empty id as authoritative fragment identity", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", id: "block", text: "A", format: "anthropic-claude-v1", index: 0 }, + { type: "reasoning.text", id: "block", text: "B", format: "anthropic-claude-v1", index: 1 }, + { type: "reasoning.text", id: "block", signature: "signed", format: "anthropic-claude-v1", index: 2 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "AB", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [{ type: "reasoning.text", id: "block", text: "AB", signature: "signed", index: 0 }], + }) + }), + ) + + it.effect("does not erase opaque fragment fields with undefined values", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "A", format: "anthropic-claude-v1", index: 0, opaque: "keep" }, + { + type: "reasoning.text", + text: "B", + signature: "signed", + format: "anthropic-claude-v1", + index: 0, + opaque: undefined, + }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "AB", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [{ type: "reasoning.text", text: "AB", signature: "signed", index: 0, opaque: "keep" }], + }) + }), + ) + + it.effect("allows defined fragment fields to replace earlier undefined values", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", text: "A", format: "anthropic-claude-v1", index: 0, opaque: undefined }, + { + type: "reasoning.text", + text: "B", + signature: "signed", + format: "anthropic-claude-v1", + index: 0, + opaque: "known", + }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "AB", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [{ type: "reasoning.text", text: "AB", signature: "signed", index: 0, opaque: "known" }], + }) + }), + ) + + it.effect("preserves an explicitly empty format from the first fragment", () => + Effect.gen(function* () { + const details = [ + { type: "reasoning.text", id: "block", text: "A", format: "", index: 0 }, + { type: "reasoning.text", id: "block", signature: "signed", format: "later", index: 1 }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "A", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [ + { type: "reasoning.text", id: "block", text: "A", signature: "signed", format: "", index: 0 }, + ], + }) + }), + ) + + it.effect("merges fragments with structurally equal opaque fields", () => + Effect.gen(function* () { + const details = [ + { + type: "reasoning.text", + text: "A", + format: "anthropic-claude-v1", + index: 0, + opaque: { version: 1 }, + }, + { + type: "reasoning.text", + signature: "signed", + format: "anthropic-claude-v1", + index: 0, + opaque: { version: 1 }, + }, + ] + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), + messages: [ + Message.assistant([ + { + type: "reasoning", + text: "A", + providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } }, + }, + ]), + ], + }), + ) + + expect(prepared.body.messages[0]).toMatchObject({ + reasoning_details: [ + { + type: "reasoning.text", + text: "A", + signature: "signed", + index: 0, + opaque: { version: 1 }, + }, + ], + }) + }), + ) }) diff --git a/packages/ai/test/recorded-scenarios.ts b/packages/ai/test/recorded-scenarios.ts index cac348da65..daf57cae24 100644 --- a/packages/ai/test/recorded-scenarios.ts +++ b/packages/ai/test/recorded-scenarios.ts @@ -120,29 +120,8 @@ export const runWeatherToolLoop = (request: LLMRequest) => throw new Error("Weather tool loop exceeded 10 steps") }) -const assistantContent = (events: ReadonlyArray) => { - const content: ContentPart[] = [] - for (const event of events) { - if (event.type === "text-delta" || event.type === "reasoning-delta") { - const type = event.type === "text-delta" ? "text" : "reasoning" - const last = content.at(-1) - if (last?.type === type) { - content[content.length - 1] = { ...last, text: `${last.text}${event.text}` } - } else { - content.push({ type, text: event.text }) - } - continue - } - if (event.type === "text-end" || event.type === "reasoning-end") { - const type = event.type === "text-end" ? "text" : "reasoning" - const last = content.at(-1) - if (last?.type === type) content[content.length - 1] = { ...last, providerMetadata: event.providerMetadata } - continue - } - if (event.type === "tool-call") content.push(event) - } - return content -} +const assistantContent = (events: ReadonlyArray) => + events.reduce(LLMResponse.reduce, LLMResponse.empty()).message.content export const expectFinish = ( events: ReadonlyArray,