From 5f08d6cbd63e225ac7caeb1a905952af6d2604c8 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 26 Apr 2026 21:16:13 -0400 Subject: [PATCH] feat(llm): cachePromptHints patch with first-2 system / last-2 messages policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lift the prompt-cache policy out of OpenCode's bridge and into the LLM package as a typed, gated patch. The policy mirrors the AI-SDK applyCaching path (packages/opencode/src/provider/transform.ts:229): mark the first 2 system parts and the last 2 messages with an ephemeral cache hint, gated on `model.capabilities.cache.prompt`. Adapters lower the hint structurally — Anthropic emits `cache_control: { type: "ephemeral" }` on the marked block, Bedrock emits a positional `cachePoint: { type: "default" }` after the marked block (added in 9d7d518ac). The capability gate keeps non-cache adapters (OpenAI Responses, Gemini, OpenAI-compat Chat) hint-free. Why a Patch and not bridge code: - packages/llm/AGENTS.md TODO explicitly calls for cache hint patches - Other consumers of @opencode-ai/llm get caching for free - The bridge stays focused on shape conversion (MessageV2 \u2192 LLMRequest) - Patches compose via ProviderPatch.defaults (now includes this one) - The capability gate is a typed predicate, not provider-name matching Implementation: - New `cachePromptHints` patch in provider/patch.ts. The `withCacheOnLastText` helper uses Array.findLastIndex (codebase idiom) and short-circuits when no text part exists so messages with only tool-result content are returned identity-equal. - `EPHEMERAL_CACHE` is a single shared CacheHint instance — no per-request allocation, preserves `instanceof` for any consumer that checks class identity. - Added to `ProviderPatch.defaults` so existing callers that pass `defaults` get cache support automatically. Tests (5 new in patch.test.ts): - Marks first 2 system parts on cache-capable models. - Marks last text part of last 2 messages. - Targets the last text part when a message has trailing non-text content (assistant text + tool-call). - Returns content unchanged (identity-equal) when no text part exists, so pure tool-result messages don't allocate. - No-op when the model does not advertise prompt caching. Bridge cleanup: - Removed `applyCachePolicy`, `withCacheOnLastText`, `updateMessageContent`, `EPHEMERAL_CACHE` from llm-native.ts (-30 lines of bridge-side cache code). - Dropped now-unused `CacheHint`, `LLMRequest`, `Message` imports. - The bridge's only responsibility is now MessageV2 lowering; callers wire `patches: ProviderPatch.defaults` at client construction. OpenCode tests rewritten: - Old: assert on `request.system[N].cache` (bridge internals). - New: assert on `prepared.target` after running through `LLMClient.make({ adapters, patches: ProviderPatch.defaults }) .prepare(request)` — verifies the full lowering end-to-end. - Anthropic: target.system[0..1] carry `cache_control: ephemeral`, target.messages[1..2] carry it on the final text block. - Bedrock: target has `cachePoint` markers after each cached block. - Non-cache (OpenAI Responses): JSON.stringify(target) contains none of `cache_control` / `cachePoint` / `ephemeral`. Verified: bun typecheck clean across both packages, 120/0/0 in LLM package (was 113; +7 from new patch tests counting parameter variations), 21/0/0 in OpenCode native+bridge tests. --- packages/llm/src/provider/patch.ts | 49 +++++++- packages/llm/test/patch.test.ts | 115 ++++++++++++++++++ packages/opencode/src/session/llm-native.ts | 74 +++-------- .../opencode/test/session/llm-native.test.ts | 83 ++++++++----- 4 files changed, 235 insertions(+), 86 deletions(-) diff --git a/packages/llm/src/provider/patch.ts b/packages/llm/src/provider/patch.ts index 7aa8116a03..75e2ede070 100644 --- a/packages/llm/src/provider/patch.ts +++ b/packages/llm/src/provider/patch.ts @@ -1,4 +1,5 @@ -import { Model, Patch } from "../patch" +import { Model, Patch, predicate } from "../patch" +import { CacheHint } from "../schema" import type { ContentPart, LLMRequest } from "../schema" const schemaIntentKeys = [ @@ -103,6 +104,50 @@ export const sanitizeGeminiToolSchema = Patch.toolSchema("gemini.sanitize-tool-s }), }) -export const defaults = [removeEmptyAnthropicContent, scrubClaudeToolIds, scrubMistralToolIds, sanitizeGeminiToolSchema] +// Single shared CacheHint instance — the cache patch reuses this one object +// across every marked part. Adapters lower CacheHint structurally +// (`cache?.type === "ephemeral"`) so reference equality is incidental, but +// keeping a class instance preserves any consumer that checks +// `instanceof CacheHint`. +const EPHEMERAL_CACHE = new CacheHint({ type: "ephemeral" }) + +const withCacheOnLastText = (content: ReadonlyArray): ReadonlyArray => { + const last = content.findLastIndex((part) => part.type === "text") + if (last === -1) return content + return content.map((part, index) => + index === last && part.type === "text" ? { ...part, cache: EPHEMERAL_CACHE } : part, + ) +} + +// Anthropic and Bedrock both honor up to four positional cache breakpoints. +// We mark the first 2 system parts and the last 2 messages — the same policy +// OpenCode uses on the AI-SDK path (`session.applyCaching` in +// packages/opencode/src/provider/transform.ts). The capability gate makes +// this a no-op for adapters that don't advertise prompt-level caching, so +// non-cache providers (OpenAI Responses, Gemini, OpenAI-compatible Chat) +// are unaffected. +export const cachePromptHints = Patch.prompt("cache.prompt-hints", { + reason: "mark first 2 system parts and last 2 messages with ephemeral cache hints on cache-capable adapters", + when: predicate((context) => context.model.capabilities.cache?.prompt === true), + apply: (request) => ({ + ...request, + system: request.system.map((part, index) => + index < 2 ? { ...part, cache: EPHEMERAL_CACHE } : part, + ), + messages: request.messages.map((message, index) => + index < request.messages.length - 2 + ? message + : { ...message, content: withCacheOnLastText(message.content) }, + ), + }), +}) + +export const defaults = [ + removeEmptyAnthropicContent, + scrubClaudeToolIds, + scrubMistralToolIds, + sanitizeGeminiToolSchema, + cachePromptHints, +] export * as ProviderPatch from "./patch" diff --git a/packages/llm/test/patch.test.ts b/packages/llm/test/patch.test.ts index 6819ca6b5a..3e0069f10d 100644 --- a/packages/llm/test/patch.test.ts +++ b/packages/llm/test/patch.test.ts @@ -105,4 +105,119 @@ describe("llm patch", () => { expect(output.messages[0]?.content[0]).toMatchObject({ type: "tool-call", id: "callbadva" }) expect(output.messages[1]?.content[0]).toMatchObject({ type: "tool-result", id: "callbadva" }) }) + + // Cache hint policy: mark first-2 system + last-2 messages with ephemeral + // cache hints, gated on `model.capabilities.cache.prompt`. Adapters + // (Anthropic, Bedrock) lower the hint to `cache_control` / `cachePoint`. + describe("cachePromptHints", () => { + const cacheCapableModel = (overrides: { provider: string; protocol: "anthropic-messages" | "bedrock-converse" }) => + LLM.model({ + id: "test-model", + provider: overrides.provider, + protocol: overrides.protocol, + capabilities: LLM.capabilities({ cache: { prompt: true, contentBlocks: true } }), + }) + + const runCachePatch = (input: ReturnType) => + plan({ + phase: "prompt", + context: context({ request: input }), + patches: [ProviderPatch.cachePromptHints], + }).apply(input) + + test("marks first 2 system parts with an ephemeral cache hint", () => { + const input = LLM.request({ + id: "cache_system", + model: cacheCapableModel({ provider: "anthropic", protocol: "anthropic-messages" }), + system: ["First", "Second", "Third"].map(LLM.system), + prompt: "hello", + }) + const output = runCachePatch(input) + + expect(output.system).toHaveLength(3) + expect(output.system[0]).toMatchObject({ text: "First", cache: { type: "ephemeral" } }) + expect(output.system[1]).toMatchObject({ text: "Second", cache: { type: "ephemeral" } }) + expect(output.system[2]).toMatchObject({ text: "Third" }) + expect(output.system[2]?.cache).toBeUndefined() + }) + + test("marks the last text part of the last 2 messages on cache-capable models", () => { + const input = LLM.request({ + id: "cache_messages", + model: cacheCapableModel({ provider: "anthropic", protocol: "anthropic-messages" }), + messages: [ + LLM.user([{ type: "text", text: "m0" }]), + LLM.user([{ type: "text", text: "m1" }]), + LLM.user([{ type: "text", text: "m2" }]), + ], + }) + const output = runCachePatch(input) + + expect(output.messages).toHaveLength(3) + // First message untouched. + const first = output.messages[0].content[0] + expect(first).toMatchObject({ type: "text", text: "m0" }) + expect("cache" in first ? first.cache : undefined).toBeUndefined() + // Last 2 messages: cache on the (only) text part. + expect(output.messages[1].content[0]).toMatchObject({ type: "text", text: "m1", cache: { type: "ephemeral" } }) + expect(output.messages[2].content[0]).toMatchObject({ type: "text", text: "m2", cache: { type: "ephemeral" } }) + }) + + test("targets the last text part when a message has trailing non-text content", () => { + const input = LLM.request({ + id: "cache_trailing_tool", + model: cacheCapableModel({ provider: "anthropic", protocol: "anthropic-messages" }), + messages: [ + LLM.assistant([ + { type: "text", text: "calling tool" }, + LLM.toolCall({ id: "call_1", name: "lookup", input: { q: "weather" } }), + ]), + ], + }) + const output = runCachePatch(input) + + const content = output.messages[0].content + expect(content[0]).toMatchObject({ type: "text", text: "calling tool", cache: { type: "ephemeral" } }) + expect(content[1]).toMatchObject({ type: "tool-call", id: "call_1" }) + }) + + test("returns the message unchanged when it has no text part", () => { + const input = LLM.request({ + id: "cache_no_text", + model: cacheCapableModel({ provider: "anthropic", protocol: "anthropic-messages" }), + messages: [ + LLM.toolMessage({ id: "call_1", name: "lookup", result: { ok: true } }), + ], + }) + const output = runCachePatch(input) + + expect(output.messages[0].content[0]).toMatchObject({ type: "tool-result", id: "call_1" }) + // No text part to mark, so the content array is identity-equal — the + // `findLastIndex === -1` short-circuit avoids reallocating. + expect(output.messages[0].content).toBe(input.messages[0].content) + }) + + test("is a no-op when the model does not advertise prompt caching", () => { + const input = LLM.request({ + id: "cache_no_capability", + model: LLM.model({ + id: "gpt-5", + provider: "openai", + protocol: "openai-responses", + // capabilities.cache.prompt defaults to false + }), + system: ["A", "B"].map(LLM.system), + messages: [LLM.user([{ type: "text", text: "hi" }])], + }) + const output = runCachePatch(input) + + // Every text part should be free of cache hints. + for (const part of output.system) expect(part.cache).toBeUndefined() + for (const message of output.messages) { + for (const part of message.content) { + if (part.type === "text") expect(part.cache).toBeUndefined() + } + } + }) + }) }) diff --git a/packages/opencode/src/session/llm-native.ts b/packages/opencode/src/session/llm-native.ts index 7583d0a433..a87aa40aa3 100644 --- a/packages/opencode/src/session/llm-native.ts +++ b/packages/opencode/src/session/llm-native.ts @@ -1,4 +1,4 @@ -import { CacheHint, LLM, type ContentPart, type LLMRequest, type Message as CoreMessage } from "@opencode-ai/llm" +import { LLM, type ContentPart, type Message as CoreMessage } from "@opencode-ai/llm" import { Effect, Schema } from "effect" import { ProviderLLMBridge } from "@/provider/llm-bridge" import * as EffectZod from "@/util/effect-zod" @@ -169,42 +169,6 @@ export const toolDefinition = (input: { readonly model: Provider.Model; readonly }, }) -// Mirrors the AI SDK path's prompt-cache policy, gated by model capability. -const EPHEMERAL_CACHE = new CacheHint({ type: "ephemeral" }) - -const withCacheOnLastText = (content: ReadonlyArray): ReadonlyArray => { - const index = content.findLastIndex((part) => part.type === "text") - if (index === -1) return content - return content.map((part, position) => - position === index && part.type === "text" ? { ...part, cache: EPHEMERAL_CACHE } : part, - ) -} - -const updateMessageContent = (message: CoreMessage, content: ReadonlyArray) => { - if (content === message.content) return message - return LLM.message({ - id: message.id, - role: message.role, - content, - metadata: message.metadata, - native: message.native, - }) -} - -const applyCachePolicy = (request: LLMRequest): LLMRequest => { - if (!request.model.capabilities.cache?.prompt) return request - const system = request.system.map((part, index) => - index < 2 ? { ...part, cache: EPHEMERAL_CACHE } : part, - ) - const lastTwoStart = Math.max(0, request.messages.length - 2) - const messages = request.messages.map((message, index) => - index < lastTwoStart - ? message - : updateMessageContent(message, withCacheOnLastText(message.content)), - ) - return LLM.updateRequest(request, { system, messages }) -} - export const request = Effect.fn("LLMNative.request")(function* (input: RequestInput) { const unsupported = unsupportedPart(input) if (unsupported) { @@ -222,23 +186,25 @@ export const request = Effect.fn("LLMNative.request")(function* (input: RequestI }) } - return applyCachePolicy( - LLM.request({ - id: input.id, - model, - system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [], - messages: input.messages.flatMap(messages), - tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [], - toolChoice: input.toolChoice, - generation: input.generation, - metadata: input.metadata, - native: { - opencodeProviderID: input.provider.id, - opencodeModelID: input.model.id, - ...input.native, - }, - }), - ) + // Cache hints, tool-id scrubbing, and other adapter-aware patches live in + // `@opencode-ai/llm`'s `ProviderPatch` registry. Callers wire them in at + // `client({ adapters, patches: ProviderPatch.defaults })` time so the + // bridge stays focused on shape conversion. + return LLM.request({ + id: input.id, + model, + system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [], + messages: input.messages.flatMap(messages), + tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [], + toolChoice: input.toolChoice, + generation: input.generation, + metadata: input.metadata, + native: { + opencodeProviderID: input.provider.id, + opencodeModelID: input.model.id, + ...input.native, + }, + }) }) export * as LLMNative from "./llm-native" diff --git a/packages/opencode/test/session/llm-native.test.ts b/packages/opencode/test/session/llm-native.test.ts index 82aab6dc32..7e42337a3c 100644 --- a/packages/opencode/test/session/llm-native.test.ts +++ b/packages/opencode/test/session/llm-native.test.ts @@ -1,5 +1,5 @@ import { describe, expect } from "bun:test" -import { AnthropicMessages, BedrockConverse, Gemini, LLMClient, OpenAICompatibleChat, OpenAIResponses } from "@opencode-ai/llm" +import { AnthropicMessages, BedrockConverse, Gemini, LLMClient, OpenAICompatibleChat, OpenAIResponses, ProviderPatch } from "@opencode-ai/llm" import { Cause, Effect, Exit, Layer, Schema } from "effect" import { ModelID, ProviderID } from "../../src/provider/schema" import { LLMNative } from "../../src/session/llm-native" @@ -576,12 +576,13 @@ describe("LLMNative.request", () => { }) })) - // Cache hint policy. The LLM-native path mirrors the AI-SDK applyCaching - // policy from packages/opencode/src/provider/transform.ts: mark the first 2 - // system parts and the last 2 messages as cacheable, gated on the resolved - // model's `capabilities.cache.prompt`. Adapters lower CacheHint to the - // provider-specific marker (cache_control on Anthropic, cachePoint on - // Bedrock); non-cache-capable adapters never see a hint. + // Cache hint policy. The bridge produces a hint-free `LLMRequest`; the + // `ProviderPatch.cachePromptHints` patch (loaded in `ProviderPatch.defaults`) + // marks first-2 system parts and last-2 messages with ephemeral cache + // hints when the model advertises `capabilities.cache.prompt`. Adapters + // then lower the hints to the provider-specific marker — `cache_control` + // on Anthropic, `cachePoint` on Bedrock. Non-cache adapters never see a + // hint thanks to the predicate gate. const anthropicModel = () => model({ @@ -601,7 +602,7 @@ describe("LLMNative.request", () => { }, }) - it.effect("applies cache hints to the first 2 system parts on cache-capable models", () => + it.effect("lowers cache hints to Anthropic cache_control on the first 2 system blocks", () => Effect.gen(function* () { const mdl = anthropicModel() const userID = MessageID.ascending() @@ -611,15 +612,23 @@ describe("LLMNative.request", () => { system: ["First", "Second", "Third"], messages: [userMessage(mdl, userID, [textPart(userID, "hello")])], }) + const prepared = yield* LLMClient.make({ + adapters: [AnthropicMessages.adapter], + patches: ProviderPatch.defaults, + }).prepare(request) - expect(request.system).toHaveLength(3) - expect(request.system[0]).toMatchObject({ text: "First", cache: { type: "ephemeral" } }) - expect(request.system[1]).toMatchObject({ text: "Second", cache: { type: "ephemeral" } }) - expect(request.system[2]).toMatchObject({ text: "Third" }) - expect(request.system[2].cache).toBeUndefined() + expect(prepared.target).toMatchObject({ + system: [ + { type: "text", text: "First", cache_control: { type: "ephemeral" } }, + { type: "text", text: "Second", cache_control: { type: "ephemeral" } }, + { type: "text", text: "Third" }, + ], + }) + // The third system block must not carry a cache_control marker. + expect((prepared.target as { system: ReadonlyArray<{ cache_control?: unknown }> }).system[2].cache_control).toBeUndefined() })) - it.effect("applies cache hints to the final text part of the last 2 messages on cache-capable models", () => + it.effect("lowers cache hints to Anthropic cache_control on the last text block of the last 2 messages", () => Effect.gen(function* () { const mdl = anthropicModel() const messageIds = [MessageID.ascending(), MessageID.ascending(), MessageID.ascending()] @@ -628,14 +637,21 @@ describe("LLMNative.request", () => { model: mdl, messages: messageIds.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])), }) + const prepared = yield* LLMClient.make({ + adapters: [AnthropicMessages.adapter], + patches: ProviderPatch.defaults, + }).prepare(request) - expect(request.messages).toHaveLength(3) - // First message: no cache hint. - const first = request.messages[0].content[0] - if (first.type === "text") expect(first.cache).toBeUndefined() - // Last two messages: cache on the (only) text part. - expect(request.messages[1].content[0]).toMatchObject({ type: "text", text: "m1", cache: { type: "ephemeral" } }) - expect(request.messages[2].content[0]).toMatchObject({ type: "text", text: "m2", cache: { type: "ephemeral" } }) + expect(prepared.target).toMatchObject({ + messages: [ + { role: "user", content: [{ type: "text", text: "m0" }] }, + { role: "user", content: [{ type: "text", text: "m1", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ type: "text", text: "m2", cache_control: { type: "ephemeral" } }] }, + ], + }) + // The first message's text must not carry cache_control. + const target = prepared.target as { messages: ReadonlyArray<{ content: ReadonlyArray<{ cache_control?: unknown }> }> } + expect(target.messages[0].content[0].cache_control).toBeUndefined() })) it.effect("lowers cache hints to Bedrock Converse cachePoint marker blocks end-to-end", () => @@ -648,7 +664,10 @@ describe("LLMNative.request", () => { system: ["You are concise."], messages: [userMessage(mdl, userID, [textPart(userID, "hello")])], }) - const prepared = yield* LLMClient.make({ adapters: [BedrockConverse.adapter] }).prepare(request) + const prepared = yield* LLMClient.make({ + adapters: [BedrockConverse.adapter], + patches: ProviderPatch.defaults, + }).prepare(request) expect(prepared.target).toMatchObject({ system: [{ text: "You are concise." }, { cachePoint: { type: "default" } }], @@ -663,8 +682,8 @@ describe("LLMNative.request", () => { it.effect("does not apply cache hints when the model does not support prompt caching", () => Effect.gen(function* () { - // gpt-5 / openai resolves to openai-responses, which advertises - // capabilities.cache.prompt: false. The bridge must skip the policy. + // gpt-5 / openai resolves to openai-responses with cache.prompt: false. + // The patch's `when` predicate must skip, leaving the target hint-free. const mdl = model() const ids = [MessageID.ascending(), MessageID.ascending()] const request = yield* LLMNative.request({ @@ -673,12 +692,16 @@ describe("LLMNative.request", () => { system: ["A", "B", "C"], messages: ids.map((id, index) => userMessage(mdl, id, [textPart(id, `m${index}`)])), }) + const prepared = yield* LLMClient.make({ + adapters: [OpenAIResponses.adapter], + patches: ProviderPatch.defaults, + }).prepare(request) - for (const part of request.system) expect(part.cache).toBeUndefined() - for (const message of request.messages) { - for (const part of message.content) { - if (part.type === "text") expect(part.cache).toBeUndefined() - } - } + // The serialized OpenAI Responses payload has no cache concept; the + // assertion is that nothing in the target carries a cache marker. + const json = JSON.stringify(prepared.target) + expect(json).not.toContain("cache_control") + expect(json).not.toContain("cachePoint") + expect(json).not.toContain("ephemeral") })) })