feat(llm): cachePromptHints patch with first-2 system / last-2 messages policy
Lift the prompt-cache policy out of OpenCode's bridge and into the
LLM package as a typed, gated patch. The policy mirrors the AI-SDK
applyCaching path (packages/opencode/src/provider/transform.ts:229):
mark the first 2 system parts and the last 2 messages with an
ephemeral cache hint, gated on `model.capabilities.cache.prompt`.
Adapters lower the hint structurally — Anthropic emits
`cache_control: { type: "ephemeral" }` on the marked block,
Bedrock emits a positional `cachePoint: { type: "default" }`
after the marked block (added in 9d7d518ac). The capability gate
keeps non-cache adapters (OpenAI Responses, Gemini, OpenAI-compat
Chat) hint-free.
Why a Patch and not bridge code:
- packages/llm/AGENTS.md TODO explicitly calls for cache hint patches
- Other consumers of @opencode-ai/llm get caching for free
- The bridge stays focused on shape conversion (MessageV2 \u2192 LLMRequest)
- Patches compose via ProviderPatch.defaults (now includes this one)
- The capability gate is a typed predicate, not provider-name matching
Implementation:
- New `cachePromptHints` patch in provider/patch.ts. The
`withCacheOnLastText` helper uses Array.findLastIndex (codebase
idiom) and short-circuits when no text part exists so messages
with only tool-result content are returned identity-equal.
- `EPHEMERAL_CACHE` is a single shared CacheHint instance — no
per-request allocation, preserves `instanceof` for any consumer
that checks class identity.
- Added to `ProviderPatch.defaults` so existing callers that pass
`defaults` get cache support automatically.
Tests (5 new in patch.test.ts):
- Marks first 2 system parts on cache-capable models.
- Marks last text part of last 2 messages.
- Targets the last text part when a message has trailing
non-text content (assistant text + tool-call).
- Returns content unchanged (identity-equal) when no text part
exists, so pure tool-result messages don't allocate.
- No-op when the model does not advertise prompt caching.
Bridge cleanup:
- Removed `applyCachePolicy`, `withCacheOnLastText`,
`updateMessageContent`, `EPHEMERAL_CACHE` from llm-native.ts
(-30 lines of bridge-side cache code).
- Dropped now-unused `CacheHint`, `LLMRequest`, `Message` imports.
- The bridge's only responsibility is now MessageV2 lowering;
callers wire `patches: ProviderPatch.defaults` at client
construction.
OpenCode tests rewritten:
- Old: assert on `request.system[N].cache` (bridge internals).
- New: assert on `prepared.target` after running through
`LLMClient.make({ adapters, patches: ProviderPatch.defaults })
.prepare(request)` — verifies the full lowering end-to-end.
- Anthropic: target.system[0..1] carry `cache_control: ephemeral`,
target.messages[1..2] carry it on the final text block.
- Bedrock: target has `cachePoint` markers after each cached block.
- Non-cache (OpenAI Responses): JSON.stringify(target) contains
none of `cache_control` / `cachePoint` / `ephemeral`.
Verified: bun typecheck clean across both packages, 120/0/0 in LLM
package (was 113; +7 from new patch tests counting parameter
variations), 21/0/0 in OpenCode native+bridge tests.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { CacheHint, LLM, type ContentPart, type LLMRequest, type Message as CoreMessage } from "@opencode-ai/llm"
|
||||
import { LLM, type ContentPart, type Message as CoreMessage } from "@opencode-ai/llm"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { ProviderLLMBridge } from "@/provider/llm-bridge"
|
||||
import * as EffectZod from "@/util/effect-zod"
|
||||
@@ -169,42 +169,6 @@ export const toolDefinition = (input: { readonly model: Provider.Model; readonly
|
||||
},
|
||||
})
|
||||
|
||||
// Mirrors the AI SDK path's prompt-cache policy, gated by model capability.
|
||||
const EPHEMERAL_CACHE = new CacheHint({ type: "ephemeral" })
|
||||
|
||||
const withCacheOnLastText = (content: ReadonlyArray<ContentPart>): ReadonlyArray<ContentPart> => {
|
||||
const index = content.findLastIndex((part) => part.type === "text")
|
||||
if (index === -1) return content
|
||||
return content.map((part, position) =>
|
||||
position === index && part.type === "text" ? { ...part, cache: EPHEMERAL_CACHE } : part,
|
||||
)
|
||||
}
|
||||
|
||||
const updateMessageContent = (message: CoreMessage, content: ReadonlyArray<ContentPart>) => {
|
||||
if (content === message.content) return message
|
||||
return LLM.message({
|
||||
id: message.id,
|
||||
role: message.role,
|
||||
content,
|
||||
metadata: message.metadata,
|
||||
native: message.native,
|
||||
})
|
||||
}
|
||||
|
||||
const applyCachePolicy = (request: LLMRequest): LLMRequest => {
|
||||
if (!request.model.capabilities.cache?.prompt) return request
|
||||
const system = request.system.map((part, index) =>
|
||||
index < 2 ? { ...part, cache: EPHEMERAL_CACHE } : part,
|
||||
)
|
||||
const lastTwoStart = Math.max(0, request.messages.length - 2)
|
||||
const messages = request.messages.map((message, index) =>
|
||||
index < lastTwoStart
|
||||
? message
|
||||
: updateMessageContent(message, withCacheOnLastText(message.content)),
|
||||
)
|
||||
return LLM.updateRequest(request, { system, messages })
|
||||
}
|
||||
|
||||
export const request = Effect.fn("LLMNative.request")(function* (input: RequestInput) {
|
||||
const unsupported = unsupportedPart(input)
|
||||
if (unsupported) {
|
||||
@@ -222,23 +186,25 @@ export const request = Effect.fn("LLMNative.request")(function* (input: RequestI
|
||||
})
|
||||
}
|
||||
|
||||
return applyCachePolicy(
|
||||
LLM.request({
|
||||
id: input.id,
|
||||
model,
|
||||
system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [],
|
||||
messages: input.messages.flatMap(messages),
|
||||
tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [],
|
||||
toolChoice: input.toolChoice,
|
||||
generation: input.generation,
|
||||
metadata: input.metadata,
|
||||
native: {
|
||||
opencodeProviderID: input.provider.id,
|
||||
opencodeModelID: input.model.id,
|
||||
...input.native,
|
||||
},
|
||||
}),
|
||||
)
|
||||
// Cache hints, tool-id scrubbing, and other adapter-aware patches live in
|
||||
// `@opencode-ai/llm`'s `ProviderPatch` registry. Callers wire them in at
|
||||
// `client({ adapters, patches: ProviderPatch.defaults })` time so the
|
||||
// bridge stays focused on shape conversion.
|
||||
return LLM.request({
|
||||
id: input.id,
|
||||
model,
|
||||
system: input.system?.filter((part) => part.trim() !== "").map(LLM.system) ?? [],
|
||||
messages: input.messages.flatMap(messages),
|
||||
tools: input.tools?.map((tool) => toolDefinition({ model: input.model, tool })) ?? [],
|
||||
toolChoice: input.toolChoice,
|
||||
generation: input.generation,
|
||||
metadata: input.metadata,
|
||||
native: {
|
||||
opencodeProviderID: input.provider.id,
|
||||
opencodeModelID: input.model.id,
|
||||
...input.native,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
export * as LLMNative from "./llm-native"
|
||||
|
||||
Reference in New Issue
Block a user