feat(llm): Bedrock Converse cache hints, image, and document blocks

Close the parity gaps deferred from the original Bedrock pass. Schema additions on the Converse target: - BedrockImageBlock for { image: { format, source: { bytes } } }. Supported formats per Converse docs: png, jpeg, gif, webp. - BedrockDocumentBlock for { document: { format, name, source: { bytes } } }. Supported formats: pdf, csv, doc, docx, xls, xlsx, html, txt, md. - BedrockCachePointBlock for the positional { cachePoint: { type } } marker. Currently emits the only Bedrock cache type, 'default'. A TODO marks where to map ttlSeconds → ttl ('5m' | '1h') once we have a recorded cassette to validate the wire shape. Lowering: - TextPart and SystemPart cache hints emit a positional cachePoint marker right after their text block. Both 'ephemeral' and 'persistent' CacheHint types map onto Bedrock's 'default' since Bedrock does not distinguish — this matches the convention the Anthropic adapter uses (cache?.type === 'ephemeral' check). - MediaPart routes by mediaType: 'image/*' → image block, everything else → document block. MIME type → format mapping is via IMAGE_FORMATS / DOCUMENT_FORMATS records typed with 'as const satisfies' so the keys stay narrow at compile time. - A small textWithCache helper collapses the 'push text, push cachePoint if cache is set' pattern that would otherwise repeat at three callsites (system, user-text, assistant-text). - Bytes are encoded via ProviderShared.mediaBytes — the shared helper Kit landed in c3346f7dc. Bug fix: lowerSystem was dead code in the previous draft. The prepare() function still inlined the pre-cache .map(...) that discarded system cache hints. prepare() now calls lowerSystem so the cachePoint markers actually flow through. Tests (7 new fixtures, all green): - Cache hint on system / user-text / assistant-text emits cachePoint after text in each context. - No cache hint → no cachePoint emitted (regression guard). - Image lowering covers png / jpeg / jpg-alias / webp. - Uint8Array image bytes are base64-encoded ([1,2,3,4,5] → AQIDBAU=). - Document lowering with filename round-trip and missing-filename fallback to 'document.<format>'. - Unsupported image MIME (image/svg+xml) is rejected with a clear error message. - Unsupported document MIME (application/x-tar) is rejected with a clear error message. Recorded cassettes for cache hints, images, and documents are still TODO — the wire shapes are exercised deterministically here and will be validated against a live model in a follow-up cassette pass. Verified: bun typecheck clean, 113 pass / 0 fail / 0 skip (was 106; +7 from the new fixture tests).
2026-04-26 20:32:37 -04:00
parent ecd73f26fc
commit 096c305a55
2 changed files with 332 additions and 10 deletions
@@ -7,9 +7,11 @@ import { Adapter } from "../adapter"
 import { capabilities, model as llmModel, type ModelInput } from "../llm"
 import {
  Usage,
+  type CacheHint,
  type FinishReason,
  type LLMEvent,
  type LLMRequest,
+  type MediaPart,
  type ProviderChunkError,
  type ToolCallPart,
  type ToolDefinition,
@@ -49,6 +51,7 @@ export type BedrockConverseModelInput = Omit<ModelInput, "provider" | "protocol"
 const BedrockTextBlock = Schema.Struct({
  text: Schema.String,
 })
+type BedrockTextBlock = Schema.Schema.Type<typeof BedrockTextBlock>

 const BedrockToolUseBlock = Schema.Struct({
  toolUse: Schema.Struct({
@@ -84,8 +87,66 @@ const BedrockReasoningBlock = Schema.Struct({
  }),
 })

-const BedrockUserBlock = Schema.Union([BedrockTextBlock, BedrockToolResultBlock])
-const BedrockAssistantBlock = Schema.Union([BedrockTextBlock, BedrockReasoningBlock, BedrockToolUseBlock])
+// Image block. Bedrock Converse accepts `format` as the file extension and
+// `source.bytes` as a base64 string (binary upload via base64 in the JSON
+// wire format). Supported formats per the Converse docs: png, jpeg, gif, webp.
+const BedrockImageFormat = Schema.Literals(["png", "jpeg", "gif", "webp"])
+type BedrockImageFormat = Schema.Schema.Type<typeof BedrockImageFormat>
+const BedrockImageBlock = Schema.Struct({
+  image: Schema.Struct({
+    format: BedrockImageFormat,
+    source: Schema.Struct({ bytes: Schema.String }),
+  }),
+})
+type BedrockImageBlock = Schema.Schema.Type<typeof BedrockImageBlock>
+
+// Document block. Required `name` is the user-facing filename so the model
+// can reference it. Supported formats per the Converse docs: pdf, csv, doc,
+// docx, xls, xlsx, html, txt, md.
+const BedrockDocumentFormat = Schema.Literals([
+  "pdf",
+  "csv",
+  "doc",
+  "docx",
+  "xls",
+  "xlsx",
+  "html",
+  "txt",
+  "md",
+])
+type BedrockDocumentFormat = Schema.Schema.Type<typeof BedrockDocumentFormat>
+const BedrockDocumentBlock = Schema.Struct({
+  document: Schema.Struct({
+    format: BedrockDocumentFormat,
+    name: Schema.String,
+    source: Schema.Struct({ bytes: Schema.String }),
+  }),
+})
+type BedrockDocumentBlock = Schema.Schema.Type<typeof BedrockDocumentBlock>
+
+// Cache breakpoint marker. Inserted positionally between content blocks (or
+// after a system text / tool spec) to mark the prefix as cacheable. Bedrock
+// Converse currently exposes `default` as the only cache-point type.
+const BedrockCachePointBlock = Schema.Struct({
+  cachePoint: Schema.Struct({ type: Schema.Literal("default") }),
+})
+type BedrockCachePointBlock = Schema.Schema.Type<typeof BedrockCachePointBlock>
+
+const BedrockUserBlock = Schema.Union([
+  BedrockTextBlock,
+  BedrockImageBlock,
+  BedrockDocumentBlock,
+  BedrockToolResultBlock,
+  BedrockCachePointBlock,
+])
+type BedrockUserBlock = Schema.Schema.Type<typeof BedrockUserBlock>
+
+const BedrockAssistantBlock = Schema.Union([
+  BedrockTextBlock,
+  BedrockReasoningBlock,
+  BedrockToolUseBlock,
+  BedrockCachePointBlock,
+])
 type BedrockAssistantBlock = Schema.Schema.Type<typeof BedrockAssistantBlock>

 const BedrockMessage = Schema.Union([
@@ -94,7 +155,8 @@ const BedrockMessage = Schema.Union([
 ])
 type BedrockMessage = Schema.Schema.Type<typeof BedrockMessage>

-const BedrockSystem = Schema.Struct({ text: Schema.String })
+const BedrockSystemBlock = Schema.Union([BedrockTextBlock, BedrockCachePointBlock])
+type BedrockSystemBlock = Schema.Schema.Type<typeof BedrockSystemBlock>

 const BedrockTool = Schema.Struct({
  toolSpec: Schema.Struct({
@@ -116,7 +178,7 @@ const BedrockToolChoice = Schema.Union([
 const BedrockTargetFields = {
  modelId: Schema.String,
  messages: Schema.Array(BedrockMessage),
-  system: Schema.optional(Schema.Array(BedrockSystem)),
+  system: Schema.optional(Schema.Array(BedrockSystemBlock)),
  inferenceConfig: Schema.optional(
    Schema.Struct({
      maxTokens: Schema.optional(Schema.Number),
@@ -246,6 +308,87 @@ const lowerTool = (tool: ToolDefinition): BedrockTool => ({
  },
 })

+// Bedrock cache markers are positional — emit a `cachePoint` block right after
+// the content the caller wants treated as a cacheable prefix. Bedrock currently
+// exposes one cache-point type (`default`); both `ephemeral` and `persistent`
+// hints from the common `CacheHint` shape map onto it. Other cache-hint types
+// (none today) would need explicit handling.
+//
+// TODO: Bedrock recently added optional `ttl: "5m" | "1h"` on cachePoint —
+// once we have a recorded cassette to validate the wire shape, map
+// `CacheHint.ttlSeconds` here.
+const CACHE_POINT_DEFAULT: BedrockCachePointBlock = { cachePoint: { type: "default" } }
+
+const cachePointBlock = (cache: CacheHint | undefined): BedrockCachePointBlock | undefined => {
+  if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined
+  return CACHE_POINT_DEFAULT
+}
+
+// Emit a text block followed by an optional positional cache marker. Used by
+// system, user-text, and assistant-text lowering — all three share the same
+// "push text, push cachePoint if cache hint is present" shape. The return type
+// is the lowest common denominator (text | cachePoint) so callers can spread
+// it into any of the three block-union arrays.
+const textWithCache = (
+  text: string,
+  cache: CacheHint | undefined,
+): Array<BedrockTextBlock | BedrockCachePointBlock> => {
+  const cachePoint = cachePointBlock(cache)
+  return cachePoint ? [{ text }, cachePoint] : [{ text }]
+}
+
+// MIME type → Bedrock format mapping. Bedrock distinguishes image vs document
+// by the top-level block type, not the mediaType, so `lowerMedia` routes by
+// the `image/` prefix and the leaf functions look up the format. `image/jpg`
+// is included as a non-standard alias commonly seen in user-supplied data.
+const IMAGE_FORMATS = {
+  "image/png": "png",
+  "image/jpeg": "jpeg",
+  "image/jpg": "jpeg",
+  "image/gif": "gif",
+  "image/webp": "webp",
+} as const satisfies Record<string, BedrockImageFormat>
+
+const DOCUMENT_FORMATS = {
+  "application/pdf": "pdf",
+  "text/csv": "csv",
+  "application/msword": "doc",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+  "application/vnd.ms-excel": "xls",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+  "text/html": "html",
+  "text/plain": "txt",
+  "text/markdown": "md",
+} as const satisfies Record<string, BedrockDocumentFormat>
+
+// Bedrock document blocks require a name; default to the filename if the
+// caller supplied one, otherwise generate a stable placeholder so the model
+// still sees a valid block.
+const lowerImage = (part: MediaPart, mime: string) => {
+  const format = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
+  if (!format) return invalid(`Bedrock Converse does not support image media type ${part.mediaType}`)
+  return Effect.succeed<BedrockImageBlock>({
+    image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
+  })
+}
+
+const lowerDocument = (part: MediaPart, mime: string) => {
+  const format = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
+  if (!format) return invalid(`Bedrock Converse does not support document media type ${part.mediaType}`)
+  return Effect.succeed<BedrockDocumentBlock>({
+    document: {
+      format,
+      name: part.filename ?? `document.${format}`,
+      source: { bytes: ProviderShared.mediaBytes(part) },
+    },
+  })
+}
+
+const lowerMedia = (part: MediaPart) => {
+  const mime = part.mediaType.toLowerCase()
+  return mime.startsWith("image/") ? lowerImage(part, mime) : lowerDocument(part, mime)
+}
+
 const lowerToolChoice = Effect.fn("BedrockConverse.lowerToolChoice")(function* (
  toolChoice: NonNullable<LLMRequest["toolChoice"]>,
 ) {
@@ -280,13 +423,17 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (requ

  for (const message of request.messages) {
    if (message.role === "user") {
-      const content: Array<Schema.Schema.Type<typeof BedrockUserBlock>> = []
+      const content: BedrockUserBlock[] = []
      for (const part of message.content) {
        if (part.type === "text") {
-          content.push({ text: part.text })
+          content.push(...textWithCache(part.text, part.cache))
          continue
        }
-        return yield* invalid("Bedrock Converse user messages only support text content for now")
+        if (part.type === "media") {
+          content.push(yield* lowerMedia(part))
+          continue
+        }
+        return yield* invalid("Bedrock Converse user messages only support text and media content for now")
      }
      messages.push({ role: "user", content })
      continue
@@ -296,7 +443,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (requ
      const content: BedrockAssistantBlock[] = []
      for (const part of message.content) {
        if (part.type === "text") {
-          content.push({ text: part.text })
+          content.push(...textWithCache(part.text, part.cache))
          continue
        }
        if (part.type === "reasoning") {
@@ -329,12 +476,17 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (requ
  return messages
 })

+// System prompts share the cache-point convention: emit the text block, then
+// optionally a positional `cachePoint` marker.
+const lowerSystem = (system: ReadonlyArray<LLMRequest["system"][number]>): BedrockSystemBlock[] =>
+  system.flatMap((part) => textWithCache(part.text, part.cache))
+
 const prepare = Effect.fn("BedrockConverse.prepare")(function* (request: LLMRequest) {
  const toolChoice = request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined
  return {
    modelId: request.model.id,
    messages: yield* lowerMessages(request),
-    system: request.system.length === 0 ? undefined : request.system.map((part) => ({ text: part.text })),
+    system: request.system.length === 0 ? undefined : lowerSystem(request.system),
    inferenceConfig:
      request.generation.maxTokens === undefined &&
      request.generation.temperature === undefined &&
@@ -2,7 +2,7 @@ import { EventStreamCodec } from "@smithy/eventstream-codec"
 import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
 import { describe, expect } from "bun:test"
 import { Effect, Layer } from "effect"
-import { LLM } from "../../src"
+import { CacheHint, LLM } from "../../src"
 import { client } from "../../src/adapter"
 import { BedrockConverse } from "../../src/provider/bedrock-converse"
 import { testEffect } from "../lib/effect"
@@ -288,6 +288,176 @@ describe("Bedrock Converse adapter", () => {
      })
    }),
  )
+
+  it.effect("emits cachePoint markers after system, user-text, and assistant-text with cache hints", () =>
+    Effect.gen(function* () {
+      const cache = new CacheHint({ type: "ephemeral" })
+      const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
+        LLM.request({
+          id: "req_cache",
+          model,
+          system: [{ type: "text", text: "System prefix.", cache }],
+          messages: [
+            LLM.user([{ type: "text", text: "User prefix.", cache }]),
+            LLM.assistant([{ type: "text", text: "Assistant prefix.", cache }]),
+          ],
+          generation: { maxTokens: 16, temperature: 0 },
+        }),
+      )
+
+      expect(prepared.target).toMatchObject({
+        // System: text block followed by cachePoint marker.
+        system: [{ text: "System prefix." }, { cachePoint: { type: "default" } }],
+        messages: [
+          {
+            role: "user",
+            content: [{ text: "User prefix." }, { cachePoint: { type: "default" } }],
+          },
+          {
+            role: "assistant",
+            content: [{ text: "Assistant prefix." }, { cachePoint: { type: "default" } }],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("does not emit cachePoint when no cache hint is set", () =>
+    Effect.gen(function* () {
+      const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(baseRequest)
+      expect(prepared.target).toMatchObject({
+        system: [{ text: "You are concise." }],
+        messages: [{ role: "user", content: [{ text: "Say hello." }] }],
+      })
+    }),
+  )
+
+  it.effect("lowers image media into Bedrock image blocks", () =>
+    Effect.gen(function* () {
+      const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
+        LLM.request({
+          id: "req_image",
+          model,
+          messages: [
+            LLM.user([
+              { type: "text", text: "What is in this image?" },
+              { type: "media", mediaType: "image/png", data: "AAAA" },
+              { type: "media", mediaType: "image/jpeg", data: "BBBB" },
+              { type: "media", mediaType: "image/jpg", data: "CCCC" },
+              { type: "media", mediaType: "image/webp", data: "DDDD" },
+            ]),
+          ],
+        }),
+      )
+
+      expect(prepared.target).toMatchObject({
+        messages: [
+          {
+            role: "user",
+            content: [
+              { text: "What is in this image?" },
+              { image: { format: "png", source: { bytes: "AAAA" } } },
+              { image: { format: "jpeg", source: { bytes: "BBBB" } } },
+              // image/jpg is a non-standard alias; we map it to jpeg.
+              { image: { format: "jpeg", source: { bytes: "CCCC" } } },
+              { image: { format: "webp", source: { bytes: "DDDD" } } },
+            ],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("base64-encodes Uint8Array image bytes", () =>
+    Effect.gen(function* () {
+      const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
+        LLM.request({
+          id: "req_image_bytes",
+          model,
+          messages: [
+            LLM.user([
+              { type: "media", mediaType: "image/png", data: new Uint8Array([1, 2, 3, 4, 5]) },
+            ]),
+          ],
+        }),
+      )
+
+      // Buffer.from([1,2,3,4,5]).toString("base64") === "AQIDBAU="
+      expect(prepared.target).toMatchObject({
+        messages: [
+          {
+            role: "user",
+            content: [{ image: { format: "png", source: { bytes: "AQIDBAU=" } } }],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("lowers document media into Bedrock document blocks with format and name", () =>
+    Effect.gen(function* () {
+      const prepared = yield* client({ adapters: [BedrockConverse.adapter] }).prepare(
+        LLM.request({
+          id: "req_doc",
+          model,
+          messages: [
+            LLM.user([
+              { type: "media", mediaType: "application/pdf", data: "PDFDATA", filename: "report.pdf" },
+              { type: "media", mediaType: "text/csv", data: "CSVDATA" },
+            ]),
+          ],
+        }),
+      )
+
+      expect(prepared.target).toMatchObject({
+        messages: [
+          {
+            role: "user",
+            content: [
+              // Filename round-trips when supplied.
+              { document: { format: "pdf", name: "report.pdf", source: { bytes: "PDFDATA" } } },
+              // Falls back to a stable placeholder when filename is missing.
+              { document: { format: "csv", name: "document.csv", source: { bytes: "CSVDATA" } } },
+            ],
+          },
+        ],
+      })
+    }),
+  )
+
+  it.effect("rejects unsupported image media types", () =>
+    Effect.gen(function* () {
+      const error = yield* client({ adapters: [BedrockConverse.adapter] })
+        .prepare(
+          LLM.request({
+            id: "req_bad_image",
+            model,
+            messages: [LLM.user([{ type: "media", mediaType: "image/svg+xml", data: "x" }])],
+          }),
+        )
+        .pipe(Effect.flip)
+
+      expect(error.message).toContain("Bedrock Converse does not support image media type image/svg+xml")
+    }),
+  )
+
+  it.effect("rejects unsupported document media types", () =>
+    Effect.gen(function* () {
+      const error = yield* client({ adapters: [BedrockConverse.adapter] })
+        .prepare(
+          LLM.request({
+            id: "req_bad_doc",
+            model,
+            messages: [
+              LLM.user([{ type: "media", mediaType: "application/x-tar", data: "x", filename: "a.tar" }]),
+            ],
+          }),
+        )
+        .pipe(Effect.flip)
+
+      expect(error.message).toContain("Bedrock Converse does not support document media type application/x-tar")
+    }),
+  )
 })

 // Live recorded integration tests. Run with `RECORD=true AWS_ACCESS_KEY_ID=...