Compare commits

...

2 Commits

Author SHA1 Message Date
Aiden Cline af6b036b22 fix(ai): address Google image audit findings 2026-07-19 16:15:33 +00:00
Aiden Cline 5969025f98 feat(ai): add Google image generation 2026-07-19 16:12:56 +00:00
6 changed files with 713 additions and 5 deletions
+31
View File
@@ -45,6 +45,37 @@ const program = Effect.gen(function* () {
})
```
Google's current Gemini image models use the same direct API:
```ts
import { Google } from "@opencode-ai/ai/providers"
const googleProgram = Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY,
image: {
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true },
},
}).image("gemini-3.1-flash-image"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
providerOptions: { google: { imageSize: "2K" } },
})
return response.images
})
```
`Google.configure(...).image(...)` is limited to Gemini `generateContent` image models (the Nano Banana family,
including `gemini-2.5-flash-image`, `gemini-3-pro-image`, and `gemini-3.1-flash-image` plus their preview
variants). Imagen model IDs use a different API and are rejected locally. Google provider options are `imageSize`,
`thinkingLevel` (`MINIMAL`, `LOW`, `MEDIUM`, or `HIGH`), and `includeThoughts`; request-level options override the
configured defaults. The API does not support a requested image count or exact pixel dimensions, so `count` and
`size` are rejected; use `aspectRatio` and the provider-specific `imageSize` tier instead. This route performs one
direct `Image.generate` request and does not provide conversational image continuation.
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
```ts
+324
View File
@@ -0,0 +1,324 @@
import { Effect, Encoding, Schema } from "effect"
import { Headers, HttpClientRequest } from "effect/unstable/http"
import {
GeneratedImage,
ImageModel,
ImageResponse,
type ImageModelDefaults,
type ImageRequest,
type ImageRoute,
} from "../image"
import { Auth, type Definition as AuthDefinition } from "../route/auth"
import {
InvalidProviderOutputReason,
LLMError,
Usage,
mergeHttpOptions,
mergeJsonRecords,
type ProviderMetadata,
} from "../schema"
import { ProviderShared } from "./shared"
const ADAPTER = "google-images"
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
export const GoogleThinkingLevel = Schema.Literals(["MINIMAL", "LOW", "MEDIUM", "HIGH"])
export type GoogleThinkingLevel = Schema.Schema.Type<typeof GoogleThinkingLevel>
export interface GoogleImageOptions {
readonly imageSize?: string
readonly thinkingLevel?: GoogleThinkingLevel
readonly includeThoughts?: boolean
}
const GoogleImageBody = Schema.Struct({
contents: Schema.Array(
Schema.Struct({
role: Schema.Literal("user"),
parts: Schema.Array(Schema.Struct({ text: Schema.String })),
}),
),
generationConfig: Schema.Struct({
responseModalities: Schema.Array(Schema.Literal("IMAGE")),
imageConfig: Schema.optional(
Schema.Struct({
aspectRatio: Schema.optional(Schema.String),
imageSize: Schema.optional(Schema.String),
}),
),
seed: Schema.optional(Schema.Number),
thinkingConfig: Schema.optional(
Schema.Struct({
thinkingLevel: Schema.optional(GoogleThinkingLevel),
includeThoughts: Schema.optional(Schema.Boolean),
}),
),
}),
})
export type GoogleImageBody = Schema.Schema.Type<typeof GoogleImageBody>
const GoogleUsage = Schema.StructWithRest(
Schema.Struct({
cachedContentTokenCount: Schema.optional(Schema.Number),
thoughtsTokenCount: Schema.optional(Schema.Number),
promptTokenCount: Schema.optional(Schema.Number),
candidatesTokenCount: Schema.optional(Schema.Number),
totalTokenCount: Schema.optional(Schema.Number),
promptTokensDetails: Schema.optional(Schema.Unknown),
candidatesTokensDetails: Schema.optional(Schema.Unknown),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
const GoogleImageResponse = Schema.Struct({
candidates: Schema.optional(
Schema.Array(
Schema.Struct({
index: Schema.optional(Schema.Number),
content: Schema.optional(
Schema.Struct({
parts: Schema.Array(
Schema.Struct({
text: Schema.optional(Schema.String),
thought: Schema.optional(Schema.Boolean),
thoughtSignature: Schema.optional(Schema.String),
inlineData: Schema.optional(
Schema.Struct({
mimeType: Schema.String,
data: Schema.String,
}),
),
}),
),
}),
),
finishReason: Schema.optional(Schema.String),
finishMessage: Schema.optional(Schema.String),
safetyRatings: Schema.optional(Schema.Unknown),
citationMetadata: Schema.optional(Schema.Unknown),
groundingMetadata: Schema.optional(Schema.Unknown),
}),
),
),
usageMetadata: Schema.optional(GoogleUsage),
modelVersion: Schema.optional(Schema.String),
responseId: Schema.optional(Schema.String),
promptFeedback: Schema.optional(Schema.Unknown),
})
export interface ModelInput {
readonly id: string
readonly auth: AuthDefinition
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly defaults?: ImageModelDefaults
}
const providerOptions = (request: ImageRequest): GoogleImageOptions => ({
...request.model.defaults?.providerOptions?.google,
...request.providerOptions?.google,
})
const body = (request: ImageRequest): GoogleImageBody => {
const options = providerOptions(request)
const image = {
aspectRatio: request.aspectRatio,
imageSize: options.imageSize,
}
const thinkingConfig = {
thinkingLevel: options.thinkingLevel,
includeThoughts: options.includeThoughts,
}
return {
contents: [{ role: "user", parts: [{ text: request.prompt }] }],
generationConfig: {
responseModalities: ["IMAGE"],
imageConfig: Object.values(image).some((value) => value !== undefined) ? image : undefined,
seed: request.seed,
thinkingConfig: Object.values(thinkingConfig).some((value) => value !== undefined) ? thinkingConfig : undefined,
},
}
}
const invalidOutput = (message: string, providerMetadata?: ProviderMetadata) =>
new LLMError({
module: ADAPTER,
method: "generate",
reason: new InvalidProviderOutputReason({ message, route: ADAPTER, providerMetadata }),
})
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
const PROTOCOL_BODY_FIELDS = new Set(["contents", "generationConfig"])
const bodyWithOverlay = Effect.fn("GoogleImages.bodyWithOverlay")(function* (
imageBody: GoogleImageBody,
overlay: Record<string, unknown> | undefined,
) {
if (!overlay) return imageBody
const reserved = Object.keys(overlay).filter((key) => PROTOCOL_BODY_FIELDS.has(key))
if (reserved.length > 0)
return yield* ProviderShared.invalidRequest(
`http.body cannot overlay protocol-owned field(s): ${reserved.join(", ")}`,
)
return mergeJsonRecords(imageBody, overlay) ?? imageBody
})
export const model = (input: ModelInput) => {
const route: ImageRoute = {
id: ADAPTER,
generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequest, execute) {
if (/^imagen(?:-|$)/i.test(request.model.id))
return yield* ProviderShared.invalidRequest(
`Google Images uses Gemini generateContent and does not support Imagen model ID ${request.model.id}`,
)
if (request.count !== undefined)
return yield* ProviderShared.invalidRequest("Google Images does not support the common count option")
if (request.size !== undefined)
return yield* ProviderShared.invalidRequest("Google Images does not support the common size option")
const requestBody = yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(GoogleImageBody))(body(request))
const http = mergeHttpOptions(request.model.defaults?.http, request.http)
const overlaidBody = yield* bodyWithOverlay(requestBody, http?.body)
const text = ProviderShared.encodeJson(overlaidBody)
const url = applyQuery(
`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}/models/${request.model.id}:generateContent`,
http?.query,
)
const headers = yield* Auth.toEffect(input.auth)({
request,
method: "POST",
url,
body: text,
headers: Headers.fromInput({ ...input.headers, ...http?.headers }),
})
const response = yield* execute(
HttpClientRequest.post(url).pipe(
HttpClientRequest.setHeaders(headers),
HttpClientRequest.bodyText(text, "application/json"),
),
)
const payload = yield* response.json.pipe(
Effect.mapError(() => invalidOutput("Failed to read the Google Images response")),
)
const decoded = yield* Schema.decodeUnknownEffect(GoogleImageResponse)(payload).pipe(
Effect.mapError(() => invalidOutput("Google Images returned an invalid response")),
)
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
index: candidate.index ?? candidateIndex,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
citationMetadata: candidate.citationMetadata,
groundingMetadata: candidate.groundingMetadata,
parts: (candidate.content?.parts ?? []).map((part) =>
part.inlineData === undefined
? {
type: "text",
text: part.text,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
}
: {
type: "inlineData",
mediaType: part.inlineData.mimeType,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
},
),
}))
const encoded = candidates.flatMap((candidate, candidateIndex) =>
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
part.inlineData === undefined || part.thought === true
? []
: [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
),
)
const images = yield* Effect.forEach(encoded, (item) =>
Effect.fromResult(Encoding.decodeBase64(item.inlineData.data)).pipe(
Effect.mapError(() =>
invalidOutput(
`Google Images candidate ${item.candidateIndex} part ${item.partIndex} contains invalid base64 data`,
),
),
Effect.map(
(data) =>
new GeneratedImage({
mediaType: item.inlineData.mimeType,
data,
providerMetadata: {
google: {
candidateIndex: item.candidate.index ?? item.candidateIndex,
partIndex: item.partIndex,
finishReason: item.candidate.finishReason,
safetyRatings: item.candidate.safetyRatings,
citationMetadata: item.candidate.citationMetadata,
groundingMetadata: item.candidate.groundingMetadata,
thoughtSignature: item.candidate.content?.parts[item.partIndex]?.thoughtSignature,
},
},
}),
),
),
)
if (images.length === 0) {
const finishReasons = candidates.flatMap((candidate) =>
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* invalidOutput(
`Google Images returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect reason.providerMetadata.google for prompt feedback and candidate details`,
{
google: {
promptFeedback: decoded.promptFeedback,
candidates: candidateMetadata,
},
},
)
}
const usage = decoded.usageMetadata
const outputTokens =
usage?.candidatesTokenCount === undefined
? undefined
: usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0)
return new ImageResponse({
images,
usage:
usage === undefined
? undefined
: new Usage({
inputTokens: usage.promptTokenCount,
outputTokens,
nonCachedInputTokens: ProviderShared.subtractTokens(
usage.promptTokenCount,
usage.cachedContentTokenCount,
),
cacheReadInputTokens: usage.cachedContentTokenCount,
reasoningTokens: usage.thoughtsTokenCount,
totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
providerMetadata: { google: usage },
}),
providerMetadata: {
google: {
modelVersion: decoded.modelVersion,
responseId: decoded.responseId,
promptFeedback: decoded.promptFeedback,
candidates: candidateMetadata,
},
},
})
}),
}
return ImageModel.make({ id: input.id, provider: "google", route, defaults: input.defaults })
}
export const GoogleImages = {
model,
} as const
+30 -4
View File
@@ -2,14 +2,25 @@ import type { RouteDefaultsInput } from "../route/client"
import { Auth } from "../route/auth"
import type { ProviderAuthOption } from "../route/auth-options"
import type { ProviderPackage } from "../provider-package"
import { ProviderID, type ModelID, type ProviderOptions } from "../schema"
import * as Gemini from "../protocols/gemini"
import { HttpOptions, ProviderID, mergeHttpOptions, type ModelID, type ProviderOptions } from "../schema"
import { Gemini } from "../protocols/gemini"
import { GoogleImages, type GoogleImageOptions } from "../protocols/google-images"
export type { GoogleImageOptions } from "../protocols/google-images"
export const id = ProviderID.make("google")
export const routes = [Gemini.route]
export type Config = RouteDefaultsInput & ProviderAuthOption<"optional"> & { readonly baseURL?: string }
export type Config = RouteDefaultsInput &
ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly image?: ImageConfig
}
export interface ImageConfig {
readonly providerOptions?: GoogleImageOptions
}
export interface Settings extends ProviderPackage.Settings {
readonly apiKey?: string
@@ -25,15 +36,28 @@ const auth = (options: ProviderAuthOption<"optional">) => {
}
const configuredRoute = (input: Config) => {
const { apiKey: _, auth: _auth, baseURL, ...rest } = input
const { apiKey: _, auth: _auth, baseURL, image: _image, ...rest } = input
return Gemini.route.with({ ...rest, endpoint: { baseURL }, auth: auth(input) })
}
export const configure = (input: Config = {}) => {
const route = configuredRoute(input)
const image = (modelID: string | ModelID) =>
GoogleImages.model({
id: modelID,
auth: auth(input),
baseURL: input.baseURL,
headers: input.headers,
defaults: {
providerOptions:
input.image?.providerOptions === undefined ? undefined : { google: { ...input.image.providerOptions } },
http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)),
},
})
return {
id,
model: (modelID: string | ModelID) => route.model({ id: modelID }),
image,
configure,
}
}
@@ -48,3 +72,5 @@ export const model: ProviderPackage.Definition<Settings>["model"] = (modelID, se
limits: settings.limits,
providerOptions: settings.providerOptions,
}).model(modelID)
export const image = provider.image
File diff suppressed because one or more lines are too long
+263 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { Image, ImageClient } from "../src"
import { OpenAI } from "../src/providers"
import { Google, OpenAI } from "../src/providers"
import { it } from "./lib/effect"
import { dynamicResponse } from "./lib/http"
@@ -92,4 +92,266 @@ describe("Image", () => {
),
),
)
it.effect("generates images through the Google generateContent API", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({
apiKey: "test",
baseURL: "https://generativelanguage.test/v1beta/",
headers: { "x-default": "yes" },
http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } },
image: {
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true },
},
}).image("gemini-3.1-flash-image"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
http: {
body: { safetySettings: [] },
headers: { "x-request": "yes" },
query: { trace: "1" },
},
})
expect(response.images).toHaveLength(3)
expect(response.images.map((image) => image.data)).toEqual([
Uint8Array.from([1, 2, 3]),
Uint8Array.from([4, 5, 6]),
Uint8Array.from([7, 8, 9]),
])
expect(response.images.map((image) => image.mediaType)).toEqual(["image/png", "image/jpeg", "image/webp"])
expect(response.images[0].providerMetadata).toMatchObject({ google: { thoughtSignature: "signature-1" } })
expect(response.images[1].providerMetadata).toMatchObject({
google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" },
})
expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } })
expect(response.usage?.inputTokens).toBe(5)
expect(response.usage?.outputTokens).toBe(10)
expect(response.usage?.reasoningTokens).toBe(3)
expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } })
expect(response.providerMetadata).toEqual({
google: {
modelVersion: "gemini-3.1-flash-image",
responseId: "response-1",
promptFeedback: undefined,
candidates: [
{
index: 0,
finishReason: "STOP",
finishMessage: undefined,
safetyRatings: [{ category: "safe" }],
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [
{
type: "inlineData",
mediaType: "image/png",
thought: undefined,
thoughtSignature: "signature-1",
},
{ type: "text", text: "planning", thought: true, thoughtSignature: "text-signature" },
{
type: "inlineData",
mediaType: "image/png",
thought: true,
thoughtSignature: "draft-signature",
},
{
type: "inlineData",
mediaType: "image/jpeg",
thought: undefined,
thoughtSignature: undefined,
},
],
},
{
index: 7,
finishReason: undefined,
finishMessage: undefined,
safetyRatings: undefined,
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [
{
type: "inlineData",
mediaType: "image/webp",
thought: undefined,
thoughtSignature: undefined,
},
],
},
],
},
})
}).pipe(
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(request.url).toBe(
"https://generativelanguage.test/v1beta/models/gemini-3.1-flash-image:generateContent?api=v1&trace=1",
)
expect(request.headers.get("x-goog-api-key")).toBe("test")
expect(request.headers.get("x-default")).toBe("yes")
expect(request.headers.get("x-request")).toBe("yes")
expect(JSON.parse(input.text)).toEqual({
contents: [{ role: "user", parts: [{ text: "A robot tending a rooftop garden" }] }],
generationConfig: {
responseModalities: ["IMAGE"],
imageConfig: { aspectRatio: "16:9", imageSize: "2K" },
seed: 42,
thinkingConfig: { thinkingLevel: "HIGH", includeThoughts: true },
},
labels: { deployment: "test" },
safetySettings: [],
})
return input.respond(
JSON.stringify({
candidates: [
{
content: {
parts: [
{
inlineData: { mimeType: "image/png", data: "AQID" },
thoughtSignature: "signature-1",
},
{ text: "planning", thought: true, thoughtSignature: "text-signature" },
{
inlineData: { mimeType: "image/png", data: "CgsM" },
thought: true,
thoughtSignature: "draft-signature",
},
{ inlineData: { mimeType: "image/jpeg", data: "BAUG" } },
],
},
finishReason: "STOP",
safetyRatings: [{ category: "safe" }],
},
{
index: 7,
content: { parts: [{ inlineData: { mimeType: "image/webp", data: "BwgJ" } }] },
},
],
usageMetadata: {
promptTokenCount: 5,
candidatesTokenCount: 7,
thoughtsTokenCount: 3,
totalTokenCount: 15,
serviceTier: "STANDARD",
},
modelVersion: "gemini-3.1-flash-image",
responseId: "response-1",
}),
{ headers: { "content-type": "application/json" } },
)
}),
),
),
),
),
),
)
it.effect("rejects unsupported Google image options locally", () =>
Image.generate({
model: Google.configure({ apiKey: "test" }).image("gemini-3.1-flash-image"),
prompt: "A robot tending a rooftop garden",
count: 2,
size: { width: 1024, height: 1024 },
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidRequest")
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid request should not reach the provider"))),
),
),
),
)
it.effect("rejects Imagen model IDs locally", () =>
Image.generate({
model: Google.configure({ apiKey: "test" }).image("imagen-4.0-generate-001"),
prompt: "A robot tending a rooftop garden",
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("does not support Imagen model ID")
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid model should not reach the provider"))),
),
),
),
)
it.effect("includes Google diagnostics when no final image is returned", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
"gemini-3.1-flash-image",
),
prompt: "A robot tending a rooftop garden",
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidProviderOutput")
if (error.reason._tag !== "InvalidProviderOutput") return
expect(error.reason.message).toContain("finish reasons: IMAGE_SAFETY")
expect(error.reason.providerMetadata).toEqual({
google: {
promptFeedback: { blockReason: "SAFETY" },
candidates: [
{
index: 0,
finishReason: "IMAGE_SAFETY",
finishMessage: "The generated image was blocked by safety filters.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [{ type: "text", text: "blocked", thought: false, thoughtSignature: undefined }],
},
],
},
})
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.succeed(
input.respond(
JSON.stringify({
candidates: [
{
content: { parts: [{ text: "blocked", thought: false }] },
finishReason: "IMAGE_SAFETY",
finishMessage: "The generated image was blocked by safety filters.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
},
],
promptFeedback: { blockReason: "SAFETY" },
}),
{ headers: { "content-type": "application/json" } },
),
),
),
),
),
),
),
)
})
@@ -0,0 +1,33 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image } from "../../src"
import { Google } from "../../src/providers"
import { recordedTests } from "../recorded-test"
const model = Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? "fixture",
}).image("gemini-3.1-flash-image")
const recorded = recordedTests({
prefix: "google-images",
provider: "google",
protocol: "google-images",
requires: ["GOOGLE_GENERATIVE_AI_API_KEY"],
})
describe("Google Images recorded", () => {
recorded.effect("generates an image", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model,
prompt: "A simple flat blue circle centered on a plain white background.",
aspectRatio: "1:1",
})
expect(response.images).toHaveLength(1)
expect(response.image?.mediaType).toMatch(/^image\//)
expect(response.image?.data).toBeInstanceOf(Uint8Array)
expect(response.image?.data.length).toBeGreaterThan(0)
}),
)
})