refactor(llm): resolve adapters from registry

This commit is contained in:
Kit Langton
2026-05-05 21:33:03 -04:00
parent 89c65949c2
commit cfe8fdb149
9 changed files with 363 additions and 288 deletions
+2 -11
View File
@@ -129,18 +129,9 @@ const FakeAdapter = Adapter.make({
})
// A provider module exports a model helper. The model helper sets provider
// identity, protocol id, and the adapter that can run this model handle.
// Serialized / revived models can still use explicit provider adapters.
// identity, protocol id, and the adapter id resolved by the registry.
const FakeEcho = {
model: (id: string) =>
Adapter.bindModel(
LLM.model({
id,
provider: "fake-echo",
protocol: "fake-echo",
}),
FakeAdapter,
),
model: (id: string) => Adapter.model(FakeAdapter, { provider: "fake-echo" })({ id }),
}
// `LLMClient.prepare` is the lower-level inspection hook: it compiles through
+60 -78
View File
@@ -9,22 +9,28 @@ import type { Protocol } from "./protocol"
import * as ProviderShared from "./protocols/shared"
import type {
AdapterID,
GenerationOptionsInput,
LLMError,
LLMEvent,
PreparedRequestOf,
ProtocolID,
} from "./schema"
import {
GenerationOptions,
HttpOptions,
LLMRequest,
LLMResponse,
ModelCapabilities,
ModelID,
ModelLimits,
ModelPolicy,
ModelRef,
NoAdapterError,
PreparedRequest,
ProviderID,
mergeGenerationOptions,
mergeHttpOptions,
mergeJsonRecords,
mergeProviderOptions,
} from "./schema"
export interface HttpContext {
@@ -56,20 +62,15 @@ export interface AdapterDefinition<Payload> extends Adapter<Payload> {}
// oxlint-disable-next-line typescript-eslint/no-explicit-any
export type AnyAdapter = AdapterDefinition<any>
const MODEL_ADAPTER = Symbol.for("@opencode-ai/llm.model-adapter")
type BoundModel = ModelRef & { readonly [MODEL_ADAPTER]?: AnyAdapter }
const adapterRegistry = new Map<string, AnyAdapter>()
const modelAdapters = new WeakMap<ModelRef, AnyAdapter>()
const modelAdapter = (model: ModelRef) => (model as BoundModel)[MODEL_ADAPTER] ?? modelAdapters.get(model)
const bindModelAdapter = (model: ModelRef, adapter: AnyAdapter) => {
if (!Object.isExtensible(model)) {
modelAdapters.set(model, adapter)
return
}
Object.defineProperty(model, MODEL_ADAPTER, { value: adapter, configurable: true })
const register = <Adapter extends AnyAdapter>(adapter: Adapter): Adapter => {
if (!adapterRegistry.has(adapter.id)) adapterRegistry.set(adapter.id, adapter)
return adapter
}
const registeredAdapter = (id: string) => adapterRegistry.get(id)
export type ModelCapabilitiesInput = {
readonly input?: Partial<ModelCapabilities["input"]>
readonly output?: Partial<ModelCapabilities["output"]>
@@ -80,18 +81,19 @@ export type ModelCapabilitiesInput = {
}
}
export type ModelPolicyInput = ModelPolicy | ConstructorParameters<typeof ModelPolicy>[0]
export type HttpOptionsInput = HttpOptions | ConstructorParameters<typeof HttpOptions>[0]
export type ModelRefInput = Omit<
ConstructorParameters<typeof ModelRef>[0],
"id" | "provider" | "adapter" | "capabilities" | "limits" | "policy"
"id" | "provider" | "adapter" | "capabilities" | "limits" | "generation" | "http"
> & {
readonly id: string | ModelID
readonly provider: string | ProviderID
readonly adapter?: string | AdapterID
readonly capabilities?: ModelCapabilities | ModelCapabilitiesInput
readonly limits?: ModelLimits | ConstructorParameters<typeof ModelLimits>[0]
readonly policy?: ModelPolicyInput
readonly generation?: GenerationOptionsInput
readonly http?: HttpOptionsInput
}
export type AdapterModelInput = Omit<ModelRefInput, "provider" | "adapter" | "protocol">
@@ -124,9 +126,14 @@ export const modelLimits = (input: ModelLimits | ConstructorParameters<typeof Mo
return new ModelLimits(input ?? {})
}
export const modelPolicy = (input: ModelPolicyInput | undefined) => {
if (input === undefined || input instanceof ModelPolicy) return input
return new ModelPolicy(input)
export const generationOptions = (input: GenerationOptionsInput | undefined) => {
if (input === undefined || input instanceof GenerationOptions) return input
return new GenerationOptions(input)
}
export const httpOptions = (input: HttpOptionsInput | undefined) => {
if (input === undefined || input instanceof HttpOptions) return input
return new HttpOptions(input)
}
export const modelRef = (input: ModelRefInput) =>
@@ -138,19 +145,10 @@ export const modelRef = (input: ModelRefInput) =>
protocol: input.protocol,
capabilities: modelCapabilities(input.capabilities),
limits: modelLimits(input.limits),
policy: modelPolicy(input.policy),
generation: generationOptions(input.generation),
http: httpOptions(input.http),
})
export const bindModel = <Model extends ModelRef>(model: Model, adapter: AnyAdapter): Model => {
if (model.adapter !== adapter.id || model.protocol !== adapter.protocol) {
throw new Error(
`Cannot bind ${adapter.id} adapter (${adapter.protocol}) to ${model.provider}/${model.id} via ${model.adapter} (${model.protocol})`,
)
}
bindModelAdapter(model, adapter)
return model
}
function model<Input extends AdapterModelInput = AdapterModelInput>(
adapter: AnyAdapter,
defaults: AdapterModelDefaults,
@@ -170,51 +168,22 @@ function model<Input extends AdapterMappedModelInput>(
const mapped = options.mapInput?.(input) ?? input
const provider = defaults.provider ?? ("provider" in mapped ? mapped.provider : undefined)
if (!provider) throw new Error(`Adapter.model(${adapter.id}) requires a provider`)
return bindModel(
modelRef({
...defaults,
...mapped,
provider,
adapter: adapter.id,
protocol: adapter.protocol,
capabilities: mapped.capabilities ?? defaults.capabilities,
limits: mapped.limits ?? defaults.limits,
}),
adapter,
)
register(adapter)
return modelRef({
...defaults,
...mapped,
provider,
adapter: adapter.id,
protocol: adapter.protocol,
capabilities: mapped.capabilities ?? defaults.capabilities,
limits: mapped.limits ?? defaults.limits,
generation: mergeGenerationOptions(defaults.generation, mapped.generation),
providerOptions: mergeProviderOptions(defaults.providerOptions, mapped.providerOptions),
http: mergeHttpOptions(httpOptions(defaults.http), httpOptions(mapped.http)),
})
}
}
export const preserveModelBinding = <Model extends ModelRef>(source: ModelRef, target: Model): Model => {
const adapter = modelAdapter(source)
if (!adapter) return target
return bindModel(target, adapter)
}
export const updateLLMRequest = (
request: LLMRequest,
patch: Partial<ConstructorParameters<typeof LLMRequest>[0]>,
) => {
const model = patch.model ?? request.model
const next = new LLMRequest({
id: request.id,
model,
system: request.system,
messages: request.messages,
tools: request.tools,
toolChoice: request.toolChoice,
generation: request.generation,
reasoning: request.reasoning,
cache: request.cache,
responseFormat: request.responseFormat,
metadata: request.metadata,
native: request.native,
...patch,
})
preserveModelBinding(model, next.model)
return next
}
export interface LLMClient {
/**
* Compile a request through protocol payload lowering, validation, and HTTP
@@ -292,12 +261,25 @@ export function make<Payload, Frame, Chunk, State>(
),
)
const buildHeaders = input.headers ?? (() => ({}))
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
if (!query) return url
const next = new URL(url)
Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value))
return next.toString()
}
const toHttp = (payload: Payload, ctx: HttpContext) =>
Effect.gen(function* () {
const url = (yield* renderEndpoint(input.endpoint, { request: ctx.request, payload })).toString()
const body = encodePayload(payload)
const merged = { ...buildHeaders({ request: ctx.request }), ...ctx.request.model.headers }
const url = applyQuery(
(yield* renderEndpoint(input.endpoint, { request: ctx.request, payload })).toString(),
ctx.request.http?.query,
)
const body = ctx.request.http?.body === undefined
? encodePayload(payload)
: ProviderShared.isRecord(payload)
? ProviderShared.encodeJson(mergeJsonRecords(payload, ctx.request.http.body) ?? {})
: yield* ProviderShared.invalidRequest("http.body can only overlay JSON object request bodies")
const merged = { ...buildHeaders({ request: ctx.request }), ...ctx.request.model.headers, ...ctx.request.http?.headers }
const headers = yield* auth({
request: ctx.request,
method: "POST",
@@ -320,14 +302,14 @@ export function make<Payload, Frame, Chunk, State>(
onHalt: protocol.onHalt,
})
return {
return register({
id: input.id,
protocol: protocol.id,
payloadSchema: protocol.payload,
toPayload: protocol.toPayload,
toHttp,
parse,
}
})
}
/**
@@ -339,7 +321,7 @@ const makeClient = (options: ClientOptions = {}): LLMClient => {
const adapters = new Map((options.adapters ?? []).map((adapter) => [adapter.id, adapter] as const))
const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) {
const adapter = adapters.get(request.model.adapter) ?? modelAdapter(request.model)
const adapter = adapters.get(request.model.adapter) ?? registeredAdapter(request.model.adapter)
if (!adapter) return yield* noAdapter(request.model)
const payload = yield* adapter.toPayload(request).pipe(
@@ -400,6 +382,6 @@ const makeClient = (options: ClientOptions = {}): LLMClient => {
return { prepare: prepare as LLMClient["prepare"], stream, generate }
}
export const Adapter = { bindModel, make, model } as const
export const Adapter = { make, model, register } as const
export const LLMClient = { make: makeClient }
+1 -1
View File
@@ -1,4 +1,4 @@
export { Adapter, LLMClient, modelCapabilities, modelLimits, modelRef, updateLLMRequest } from "./adapter"
export { Adapter, LLMClient, modelCapabilities, modelLimits, modelRef } from "./adapter"
export type {
Adapter as AdapterShape,
AdapterDefinition,
+27 -73
View File
@@ -4,7 +4,6 @@ import {
modelCapabilities,
modelLimits,
modelRef,
preserveModelBinding,
type ModelCapabilitiesInput,
type ModelRefInput,
} from "./adapter"
@@ -13,19 +12,20 @@ import { type Tools } from "./tool"
import { ToolRuntime, type RunOptions } from "./tool-runtime"
import {
GenerationOptions,
CacheIntent,
HttpOptions,
LLMEvent,
LLMRequest,
LLMResponse,
Message,
ReasoningIntent,
ToolChoice,
ToolDefinition,
type ContentPart,
type SystemPart,
type ToolCallPart,
type ToolResultPart,
type ToolResultValue,
ToolCallPart,
ToolResultPart,
mergeGenerationOptions,
mergeHttpOptions,
mergeProviderOptions,
} from "./schema"
import type { LLMError } from "./schema"
@@ -80,21 +80,16 @@ export type CapabilitiesInput = ModelCapabilitiesInput
export type ModelInput = ModelRefInput
export type MessageInput = Omit<ConstructorParameters<typeof Message>[0], "content"> & {
readonly content: string | ContentPart | ReadonlyArray<ContentPart>
}
export type MessageInput = Message.Input
export type ToolChoiceInput = ToolChoice | ConstructorParameters<typeof ToolChoice>[0] | ToolDefinition | string
export type ToolChoiceMode = Exclude<ToolChoice["type"], "tool">
export type ToolResultInput = Omit<ToolResultPart, "type" | "result"> & {
readonly result: unknown
readonly resultType?: ToolResultValue["type"]
}
export type ToolResultInput = Parameters<typeof ToolResultPart.make>[0]
export type RequestInput = Omit<
ConstructorParameters<typeof LLMRequest>[0],
"system" | "messages" | "tools" | "toolChoice" | "generation"
"system" | "messages" | "tools" | "toolChoice" | "generation" | "http"
> & {
readonly system?: string | SystemPart | ReadonlyArray<SystemPart>
readonly prompt?: string | ContentPart | ReadonlyArray<ContentPart>
@@ -102,33 +97,27 @@ export type RequestInput = Omit<
readonly tools?: ReadonlyArray<ToolDefinition | ConstructorParameters<typeof ToolDefinition>[0]>
readonly toolChoice?: ToolChoiceInput
readonly generation?: GenerationOptions | ConstructorParameters<typeof GenerationOptions>[0]
readonly http?: HttpOptions | ConstructorParameters<typeof HttpOptions>[0]
}
export const capabilities = modelCapabilities
export const limits = modelLimits
export const text = (value: string): ContentPart => ({ type: "text", text: value })
export const text = Message.text
export const system = (value: string): SystemPart => ({ type: "text", text: value })
const contentParts = (input: string | ContentPart | ReadonlyArray<ContentPart>) =>
typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input]
const systemParts = (input?: string | SystemPart | ReadonlyArray<SystemPart>) => {
if (input === undefined) return []
return typeof input === "string" ? [system(input)] : Array.isArray(input) ? [...input] : [input]
}
export const message = (input: Message | MessageInput) => {
if (input instanceof Message) return input
return new Message({ ...input, content: contentParts(input.content) })
}
export const message = Message.make
export const user = (content: string | ContentPart | ReadonlyArray<ContentPart>) => message({ role: "user", content })
export const user = Message.user
export const assistant = (content: string | ContentPart | ReadonlyArray<ContentPart>) =>
message({ role: "assistant", content })
export const assistant = Message.assistant
export const model = modelRef
@@ -137,30 +126,11 @@ export const toolDefinition = (input: ToolDefinition | ConstructorParameters<typ
return new ToolDefinition(input)
}
export const toolCall = (input: Omit<ToolCallPart, "type">): ToolCallPart => ({ type: "tool-call", ...input })
export const toolCall = ToolCallPart.make
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value)
export const toolResult = ToolResultPart.make
const isToolResultValue = (value: unknown): value is ToolResultValue =>
isRecord(value) && (value.type === "text" || value.type === "json" || value.type === "error") && "value" in value
const toolResultValue = (value: unknown, type: ToolResultValue["type"] = "json"): ToolResultValue => {
if (isToolResultValue(value)) return value
return { type, value }
}
export const toolResult = (input: ToolResultInput): ToolResultPart => ({
type: "tool-result",
id: input.id,
name: input.name,
result: toolResultValue(input.result, input.resultType),
providerExecuted: input.providerExecuted,
metadata: input.metadata,
})
export const toolMessage = (input: ToolResultPart | ToolResultInput) =>
message({ role: "tool", content: ["type" in input ? input : toolResult(input)] })
export const toolMessage = Message.tool
export const toolChoiceName = (name: string) => new ToolChoice({ type: "tool", name })
@@ -180,29 +150,13 @@ export const generation = (input: GenerationOptions | ConstructorParameters<type
return new GenerationOptions(input)
}
const reasoning = (input: ReasoningIntent | ConstructorParameters<typeof ReasoningIntent>[0] | undefined) => {
if (input === undefined || input instanceof ReasoningIntent) return input
return new ReasoningIntent(input)
}
const cache = (input: CacheIntent | ConstructorParameters<typeof CacheIntent>[0] | undefined) => {
if (input === undefined || input instanceof CacheIntent) return input
return new CacheIntent(input)
const http = (input: HttpOptions | ConstructorParameters<typeof HttpOptions>[0] | undefined) => {
if (input === undefined || input instanceof HttpOptions) return input
return new HttpOptions(input)
}
export const requestInput = (input: LLMRequest): RequestInput => ({
id: input.id,
model: input.model,
system: input.system,
messages: input.messages,
tools: input.tools,
toolChoice: input.toolChoice,
generation: input.generation,
reasoning: input.reasoning,
cache: input.cache,
responseFormat: input.responseFormat,
metadata: input.metadata,
native: input.native,
...LLMRequest.input(input),
})
export const request = (input: RequestInput) => {
@@ -213,20 +167,20 @@ export const request = (input: RequestInput) => {
tools,
toolChoice: requestToolChoice,
generation: requestGeneration,
providerOptions: requestProviderOptions,
http: requestHttp,
...rest
} = input
const result = new LLMRequest({
return new LLMRequest({
...rest,
system: systemParts(requestSystem),
messages: [...(messages?.map(message) ?? []), ...(prompt === undefined ? [] : [user(prompt)])],
tools: tools?.map(toolDefinition) ?? [],
toolChoice: requestToolChoice ? toolChoice(requestToolChoice) : undefined,
generation: generation(requestGeneration),
reasoning: reasoning(rest.reasoning),
cache: cache(rest.cache),
generation: mergeGenerationOptions(input.model.generation, generation(requestGeneration)) ?? generation(),
providerOptions: mergeProviderOptions(input.model.providerOptions, requestProviderOptions),
http: mergeHttpOptions(input.model.http, http(requestHttp)),
})
preserveModelBinding(input.model, result.model)
return result
}
export const updateRequest = (input: LLMRequest, patch: Partial<RequestInput>) =>
@@ -1,46 +1,55 @@
import { Schema } from "effect"
import type { LLMRequest, ReasoningEffort } from "../../schema"
import { ReasoningEfforts, TextVerbosity } from "../../schema"
import type { LLMRequest, ReasoningEffort, TextVerbosity as TextVerbosityValue } from "../../schema"
import { ReasoningEfforts, TextVerbosity, mergeProviderOptions } from "../../schema"
export const OpenAIReasoningEfforts = ReasoningEfforts.filter(
(effort): effort is Exclude<ReasoningEffort, "max"> => effort !== "max",
)
export type OpenAIReasoningEffort = typeof OpenAIReasoningEfforts[number]
const OPENAI_REASONING_EFFORTS = new Set<ReasoningEffort>(OpenAIReasoningEfforts)
const REASONING_EFFORTS = new Set<string>(ReasoningEfforts)
const OPENAI_REASONING_EFFORTS = new Set<string>(OpenAIReasoningEfforts)
const TEXT_VERBOSITY = new Set<string>(["low", "medium", "high"])
export const OpenAIReasoningEffort = Schema.Literals(OpenAIReasoningEfforts)
export const OpenAITextVerbosity = TextVerbosity
export const isReasoningEffort = (effort: ReasoningEffort): effort is OpenAIReasoningEffort =>
OPENAI_REASONING_EFFORTS.has(effort)
const isAnyReasoningEffort = (effort: unknown): effort is ReasoningEffort =>
typeof effort === "string" && REASONING_EFFORTS.has(effort)
export const store = (request: LLMRequest) =>
typeof request.model.policy?.retention?.store === "boolean" ? request.model.policy.retention.store : undefined
export const isReasoningEffort = (effort: unknown): effort is OpenAIReasoningEffort =>
typeof effort === "string" && OPENAI_REASONING_EFFORTS.has(effort)
const isTextVerbosity = (value: unknown): value is TextVerbosityValue =>
typeof value === "string" && TEXT_VERBOSITY.has(value)
const options = (request: LLMRequest) => mergeProviderOptions(request.model.providerOptions, request.providerOptions)?.openai
export const store = (request: LLMRequest): boolean | undefined => {
const value = options(request)?.store
return typeof value === "boolean" ? value : undefined
}
export const reasoningEffort = (request: LLMRequest): ReasoningEffort | undefined => {
if (request.reasoning?.enabled === false) return undefined
return request.reasoning?.effort ?? request.model.policy?.reasoning?.effort
const value = options(request)?.reasoningEffort
return isAnyReasoningEffort(value) ? value : undefined
}
export const reasoningSummary = (request: LLMRequest): "auto" | undefined => {
if (request.reasoning?.enabled === false) return undefined
if (request.reasoning?.summary !== undefined) return request.reasoning.summary ? "auto" : undefined
const summary = request.model.policy?.reasoning?.summary
return summary === true || summary === "auto" ? "auto" : undefined
return options(request)?.reasoningSummary === "auto" ? "auto" : undefined
}
export const encryptedReasoning = (request: LLMRequest) => {
if (request.reasoning?.enabled === false) return undefined
if (request.reasoning?.encryptedContent !== undefined) return request.reasoning.encryptedContent
return request.model.policy?.reasoning?.encryptedState
}
export const encryptedReasoning = (request: LLMRequest) =>
options(request)?.includeEncryptedReasoning === true ? true : undefined
export const promptCacheKey = (request: LLMRequest) => {
if (request.cache?.enabled === false) return undefined
return request.cache?.key ?? request.model.policy?.cache?.promptKey
const value = options(request)?.promptCacheKey
return typeof value === "string" ? value : undefined
}
export const textVerbosity = (request: LLMRequest) => request.model.policy?.text?.verbosity
export const textVerbosity = (request: LLMRequest) => {
const value = options(request)?.textVerbosity
return isTextVerbosity(value) ? value : undefined
}
export * as OpenAIOptions from "./openai-options"
+183 -61
View File
@@ -26,9 +26,6 @@ export type ReasoningEffort = Schema.Schema.Type<typeof ReasoningEffort>
export const TextVerbosity = Schema.Literals(["low", "medium", "high"])
export type TextVerbosity = Schema.Schema.Type<typeof TextVerbosity>
export const TransformPhase = Schema.Literals(["request", "prompt", "tool-schema", "payload", "stream"])
export type TransformPhase = Schema.Schema.Type<typeof TransformPhase>
export const MessageRole = Schema.Literals(["user", "assistant", "tool"])
export type MessageRole = Schema.Schema.Type<typeof MessageRole>
@@ -38,6 +35,105 @@ export type FinishReason = Schema.Schema.Type<typeof FinishReason>
export const JsonSchema = Schema.Record(Schema.String, Schema.Unknown)
export type JsonSchema = Schema.Schema.Type<typeof JsonSchema>
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value)
export const mergeJsonRecords = (...items: ReadonlyArray<Record<string, unknown> | undefined>): Record<string, unknown> | undefined => {
const result: Record<string, unknown> = items.reduce<Record<string, unknown>>((acc, item) => {
if (!item) return acc
return Object.entries(item).reduce<Record<string, unknown>>((next, [key, value]) => {
if (value === undefined) return next
return {
...next,
[key]: isRecord(next[key]) && isRecord(value) ? mergeJsonRecords(next[key], value) : value,
}
}, acc)
}, {})
return Object.keys(result).length === 0 ? undefined : result
}
const mergeStringRecords = (...items: ReadonlyArray<Record<string, string> | undefined>): Record<string, string> | undefined => {
const result = Object.fromEntries(
items.flatMap((item) => Object.entries(item ?? {}).filter((entry): entry is [string, string] => entry[1] !== undefined)),
)
return Object.keys(result).length === 0 ? undefined : result
}
export const ProviderOptions = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))
export type ProviderOptions = Schema.Schema.Type<typeof ProviderOptions>
export const mergeProviderOptions = (...items: ReadonlyArray<ProviderOptions | undefined>): ProviderOptions | undefined => {
const result = Object.fromEntries(
Object.entries(
items.reduce<Record<string, Record<string, unknown>>>((acc, item) => {
if (!item) return acc
return Object.entries(item).reduce<Record<string, Record<string, unknown>>>((next, [provider, options]) => ({
...next,
[provider]: mergeJsonRecords(next[provider], options) ?? {},
}), acc)
}, {}),
).filter((entry) => Object.keys(entry[1]).length > 0),
)
return Object.keys(result).length === 0 ? undefined : result
}
export class HttpOptions extends Schema.Class<HttpOptions>("LLM.HttpOptions")({
body: Schema.optional(JsonSchema),
headers: Schema.optional(Schema.Record(Schema.String, Schema.String)),
query: Schema.optional(Schema.Record(Schema.String, Schema.String)),
}) {}
export const mergeHttpOptions = (...items: ReadonlyArray<HttpOptions | undefined>): HttpOptions | undefined => {
const body = mergeJsonRecords(...items.map((item) => item?.body))
const headers = mergeStringRecords(...items.map((item) => item?.headers))
const query = mergeStringRecords(...items.map((item) => item?.query))
if (!body && !headers && !query) return undefined
return new HttpOptions({ body, headers, query })
}
export class GenerationOptions extends Schema.Class<GenerationOptions>("LLM.GenerationOptions")({
maxTokens: Schema.optional(Schema.Number),
temperature: Schema.optional(Schema.Number),
topP: Schema.optional(Schema.Number),
topK: Schema.optional(Schema.Number),
frequencyPenalty: Schema.optional(Schema.Number),
presencePenalty: Schema.optional(Schema.Number),
seed: Schema.optional(Schema.Number),
stop: Schema.optional(Schema.Array(Schema.String)),
}) {}
export type GenerationOptionsFields = {
readonly maxTokens?: number
readonly temperature?: number
readonly topP?: number
readonly topK?: number
readonly frequencyPenalty?: number
readonly presencePenalty?: number
readonly seed?: number
readonly stop?: ReadonlyArray<string>
}
export type GenerationOptionsInput = GenerationOptions | GenerationOptionsFields
const latestGeneration = <Key extends keyof GenerationOptionsFields>(
items: ReadonlyArray<GenerationOptionsInput | undefined>,
key: Key,
) => items.findLast((item) => item?.[key] !== undefined)?.[key]
export const mergeGenerationOptions = (...items: ReadonlyArray<GenerationOptionsInput | undefined>) => {
const result = new GenerationOptions({
maxTokens: latestGeneration(items, "maxTokens"),
temperature: latestGeneration(items, "temperature"),
topP: latestGeneration(items, "topP"),
topK: latestGeneration(items, "topK"),
frequencyPenalty: latestGeneration(items, "frequencyPenalty"),
presencePenalty: latestGeneration(items, "presencePenalty"),
seed: latestGeneration(items, "seed"),
stop: latestGeneration(items, "stop"),
})
return Object.values(result).some((value) => value !== undefined) ? result : undefined
}
export class ModelCapabilities extends Schema.Class<ModelCapabilities>("LLM.ModelCapabilities")({
input: Schema.Struct({
text: Schema.Boolean,
@@ -72,30 +168,6 @@ export class ModelLimits extends Schema.Class<ModelLimits>("LLM.ModelLimits")({
output: Schema.optional(Schema.Number),
}) {}
export class ModelPolicy extends Schema.Class<ModelPolicy>("LLM.ModelPolicy")({
retention: Schema.optional(Schema.Struct({
store: Schema.optional(Schema.Boolean),
dataCollection: Schema.optional(Schema.Literals(["allow", "deny"])),
})),
reasoning: Schema.optional(Schema.Struct({
effort: Schema.optional(ReasoningEffort),
summary: Schema.optional(Schema.Union([Schema.Boolean, Schema.Literal("auto")])),
encryptedState: Schema.optional(Schema.Boolean),
display: Schema.optional(Schema.Literals(["summarized", "omitted"])),
})),
text: Schema.optional(Schema.Struct({
verbosity: Schema.optional(TextVerbosity),
})),
cache: Schema.optional(Schema.Struct({
promptKey: Schema.optional(Schema.String),
ttl: Schema.optional(Schema.Literals(["5m", "1h"])),
})),
usage: Schema.optional(Schema.Struct({
include: Schema.optional(Schema.Boolean),
includeCost: Schema.optional(Schema.Boolean),
})),
}) {}
export class ModelRef extends Schema.Class<ModelRef>("LLM.ModelRef")({
id: ModelID,
provider: ProviderID,
@@ -118,13 +190,12 @@ export class ModelRef extends Schema.Class<ModelRef>("LLM.ModelRef")({
queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)),
capabilities: ModelCapabilities,
limits: ModelLimits,
/**
* Provider-agnostic defaults and policy that protocols can lower into their
* native fields. Request-level options override these defaults.
*/
policy: Schema.optional(ModelPolicy),
/** Provider-neutral generation defaults. Request-level values override them. */
generation: Schema.optional(GenerationOptions),
/** Provider-owned typed-at-the-facade options for non-portable knobs. */
providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
providerOptions: Schema.optional(ProviderOptions),
/** Serializable raw HTTP overlays applied to the final outgoing request. */
http: Schema.optional(HttpOptions),
/**
* Provider-specific opaque options. Reach for this only when the value is
* genuinely provider-private and does not fit a typed axis (e.g. Bedrock's
@@ -164,30 +235,50 @@ export const MediaPart = Schema.Struct({
}).annotate({ identifier: "LLM.Content.Media" })
export type MediaPart = Schema.Schema.Type<typeof MediaPart>
export const ToolResultValue = Schema.Struct({
const isToolResultValue = (value: unknown): value is ToolResultValue =>
isRecord(value) && (value.type === "text" || value.type === "json" || value.type === "error") && "value" in value
export const ToolResultValue = Object.assign(Schema.Struct({
type: Schema.Literals(["json", "text", "error"]),
value: Schema.Unknown,
}).annotate({ identifier: "LLM.ToolResult" })
}).annotate({ identifier: "LLM.ToolResult" }), {
make: (value: unknown, type: ToolResultValue["type"] = "json"): ToolResultValue =>
isToolResultValue(value) ? value : { type, value },
})
export type ToolResultValue = Schema.Schema.Type<typeof ToolResultValue>
export const ToolCallPart = Schema.Struct({
export const ToolCallPart = Object.assign(Schema.Struct({
type: Schema.Literal("tool-call"),
id: Schema.String,
name: Schema.String,
input: Schema.Unknown,
providerExecuted: Schema.optional(Schema.Boolean),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}).annotate({ identifier: "LLM.Content.ToolCall" })
}).annotate({ identifier: "LLM.Content.ToolCall" }), {
make: (input: Omit<ToolCallPart, "type">): ToolCallPart => ({ type: "tool-call", ...input }),
})
export type ToolCallPart = Schema.Schema.Type<typeof ToolCallPart>
export const ToolResultPart = Schema.Struct({
export const ToolResultPart = Object.assign(Schema.Struct({
type: Schema.Literal("tool-result"),
id: Schema.String,
name: Schema.String,
result: ToolResultValue,
providerExecuted: Schema.optional(Schema.Boolean),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}).annotate({ identifier: "LLM.Content.ToolResult" })
}).annotate({ identifier: "LLM.Content.ToolResult" }), {
make: (input: Omit<ToolResultPart, "type" | "result"> & {
readonly result: unknown
readonly resultType?: ToolResultValue["type"]
}): ToolResultPart => ({
type: "tool-result",
id: input.id,
name: input.name,
result: ToolResultValue.make(input.result, input.resultType),
providerExecuted: input.providerExecuted,
metadata: input.metadata,
}),
})
export type ToolResultPart = Schema.Schema.Type<typeof ToolResultPart>
export const ReasoningPart = Schema.Struct({
@@ -211,6 +302,30 @@ export class Message extends Schema.Class<Message>("LLM.Message")({
native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
export namespace Message {
export type ContentInput = string | ContentPart | ReadonlyArray<ContentPart>
export type Input = Omit<ConstructorParameters<typeof Message>[0], "content"> & {
readonly content: ContentInput
}
export const text = (value: string): ContentPart => ({ type: "text", text: value })
export const content = (input: ContentInput) =>
typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input]
export const make = (input: Message | Input) => {
if (input instanceof Message) return input
return new Message({ ...input, content: content(input.content) })
}
export const user = (content: ContentInput) => make({ role: "user", content })
export const assistant = (content: ContentInput) => make({ role: "assistant", content })
export const tool = (result: ToolResultPart | Parameters<typeof ToolResultPart.make>[0]) =>
make({ role: "tool", content: ["type" in result ? result : ToolResultPart.make(result)] })
}
export class ToolDefinition extends Schema.Class<ToolDefinition>("LLM.ToolDefinition")({
name: Schema.String,
description: Schema.String,
@@ -224,25 +339,6 @@ export class ToolChoice extends Schema.Class<ToolChoice>("LLM.ToolChoice")({
name: Schema.optional(Schema.String),
}) {}
export class GenerationOptions extends Schema.Class<GenerationOptions>("LLM.GenerationOptions")({
maxTokens: Schema.optional(Schema.Number),
temperature: Schema.optional(Schema.Number),
topP: Schema.optional(Schema.Number),
stop: Schema.optional(Schema.Array(Schema.String)),
}) {}
export class ReasoningIntent extends Schema.Class<ReasoningIntent>("LLM.ReasoningIntent")({
enabled: Schema.Boolean,
effort: Schema.optional(ReasoningEffort),
summary: Schema.optional(Schema.Boolean),
encryptedContent: Schema.optional(Schema.Boolean),
}) {}
export class CacheIntent extends Schema.Class<CacheIntent>("LLM.CacheIntent")({
enabled: Schema.Boolean,
key: Schema.optional(Schema.String),
}) {}
export const ResponseFormat = Schema.Union([
Schema.Struct({ type: Schema.Literal("text") }),
Schema.Struct({ type: Schema.Literal("json"), schema: JsonSchema }),
@@ -258,13 +354,39 @@ export class LLMRequest extends Schema.Class<LLMRequest>("LLM.Request")({
tools: Schema.Array(ToolDefinition),
toolChoice: Schema.optional(ToolChoice),
generation: GenerationOptions,
reasoning: Schema.optional(ReasoningIntent),
cache: Schema.optional(CacheIntent),
providerOptions: Schema.optional(ProviderOptions),
http: Schema.optional(HttpOptions),
responseFormat: Schema.optional(ResponseFormat),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
export namespace LLMRequest {
export type Input = ConstructorParameters<typeof LLMRequest>[0]
export const input = (request: LLMRequest): Input => ({
id: request.id,
model: request.model,
system: request.system,
messages: request.messages,
tools: request.tools,
toolChoice: request.toolChoice,
generation: request.generation,
providerOptions: request.providerOptions,
http: request.http,
responseFormat: request.responseFormat,
metadata: request.metadata,
})
export const update = (request: LLMRequest, patch: Partial<Input>) => {
if (Object.keys(patch).length === 0) return request
return new LLMRequest({
...input(request),
...patch,
model: patch.model ?? request.model,
})
}
}
export class Usage extends Schema.Class<Usage>("LLM.Usage")({
inputTokens: Schema.optional(Schema.Number),
outputTokens: Schema.optional(Schema.Number),
+17 -36
View File
@@ -1,6 +1,6 @@
import { Effect, Stream } from "effect"
import type { Concurrency } from "effect/Types"
import { updateLLMRequest, type LLMClient } from "./adapter"
import type { LLMClient } from "./adapter"
import type { RequestExecutor } from "./executor"
import {
type ContentPart,
@@ -9,8 +9,9 @@ import {
type LLMEvent,
LLMRequest,
Message,
type ToolCallPart,
type ToolResultValue,
ToolCallPart,
ToolResultPart,
} from "./schema"
import { ToolFailure } from "./schema"
import { type AnyTool, type Tools, toDefinitions } from "./tool"
@@ -64,12 +65,15 @@ export const run = <T extends Tools>(
const tools = options.tools as Tools
const runtimeTools = toDefinitions(tools)
const runtimeToolNames = new Set(runtimeTools.map((tool) => tool.name))
const initialRequest = updateLLMRequest(options.request, {
tools: [
...options.request.tools.filter((tool) => !runtimeToolNames.has(tool.name)),
...runtimeTools,
],
})
const initialRequest =
runtimeTools.length === 0
? options.request
: LLMRequest.update(options.request, {
tools: [
...options.request.tools.filter((tool) => !runtimeToolNames.has(tool.name)),
...runtimeTools,
],
})
const loop = (request: LLMRequest, step: number): Stream.Stream<LLMEvent, LLMError, RequestExecutor.Service> =>
Stream.unwrap(
@@ -91,12 +95,12 @@ export const run = <T extends Tools>(
(call) => dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)),
{ concurrency },
)
const followUp = updateLLMRequest(request, {
const followUp = LLMRequest.update(request, {
messages: [
...request.messages,
assistant(state.assistantContent),
Message.assistant(state.assistantContent),
...dispatched.map(([call, result]) =>
toolMessage({ id: call.id, name: call.name, result }),
Message.tool({ id: call.id, name: call.name, result }),
),
],
})
@@ -130,7 +134,7 @@ const accumulate = (state: StepState, event: LLMEvent) => {
return
}
if (event.type === "tool-call") {
const part = toolCall({
const part = ToolCallPart.make({
id: event.id,
name: event.name,
input: event.input,
@@ -145,7 +149,7 @@ const accumulate = (state: StepState, event: LLMEvent) => {
return
}
if (event.type === "tool-result" && event.providerExecuted) {
state.assistantContent.push(toolResult({
state.assistantContent.push(ToolResultPart.make({
id: event.id,
name: event.name,
result: event.result,
@@ -167,29 +171,6 @@ const appendStreamingText = (state: StepState, type: "text" | "reasoning", text:
state.assistantContent.push({ type, text })
}
const assistant = (content: ReadonlyArray<ContentPart>) => new Message({ role: "assistant", content })
const toolCall = (input: Omit<ToolCallPart, "type">): ToolCallPart => ({ type: "tool-call", ...input })
const toolResult = (input: {
readonly id: string
readonly name: string
readonly result: ToolResultValue
readonly providerExecuted?: boolean
}): ContentPart => ({
type: "tool-result",
id: input.id,
name: input.name,
result: input.result,
providerExecuted: input.providerExecuted,
})
const toolMessage = (input: {
readonly id: string
readonly name: string
readonly result: ToolResultValue
}) => new Message({ role: "tool", content: [toolResult(input)] })
const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect<ToolResultValue> => {
const tool = tools[call.name]
if (!tool) return Effect.succeed({ type: "error" as const, value: `Unknown tool: ${call.name}` })
+3 -7
View File
@@ -133,12 +133,10 @@ describe("llm adapter", () => {
}),
)
it.effect("falls back to adapter bound to model", () =>
it.effect("uses registered adapters by model adapter id", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.make({ adapters: [] }).prepare(
LLM.updateRequest(request, {
model: Adapter.bindModel(updateModel(request.model, { adapter: "gemini-fake" }), gemini),
}),
LLM.updateRequest(request, { model: updateModel(request.model, { adapter: "gemini-fake" }) }),
)
expect(prepared.adapter).toBe("gemini-fake")
@@ -174,9 +172,7 @@ describe("llm adapter", () => {
framing: fakeFraming,
})
const response = yield* LLMClient.make({ adapters: [override] }).generate(
LLM.updateRequest(request, { model: Adapter.bindModel(updateModel(request.model, { adapter: "fake" }), fake) }),
)
const response = yield* LLMClient.make({ adapters: [override] }).generate(request)
expect(response.text).toBe('echo:{"body":"override"}')
}),
+40
View File
@@ -38,6 +38,46 @@ describe("llm constructors", () => {
expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
})
test("merges model defaults with call options", () => {
const request = LLM.request({
model: LLM.model({
id: "fake-model",
provider: "fake",
protocol: "openai-chat",
generation: { maxTokens: 100, temperature: 1 },
providerOptions: { openai: { store: false, metadata: { model: true } } },
http: { body: { metadata: { model: true } }, headers: { "x-shared": "model" }, query: { model: "1" } },
}),
prompt: "Say hello.",
generation: { temperature: 0 },
providerOptions: { openai: { store: true, metadata: { request: true } } },
http: { body: { metadata: { request: true } }, headers: { "x-shared": "request" }, query: { request: "1" } },
})
expect(request.generation).toEqual({ maxTokens: 100, temperature: 0 })
expect(request.providerOptions).toEqual({ openai: { store: true, metadata: { model: true, request: true } } })
expect(request.http).toEqual({
body: { metadata: { model: true, request: true } },
headers: { "x-shared": "request" },
query: { model: "1", request: "1" },
})
})
test("updates canonical requests from the request datatype", () => {
const base = LLM.request({
id: "req_1",
model: LLM.model({ id: "fake-model", provider: "fake", protocol: "openai-chat" }),
prompt: "Say hello.",
})
const updated = LLMRequest.update(base, { messages: [...base.messages, LLM.assistant("Hi.")] })
expect(updated).toBeInstanceOf(LLMRequest)
expect(updated.id).toBe("req_1")
expect(LLMRequest.input(updated).id).toBe("req_1")
expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"])
expect(LLMRequest.update(updated, {})).toBe(updated)
})
test("builds tool choices from names and tools", () => {
const tool = LLM.toolDefinition({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })