From cfe8fdb149d28deb486c16622860f2f8f56cf16b Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 5 May 2026 21:33:03 -0400 Subject: [PATCH] refactor(llm): resolve adapters from registry --- packages/llm/example/tutorial.ts | 13 +- packages/llm/src/adapter.ts | 138 +++++----- packages/llm/src/index.ts | 2 +- packages/llm/src/llm.ts | 100 ++----- .../llm/src/protocols/utils/openai-options.ts | 51 ++-- packages/llm/src/schema.ts | 244 +++++++++++++----- packages/llm/src/tool-runtime.ts | 53 ++-- packages/llm/test/adapter.test.ts | 10 +- packages/llm/test/llm.test.ts | 40 +++ 9 files changed, 363 insertions(+), 288 deletions(-) diff --git a/packages/llm/example/tutorial.ts b/packages/llm/example/tutorial.ts index 6e60f96c44..66733f21c3 100644 --- a/packages/llm/example/tutorial.ts +++ b/packages/llm/example/tutorial.ts @@ -129,18 +129,9 @@ const FakeAdapter = Adapter.make({ }) // A provider module exports a model helper. The model helper sets provider -// identity, protocol id, and the adapter that can run this model handle. -// Serialized / revived models can still use explicit provider adapters. +// identity, protocol id, and the adapter id resolved by the registry. const FakeEcho = { - model: (id: string) => - Adapter.bindModel( - LLM.model({ - id, - provider: "fake-echo", - protocol: "fake-echo", - }), - FakeAdapter, - ), + model: (id: string) => Adapter.model(FakeAdapter, { provider: "fake-echo" })({ id }), } // `LLMClient.prepare` is the lower-level inspection hook: it compiles through diff --git a/packages/llm/src/adapter.ts b/packages/llm/src/adapter.ts index f1d94225cd..fb05302236 100644 --- a/packages/llm/src/adapter.ts +++ b/packages/llm/src/adapter.ts @@ -9,22 +9,28 @@ import type { Protocol } from "./protocol" import * as ProviderShared from "./protocols/shared" import type { AdapterID, + GenerationOptionsInput, LLMError, LLMEvent, PreparedRequestOf, ProtocolID, } from "./schema" import { + GenerationOptions, + HttpOptions, LLMRequest, LLMResponse, ModelCapabilities, ModelID, ModelLimits, - ModelPolicy, ModelRef, NoAdapterError, PreparedRequest, ProviderID, + mergeGenerationOptions, + mergeHttpOptions, + mergeJsonRecords, + mergeProviderOptions, } from "./schema" export interface HttpContext { @@ -56,20 +62,15 @@ export interface AdapterDefinition extends Adapter {} // oxlint-disable-next-line typescript-eslint/no-explicit-any export type AnyAdapter = AdapterDefinition -const MODEL_ADAPTER = Symbol.for("@opencode-ai/llm.model-adapter") -type BoundModel = ModelRef & { readonly [MODEL_ADAPTER]?: AnyAdapter } +const adapterRegistry = new Map() -const modelAdapters = new WeakMap() - -const modelAdapter = (model: ModelRef) => (model as BoundModel)[MODEL_ADAPTER] ?? modelAdapters.get(model) -const bindModelAdapter = (model: ModelRef, adapter: AnyAdapter) => { - if (!Object.isExtensible(model)) { - modelAdapters.set(model, adapter) - return - } - Object.defineProperty(model, MODEL_ADAPTER, { value: adapter, configurable: true }) +const register = (adapter: Adapter): Adapter => { + if (!adapterRegistry.has(adapter.id)) adapterRegistry.set(adapter.id, adapter) + return adapter } +const registeredAdapter = (id: string) => adapterRegistry.get(id) + export type ModelCapabilitiesInput = { readonly input?: Partial readonly output?: Partial @@ -80,18 +81,19 @@ export type ModelCapabilitiesInput = { } } -export type ModelPolicyInput = ModelPolicy | ConstructorParameters[0] +export type HttpOptionsInput = HttpOptions | ConstructorParameters[0] export type ModelRefInput = Omit< ConstructorParameters[0], - "id" | "provider" | "adapter" | "capabilities" | "limits" | "policy" + "id" | "provider" | "adapter" | "capabilities" | "limits" | "generation" | "http" > & { readonly id: string | ModelID readonly provider: string | ProviderID readonly adapter?: string | AdapterID readonly capabilities?: ModelCapabilities | ModelCapabilitiesInput readonly limits?: ModelLimits | ConstructorParameters[0] - readonly policy?: ModelPolicyInput + readonly generation?: GenerationOptionsInput + readonly http?: HttpOptionsInput } export type AdapterModelInput = Omit @@ -124,9 +126,14 @@ export const modelLimits = (input: ModelLimits | ConstructorParameters { - if (input === undefined || input instanceof ModelPolicy) return input - return new ModelPolicy(input) +export const generationOptions = (input: GenerationOptionsInput | undefined) => { + if (input === undefined || input instanceof GenerationOptions) return input + return new GenerationOptions(input) +} + +export const httpOptions = (input: HttpOptionsInput | undefined) => { + if (input === undefined || input instanceof HttpOptions) return input + return new HttpOptions(input) } export const modelRef = (input: ModelRefInput) => @@ -138,19 +145,10 @@ export const modelRef = (input: ModelRefInput) => protocol: input.protocol, capabilities: modelCapabilities(input.capabilities), limits: modelLimits(input.limits), - policy: modelPolicy(input.policy), + generation: generationOptions(input.generation), + http: httpOptions(input.http), }) -export const bindModel = (model: Model, adapter: AnyAdapter): Model => { - if (model.adapter !== adapter.id || model.protocol !== adapter.protocol) { - throw new Error( - `Cannot bind ${adapter.id} adapter (${adapter.protocol}) to ${model.provider}/${model.id} via ${model.adapter} (${model.protocol})`, - ) - } - bindModelAdapter(model, adapter) - return model -} - function model( adapter: AnyAdapter, defaults: AdapterModelDefaults, @@ -170,51 +168,22 @@ function model( const mapped = options.mapInput?.(input) ?? input const provider = defaults.provider ?? ("provider" in mapped ? mapped.provider : undefined) if (!provider) throw new Error(`Adapter.model(${adapter.id}) requires a provider`) - return bindModel( - modelRef({ - ...defaults, - ...mapped, - provider, - adapter: adapter.id, - protocol: adapter.protocol, - capabilities: mapped.capabilities ?? defaults.capabilities, - limits: mapped.limits ?? defaults.limits, - }), - adapter, - ) + register(adapter) + return modelRef({ + ...defaults, + ...mapped, + provider, + adapter: adapter.id, + protocol: adapter.protocol, + capabilities: mapped.capabilities ?? defaults.capabilities, + limits: mapped.limits ?? defaults.limits, + generation: mergeGenerationOptions(defaults.generation, mapped.generation), + providerOptions: mergeProviderOptions(defaults.providerOptions, mapped.providerOptions), + http: mergeHttpOptions(httpOptions(defaults.http), httpOptions(mapped.http)), + }) } } -export const preserveModelBinding = (source: ModelRef, target: Model): Model => { - const adapter = modelAdapter(source) - if (!adapter) return target - return bindModel(target, adapter) -} - -export const updateLLMRequest = ( - request: LLMRequest, - patch: Partial[0]>, -) => { - const model = patch.model ?? request.model - const next = new LLMRequest({ - id: request.id, - model, - system: request.system, - messages: request.messages, - tools: request.tools, - toolChoice: request.toolChoice, - generation: request.generation, - reasoning: request.reasoning, - cache: request.cache, - responseFormat: request.responseFormat, - metadata: request.metadata, - native: request.native, - ...patch, - }) - preserveModelBinding(model, next.model) - return next -} - export interface LLMClient { /** * Compile a request through protocol payload lowering, validation, and HTTP @@ -292,12 +261,25 @@ export function make( ), ) const buildHeaders = input.headers ?? (() => ({})) + const applyQuery = (url: string, query: Record | undefined) => { + if (!query) return url + const next = new URL(url) + Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value)) + return next.toString() + } const toHttp = (payload: Payload, ctx: HttpContext) => Effect.gen(function* () { - const url = (yield* renderEndpoint(input.endpoint, { request: ctx.request, payload })).toString() - const body = encodePayload(payload) - const merged = { ...buildHeaders({ request: ctx.request }), ...ctx.request.model.headers } + const url = applyQuery( + (yield* renderEndpoint(input.endpoint, { request: ctx.request, payload })).toString(), + ctx.request.http?.query, + ) + const body = ctx.request.http?.body === undefined + ? encodePayload(payload) + : ProviderShared.isRecord(payload) + ? ProviderShared.encodeJson(mergeJsonRecords(payload, ctx.request.http.body) ?? {}) + : yield* ProviderShared.invalidRequest("http.body can only overlay JSON object request bodies") + const merged = { ...buildHeaders({ request: ctx.request }), ...ctx.request.model.headers, ...ctx.request.http?.headers } const headers = yield* auth({ request: ctx.request, method: "POST", @@ -320,14 +302,14 @@ export function make( onHalt: protocol.onHalt, }) - return { + return register({ id: input.id, protocol: protocol.id, payloadSchema: protocol.payload, toPayload: protocol.toPayload, toHttp, parse, - } + }) } /** @@ -339,7 +321,7 @@ const makeClient = (options: ClientOptions = {}): LLMClient => { const adapters = new Map((options.adapters ?? []).map((adapter) => [adapter.id, adapter] as const)) const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) { - const adapter = adapters.get(request.model.adapter) ?? modelAdapter(request.model) + const adapter = adapters.get(request.model.adapter) ?? registeredAdapter(request.model.adapter) if (!adapter) return yield* noAdapter(request.model) const payload = yield* adapter.toPayload(request).pipe( @@ -400,6 +382,6 @@ const makeClient = (options: ClientOptions = {}): LLMClient => { return { prepare: prepare as LLMClient["prepare"], stream, generate } } -export const Adapter = { bindModel, make, model } as const +export const Adapter = { make, model, register } as const export const LLMClient = { make: makeClient } diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index 6c8ecabb43..b7f7b0874a 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -1,4 +1,4 @@ -export { Adapter, LLMClient, modelCapabilities, modelLimits, modelRef, updateLLMRequest } from "./adapter" +export { Adapter, LLMClient, modelCapabilities, modelLimits, modelRef } from "./adapter" export type { Adapter as AdapterShape, AdapterDefinition, diff --git a/packages/llm/src/llm.ts b/packages/llm/src/llm.ts index bcf9b648d7..bed99f54d3 100644 --- a/packages/llm/src/llm.ts +++ b/packages/llm/src/llm.ts @@ -4,7 +4,6 @@ import { modelCapabilities, modelLimits, modelRef, - preserveModelBinding, type ModelCapabilitiesInput, type ModelRefInput, } from "./adapter" @@ -13,19 +12,20 @@ import { type Tools } from "./tool" import { ToolRuntime, type RunOptions } from "./tool-runtime" import { GenerationOptions, - CacheIntent, + HttpOptions, LLMEvent, LLMRequest, LLMResponse, Message, - ReasoningIntent, ToolChoice, ToolDefinition, type ContentPart, type SystemPart, - type ToolCallPart, - type ToolResultPart, - type ToolResultValue, + ToolCallPart, + ToolResultPart, + mergeGenerationOptions, + mergeHttpOptions, + mergeProviderOptions, } from "./schema" import type { LLMError } from "./schema" @@ -80,21 +80,16 @@ export type CapabilitiesInput = ModelCapabilitiesInput export type ModelInput = ModelRefInput -export type MessageInput = Omit[0], "content"> & { - readonly content: string | ContentPart | ReadonlyArray -} +export type MessageInput = Message.Input export type ToolChoiceInput = ToolChoice | ConstructorParameters[0] | ToolDefinition | string export type ToolChoiceMode = Exclude -export type ToolResultInput = Omit & { - readonly result: unknown - readonly resultType?: ToolResultValue["type"] -} +export type ToolResultInput = Parameters[0] export type RequestInput = Omit< ConstructorParameters[0], - "system" | "messages" | "tools" | "toolChoice" | "generation" + "system" | "messages" | "tools" | "toolChoice" | "generation" | "http" > & { readonly system?: string | SystemPart | ReadonlyArray readonly prompt?: string | ContentPart | ReadonlyArray @@ -102,33 +97,27 @@ export type RequestInput = Omit< readonly tools?: ReadonlyArray[0]> readonly toolChoice?: ToolChoiceInput readonly generation?: GenerationOptions | ConstructorParameters[0] + readonly http?: HttpOptions | ConstructorParameters[0] } export const capabilities = modelCapabilities export const limits = modelLimits -export const text = (value: string): ContentPart => ({ type: "text", text: value }) +export const text = Message.text export const system = (value: string): SystemPart => ({ type: "text", text: value }) -const contentParts = (input: string | ContentPart | ReadonlyArray) => - typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input] - const systemParts = (input?: string | SystemPart | ReadonlyArray) => { if (input === undefined) return [] return typeof input === "string" ? [system(input)] : Array.isArray(input) ? [...input] : [input] } -export const message = (input: Message | MessageInput) => { - if (input instanceof Message) return input - return new Message({ ...input, content: contentParts(input.content) }) -} +export const message = Message.make -export const user = (content: string | ContentPart | ReadonlyArray) => message({ role: "user", content }) +export const user = Message.user -export const assistant = (content: string | ContentPart | ReadonlyArray) => - message({ role: "assistant", content }) +export const assistant = Message.assistant export const model = modelRef @@ -137,30 +126,11 @@ export const toolDefinition = (input: ToolDefinition | ConstructorParameters): ToolCallPart => ({ type: "tool-call", ...input }) +export const toolCall = ToolCallPart.make -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +export const toolResult = ToolResultPart.make -const isToolResultValue = (value: unknown): value is ToolResultValue => - isRecord(value) && (value.type === "text" || value.type === "json" || value.type === "error") && "value" in value - -const toolResultValue = (value: unknown, type: ToolResultValue["type"] = "json"): ToolResultValue => { - if (isToolResultValue(value)) return value - return { type, value } -} - -export const toolResult = (input: ToolResultInput): ToolResultPart => ({ - type: "tool-result", - id: input.id, - name: input.name, - result: toolResultValue(input.result, input.resultType), - providerExecuted: input.providerExecuted, - metadata: input.metadata, -}) - -export const toolMessage = (input: ToolResultPart | ToolResultInput) => - message({ role: "tool", content: ["type" in input ? input : toolResult(input)] }) +export const toolMessage = Message.tool export const toolChoiceName = (name: string) => new ToolChoice({ type: "tool", name }) @@ -180,29 +150,13 @@ export const generation = (input: GenerationOptions | ConstructorParameters[0] | undefined) => { - if (input === undefined || input instanceof ReasoningIntent) return input - return new ReasoningIntent(input) -} - -const cache = (input: CacheIntent | ConstructorParameters[0] | undefined) => { - if (input === undefined || input instanceof CacheIntent) return input - return new CacheIntent(input) +const http = (input: HttpOptions | ConstructorParameters[0] | undefined) => { + if (input === undefined || input instanceof HttpOptions) return input + return new HttpOptions(input) } export const requestInput = (input: LLMRequest): RequestInput => ({ - id: input.id, - model: input.model, - system: input.system, - messages: input.messages, - tools: input.tools, - toolChoice: input.toolChoice, - generation: input.generation, - reasoning: input.reasoning, - cache: input.cache, - responseFormat: input.responseFormat, - metadata: input.metadata, - native: input.native, + ...LLMRequest.input(input), }) export const request = (input: RequestInput) => { @@ -213,20 +167,20 @@ export const request = (input: RequestInput) => { tools, toolChoice: requestToolChoice, generation: requestGeneration, + providerOptions: requestProviderOptions, + http: requestHttp, ...rest } = input - const result = new LLMRequest({ + return new LLMRequest({ ...rest, system: systemParts(requestSystem), messages: [...(messages?.map(message) ?? []), ...(prompt === undefined ? [] : [user(prompt)])], tools: tools?.map(toolDefinition) ?? [], toolChoice: requestToolChoice ? toolChoice(requestToolChoice) : undefined, - generation: generation(requestGeneration), - reasoning: reasoning(rest.reasoning), - cache: cache(rest.cache), + generation: mergeGenerationOptions(input.model.generation, generation(requestGeneration)) ?? generation(), + providerOptions: mergeProviderOptions(input.model.providerOptions, requestProviderOptions), + http: mergeHttpOptions(input.model.http, http(requestHttp)), }) - preserveModelBinding(input.model, result.model) - return result } export const updateRequest = (input: LLMRequest, patch: Partial) => diff --git a/packages/llm/src/protocols/utils/openai-options.ts b/packages/llm/src/protocols/utils/openai-options.ts index 0181d5e833..13b7f3318b 100644 --- a/packages/llm/src/protocols/utils/openai-options.ts +++ b/packages/llm/src/protocols/utils/openai-options.ts @@ -1,46 +1,55 @@ import { Schema } from "effect" -import type { LLMRequest, ReasoningEffort } from "../../schema" -import { ReasoningEfforts, TextVerbosity } from "../../schema" +import type { LLMRequest, ReasoningEffort, TextVerbosity as TextVerbosityValue } from "../../schema" +import { ReasoningEfforts, TextVerbosity, mergeProviderOptions } from "../../schema" export const OpenAIReasoningEfforts = ReasoningEfforts.filter( (effort): effort is Exclude => effort !== "max", ) export type OpenAIReasoningEffort = typeof OpenAIReasoningEfforts[number] -const OPENAI_REASONING_EFFORTS = new Set(OpenAIReasoningEfforts) +const REASONING_EFFORTS = new Set(ReasoningEfforts) +const OPENAI_REASONING_EFFORTS = new Set(OpenAIReasoningEfforts) +const TEXT_VERBOSITY = new Set(["low", "medium", "high"]) export const OpenAIReasoningEffort = Schema.Literals(OpenAIReasoningEfforts) export const OpenAITextVerbosity = TextVerbosity -export const isReasoningEffort = (effort: ReasoningEffort): effort is OpenAIReasoningEffort => - OPENAI_REASONING_EFFORTS.has(effort) +const isAnyReasoningEffort = (effort: unknown): effort is ReasoningEffort => + typeof effort === "string" && REASONING_EFFORTS.has(effort) -export const store = (request: LLMRequest) => - typeof request.model.policy?.retention?.store === "boolean" ? request.model.policy.retention.store : undefined +export const isReasoningEffort = (effort: unknown): effort is OpenAIReasoningEffort => + typeof effort === "string" && OPENAI_REASONING_EFFORTS.has(effort) + +const isTextVerbosity = (value: unknown): value is TextVerbosityValue => + typeof value === "string" && TEXT_VERBOSITY.has(value) + +const options = (request: LLMRequest) => mergeProviderOptions(request.model.providerOptions, request.providerOptions)?.openai + +export const store = (request: LLMRequest): boolean | undefined => { + const value = options(request)?.store + return typeof value === "boolean" ? value : undefined +} export const reasoningEffort = (request: LLMRequest): ReasoningEffort | undefined => { - if (request.reasoning?.enabled === false) return undefined - return request.reasoning?.effort ?? request.model.policy?.reasoning?.effort + const value = options(request)?.reasoningEffort + return isAnyReasoningEffort(value) ? value : undefined } export const reasoningSummary = (request: LLMRequest): "auto" | undefined => { - if (request.reasoning?.enabled === false) return undefined - if (request.reasoning?.summary !== undefined) return request.reasoning.summary ? "auto" : undefined - const summary = request.model.policy?.reasoning?.summary - return summary === true || summary === "auto" ? "auto" : undefined + return options(request)?.reasoningSummary === "auto" ? "auto" : undefined } -export const encryptedReasoning = (request: LLMRequest) => { - if (request.reasoning?.enabled === false) return undefined - if (request.reasoning?.encryptedContent !== undefined) return request.reasoning.encryptedContent - return request.model.policy?.reasoning?.encryptedState -} +export const encryptedReasoning = (request: LLMRequest) => + options(request)?.includeEncryptedReasoning === true ? true : undefined export const promptCacheKey = (request: LLMRequest) => { - if (request.cache?.enabled === false) return undefined - return request.cache?.key ?? request.model.policy?.cache?.promptKey + const value = options(request)?.promptCacheKey + return typeof value === "string" ? value : undefined } -export const textVerbosity = (request: LLMRequest) => request.model.policy?.text?.verbosity +export const textVerbosity = (request: LLMRequest) => { + const value = options(request)?.textVerbosity + return isTextVerbosity(value) ? value : undefined +} export * as OpenAIOptions from "./openai-options" diff --git a/packages/llm/src/schema.ts b/packages/llm/src/schema.ts index 73b0be4515..2a1427dcf6 100644 --- a/packages/llm/src/schema.ts +++ b/packages/llm/src/schema.ts @@ -26,9 +26,6 @@ export type ReasoningEffort = Schema.Schema.Type export const TextVerbosity = Schema.Literals(["low", "medium", "high"]) export type TextVerbosity = Schema.Schema.Type -export const TransformPhase = Schema.Literals(["request", "prompt", "tool-schema", "payload", "stream"]) -export type TransformPhase = Schema.Schema.Type - export const MessageRole = Schema.Literals(["user", "assistant", "tool"]) export type MessageRole = Schema.Schema.Type @@ -38,6 +35,105 @@ export type FinishReason = Schema.Schema.Type export const JsonSchema = Schema.Record(Schema.String, Schema.Unknown) export type JsonSchema = Schema.Schema.Type +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +export const mergeJsonRecords = (...items: ReadonlyArray | undefined>): Record | undefined => { + const result: Record = items.reduce>((acc, item) => { + if (!item) return acc + return Object.entries(item).reduce>((next, [key, value]) => { + if (value === undefined) return next + return { + ...next, + [key]: isRecord(next[key]) && isRecord(value) ? mergeJsonRecords(next[key], value) : value, + } + }, acc) + }, {}) + return Object.keys(result).length === 0 ? undefined : result +} + +const mergeStringRecords = (...items: ReadonlyArray | undefined>): Record | undefined => { + const result = Object.fromEntries( + items.flatMap((item) => Object.entries(item ?? {}).filter((entry): entry is [string, string] => entry[1] !== undefined)), + ) + return Object.keys(result).length === 0 ? undefined : result +} + +export const ProviderOptions = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)) +export type ProviderOptions = Schema.Schema.Type + +export const mergeProviderOptions = (...items: ReadonlyArray): ProviderOptions | undefined => { + const result = Object.fromEntries( + Object.entries( + items.reduce>>((acc, item) => { + if (!item) return acc + return Object.entries(item).reduce>>((next, [provider, options]) => ({ + ...next, + [provider]: mergeJsonRecords(next[provider], options) ?? {}, + }), acc) + }, {}), + ).filter((entry) => Object.keys(entry[1]).length > 0), + ) + return Object.keys(result).length === 0 ? undefined : result +} + +export class HttpOptions extends Schema.Class("LLM.HttpOptions")({ + body: Schema.optional(JsonSchema), + headers: Schema.optional(Schema.Record(Schema.String, Schema.String)), + query: Schema.optional(Schema.Record(Schema.String, Schema.String)), +}) {} + +export const mergeHttpOptions = (...items: ReadonlyArray): HttpOptions | undefined => { + const body = mergeJsonRecords(...items.map((item) => item?.body)) + const headers = mergeStringRecords(...items.map((item) => item?.headers)) + const query = mergeStringRecords(...items.map((item) => item?.query)) + if (!body && !headers && !query) return undefined + return new HttpOptions({ body, headers, query }) +} + +export class GenerationOptions extends Schema.Class("LLM.GenerationOptions")({ + maxTokens: Schema.optional(Schema.Number), + temperature: Schema.optional(Schema.Number), + topP: Schema.optional(Schema.Number), + topK: Schema.optional(Schema.Number), + frequencyPenalty: Schema.optional(Schema.Number), + presencePenalty: Schema.optional(Schema.Number), + seed: Schema.optional(Schema.Number), + stop: Schema.optional(Schema.Array(Schema.String)), +}) {} + +export type GenerationOptionsFields = { + readonly maxTokens?: number + readonly temperature?: number + readonly topP?: number + readonly topK?: number + readonly frequencyPenalty?: number + readonly presencePenalty?: number + readonly seed?: number + readonly stop?: ReadonlyArray +} + +export type GenerationOptionsInput = GenerationOptions | GenerationOptionsFields + +const latestGeneration = ( + items: ReadonlyArray, + key: Key, +) => items.findLast((item) => item?.[key] !== undefined)?.[key] + +export const mergeGenerationOptions = (...items: ReadonlyArray) => { + const result = new GenerationOptions({ + maxTokens: latestGeneration(items, "maxTokens"), + temperature: latestGeneration(items, "temperature"), + topP: latestGeneration(items, "topP"), + topK: latestGeneration(items, "topK"), + frequencyPenalty: latestGeneration(items, "frequencyPenalty"), + presencePenalty: latestGeneration(items, "presencePenalty"), + seed: latestGeneration(items, "seed"), + stop: latestGeneration(items, "stop"), + }) + return Object.values(result).some((value) => value !== undefined) ? result : undefined +} + export class ModelCapabilities extends Schema.Class("LLM.ModelCapabilities")({ input: Schema.Struct({ text: Schema.Boolean, @@ -72,30 +168,6 @@ export class ModelLimits extends Schema.Class("LLM.ModelLimits")({ output: Schema.optional(Schema.Number), }) {} -export class ModelPolicy extends Schema.Class("LLM.ModelPolicy")({ - retention: Schema.optional(Schema.Struct({ - store: Schema.optional(Schema.Boolean), - dataCollection: Schema.optional(Schema.Literals(["allow", "deny"])), - })), - reasoning: Schema.optional(Schema.Struct({ - effort: Schema.optional(ReasoningEffort), - summary: Schema.optional(Schema.Union([Schema.Boolean, Schema.Literal("auto")])), - encryptedState: Schema.optional(Schema.Boolean), - display: Schema.optional(Schema.Literals(["summarized", "omitted"])), - })), - text: Schema.optional(Schema.Struct({ - verbosity: Schema.optional(TextVerbosity), - })), - cache: Schema.optional(Schema.Struct({ - promptKey: Schema.optional(Schema.String), - ttl: Schema.optional(Schema.Literals(["5m", "1h"])), - })), - usage: Schema.optional(Schema.Struct({ - include: Schema.optional(Schema.Boolean), - includeCost: Schema.optional(Schema.Boolean), - })), -}) {} - export class ModelRef extends Schema.Class("LLM.ModelRef")({ id: ModelID, provider: ProviderID, @@ -118,13 +190,12 @@ export class ModelRef extends Schema.Class("LLM.ModelRef")({ queryParams: Schema.optional(Schema.Record(Schema.String, Schema.String)), capabilities: ModelCapabilities, limits: ModelLimits, - /** - * Provider-agnostic defaults and policy that protocols can lower into their - * native fields. Request-level options override these defaults. - */ - policy: Schema.optional(ModelPolicy), + /** Provider-neutral generation defaults. Request-level values override them. */ + generation: Schema.optional(GenerationOptions), /** Provider-owned typed-at-the-facade options for non-portable knobs. */ - providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerOptions: Schema.optional(ProviderOptions), + /** Serializable raw HTTP overlays applied to the final outgoing request. */ + http: Schema.optional(HttpOptions), /** * Provider-specific opaque options. Reach for this only when the value is * genuinely provider-private and does not fit a typed axis (e.g. Bedrock's @@ -164,30 +235,50 @@ export const MediaPart = Schema.Struct({ }).annotate({ identifier: "LLM.Content.Media" }) export type MediaPart = Schema.Schema.Type -export const ToolResultValue = Schema.Struct({ +const isToolResultValue = (value: unknown): value is ToolResultValue => + isRecord(value) && (value.type === "text" || value.type === "json" || value.type === "error") && "value" in value + +export const ToolResultValue = Object.assign(Schema.Struct({ type: Schema.Literals(["json", "text", "error"]), value: Schema.Unknown, -}).annotate({ identifier: "LLM.ToolResult" }) +}).annotate({ identifier: "LLM.ToolResult" }), { + make: (value: unknown, type: ToolResultValue["type"] = "json"): ToolResultValue => + isToolResultValue(value) ? value : { type, value }, +}) export type ToolResultValue = Schema.Schema.Type -export const ToolCallPart = Schema.Struct({ +export const ToolCallPart = Object.assign(Schema.Struct({ type: Schema.Literal("tool-call"), id: Schema.String, name: Schema.String, input: Schema.Unknown, providerExecuted: Schema.optional(Schema.Boolean), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), -}).annotate({ identifier: "LLM.Content.ToolCall" }) +}).annotate({ identifier: "LLM.Content.ToolCall" }), { + make: (input: Omit): ToolCallPart => ({ type: "tool-call", ...input }), +}) export type ToolCallPart = Schema.Schema.Type -export const ToolResultPart = Schema.Struct({ +export const ToolResultPart = Object.assign(Schema.Struct({ type: Schema.Literal("tool-result"), id: Schema.String, name: Schema.String, result: ToolResultValue, providerExecuted: Schema.optional(Schema.Boolean), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), -}).annotate({ identifier: "LLM.Content.ToolResult" }) +}).annotate({ identifier: "LLM.Content.ToolResult" }), { + make: (input: Omit & { + readonly result: unknown + readonly resultType?: ToolResultValue["type"] + }): ToolResultPart => ({ + type: "tool-result", + id: input.id, + name: input.name, + result: ToolResultValue.make(input.result, input.resultType), + providerExecuted: input.providerExecuted, + metadata: input.metadata, + }), +}) export type ToolResultPart = Schema.Schema.Type export const ReasoningPart = Schema.Struct({ @@ -211,6 +302,30 @@ export class Message extends Schema.Class("LLM.Message")({ native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} +export namespace Message { + export type ContentInput = string | ContentPart | ReadonlyArray + export type Input = Omit[0], "content"> & { + readonly content: ContentInput + } + + export const text = (value: string): ContentPart => ({ type: "text", text: value }) + + export const content = (input: ContentInput) => + typeof input === "string" ? [text(input)] : Array.isArray(input) ? [...input] : [input] + + export const make = (input: Message | Input) => { + if (input instanceof Message) return input + return new Message({ ...input, content: content(input.content) }) + } + + export const user = (content: ContentInput) => make({ role: "user", content }) + + export const assistant = (content: ContentInput) => make({ role: "assistant", content }) + + export const tool = (result: ToolResultPart | Parameters[0]) => + make({ role: "tool", content: ["type" in result ? result : ToolResultPart.make(result)] }) +} + export class ToolDefinition extends Schema.Class("LLM.ToolDefinition")({ name: Schema.String, description: Schema.String, @@ -224,25 +339,6 @@ export class ToolChoice extends Schema.Class("LLM.ToolChoice")({ name: Schema.optional(Schema.String), }) {} -export class GenerationOptions extends Schema.Class("LLM.GenerationOptions")({ - maxTokens: Schema.optional(Schema.Number), - temperature: Schema.optional(Schema.Number), - topP: Schema.optional(Schema.Number), - stop: Schema.optional(Schema.Array(Schema.String)), -}) {} - -export class ReasoningIntent extends Schema.Class("LLM.ReasoningIntent")({ - enabled: Schema.Boolean, - effort: Schema.optional(ReasoningEffort), - summary: Schema.optional(Schema.Boolean), - encryptedContent: Schema.optional(Schema.Boolean), -}) {} - -export class CacheIntent extends Schema.Class("LLM.CacheIntent")({ - enabled: Schema.Boolean, - key: Schema.optional(Schema.String), -}) {} - export const ResponseFormat = Schema.Union([ Schema.Struct({ type: Schema.Literal("text") }), Schema.Struct({ type: Schema.Literal("json"), schema: JsonSchema }), @@ -258,13 +354,39 @@ export class LLMRequest extends Schema.Class("LLM.Request")({ tools: Schema.Array(ToolDefinition), toolChoice: Schema.optional(ToolChoice), generation: GenerationOptions, - reasoning: Schema.optional(ReasoningIntent), - cache: Schema.optional(CacheIntent), + providerOptions: Schema.optional(ProviderOptions), + http: Schema.optional(HttpOptions), responseFormat: Schema.optional(ResponseFormat), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), - native: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} +export namespace LLMRequest { + export type Input = ConstructorParameters[0] + + export const input = (request: LLMRequest): Input => ({ + id: request.id, + model: request.model, + system: request.system, + messages: request.messages, + tools: request.tools, + toolChoice: request.toolChoice, + generation: request.generation, + providerOptions: request.providerOptions, + http: request.http, + responseFormat: request.responseFormat, + metadata: request.metadata, + }) + + export const update = (request: LLMRequest, patch: Partial) => { + if (Object.keys(patch).length === 0) return request + return new LLMRequest({ + ...input(request), + ...patch, + model: patch.model ?? request.model, + }) + } +} + export class Usage extends Schema.Class("LLM.Usage")({ inputTokens: Schema.optional(Schema.Number), outputTokens: Schema.optional(Schema.Number), diff --git a/packages/llm/src/tool-runtime.ts b/packages/llm/src/tool-runtime.ts index a266bd7f58..951d6a484d 100644 --- a/packages/llm/src/tool-runtime.ts +++ b/packages/llm/src/tool-runtime.ts @@ -1,6 +1,6 @@ import { Effect, Stream } from "effect" import type { Concurrency } from "effect/Types" -import { updateLLMRequest, type LLMClient } from "./adapter" +import type { LLMClient } from "./adapter" import type { RequestExecutor } from "./executor" import { type ContentPart, @@ -9,8 +9,9 @@ import { type LLMEvent, LLMRequest, Message, - type ToolCallPart, type ToolResultValue, + ToolCallPart, + ToolResultPart, } from "./schema" import { ToolFailure } from "./schema" import { type AnyTool, type Tools, toDefinitions } from "./tool" @@ -64,12 +65,15 @@ export const run = ( const tools = options.tools as Tools const runtimeTools = toDefinitions(tools) const runtimeToolNames = new Set(runtimeTools.map((tool) => tool.name)) - const initialRequest = updateLLMRequest(options.request, { - tools: [ - ...options.request.tools.filter((tool) => !runtimeToolNames.has(tool.name)), - ...runtimeTools, - ], - }) + const initialRequest = + runtimeTools.length === 0 + ? options.request + : LLMRequest.update(options.request, { + tools: [ + ...options.request.tools.filter((tool) => !runtimeToolNames.has(tool.name)), + ...runtimeTools, + ], + }) const loop = (request: LLMRequest, step: number): Stream.Stream => Stream.unwrap( @@ -91,12 +95,12 @@ export const run = ( (call) => dispatch(tools, call).pipe(Effect.map((result) => [call, result] as const)), { concurrency }, ) - const followUp = updateLLMRequest(request, { + const followUp = LLMRequest.update(request, { messages: [ ...request.messages, - assistant(state.assistantContent), + Message.assistant(state.assistantContent), ...dispatched.map(([call, result]) => - toolMessage({ id: call.id, name: call.name, result }), + Message.tool({ id: call.id, name: call.name, result }), ), ], }) @@ -130,7 +134,7 @@ const accumulate = (state: StepState, event: LLMEvent) => { return } if (event.type === "tool-call") { - const part = toolCall({ + const part = ToolCallPart.make({ id: event.id, name: event.name, input: event.input, @@ -145,7 +149,7 @@ const accumulate = (state: StepState, event: LLMEvent) => { return } if (event.type === "tool-result" && event.providerExecuted) { - state.assistantContent.push(toolResult({ + state.assistantContent.push(ToolResultPart.make({ id: event.id, name: event.name, result: event.result, @@ -167,29 +171,6 @@ const appendStreamingText = (state: StepState, type: "text" | "reasoning", text: state.assistantContent.push({ type, text }) } -const assistant = (content: ReadonlyArray) => new Message({ role: "assistant", content }) - -const toolCall = (input: Omit): ToolCallPart => ({ type: "tool-call", ...input }) - -const toolResult = (input: { - readonly id: string - readonly name: string - readonly result: ToolResultValue - readonly providerExecuted?: boolean -}): ContentPart => ({ - type: "tool-result", - id: input.id, - name: input.name, - result: input.result, - providerExecuted: input.providerExecuted, -}) - -const toolMessage = (input: { - readonly id: string - readonly name: string - readonly result: ToolResultValue -}) => new Message({ role: "tool", content: [toolResult(input)] }) - const dispatch = (tools: Tools, call: ToolCallPart): Effect.Effect => { const tool = tools[call.name] if (!tool) return Effect.succeed({ type: "error" as const, value: `Unknown tool: ${call.name}` }) diff --git a/packages/llm/test/adapter.test.ts b/packages/llm/test/adapter.test.ts index 72afebf5af..b8912a2366 100644 --- a/packages/llm/test/adapter.test.ts +++ b/packages/llm/test/adapter.test.ts @@ -133,12 +133,10 @@ describe("llm adapter", () => { }), ) - it.effect("falls back to adapter bound to model", () => + it.effect("uses registered adapters by model adapter id", () => Effect.gen(function* () { const prepared = yield* LLMClient.make({ adapters: [] }).prepare( - LLM.updateRequest(request, { - model: Adapter.bindModel(updateModel(request.model, { adapter: "gemini-fake" }), gemini), - }), + LLM.updateRequest(request, { model: updateModel(request.model, { adapter: "gemini-fake" }) }), ) expect(prepared.adapter).toBe("gemini-fake") @@ -174,9 +172,7 @@ describe("llm adapter", () => { framing: fakeFraming, }) - const response = yield* LLMClient.make({ adapters: [override] }).generate( - LLM.updateRequest(request, { model: Adapter.bindModel(updateModel(request.model, { adapter: "fake" }), fake) }), - ) + const response = yield* LLMClient.make({ adapters: [override] }).generate(request) expect(response.text).toBe('echo:{"body":"override"}') }), diff --git a/packages/llm/test/llm.test.ts b/packages/llm/test/llm.test.ts index 38d82dc2ba..f0348fa525 100644 --- a/packages/llm/test/llm.test.ts +++ b/packages/llm/test/llm.test.ts @@ -38,6 +38,46 @@ describe("llm constructors", () => { expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"]) }) + test("merges model defaults with call options", () => { + const request = LLM.request({ + model: LLM.model({ + id: "fake-model", + provider: "fake", + protocol: "openai-chat", + generation: { maxTokens: 100, temperature: 1 }, + providerOptions: { openai: { store: false, metadata: { model: true } } }, + http: { body: { metadata: { model: true } }, headers: { "x-shared": "model" }, query: { model: "1" } }, + }), + prompt: "Say hello.", + generation: { temperature: 0 }, + providerOptions: { openai: { store: true, metadata: { request: true } } }, + http: { body: { metadata: { request: true } }, headers: { "x-shared": "request" }, query: { request: "1" } }, + }) + + expect(request.generation).toEqual({ maxTokens: 100, temperature: 0 }) + expect(request.providerOptions).toEqual({ openai: { store: true, metadata: { model: true, request: true } } }) + expect(request.http).toEqual({ + body: { metadata: { model: true, request: true } }, + headers: { "x-shared": "request" }, + query: { model: "1", request: "1" }, + }) + }) + + test("updates canonical requests from the request datatype", () => { + const base = LLM.request({ + id: "req_1", + model: LLM.model({ id: "fake-model", provider: "fake", protocol: "openai-chat" }), + prompt: "Say hello.", + }) + const updated = LLMRequest.update(base, { messages: [...base.messages, LLM.assistant("Hi.")] }) + + expect(updated).toBeInstanceOf(LLMRequest) + expect(updated.id).toBe("req_1") + expect(LLMRequest.input(updated).id).toBe("req_1") + expect(updated.messages.map((message) => message.role)).toEqual(["user", "assistant"]) + expect(LLMRequest.update(updated, {})).toBe(updated) + }) + test("builds tool choices from names and tools", () => { const tool = LLM.toolDefinition({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })