diff --git a/packages/core/src/model-resolver.ts b/packages/core/src/model-resolver.ts index a8840a35fc..d25be4589b 100644 --- a/packages/core/src/model-resolver.ts +++ b/packages/core/src/model-resolver.ts @@ -68,6 +68,8 @@ export interface Resolved { readonly capabilities: Capabilities /** Catalog pricing in dollars per million tokens. */ readonly cost: Info["cost"] + /** Catalog token limits used by Core for context management. */ + readonly limit: Info["limit"] } export interface Interface { @@ -297,6 +299,7 @@ export const layer = Layer.effect( }), capabilities: selected.capabilities, cost: selected.cost, + limit: selected.limit, } }) return Service.of({ diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 970c9ef1be..e3385644d9 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -1,6 +1,6 @@ export * as SessionCompaction from "./compaction.js" -import { LLM, LLMClient, AIError, LLMEvent, Message, type LLMRequest, type LanguageModel } from "@opencode-ai/ai" +import { LLM, LLMClient, AIError, LLMEvent, Message, type LLMRequest } from "@opencode-ai/ai" import type { StreamOptions } from "@opencode-ai/ai/route" import { SessionError } from "@opencode-ai/schema/session-error" import { Context, Effect, Layer, Stream } from "effect" @@ -18,7 +18,6 @@ import { SessionRunnerModel } from "./runner/model.js" import { SessionSchema } from "./schema.js" import { toSessionError } from "./to-session-error.js" import { Token } from "../util/token.js" -import type { Info, Ref } from "../model.js" import { SessionUsage } from "./usage.js" import { PluginHooks } from "../plugin/hooks.js" import { Agent } from "../agent.js" @@ -83,9 +82,7 @@ type Dependencies = { export type AutoInput = { readonly session: SessionSchema.Info readonly messages: readonly SessionMessage.Info[] - readonly model: LanguageModel - readonly ref: Ref - readonly cost: Info["cost"] + readonly resolved: SessionRunnerModel.Resolved } export type ManualInput = { @@ -95,13 +92,11 @@ export type ManualInput = { readonly started?: boolean } -type RequiredInput = Omit +type RequiredInput = Pick type Plan = { readonly session: SessionSchema.Info - readonly model: LanguageModel - readonly ref: Ref - readonly cost: Info["cost"] + readonly resolved: SessionRunnerModel.Resolved readonly reason: SessionMessage.Compaction["reason"] readonly prompt: string readonly recent: string @@ -273,9 +268,9 @@ const make = (dependencies: Dependencies) => { ) const request = yield* SessionModelHook.apply( dependencies.hooks, - { sessionID: plan.session.id, agent: Agent.ID.make("compaction"), model: plan.ref }, + { sessionID: plan.session.id, agent: Agent.ID.make("compaction"), model: plan.resolved.ref }, LLM.request({ - model: plan.model, + model: plan.resolved.model, promptCacheKey: SessionPromptCacheKey.make(plan.session.id), http: { headers: SessionModelHeaders.make(plan.session, dependencies.app) }, messages: [Message.user(plan.prompt)], @@ -287,7 +282,7 @@ const make = (dependencies: Dependencies) => { http: SessionModelHttp.middleware(dependencies.hooks, { sessionID: plan.session.id, agent: Agent.ID.make("compaction"), - model: plan.ref, + model: plan.resolved.ref, }), }) .pipe( @@ -305,7 +300,7 @@ const make = (dependencies: Dependencies) => { }) } if (LLMEvent.is.stepFinish(event)) { - const step = SessionUsage.record(event.usage, plan.cost) + const step = SessionUsage.record(event.usage, plan.resolved.cost) usage = usage ? SessionUsage.add(usage, step) : step } return Effect.void @@ -354,9 +349,7 @@ const make = (dependencies: Dependencies) => { if (content) return yield* execute({ session: input.session, - model: input.model, - ref: input.ref, - cost: input.cost, + resolved: input.resolved, reason: "auto", ...content, }) @@ -370,17 +363,17 @@ const make = (dependencies: Dependencies) => { const required = (input: RequiredInput) => { const config = state.get() if (!config.auto) return false - const context = input.model.route.defaults.limits?.context - if (context === undefined || context <= 0) return false + const limit = input.resolved.limit + const context = limit.context + if (context <= 0) return false const last = input.messages.findLast( (message): message is SessionMessage.Assistant & { tokens: NonNullable } => message.type === "assistant" && message.tokens !== undefined, ) if (!last) return false - const limits = input.model.route.defaults.limits - const output = Math.min(limits?.output ?? 0, OUTPUT_TOKEN_MAX) + const output = Math.min(limit.output, OUTPUT_TOKEN_MAX) const promptCeiling = Math.min( - limits?.input === undefined ? Number.POSITIVE_INFINITY : limits.input - config.buffer, + limit.input === undefined ? Number.POSITIVE_INFINITY : limit.input - config.buffer, context - Math.max(output, config.buffer), ) const used = @@ -410,9 +403,7 @@ const make = (dependencies: Dependencies) => { if ("status" in resolved) return resolved return yield* execute({ session: input.session, - model: resolved.model, - ref: resolved.ref, - cost: resolved.cost, + resolved, reason: "manual", inputID: input.inputID, started: input.started, diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 0e721f4913..48f33911e6 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -318,7 +318,7 @@ const layer = Layer.effect( const model = resolved.model // Make room: history must fit the context window before the call. A pending manual // compaction owns this instead; the runner executes it between steps. - const compactionInput = { session, messages: loaded.messages, model, ref: resolved.ref, cost: resolved.cost } + const compactionInput = { session, messages: loaded.messages, resolved } if (compaction.required(compactionInput)) { const compacted = yield* compaction.compact(compactionInput) if (compacted.status === "completed") diff --git a/packages/core/src/session/runner/model.ts b/packages/core/src/session/runner/model.ts index 00205e716c..091880452a 100644 --- a/packages/core/src/session/runner/model.ts +++ b/packages/core/src/session/runner/model.ts @@ -53,6 +53,7 @@ export const resolved = ( readonly capabilities: Capabilities readonly variant?: VariantID readonly cost: Info["cost"] + readonly limit: Info["limit"] }, ): Resolved => ({ model, @@ -63,6 +64,7 @@ export const resolved = ( }), capabilities: options.capabilities, cost: options.cost, + limit: options.limit, }) const layer = Layer.effect( diff --git a/packages/core/test/config/compaction.test.ts b/packages/core/test/config/compaction.test.ts index 0010c6871c..eed605633e 100644 --- a/packages/core/test/config/compaction.test.ts +++ b/packages/core/test/config/compaction.test.ts @@ -26,7 +26,13 @@ import { host } from "../plugin/host" const model = LanguageModel.make({ id: "test-model", provider: "test-provider", - route: OpenAIChat.route.with({ limits: { context: 100_000, output: 1_000 } }), + route: OpenAIChat.route, +}) +const limit = { context: 100_000, output: 1_000 } +const resolved = SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + cost: [], + limit, }) const config = Config.testLayer() const it = testEffect( @@ -42,13 +48,7 @@ const it = testEffect( [ SessionRunnerModel.node, Layer.mock(SessionRunnerModel.Service)({ - resolve: () => - Effect.succeed( - SessionRunnerModel.resolved(model, { - capabilities: { tools: true, input: ["text"], output: ["text"] }, - cost: [], - }), - ), + resolve: () => Effect.succeed(resolved), }), ], [Config.node, config], @@ -150,8 +150,7 @@ const session = Session.Info.make({ }) const input = (tokens: number) => ({ session, - model, - cost: [], + resolved, messages: [ Schema.decodeUnknownSync(SessionMessage.Assistant)({ id: SessionMessage.ID.make("msg_compaction_config"), diff --git a/packages/core/test/generate.test.ts b/packages/core/test/generate.test.ts index b36d9c5303..46ecba66e0 100644 --- a/packages/core/test/generate.test.ts +++ b/packages/core/test/generate.test.ts @@ -92,6 +92,7 @@ resolverIt.effect("resolves dynamic models with their catalog metadata", () => ref: Ref.make({ providerID: selected.providerID, id: selected.id }), capabilities: selected.capabilities, cost: selected.cost, + limit: selected.limit, }) }), ) diff --git a/packages/core/test/model-resolver.test.ts b/packages/core/test/model-resolver.test.ts index 772abab089..7d7ee3bafe 100644 --- a/packages/core/test/model-resolver.test.ts +++ b/packages/core/test/model-resolver.test.ts @@ -346,6 +346,7 @@ describe("ModelResolver", () => { const resolver = yield* ModelResolver.Service const resolved = yield* resolver.resolveModel(selected) + expect(resolved.limit).toEqual(selected.limit) const headers = yield* resolved.model.route.auth.apply({ request: LLM.request({ model: resolved.model, prompt: "Hello" }), method: "POST", diff --git a/packages/core/test/plugin/provider-openai.test.ts b/packages/core/test/plugin/provider-openai.test.ts index e1e0130382..a19eda022a 100644 --- a/packages/core/test/plugin/provider-openai.test.ts +++ b/packages/core/test/plugin/provider-openai.test.ts @@ -224,6 +224,7 @@ describe("OpenAIPlugin", () => { const model = SessionRunnerModel.resolved(OpenAIResponses.route.model({ id: "gpt-5.5" }), { capabilities: { tools: true, input: ["text"], output: ["text"] }, cost: [], + limit: { context: 200_000, output: 32_000 }, }) const program = Effect.gen(function* () { const requests = yield* SessionModelRequest.Service diff --git a/packages/core/test/session-compact.test.ts b/packages/core/test/session-compact.test.ts index d23a3f0222..724713ed71 100644 --- a/packages/core/test/session-compact.test.ts +++ b/packages/core/test/session-compact.test.ts @@ -45,6 +45,7 @@ const models = Layer.mock(SessionRunnerModel.Service)({ SessionRunnerModel.resolved(model, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost: [], + limit: { context: 10_000, output: 1_000 }, }), ), }) diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 08033ba8aa..295db646ee 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -66,14 +66,13 @@ const client = Layer.mock(LLMClient.Service)({ }, generate: () => Effect.die("unused"), }) +const resolved = SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost, + limit: { context: 200_000, output: 32_000 }, +}) const models = Layer.mock(SessionRunnerModel.Service)({ - resolve: () => - Effect.succeed( - SessionRunnerModel.resolved(model, { - capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, - cost, - }), - ), + resolve: () => Effect.succeed(resolved), }) const it = testEffect( AppNodeBuilder.build( @@ -141,14 +140,13 @@ it.effect("auto compaction reserves a buffer below the prompt ceiling", () => time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) }, location: Location.Ref.make({ directory: AbsolutePath.make("/tmp") }), }) - const input = (tokens: number, limits: { context: number; input?: number; output: number }) => ({ + const input = (tokens: number, limit: { context: number; input?: number; output: number }) => ({ session, - model: LanguageModel.make({ - id: "test-model", - provider: "test-provider", - route: OpenAIChat.route.with({ limits }), + resolved: SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost: [], + limit, }), - cost: [], messages: [ Schema.decodeUnknownSync(SessionMessage.Assistant)({ id: SessionMessage.ID.make("msg_assistant"), diff --git a/packages/core/test/session-generate.test.ts b/packages/core/test/session-generate.test.ts index fbf6524103..8313800f12 100644 --- a/packages/core/test/session-generate.test.ts +++ b/packages/core/test/session-generate.test.ts @@ -85,6 +85,7 @@ const models = Layer.mock(SessionRunnerModel.Service)({ SessionRunnerModel.resolved(model, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost: [], + limit: { context: 200_000, output: 32_000 }, }), ), }) diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index 7a9f83bf45..1d29c9bddf 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -70,6 +70,7 @@ const models = Layer.mock(SessionRunnerModel.Service)({ SessionRunnerModel.resolved(model, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost: [], + limit: { context: 200_000, output: 20 }, }), ), }) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 13b235d5a0..c1484259db 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -146,29 +146,20 @@ const modelTransport = Layer.succeed( closeAll: Effect.void, }), ) -const model = LanguageModel.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }) +type ModelLimit = { readonly context: number; readonly input?: number; readonly output: number } +const defaultModelLimit = { context: 200_000, output: 32_000 } +const modelLimits = new Map() +const testModel = (id: string, limit: ModelLimit = defaultModelLimit) => { + modelLimits.set(id, limit) + return LanguageModel.make({ id, provider: "fake", route: OpenAIChat.route }) +} +const model = testModel("fake-model") const defaultSystem = SessionSystemPrompt.make([]) -const replacementModel = LanguageModel.make({ id: "replacement", provider: "fake", route: OpenAIChat.route }) -const compactModel = LanguageModel.make({ - id: "compact", - provider: "fake", - route: OpenAIChat.route.with({ limits: { context: 4_000, output: 50 } }), -}) -const fullOutputModel = LanguageModel.make({ - id: "full-output", - provider: "fake", - route: OpenAIChat.route.with({ limits: { context: 262_144, output: 262_144 } }), -}) -const undersizedContextModel = LanguageModel.make({ - id: "undersized-context", - provider: "fake", - route: OpenAIChat.route.with({ limits: { context: 1, output: 1_000 } }), -}) -const recoveryModel = LanguageModel.make({ - id: "recovery", - provider: "fake", - route: OpenAIChat.route.with({ limits: { context: 20_000, output: 1_000 } }), -}) +const replacementModel = testModel("replacement") +const compactModel = testModel("compact", { context: 4_000, output: 50 }) +const fullOutputModel = testModel("full-output", { context: 262_144, output: 262_144 }) +const undersizedContextModel = testModel("undersized-context", { context: 1, output: 1_000 }) +const recoveryModel = testModel("recovery", { context: 20_000, output: 1_000 }) test("calculates step cost using the matching context tier", () => { expect( @@ -304,13 +295,15 @@ let currentModel = model const models = Layer.mock(SessionRunnerModel.Service)({ resolve: (session) => modelResolveHook.pipe( - Effect.as( - SessionRunnerModel.resolved(session.model?.id === "replacement" ? replacementModel : currentModel, { + Effect.map(() => { + const selected = session.model?.id === "replacement" ? replacementModel : currentModel + return SessionRunnerModel.resolved(selected, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost: [], + limit: modelLimits.get(String(selected.id)) ?? defaultModelLimit, variant: session.model?.variant, - }), - ), + }) + }), ), }) const systemContextKey = Instructions.Key.make("test/context") diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index b32e98e5ea..2cf58fef5b 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -72,6 +72,7 @@ const models = Layer.mock(SessionRunnerModel.Service)({ SessionRunnerModel.resolved(model, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost, + limit: { context: 200_000, output: 32_000 }, }), ), })