From d19f58c5df0a5362fa1c469c748ac558f79277dd Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:28:27 -0500 Subject: [PATCH] fix(core): use small model for titles (#43702) --- packages/core/src/catalog.ts | 43 +----- packages/core/src/session/title.ts | 135 +++++++++++------ packages/core/test/catalog.test.ts | 58 ++++---- .../test/plugin/provider-opencode.test.ts | 27 ---- packages/core/test/session-title.test.ts | 137 +++++++++++++++++- 5 files changed, 256 insertions(+), 144 deletions(-) diff --git a/packages/core/src/catalog.ts b/packages/core/src/catalog.ts index fd7485ecc8..bf521dbe46 100644 --- a/packages/core/src/catalog.ts +++ b/packages/core/src/catalog.ts @@ -209,19 +209,7 @@ const layer = Layer.effect( small: Effect.fn("Catalog.model.small")(function* (providerID) { const record = state.get().providers.get(providerID) if (!record) return - const provider = record.provider - - // TODO: Remove these provider-specific assumptions once model syncing reliably reports available deployments. - if (providerID === Provider.ID.azure) { - return - } - - if (providerID === Provider.ID.opencode) { - const gpt5Nano = record.models.get(Model.ID.make("gpt-5-nano")) - if (gpt5Nano?.enabled && gpt5Nano.status === "active") return projectModel(gpt5Nano, provider) - } - - const candidates = pipe( + const models = pipe( Array.fromIterable(record.models.values()), Array.filter( (model) => @@ -231,31 +219,12 @@ const layer = Layer.effect( model.capabilities.input.some((item) => item.startsWith("text")) && model.capabilities.output.some((item) => item.startsWith("text")), ), - Array.map((model) => ({ - model, - cost: model.cost[0] ? model.cost[0].input + model.cost[0].output : 999, - age: (Date.now() - model.time.released) / (1000 * 60 * 60 * 24 * 30), - small: SMALL_MODEL_RE.test(`${model.id} ${model.family ?? ""} ${model.name}`.toLowerCase()), - })), - Array.filter((item) => item.cost > 0 && item.age <= 18), + Array.sortWith((model) => model.time.released, Order.flip(Order.Number)), ) - - const pick = (items: typeof candidates) => { - if (!Array.isReadonlyArrayNonEmpty(items)) return - const maxCost = Math.max(...items.map((item) => item.cost), 0.01) - const maxAge = Math.max(...items.map((item) => item.age), 0.01) - const selected = Array.min( - items, - Order.mapInput( - Order.Number, - (item: (typeof candidates)[number]) => (item.cost / maxCost) * 0.8 + (item.age / maxAge) * 0.2, - ), - ) - return projectModel(selected.model, provider) + for (const family of SMALL_MODEL_FAMILY_PRIORITY) { + const selected = models.find((model) => model.family === family) + if (selected) return projectModel(selected, record.provider) } - - const small = candidates.filter((item) => item.small) - return pick(small.length > 0 ? small : candidates) }), }, } @@ -264,6 +233,6 @@ const layer = Layer.effect( }), ) -const SMALL_MODEL_RE = /\b(nano|flash|lite|mini|haiku|small|fast)\b/ +const SMALL_MODEL_FAMILY_PRIORITY = ["gpt-luna", "gemini-flash-lite", "gemini-flash", "claude-haiku"] export const node = makeLocationNode({ service: Service, layer, deps: [Bus.node, Integration.node] }) diff --git a/packages/core/src/session/title.ts b/packages/core/src/session/title.ts index f4fda46727..fec348d1a3 100644 --- a/packages/core/src/session/title.ts +++ b/packages/core/src/session/title.ts @@ -1,14 +1,17 @@ export * as SessionTitle from "./title.js" +import { isDeepStrictEqual } from "node:util" import { LLMClient, AIError, LLMEvent, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai" import type { StreamOptions } from "@opencode-ai/ai/route" import { Context, DateTime, Effect, Layer, Stream } from "effect" import { Agent } from "../agent.js" +import { Catalog } from "../catalog.js" import { Database } from "../database/database.js" import { Bus } from "../bus.js" import { makeLocationNode } from "@opencode-ai/util/effect/app-node" import { isExactRootFallback } from "@opencode-ai/util/session-title-fallback" import { llmClient } from "../effect/app-node-platform.js" +import { Model } from "../model.js" import { SessionEvent } from "./event.js" import { SessionHistory } from "./history.js" import { SessionModelRequest } from "./model-request.js" @@ -26,6 +29,7 @@ type Dependencies = { readonly stream: (request: LLMRequest, options?: StreamOptions) => Stream.Stream } readonly agents: Agent.Interface + readonly catalog: Catalog.Interface readonly models: SessionRunnerModel.Interface readonly modelRequests: SessionModelRequest.Interface readonly store: SessionStore.Interface @@ -45,6 +49,61 @@ const isUntitled = (session: SessionSchema.Info) => time: { created: DateTime.toEpochMillis(session.time.created) }, }) +const attempt = Effect.fn("SessionTitle.attempt")(function* ( + dependencies: Dependencies, + input: { + readonly session: SessionSchema.Info + readonly agent: Agent.Info + readonly text: string + readonly model: SessionRunnerModel.Resolved + }, +) { + const chunks: string[] = [] + let failed = false + let usage: SessionUsage.Recorded | undefined + const recordUsage = Effect.suspend(() => + usage + ? dependencies.bus.publish(SessionEvent.UsageRecorded, { + sessionID: input.session.id, + source: "title", + ...usage, + }) + : Effect.void, + ) + const prepared = yield* dependencies.modelRequests.prepare({ + scope: { session: input.session, agentID: input.agent.id, model: input.model }, + transcript: { + system: input.agent.system ? [SystemPart.make(input.agent.system)] : [], + messages: [Message.user(input.text)], + }, + contextHooks: false, + }) + const streamed = yield* dependencies.llm.stream(prepared.request, prepared.options).pipe( + Stream.runForEach((event) => { + if (LLMEvent.is.providerError(event)) failed = true + if (LLMEvent.is.textDelta(event)) chunks.push(event.text) + if (LLMEvent.is.stepFinish(event)) { + const step = SessionUsage.record(event.usage, input.model.cost) + usage = usage ? SessionUsage.add(usage, step) : step + } + return Effect.void + }), + Effect.as(true), + Effect.catchTag("AI.Error", () => Effect.succeed(false)), + Effect.onInterrupt(() => recordUsage.pipe(Effect.asVoid)), + ) + yield* recordUsage + if (!streamed || failed) return + return chunks + .join("") + .split("\n") + .map((line) => line.trim()) + .find((line) => line.length > 0) +}) + +/** Variant IDs that minimize reasoning output, in preference order. */ +const MINIMAL_REASONING_VARIANTS = ["none", "minimal", "low"].map((id) => Model.VariantID.make(id)) + const make = (dependencies: Dependencies) => { const generateForFirstPrompt = Effect.fn("SessionTitle.generateForFirstPrompt")(function* ( db: Database.Interface["db"], @@ -58,53 +117,33 @@ const make = (dependencies: Dependencies) => { if (!firstUser) return const agent = yield* dependencies.agents.get(Agent.ID.make("title")) if (!agent) return - const resolved = yield* ( - agent.model - ? dependencies.models.resolve({ ...session, model: agent.model }) - : dependencies.models.resolve(session) - ).pipe(Effect.catch(() => Effect.succeed(undefined))) - if (!resolved) return - const chunks: string[] = [] - let failed = false - let usage: SessionUsage.Recorded | undefined - const recordUsage = Effect.suspend(() => - usage - ? dependencies.bus.publish(SessionEvent.UsageRecorded, { - sessionID: session.id, - source: "title", - ...usage, - }) - : Effect.void, - ) - const prepared = yield* dependencies.modelRequests.prepare({ - scope: { session, agentID: agent.id, model: resolved }, - transcript: { - system: agent.system ? [SystemPart.make(agent.system)] : [], - messages: [Message.user(firstUser.text)], - }, - contextHooks: false, + const primary = yield* dependencies.models.resolve(session).pipe(Effect.catch(() => Effect.succeed(undefined))) + const info = yield* Effect.gen(function* () { + if (agent.model) return yield* dependencies.catalog.model.get(agent.model.providerID, agent.model.id) + if (!primary) return + return yield* dependencies.catalog.model.small(primary.ref.providerID) }) - const streamed = yield* dependencies.llm.stream(prepared.request, prepared.options).pipe( - Stream.runForEach((event) => { - if (LLMEvent.is.providerError(event)) failed = true - if (LLMEvent.is.textDelta(event)) chunks.push(event.text) - if (LLMEvent.is.stepFinish(event)) { - const step = SessionUsage.record(event.usage, resolved.cost) - usage = usage ? SessionUsage.add(usage, step) : step - } - return Effect.void - }), - Effect.as(true), - Effect.catchTag("AI.Error", () => Effect.succeed(false)), - Effect.onInterrupt(() => recordUsage.pipe(Effect.asVoid)), - ) - yield* recordUsage - if (!streamed || failed) return - const title = chunks - .join("") - .split("\n") - .map((line) => line.trim()) - .find((line) => line.length > 0) + const variant = + agent.model?.variant ?? MINIMAL_REASONING_VARIANTS.find((id) => info?.variants.some((item) => item.id === id)) + const preferred = + info && + (yield* dependencies.models + .resolve({ + ...session, + model: Model.Ref.make({ + providerID: info.providerID, + id: info.id, + ...(variant ? { variant } : {}), + }), + }) + .pipe(Effect.catch(() => Effect.succeed(undefined)))) + const selected = preferred ?? primary + if (!selected) return + const title = + (yield* attempt(dependencies, { session, agent, text: firstUser.text, model: selected })) ?? + (primary && !isDeepStrictEqual(selected.ref, primary.ref) + ? yield* attempt(dependencies, { session, agent, text: firstUser.text, model: primary }) + : undefined) if (!title) return const expectedSequence = (yield* Bus.latestSequence(db, sessionID)) + 1 const current = yield* dependencies.store.get(sessionID) @@ -129,11 +168,12 @@ export const layer = Layer.effect( const bus = yield* Bus.Service const llm = yield* LLMClient.Service const agents = yield* Agent.Service + const catalog = yield* Catalog.Service const models = yield* SessionRunnerModel.Service const modelRequests = yield* SessionModelRequest.Service const store = yield* SessionStore.Service const database = yield* Database.Service - const title = make({ bus, llm, agents, models, modelRequests, store }) + const title = make({ bus, llm, agents, catalog, models, modelRequests, store }) return Service.of({ generateForFirstPrompt: (sessionID) => title.generateForFirstPrompt(database.db, sessionID), }) @@ -147,6 +187,7 @@ export const node = makeLocationNode({ Bus.node, llmClient, Agent.node, + Catalog.node, SessionRunnerModel.node, SessionModelRequest.node, SessionStore.node, diff --git a/packages/core/test/catalog.test.ts b/packages/core/test/catalog.test.ts index 0d0e8a65b0..e10c00bcd6 100644 --- a/packages/core/test/catalog.test.ts +++ b/packages/core/test/catalog.test.ts @@ -1,5 +1,4 @@ import { describe, expect } from "bun:test" -import { Money } from "@opencode-ai/schema/money" import { Effect, Fiber, Layer, Stream } from "effect" import { TestClock } from "effect/testing" import { Catalog } from "@opencode-ai/core/catalog" @@ -322,45 +321,50 @@ describe("Catalog", () => { }), ) - it.effect("small model prefers small keyword candidates before cost scoring", () => + it.effect("small model uses the newest release in the first matching family", () => Effect.gen(function* () { const catalog = yield* Catalog.Service const providerID = Provider.ID.make("test") yield* catalog.transform((catalog) => { catalog.provider.update(providerID, () => {}) - catalog.model.update(providerID, Model.ID.make("cheap-large"), (model) => { + catalog.model.update(providerID, Model.ID.make("newer-flash"), (model) => { + model.family = Model.Family.make("gemini-flash") model.capabilities.input = ["text"] model.capabilities.output = ["text"] - model.cost = [ - { - input: Money.USDPerMillionTokens.make(1), - output: Money.USDPerMillionTokens.make(1), - cache: { - read: Money.USDPerMillionTokens.zero, - write: Money.USDPerMillionTokens.zero, - }, - }, - ] - model.time.released = Date.now() + model.time.released = 3000 }) - catalog.model.update(providerID, Model.ID.make("expensive-mini"), (model) => { + catalog.model.update(providerID, Model.ID.make("older-luna"), (model) => { + model.family = Model.Family.make("gpt-luna") model.capabilities.input = ["text"] model.capabilities.output = ["text"] - model.cost = [ - { - input: Money.USDPerMillionTokens.make(10), - output: Money.USDPerMillionTokens.make(10), - cache: { - read: Money.USDPerMillionTokens.zero, - write: Money.USDPerMillionTokens.zero, - }, - }, - ] - model.time.released = Date.now() + model.time.released = 1000 + }) + catalog.model.update(providerID, Model.ID.make("newer-luna"), (model) => { + model.family = Model.Family.make("gpt-luna") + model.capabilities.input = ["text"] + model.capabilities.output = ["text"] + model.time.released = 2000 }) }) - expect((yield* catalog.model.small(providerID))?.id).toMatch("expensive-mini") + expect((yield* catalog.model.small(providerID))?.id).toBe(Model.ID.make("newer-luna")) + }), + ) + + it.effect("small model returns undefined without a matching family", () => + Effect.gen(function* () { + const catalog = yield* Catalog.Service + const providerID = Provider.ID.make("test") + yield* catalog.transform((catalog) => { + catalog.provider.update(providerID, () => {}) + catalog.model.update(providerID, Model.ID.make("large"), (model) => { + model.family = Model.Family.make("gpt") + model.capabilities.input = ["text"] + model.capabilities.output = ["text"] + }) + }) + + expect(yield* catalog.model.small(providerID)).toBeUndefined() }), ) }) diff --git a/packages/core/test/plugin/provider-opencode.test.ts b/packages/core/test/plugin/provider-opencode.test.ts index 64ce6f4e1e..784196cc7d 100644 --- a/packages/core/test/plugin/provider-opencode.test.ts +++ b/packages/core/test/plugin/provider-opencode.test.ts @@ -501,31 +501,4 @@ describe("OpencodePlugin", () => { }), ), ) - - it.effect("prefers gpt-5-nano as the opencode small model", () => - Effect.gen(function* () { - const catalog = yield* Catalog.Service - const providerID = Provider.ID.opencode - - yield* catalog.transform((catalog) => { - catalog.provider.update(providerID, () => {}) - catalog.model.update(providerID, Model.ID.make("cheap-mini"), (model) => { - model.capabilities.input = ["text"] - model.capabilities.output = ["text"] - model.cost = [...cost(1, 1)] - model.time.released = Date.now() - }) - catalog.model.update(providerID, Model.ID.make("gpt-5-nano"), (model) => { - model.capabilities.input = ["text"] - model.capabilities.output = ["text"] - model.cost = [...cost(10, 10)] - model.time.released = Date.now() - }) - }) - - const selected = yield* catalog.model.small(providerID) - - expect(selected?.id).toBe(Model.ID.make("gpt-5-nano")) - }), - ) }) diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index 6caae49de7..701bbb9d04 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -1,7 +1,8 @@ -import { expect } from "bun:test" +import { beforeEach, expect } from "bun:test" import { LLMClient, LLMEvent, LanguageModel, SystemPart, type LLMRequest } from "@opencode-ai/ai" import { OpenAIChat } from "@opencode-ai/ai/protocols" import { Agent } from "@opencode-ai/core/agent" +import { Catalog } from "@opencode-ai/core/catalog" import { Database } from "@opencode-ai/core/database/database" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { llmClient } from "@opencode-ai/core/effect/app-node-platform" @@ -19,17 +20,26 @@ import { Session } from "@opencode-ai/core/session" import { Project } from "@opencode-ai/core/project" import { ProjectTable } from "@opencode-ai/core/project/sql" import { App } from "@opencode-ai/core/app" +import { Model } from "@opencode-ai/core/model" +import { Provider } from "@opencode-ai/core/provider" import { AbsolutePath } from "@opencode-ai/core/schema" import { Money } from "@opencode-ai/schema/money" import { Deferred, Effect, Fiber, Layer, Stream } from "effect" import { testEffect } from "./lib/effect" let requests: LLMRequest[] = [] +let selectedSmall: Model.Info | undefined +let selections: Array = [] const model = LanguageModel.make({ id: "title-model", provider: "test", route: OpenAIChat.route, }) +const smallModel = LanguageModel.make({ + id: "title-small", + provider: "test", + route: OpenAIChat.route, +}) const cost = [ { input: Money.USDPerMillionTokens.make(1), @@ -68,14 +78,31 @@ const client = Layer.mock(LLMClient.Service)({ generate: () => Effect.die("unused"), }) const models = Layer.mock(SessionRunnerModel.Service)({ - resolve: () => - Effect.succeed( - SessionRunnerModel.resolved(model, { + resolve: (session) => { + selections.push(session.model) + return Effect.succeed( + SessionRunnerModel.resolved(session.model?.id === "title-small" ? smallModel : model, { capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, cost, limit: { context: 200_000, output: 32_000 }, + variant: session.model?.variant, }), - ), + ) + }, +}) +const catalog = Layer.mock(Catalog.Service, { + provider: { + get: () => Effect.die("unused"), + all: () => Effect.die("unused"), + available: () => Effect.die("unused"), + }, + model: { + get: () => Effect.die("unused"), + all: () => Effect.die("unused"), + available: () => Effect.die("unused"), + default: () => Effect.die("unused"), + small: () => Effect.succeed(selectedSmall), + }, }) const it = testEffect( AppNodeBuilder.build( @@ -90,12 +117,13 @@ const it = testEffect( ]), [ [llmClient, client], + [Catalog.node, catalog], [SessionRunnerModel.node, models], ], ), ) -const insertSession = (id: Session.ID, title?: string, created?: number) => +const insertSession = (id: Session.ID, title?: string, created?: number, model?: Model.Ref) => Effect.gen(function* () { const { db } = yield* Database.Service yield* db @@ -112,6 +140,7 @@ const insertSession = (id: Session.ID, title?: string, created?: number) => slug: id, directory: "/project", title, + model, time_created: created, version: "test", }) @@ -135,6 +164,28 @@ const prompt = (sessionID: Session.ID, text: string) => }) }) +const small = Model.Info.make({ + ...Model.Info.default(Provider.ID.make("test"), Model.ID.make("title-small")), + family: Model.Family.make("gpt-luna"), + capabilities: { tools: false, input: ["text"], output: ["text"] }, + variants: [ + { id: Model.VariantID.make("low") }, + { id: Model.VariantID.make("none") }, + { id: Model.VariantID.make("high") }, + ], +}) +const lowSmall = Model.Info.make({ + ...small, + variants: [{ id: Model.VariantID.make("low") }, { id: Model.VariantID.make("high") }], +}) + +beforeEach(() => { + requests = [] + selectedSmall = undefined + selections = [] + titleStream = successfulTitle +}) + it.effect("generates a title from the sole user message and renames the session", () => Effect.gen(function* () { requests = [] @@ -175,6 +226,80 @@ it.effect("generates a title from the sole user message and renames the session" }), ) +it.effect("uses a small model from the primary provider", () => + Effect.gen(function* () { + requests = [] + titleStream = successfulTitle + selectedSmall = small + const agentService = yield* Agent.Service + yield* agentService.transform((editor) => { + editor.update(Agent.ID.make("title"), (agent) => { + agent.mode = "primary" + agent.hidden = true + agent.system = "You are a title generator." + }) + }) + const sessionID = Session.ID.make("ses_title_small_model") + yield* insertSession(sessionID) + yield* prompt(sessionID, "Use a small model for this title") + + const title = yield* SessionTitle.Service + yield* title.generateForFirstPrompt(sessionID) + + expect(requests.map((request) => String(request.model.id))).toEqual(["title-small"]) + expect(selections[1]?.variant).toBe(Model.VariantID.make("none")) + const store = yield* SessionStore.Service + expect((yield* store.get(sessionID))?.title).toBe("Generated Title") + }), +) + +it.effect("falls back to the primary model when the small model fails", () => + Effect.gen(function* () { + requests = [] + titleStream = () => + requests.length === 1 + ? Stream.make(LLMEvent.providerError({ message: "Small model unavailable" })) + : successfulTitle() + selectedSmall = lowSmall + const agentService = yield* Agent.Service + yield* agentService.transform((editor) => { + editor.update(Agent.ID.make("title"), (agent) => { + agent.mode = "primary" + agent.hidden = true + agent.system = "You are a title generator." + }) + }) + const sessionID = Session.ID.make("ses_title_small_fallback") + yield* insertSession( + sessionID, + undefined, + undefined, + Model.Ref.make({ + providerID: Provider.ID.make("test"), + id: Model.ID.make("title-model"), + variant: Model.VariantID.make("high"), + }), + ) + yield* prompt(sessionID, "Fall back when title generation fails") + + const attempted: Model.Ref[] = [] + const hooks = yield* PluginHooks.Service + yield* hooks.register("session", "model.request", (event) => + Effect.sync(() => { + attempted.push(event.model) + }), + ) + + const title = yield* SessionTitle.Service + yield* title.generateForFirstPrompt(sessionID) + + expect(requests.map((request) => String(request.model.id))).toEqual(["title-small", "title-model"]) + expect(attempted.map((model) => String(model.variant))).toEqual(["low", "high"]) + const store = yield* SessionStore.Service + expect((yield* store.get(sessionID))?.title).toBe("Generated Title") + }), +) + it.effect("generates from the first user message after later messages exist", () => Effect.gen(function* () { requests = []