diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index c631cd470a..a5b2c626b6 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -142,8 +142,7 @@ const serialize = (message: SessionMessage.Info) => { .join("\n") } if (message.type === "system") return `[System update]: ${message.text}` - if (message.type === "synthetic") - return message.metadata?.modelVisible === false ? "" : `[Synthetic context]: ${message.text}` + if (message.type === "synthetic") return `[Synthetic context]: ${message.text}` if (message.type === "skill") return `[Skill activated: ${message.name}]\n${message.text}` if (message.type === "shell") return `[Shell]: ${message.command}\n${truncate(message.output?.output ?? "")}` return "" diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 9312d2c06a..ad792ce1c7 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -102,15 +102,10 @@ const layer = Layer.effect( const agent = loaded.agent const resolved = loaded.model const model = resolved.model - const previousCacheRead = loaded.messages.findLast( - (message): message is SessionMessage.Assistant & { tokens: NonNullable } => - message.type === "assistant" && message.tokens !== undefined, - )?.tokens.cache.read const compactionInput = { session, messages: loaded.messages, model, cost: resolved.cost } if (compaction.required(compactionInput) && !(yield* SessionPending.compaction(db, session.id))) { const compacted = yield* compaction.compact(compactionInput) - if (compacted.status === "completed") - return { _tag: "RestartAfterCompaction", step: currentStep, previousCacheRead } as const + if (compacted.status === "completed") return { _tag: "RestartAfterCompaction", step: currentStep } as const return yield* new StepFailedError({ error: compacted.error }) } const prepared = yield* modelRequests.prepare({ @@ -238,7 +233,7 @@ const layer = Layer.effect( (yield* restore(recoverOverflow({ session, messages: loaded.messages, model, cost: resolved.cost }))) .status === "completed" ) - return { _tag: "RestartAfterOverflowCompaction", step: currentStep, previousCacheRead } as const + return { _tag: "RestartAfterOverflowCompaction", step: currentStep } as const // An unrecovered held-back overflow becomes the step's durable provider error. A // thrown LLM failure records the assistant failure unless a provider error was @@ -346,14 +341,6 @@ const layer = Layer.effect( _tag: "Completed", needsContinuation, step: currentStep, - previousCacheRead, - settlement: - stepSettlement - ? { - finish: stepSettlement.finish, - tokens: stepSettlement.usageAvailable ? stepSettlement.tokens : undefined, - } - : undefined, } as const }), ) @@ -371,7 +358,6 @@ const layer = Layer.effect( let currentPromotion = promotion let currentStep = step let assistantMessageID: SessionMessage.ID | undefined - let previousCacheRead: number | undefined while (true) { const attempt = yield* Effect.suspend(() => attemptStep(sessionID, currentPromotion, currentStep, recoverOverflow, assistantMessageID), @@ -396,13 +382,10 @@ const layer = Layer.effect( .pipe(Effect.andThen(Effect.fail(error.cause))) }), ) - previousCacheRead ??= attempt.previousCacheRead if (attempt._tag === "Completed") return { needsContinuation: attempt.needsContinuation, step: attempt.step, - previousCacheRead, - settlement: attempt.settlement, } if (attempt._tag === "RestartAfterOverflowCompaction") recoverOverflow = undefined yield* Effect.yieldNow @@ -461,14 +444,6 @@ const layer = Layer.effect( while (shouldRun) { let needsContinuation = true let step = 1 - const steps: Array<{ - readonly label: string - readonly newTokens: number - readonly cached: number - readonly total: number - readonly breakdown: string - readonly cacheBust?: number - }> = [] // Repeat steps while continuation is needed. A step needs continuation only // when it recorded local tool calls whose results the model has not yet seen; // a provider error suppresses it. Pending steers also continue the loop so @@ -481,30 +456,6 @@ const layer = Layer.effect( titleAttempted.add(input.sessionID) forkTitle(title.generateForFirstPrompt(yield* getSession(input.sessionID)).pipe(Effect.ignore)) } - if (result.settlement) { - const tokens = result.settlement.tokens - if (tokens) { - const breakdown = [ - tokens.input > 0 ? `${tokens.input.toLocaleString("en-US")} input` : undefined, - tokens.output > 0 ? `${tokens.output.toLocaleString("en-US")} output` : undefined, - tokens.reasoning > 0 ? `${tokens.reasoning.toLocaleString("en-US")} reasoning` : undefined, - tokens.cache.write > 0 ? `${tokens.cache.write.toLocaleString("en-US")} cache write` : undefined, - ] - .filter((value): value is string => value !== undefined) - .join(", ") - const newTokens = tokens.input + tokens.output + tokens.reasoning + tokens.cache.write - steps.push({ - label: `Step ${steps.length + 1} [${result.settlement.finish}]`, - newTokens, - cached: tokens.cache.read, - total: newTokens + tokens.cache.read, - breakdown, - ...(result.previousCacheRead !== undefined && tokens.cache.read < result.previousCacheRead - ? { cacheBust: result.previousCacheRead - tokens.cache.read } - : {}), - }) - } - } needsContinuation = result.needsContinuation step = result.step + 1 if (needsContinuation) { @@ -516,24 +467,6 @@ const layer = Layer.effect( promotion = "steer" needsContinuation = yield* SessionPending.has(db, input.sessionID, "steer") } - if (steps.length > 0) { - const lines = steps.flatMap((item) => [ - `${item.label}: New tokens: ${item.newTokens.toLocaleString("en-US")} · Cached: ${item.cached.toLocaleString("en-US")} · Total: ${item.total.toLocaleString("en-US")}${item.breakdown ? ` [${item.breakdown}]` : ""}`, - ...(item.cacheBust === undefined - ? [] - : [`! Cache bust: ${item.cacheBust.toLocaleString("en-US")} fewer cached tokens than the previous step`]), - ]) - yield* events.publish(SessionEvent.Synthetic, { - sessionID: input.sessionID, - text: "", - description: `Turn token usage:\n${lines.join("\n")}`, - metadata: { - kind: "turn-token-usage", - modelVisible: false, - steps, - }, - }) - } yield* runPendingCompaction(input.sessionID) const hasSteer = yield* SessionPending.has(db, input.sessionID, "steer") const hasQueue = hasSteer ? false : yield* SessionPending.has(db, input.sessionID, "queue") diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 32f2e774e1..df3e950895 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -76,7 +76,6 @@ export const createLLMEventPublisher = (events: Pick["reason"] readonly tokens: ReturnType - readonly usageAvailable: boolean } | undefined @@ -470,11 +469,7 @@ export const createLLMEventPublisher = (events: Pick { text: "Synthetic context", time: { created }, }), - SessionMessage.Synthetic.make({ - id: id("hidden-synthetic"), - type: "synthetic", - text: "Internal accounting", - metadata: { modelVisible: false }, - time: { created }, - }), SessionMessage.Shell.make({ id: id("shell"), type: "shell", diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index 02423b9330..d26fa37df4 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -223,22 +223,12 @@ describe("SessionRunnerLLM recorded", () => { yield* session.resume(sessionID) const messages = yield* session.context(sessionID) - expect(messages).toHaveLength(3) + expect(messages).toHaveLength(2) expect(messages[0]).toMatchObject({ id: prompt.id, type: "user", text: "Say hello in one short sentence." }) expect(messages[1]).toMatchObject({ type: "assistant", agent: "build", finish: "stop" }) expect(messages[1]?.type === "assistant" ? messages[1].content : []).toMatchObject([ { type: "text", text: "Hello!" }, ]) - expect(messages[2]).toMatchObject({ - type: "synthetic", - description: - "Turn token usage:\nStep 1 [stop]: New tokens: 24 · Cached: 0 · Total: 24 [22 input, 2 output]", - metadata: { - kind: "turn-token-usage", - modelVisible: false, - steps: [{ label: "Step 1 [stop]", newTokens: 24, cached: 0, total: 24, breakdown: "22 input, 2 output" }], - }, - }) expect( (yield* db .select({ type: EventTable.type }) @@ -254,7 +244,6 @@ describe("SessionRunnerLLM recorded", () => { "session.text.started.1", "session.text.ended.1", "session.step.ended.1", - "session.synthetic.1", ]) }), ) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 357338d91e..6ce9cf0f20 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -2016,7 +2016,7 @@ describe("SessionRunnerLLM", () => { expect(userTexts(requests[1])[0]).toContain(`[User]: ${"Recent exact request ".repeat(180)}`) const context = yield* (yield* SessionStore.Service).context(sessionID) - expect(context.map((message) => message.type)).toEqual(["compaction", "assistant", "synthetic"]) + expect(context.map((message) => message.type)).toEqual(["compaction", "assistant"]) expect(context[0]).toMatchObject({ type: "compaction", summary: "## Objective\n- Preserve the task", @@ -2385,24 +2385,6 @@ describe("SessionRunnerLLM", () => { }, ], }, - { - type: "synthetic", - description: - "Turn token usage:\nStep 1 [tool-calls]: New tokens: 12 · Cached: 2 · Total: 14 [8 input, 3 output, 1 reasoning]", - metadata: { - kind: "turn-token-usage", - modelVisible: false, - steps: [ - { - label: "Step 1 [tool-calls]", - newTokens: 12, - cached: 2, - total: 14, - breakdown: "8 input, 3 output, 1 reasoning", - }, - ], - }, - }, ]) }), ) @@ -2452,112 +2434,6 @@ describe("SessionRunnerLLM", () => { }), ) - it.effect("appends each step's token usage after a completed turn", () => - Effect.gen(function* () { - const session = yield* setup - yield* admit(session, "Use a tool") - responses = [ - [ - LLMEvent.stepStart({ index: 0 }), - LLMEvent.toolCall({ id: "call-token-usage", name: "echo", input: { text: "hello" } }), - LLMEvent.stepFinish({ - index: 0, - reason: "tool-calls", - usage: { - inputTokens: 10, - nonCachedInputTokens: 8, - outputTokens: 4, - reasoningTokens: 1, - cacheReadInputTokens: 2, - }, - }), - LLMEvent.finish({ reason: "tool-calls" }), - ], - [ - LLMEvent.stepStart({ index: 0 }), - LLMEvent.stepFinish({ - index: 0, - reason: "stop", - usage: { - inputTokens: 20, - nonCachedInputTokens: 15, - outputTokens: 6, - reasoningTokens: 2, - cacheReadInputTokens: 5, - }, - }), - LLMEvent.finish({ reason: "stop" }), - ], - ] - - yield* session.resume(sessionID) - - expect(yield* session.context(sessionID)).toMatchObject([ - { type: "user", text: "Use a tool" }, - { type: "assistant", tokens: { input: 8, output: 3, reasoning: 1, cache: { read: 2, write: 0 } } }, - { type: "assistant", tokens: { input: 15, output: 4, reasoning: 2, cache: { read: 5, write: 0 } } }, - { - type: "synthetic", - text: "", - description: - "Turn token usage:\nStep 1 [tool-calls]: New tokens: 12 · Cached: 2 · Total: 14 [8 input, 3 output, 1 reasoning]\nStep 2 [stop]: New tokens: 21 · Cached: 5 · Total: 26 [15 input, 4 output, 2 reasoning]", - metadata: { - kind: "turn-token-usage", - modelVisible: false, - steps: [ - { - label: "Step 1 [tool-calls]", - newTokens: 12, - cached: 2, - total: 14, - breakdown: "8 input, 3 output, 1 reasoning", - }, - { - label: "Step 2 [stop]", - newTokens: 21, - cached: 5, - total: 26, - breakdown: "15 input, 4 output, 2 reasoning", - }, - ], - }, - }, - ]) - - response = [ - LLMEvent.stepStart({ index: 0 }), - LLMEvent.stepFinish({ - index: 0, - reason: "stop", - usage: { inputTokens: 21, nonCachedInputTokens: 20, outputTokens: 5, cacheReadInputTokens: 1 }, - }), - LLMEvent.finish({ reason: "stop" }), - ] - responses = undefined - requests.length = 0 - yield* admit(session, "Continue") - yield* session.resume(sessionID) - - expect(userTexts(requests[0]!)).toEqual(["Use a tool", "Continue"]) - expect((yield* session.context(sessionID)).findLast((message) => message.type === "synthetic")).toMatchObject({ - description: - "Turn token usage:\nStep 1 [stop]: New tokens: 25 · Cached: 1 · Total: 26 [20 input, 5 output]\n! Cache bust: 4 fewer cached tokens than the previous step", - metadata: { - steps: [ - { - label: "Step 1 [stop]", - newTokens: 25, - cached: 1, - total: 26, - breakdown: "20 input, 5 output", - cacheBust: 4, - }, - ], - }, - }) - }), - ) - it.effect("reloads a model switch before a tool-driven continuation step", () => Effect.gen(function* () { const session = yield* setup diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 6dc9535ad3..6209a4779e 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -1034,8 +1034,10 @@ function SessionRowView(props: { message: (messageID: string) => SessionMessageInfo | undefined boundaryID?: string }) { + const config = useConfig() + const hidden = () => props.row.type === "turn-usage" && config.data.debug?.turn_tokens !== true return ( - + {(row) => ( @@ -1072,11 +1074,95 @@ function SessionRowView(props: { )} + + {(row) => ( + + )} + ) } +function TurnTokenUsage(props: { + messageIDs: string[] + previousCacheRead?: number + message: (messageID: string) => SessionMessageInfo | undefined +}) { + const config = useConfig() + const { themeV2 } = useTheme() + const steps = createMemo(() => { + let previousCacheRead = props.previousCacheRead + return props.messageIDs.flatMap((messageID, index) => { + const message = props.message(messageID) + if (message?.type !== "assistant" || !message.tokens) return [] + const total = + message.tokens.input + + message.tokens.output + + message.tokens.reasoning + + message.tokens.cache.read + + message.tokens.cache.write + if (total === 0) return [] + const newTokens = total - message.tokens.cache.read + const breakdown = [ + message.tokens.input > 0 ? `${message.tokens.input.toLocaleString()} input` : undefined, + message.tokens.output > 0 ? `${message.tokens.output.toLocaleString()} output` : undefined, + message.tokens.reasoning > 0 ? `${message.tokens.reasoning.toLocaleString()} reasoning` : undefined, + message.tokens.cache.write > 0 ? `${message.tokens.cache.write.toLocaleString()} cache write` : undefined, + ] + .filter((value): value is string => value !== undefined) + .join(", ") + const cacheBust = + previousCacheRead !== undefined && message.tokens.cache.read < previousCacheRead + ? previousCacheRead - message.tokens.cache.read + : undefined + previousCacheRead = message.tokens.cache.read + return [ + { + label: `Step ${index + 1} [${message.finish ?? "unknown"}]`, + newTokens, + cached: message.tokens.cache.read, + total, + breakdown, + cacheBust, + }, + ] + }) + }) + return ( + 0}> + + + + ◈ + + Turn token usage: + + + {(item) => ( + + + {item.label}:{" "} + New tokens: {item.newTokens.toLocaleString()} + {` · Cached: ${item.cached.toLocaleString()} · Total: ${item.total.toLocaleString()}${item.breakdown ? ` [${item.breakdown}]` : ""}`} + + + + ! Cache bust: {item.cacheBust?.toLocaleString()} fewer cached tokens than the previous step + + + + )} + + + + ) +} + function BackgroundToolHint(props: { messages: SessionMessageInfo[] }) { const { themeV2 } = useTheme() const shortcut = Keymap.useShortcut("session.background") @@ -1414,11 +1500,8 @@ function SessionSwitchMessageV2(props: { message: SessionMessageInfo }) { function SessionNoticeMessageV2(props: { message: SessionMessageInfo }) { const ctx = use() - const config = useConfig() const { themeV2 } = useTheme() const metadata = () => (props.message.type === "synthetic" ? props.message.metadata : undefined) - const turnTokenUsage = () => readTurnTokenUsage(metadata()) - const visible = () => metadata()?.kind !== "turn-token-usage" || config.data.debug?.turn_tokens === true const source = () => stringValue(metadata()?.source) const completion = () => source() === "subagent" || source() === "shell" const state = () => stringValue(metadata()?.state) @@ -1442,101 +1525,24 @@ function SessionNoticeMessageV2(props: { message: SessionMessageInfo }) { return themeV2.text.feedback.info.default } return ( - - - {text()} - - } - > - - - {heading()} - {suffix()} - - - - } - > - {(steps) => ( - - - - ◈ - - Turn token usage: - - - {(item) => ( - - - {item.label}:{" "} - - New tokens: {item.newTokens.toLocaleString()} - - {` · Cached: ${item.cached.toLocaleString()} · Total: ${item.total.toLocaleString()}${item.breakdown ? ` [${item.breakdown}]` : ""}`} - - - - ! Cache bust: {item.cacheBust?.toLocaleString()} fewer cached tokens than the previous step - - - - )} - - - )} - + + {text()} + + } + > + + + {heading()} + {suffix()} + + ) } -type TurnTokenStep = { - readonly label: string - readonly newTokens: number - readonly cached: number - readonly total: number - readonly breakdown: string - readonly cacheBust?: number -} - -function readTurnTokenUsage(metadata: Record | undefined) { - if (metadata?.kind !== "turn-token-usage" || !Array.isArray(metadata.steps)) return - const steps = metadata.steps.flatMap((value): TurnTokenStep[] => { - if (typeof value !== "object" || value === null || Array.isArray(value)) return [] - const step = value as Record - const newTokens = finiteNumber(step.newTokens) - const cached = finiteNumber(step.cached) - const total = finiteNumber(step.total) - const cacheBust = step.cacheBust === undefined ? undefined : finiteNumber(step.cacheBust) - if ( - typeof step.label !== "string" || - newTokens === undefined || - cached === undefined || - total === undefined || - typeof step.breakdown !== "string" || - (step.cacheBust !== undefined && cacheBust === undefined) - ) - return [] - return [ - { - label: step.label, - newTokens, - cached, - total, - breakdown: step.breakdown, - ...(cacheBust === undefined ? {} : { cacheBust }), - }, - ] - }) - return steps.length === metadata.steps.length ? steps : undefined -} - function SessionSkillMessage(props: { message: Extract }) { const { themeV2 } = useTheme() return ( diff --git a/packages/tui/src/routes/session/rows.ts b/packages/tui/src/routes/session/rows.ts index dc4a681bb9..837725faf8 100644 --- a/packages/tui/src/routes/session/rows.ts +++ b/packages/tui/src/routes/session/rows.ts @@ -27,6 +27,7 @@ export type SessionRow = completed: boolean } | { type: "assistant-footer"; messageID: string } + | { type: "turn-usage"; messageIDs: string[]; previousCacheRead?: number } export function createSessionRows(sessionID: Accessor) { const data = useData() @@ -127,6 +128,26 @@ export function createSessionRows(sessionID: Accessor) { ), ) + createEffect( + on( + () => + data.session.message.list(sessionID()).flatMap((message) => + message.type === "assistant" + ? [ + { + id: message.id, + finish: message.finish, + error: message.error, + retry: message.retry, + tokens: message.tokens, + }, + ] + : [], + ), + () => setRows(reconcile(reduce())), + ), + ) + const appendMessage = (messageID: string) => setRows( produce((draft) => { @@ -260,6 +281,10 @@ export function reduceSessionRows(messages: SessionMessageInfo[], inputs = new S const isInput = (message: SessionMessageInfo) => inputs.has(message.id) const pendingCompactions = messages.filter((message) => message.type === "compaction" && message.status === "running") const pending = new Set([...pendingCompactions.map((message) => message.id), ...inputs]) + const steps: string[] = [] + let previousCacheRead: number | undefined + let turnPreviousCacheRead: number | undefined + let measured = false return [ ...messages.filter((message) => !pending.has(message.id)), ...pendingCompactions, @@ -271,20 +296,42 @@ export function reduceSessionRows(messages: SessionMessageInfo[], inputs = new S rows.push({ type: "message", messageID: message.id }) return rows } + if (steps.length === 0) turnPreviousCacheRead = previousCacheRead + steps.push(message.id) + if (message.tokens && tokenTotal(message.tokens) > 0) { + previousCacheRead = message.tokens.cache.read + measured = true + } const ordinals = { text: 0, reasoning: 0 } message.content.forEach((part) => { const partID = part.type === "tool" ? part.id : `${part.type}:${ordinals[part.type]++}` if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return append(rows, { messageID: message.id, partID }, part) }) - if ((message.finish && !["tool-calls", "unknown"].includes(message.finish)) || message.error || message.retry) { + const terminal = (message.finish && !["tool-calls", "unknown"].includes(message.finish)) || message.error + if (terminal || message.retry) { completePrevious(rows) rows.push({ type: "assistant-footer", messageID: message.id }) } + if (terminal) { + if (measured) + rows.push({ + type: "turn-usage", + messageIDs: [...steps], + ...(turnPreviousCacheRead === undefined ? {} : { previousCacheRead: turnPreviousCacheRead }), + }) + steps.length = 0 + turnPreviousCacheRead = undefined + measured = false + } return rows }, []) } +function tokenTotal(tokens: NonNullable) { + return tokens.input + tokens.output + tokens.reasoning + tokens.cache.read + tokens.cache.write +} + export function messageBoundaryIDs(rows: SessionRow[], messages: SessionMessageInfo[]) { const byID = new Map(messages.map((message) => [message.id, message])) const seen = new Set() @@ -309,6 +356,8 @@ function rowBoundaryMessageID(row: SessionRow, messages: Map { ]) }) +test("derives turn usage rows from completed assistant steps", () => { + const usage = (read: number) => ({ + input: 100, + output: 16, + reasoning: 0, + cache: { read, write: 0 }, + }) + const messages: SessionMessageInfo[] = [ + { type: "user", id: "user-1", text: "First", time: { created: 0 } }, + assistant("assistant-1", [], { finish: "stop", tokens: usage(10) }), + { type: "user", id: "user-2", text: "Second", time: { created: 2 } }, + assistant("assistant-2", [], { finish: "tool-calls", tokens: usage(8) }), + assistant("assistant-3", [], { finish: "stop", tokens: usage(12) }), + ] + + expect(reduceSessionRows(messages)).toEqual([ + { type: "message", messageID: "user-1" }, + { type: "assistant-footer", messageID: "assistant-1" }, + { type: "turn-usage", messageIDs: ["assistant-1"] }, + { type: "message", messageID: "user-2" }, + { type: "assistant-footer", messageID: "assistant-3" }, + { + type: "turn-usage", + messageIDs: ["assistant-2", "assistant-3"], + previousCacheRead: 10, + }, + ]) +}) + test("renders a footer for a pre-output retry assistant after replay", () => { const message = assistant("assistant-retry", []) message.retry = { @@ -294,7 +323,11 @@ test("places a running compaction barrier before every queued user message", () ]) }) -function assistant(id: string, content: SessionMessageAssistant["content"]): SessionMessageAssistant { +function assistant( + id: string, + content: SessionMessageAssistant["content"], + info: Partial = {}, +): SessionMessageAssistant { return { type: "assistant", id, @@ -302,6 +335,7 @@ function assistant(id: string, content: SessionMessageAssistant["content"]): Ses model: { id: "model", providerID: "provider" }, content, time: { created: 1 }, + ...info, } }