diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 3c64fc6078..20fee1df11 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -1058,7 +1058,37 @@ export type SessionContextOutput = { | { readonly type: "file"; readonly uri: string; readonly mime: string; readonly name?: string } > readonly structured: { readonly [x: string]: JsonValue } - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { + readonly type: "provider.rate-limit" + readonly message: string + readonly retryAfterMs?: number + } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } readonly result?: JsonValue } readonly time: { @@ -1070,7 +1100,7 @@ export type SessionContextOutput = { } > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } - readonly finish?: string + readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" readonly cost?: number readonly tokens?: { readonly input: number @@ -1078,7 +1108,56 @@ export type SessionContextOutput = { readonly reasoning: number readonly cache: { readonly read: number; readonly write: number } } - readonly error?: { readonly type: "unknown"; readonly message: string } + readonly error?: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + readonly retry?: { + readonly attempt: number + readonly at: number + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + } } | { readonly type: "compaction" @@ -1209,6 +1288,71 @@ export type SessionLogOutput = readonly delivery: "steer" | "queue" } } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.started" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string } + } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.succeeded" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string } + } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.failed" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { + readonly sessionID: string + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + } + } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.interrupted" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string; readonly reason: "user" | "shutdown" | "superseded" } + } | { readonly id: string readonly created: number @@ -1318,7 +1462,7 @@ export type SessionLogOutput = readonly data: { readonly sessionID: string readonly assistantMessageID: string - readonly finish: string + readonly finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" readonly cost: number readonly tokens: { readonly input: number @@ -1340,7 +1484,33 @@ export type SessionLogOutput = readonly data: { readonly sessionID: string readonly assistantMessageID: string - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } } } | { @@ -1483,7 +1653,33 @@ export type SessionLogOutput = readonly sessionID: string readonly assistantMessageID: string readonly callID: string - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } readonly result?: unknown readonly executed: boolean readonly resultState?: { readonly [x: string]: unknown } @@ -1493,20 +1689,41 @@ export type SessionLogOutput = readonly id: string readonly created: number readonly metadata?: { readonly [x: string]: unknown } - readonly type: "session.retried" + readonly type: "session.retry.scheduled" readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } readonly location?: { readonly directory: string; readonly workspaceID?: string } readonly data: { readonly sessionID: string + readonly assistantMessageID: string readonly attempt: number - readonly error: { - readonly message: string - readonly statusCode?: number - readonly isRetryable: boolean - readonly responseHeaders?: { readonly [x: string]: string } - readonly responseBody?: string - readonly metadata?: { readonly [x: string]: string } - } + readonly at: number + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } } } | { @@ -1735,7 +1952,37 @@ export type SessionMessageOutput = { | { readonly type: "file"; readonly uri: string; readonly mime: string; readonly name?: string } > readonly structured: { readonly [x: string]: JsonValue } - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { + readonly type: "provider.rate-limit" + readonly message: string + readonly retryAfterMs?: number + } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } readonly result?: JsonValue } readonly time: { @@ -1747,7 +1994,7 @@ export type SessionMessageOutput = { } > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } - readonly finish?: string + readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" readonly cost?: number readonly tokens?: { readonly input: number @@ -1755,7 +2002,56 @@ export type SessionMessageOutput = { readonly reasoning: number readonly cache: { readonly read: number; readonly write: number } } - readonly error?: { readonly type: "unknown"; readonly message: string } + readonly error?: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + readonly retry?: { + readonly attempt: number + readonly at: number + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + } } | { readonly type: "compaction" @@ -1932,7 +2228,37 @@ export type MessageListOutput = { | { readonly type: "file"; readonly uri: string; readonly mime: string; readonly name?: string } > readonly structured: { readonly [x: string]: JsonValue } - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { + readonly type: "provider.rate-limit" + readonly message: string + readonly retryAfterMs?: number + } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { + readonly type: "aborted" + readonly message: string + readonly reason?: "user" | "shutdown" | "timeout" + } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } readonly result?: JsonValue } readonly time: { @@ -1944,7 +2270,7 @@ export type MessageListOutput = { } > readonly snapshot?: { readonly start?: string; readonly end?: string; readonly files?: ReadonlyArray } - readonly finish?: string + readonly finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" readonly cost?: number readonly tokens?: { readonly input: number @@ -1952,7 +2278,56 @@ export type MessageListOutput = { readonly reasoning: number readonly cache: { readonly read: number; readonly write: number } } - readonly error?: { readonly type: "unknown"; readonly message: string } + readonly error?: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + readonly retry?: { + readonly attempt: number + readonly at: number + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } + } } | { readonly type: "compaction" @@ -4465,14 +4840,63 @@ export type EventSubscribeOutput = readonly id: string readonly created: number readonly metadata?: { readonly [x: string]: unknown } - readonly type: "session.execution.settled" + readonly type: "session.execution.started" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string } + } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.succeeded" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string } + } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.failed" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } readonly location?: { readonly directory: string; readonly workspaceID?: string } readonly data: { readonly sessionID: string - readonly outcome: "success" | "failure" | "interrupted" - readonly error?: { readonly type: "unknown"; readonly message: string } + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } } } + | { + readonly id: string + readonly created: number + readonly metadata?: { readonly [x: string]: unknown } + readonly type: "session.execution.interrupted" + readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } + readonly location?: { readonly directory: string; readonly workspaceID?: string } + readonly data: { readonly sessionID: string; readonly reason: "user" | "shutdown" | "superseded" } + } | { readonly id: string readonly created: number @@ -4582,7 +5006,7 @@ export type EventSubscribeOutput = readonly data: { readonly sessionID: string readonly assistantMessageID: string - readonly finish: string + readonly finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" readonly cost: number readonly tokens: { readonly input: number @@ -4604,7 +5028,29 @@ export type EventSubscribeOutput = readonly data: { readonly sessionID: string readonly assistantMessageID: string - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } } } | { @@ -4776,7 +5222,29 @@ export type EventSubscribeOutput = readonly sessionID: string readonly assistantMessageID: string readonly callID: string - readonly error: { readonly type: "unknown"; readonly message: string } + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } readonly result?: unknown readonly executed: boolean readonly resultState?: { readonly [x: string]: unknown } @@ -4786,20 +5254,37 @@ export type EventSubscribeOutput = readonly id: string readonly created: number readonly metadata?: { readonly [x: string]: unknown } - readonly type: "session.retried" + readonly type: "session.retry.scheduled" readonly durable: { readonly aggregateID: string; readonly seq: number; readonly version: number } readonly location?: { readonly directory: string; readonly workspaceID?: string } readonly data: { readonly sessionID: string + readonly assistantMessageID: string readonly attempt: number - readonly error: { - readonly message: string - readonly statusCode?: number - readonly isRetryable: boolean - readonly responseHeaders?: { readonly [x: string]: string } - readonly responseBody?: string - readonly metadata?: { readonly [x: string]: string } - } + readonly at: number + readonly error: + | { readonly type: "provider.rate-limit"; readonly message: string; readonly retryAfterMs?: number } + | { readonly type: "provider.auth"; readonly message: string } + | { readonly type: "provider.quota"; readonly message: string } + | { readonly type: "provider.content-filter"; readonly message: string } + | { readonly type: "provider.transport"; readonly message: string } + | { readonly type: "provider.internal"; readonly message: string } + | { readonly type: "provider.invalid-output"; readonly message: string } + | { readonly type: "provider.invalid-request"; readonly message: string } + | { readonly type: "provider.no-route"; readonly message: string } + | { readonly type: "provider.unknown"; readonly message: string } + | { + readonly type: "permission.rejected" + readonly message: string + readonly permission: string + readonly resources: ReadonlyArray + } + | { readonly type: "tool.unknown"; readonly message: string; readonly name: string } + | { readonly type: "tool.stale"; readonly message: string; readonly name?: string } + | { readonly type: "tool.execution"; readonly message: string } + | { readonly type: "tool.result-missing"; readonly message: string; readonly callID?: string } + | { readonly type: "aborted"; readonly message: string; readonly reason?: "user" | "shutdown" | "timeout" } + | { readonly type: "unknown"; readonly message: string; readonly agent?: string } } } | { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 00e80fda51..2e584a7873 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -45,5 +45,6 @@ export const migrations = ( import("./migration/20260703181610_event_created_column"), import("./migration/20260703190000_reset_v2_shell_event_payloads"), import("./migration/20260703200000_reset_v2_event_fragments"), + import("./migration/20260703210000_reset_v2_execution_errors"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260703210000_reset_v2_execution_errors.ts b/packages/core/src/database/migration/20260703210000_reset_v2_execution_errors.ts new file mode 100644 index 0000000000..bcb96adbd7 --- /dev/null +++ b/packages/core/src/database/migration/20260703210000_reset_v2_execution_errors.ts @@ -0,0 +1,14 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260703210000_reset_v2_execution_errors", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`DELETE FROM \`session_input\`;`) + yield* tx.run(`DELETE FROM \`session_message\`;`) + yield* tx.run(`DELETE FROM \`event\`;`) + yield* tx.run(`DELETE FROM \`event_sequence\`;`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/permission.ts b/packages/core/src/permission.ts index 3a8dc12b25..05b5e674a9 100644 --- a/packages/core/src/permission.ts +++ b/packages/core/src/permission.ts @@ -59,7 +59,14 @@ export type AskResult = typeof AskResult.Type export const Event = Permission.Event -export class RejectedError extends Schema.TaggedErrorClass()("PermissionV2.RejectedError", {}) {} +export class RejectedError extends Schema.TaggedErrorClass()("PermissionV2.RejectedError", { + permission: Schema.String, + resources: Schema.Array(Schema.String), +}) { + override get message() { + return `Permission rejected: ${this.permission}` + } +} export class CorrectedError extends Schema.TaggedErrorClass()("PermissionV2.CorrectedError", { feedback: Schema.String, @@ -119,9 +126,17 @@ const layer = Layer.effect( const pending = new Map() yield* Effect.addFinalizer(() => - Effect.forEach(pending.values(), (item) => Deferred.fail(item.deferred, new RejectedError()), { - discard: true, - }).pipe( + Effect.forEach( + pending.values(), + (item) => + Deferred.fail( + item.deferred, + new RejectedError({ permission: item.request.action, resources: [...item.request.resources] }), + ), + { + discard: true, + }, + ).pipe( Effect.ensuring( Effect.sync(() => { pending.clear() @@ -230,7 +245,12 @@ const layer = Layer.effect( if (input.reply === "reject") { yield* Deferred.fail( existing.deferred, - input.message ? new CorrectedError({ feedback: input.message }) : new RejectedError(), + input.message + ? new CorrectedError({ feedback: input.message }) + : new RejectedError({ + permission: existing.request.action, + resources: [...existing.request.resources], + }), ) pending.delete(input.requestID) for (const [id, item] of pending) { @@ -240,7 +260,10 @@ const layer = Layer.effect( requestID: item.request.id, reply: "reject", }) - yield* Deferred.fail(item.deferred, new RejectedError()) + yield* Deferred.fail( + item.deferred, + new RejectedError({ permission: item.request.action, resources: [...item.request.resources] }), + ) pending.delete(id) } return diff --git a/packages/core/src/session/error.ts b/packages/core/src/session/error.ts index 68f4fd32a4..7842390f3b 100644 --- a/packages/core/src/session/error.ts +++ b/packages/core/src/session/error.ts @@ -1,6 +1,7 @@ import { Schema } from "effect" import { SessionMessage } from "./message" import { SessionSchema } from "./schema" +import { SessionError } from "@opencode-ai/schema/session-error" export class MessageDecodeError extends Schema.TaggedErrorClass()("Session.MessageDecodeError", { sessionID: SessionSchema.ID, @@ -10,3 +11,20 @@ export class MessageDecodeError extends Schema.TaggedErrorClass()("Session.StepFailedError", { + error: SessionError.Error, +}) { + override get message() { + return this.error.message + } +} + +export class UserInterruptedError extends Schema.TaggedErrorClass()( + "Session.UserInterruptedError", + {}, +) { + override get message() { + return "Session interrupted by user" + } +} diff --git a/packages/core/src/session/execution/local.ts b/packages/core/src/session/execution/local.ts index 4e68e6c3d9..c925ee655c 100644 --- a/packages/core/src/session/execution/local.ts +++ b/packages/core/src/session/execution/local.ts @@ -1,4 +1,4 @@ -import { Cause, DateTime, Effect, Exit, Layer } from "effect" +import { Cause, Effect, Exit, Layer } from "effect" import { EventV2 } from "../../event" import { LocationServiceMap } from "../../location-service-map" import { makeGlobalNode } from "../../effect/app-node" @@ -8,6 +8,16 @@ import { SessionRunner } from "../runner" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionExecution } from "../execution" +import { toSessionError } from "../to-session-error" +import { UserInterruptedError } from "../error" + +export function terminal(exit: Exit.Exit, reason?: "user" | "shutdown" | "superseded") { + if (Exit.isSuccess(exit)) return { type: "succeeded" as const } + if (Cause.hasInterrupts(exit.cause)) return { type: "interrupted" as const, reason: reason ?? "shutdown" } + const failure = Cause.squash(exit.cause) + if (failure instanceof UserInterruptedError) return { type: "interrupted" as const, reason: "user" as const } + return { type: "failed" as const, error: toSessionError(failure) } +} /** Current-process routing for implicit-local Locations. Future remote placement belongs here. */ const layer = Layer.effect( @@ -16,7 +26,23 @@ const layer = Layer.effect( const store = yield* SessionStore.Service const locations = yield* LocationServiceMap.Service const events = yield* EventV2.Service - const coordinator = yield* SessionRunCoordinator.make({ + const reportLifecycle = (sessionID: SessionSchema.ID, effect: Effect.Effect) => + effect.pipe( + Effect.tapCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : Effect.logError("Failed to publish Session execution lifecycle", cause).pipe( + Effect.annotateLogs({ sessionID }), + ), + ), + Effect.asVoid, + ) + const coordinator = yield* SessionRunCoordinator.make< + SessionSchema.ID, + SessionRunner.RunError, + "user" | "shutdown" | "superseded" + >({ + started: (sessionID) => reportLifecycle(sessionID, events.publish(SessionEvent.Execution.Started, { sessionID })), drain: Effect.fnUntraced(function* (sessionID: SessionSchema.ID, force) { const session = yield* store.get(sessionID) if (!session) return yield* Effect.die(new Error(`Session not found: ${sessionID}`)) @@ -29,28 +55,31 @@ const layer = Layer.effect( ), ) }), - // One ExecutionSettled per execution (busy period), covering every coalesced drain. - settled: (sessionID, exit) => - Effect.gen(function* () { - const failure = - Exit.isFailure(exit) && !Cause.hasInterrupts(exit.cause) ? Cause.squash(exit.cause) : undefined - yield* events.publish(SessionEvent.ExecutionSettled, { - sessionID, - outcome: Exit.isSuccess(exit) ? "success" : Cause.hasInterrupts(exit.cause) ? "interrupted" : "failure", - error: - failure !== undefined - ? { type: "unknown", message: failure instanceof Error ? failure.message : String(failure) } - : undefined, - }) - }).pipe( - Effect.catchCause(() => Effect.void), - Effect.asVoid, + // One terminal observation per busy period, covering every coalesced drain. + settled: (sessionID, exit, reason) => + reportLifecycle( + sessionID, + Effect.gen(function* () { + const outcome = terminal(exit, reason) + if (outcome.type === "succeeded") { + yield* events.publish(SessionEvent.Execution.Succeeded, { sessionID }) + return + } + if (outcome.type === "interrupted") { + yield* events.publish(SessionEvent.Execution.Interrupted, { sessionID, reason: outcome.reason }) + return + } + yield* events.publish(SessionEvent.Execution.Failed, { + sessionID, + error: outcome.error, + }) + }), ), }) return SessionExecution.Service.of({ active: coordinator.active, - interrupt: coordinator.interrupt, + interrupt: (sessionID) => coordinator.interrupt(sessionID, "user"), resume: coordinator.run, wake: coordinator.wake, awaitIdle: coordinator.awaitIdle, diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index 258bcaf66b..0fc9b75f6a 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -1,5 +1,5 @@ import { castDraft, produce, type WritableDraft } from "immer" -import { Effect } from "effect" +import { DateTime, Effect } from "effect" import { SessionEvent } from "./event" import { SessionMessage } from "./message" @@ -111,6 +111,17 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { if (assistant) yield* adapter.updateAssistant(produce(assistant, recipe)) }) + const clearCurrentRetry = Effect.gen(function* () { + const assistant = yield* adapter.getCurrentAssistant() + if (assistant?.retry) { + yield* adapter.updateAssistant( + produce(assistant, (draft) => { + draft.retry = undefined + }), + ) + } + }) + return Effect.gen(function* () { yield* SessionEvent.All.match(event, { "session.agent.selected": (event) => { @@ -144,7 +155,10 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { "session.forked": () => Effect.void, "session.prompt.promoted": () => Effect.void, "session.prompt.admitted": () => Effect.void, - "session.execution.settled": () => Effect.void, + "session.execution.started": () => Effect.void, + "session.execution.succeeded": () => clearCurrentRetry, + "session.execution.failed": () => clearCurrentRetry, + "session.execution.interrupted": () => clearCurrentRetry, "session.context.updated": (event) => adapter.appendMessage( SessionMessage.System.make({ @@ -205,10 +219,26 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { }, "session.step.started": (event) => { return Effect.gen(function* () { + const existing = yield* adapter.getAssistant(event.data.assistantMessageID) + if (existing) { + yield* adapter.updateAssistant( + produce(existing, (draft) => { + draft.agent = event.data.agent + draft.model = castDraft(event.data.model) + draft.retry = undefined + draft.error = undefined + draft.finish = undefined + draft.time.completed = undefined + if (event.data.snapshot) draft.snapshot = { ...draft.snapshot, start: event.data.snapshot } + }), + ) + return + } const currentAssistant = yield* adapter.getCurrentAssistant() if (currentAssistant) { yield* adapter.updateAssistant( produce(currentAssistant, (draft) => { + draft.retry = undefined draft.time.completed = event.created }), ) @@ -244,7 +274,8 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { draft.time.completed = event.created draft.finish = "error" - draft.error = event.data.error + draft.error = castDraft(event.data.error) + draft.retry = undefined }) }, "session.text.started": (event) => { @@ -383,7 +414,15 @@ export function update(adapter: Adapter, event: SessionEvent.Event) { } }) }, - "session.retried": () => Effect.void, + "session.retry.scheduled": (event) => { + return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { + draft.retry = { + attempt: event.data.attempt, + at: DateTime.makeUnsafe(event.data.at), + error: castDraft(event.data.error), + } + }) + }, "session.compaction.started": () => Effect.void, "session.compaction.delta": () => Effect.void, "session.compaction.ended": (event) => { diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index fa67dd2162..92c3b32c07 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -634,6 +634,9 @@ const layer = Layer.effectDiscard( }) }), ) + yield* events.project(SessionEvent.Execution.Succeeded, (event) => run(db, event)) + yield* events.project(SessionEvent.Execution.Failed, (event) => run(db, event)) + yield* events.project(SessionEvent.Execution.Interrupted, (event) => run(db, event)) yield* events.project(SessionEvent.ContextUpdated, (event) => run(db, event)) yield* events.project(SessionEvent.Synthetic, (event) => run(db, event)) yield* events.project(SessionEvent.Skill.Activated, (event) => @@ -660,7 +663,7 @@ const layer = Layer.effectDiscard( yield* events.project(SessionEvent.Tool.Failed, (event) => run(db, event)) yield* events.project(SessionEvent.Reasoning.Started, (event) => run(db, event)) yield* events.project(SessionEvent.Reasoning.Ended, (event) => run(db, event)) - // yield* events.project(SessionEvent.Retried, (event) => run(db, event)) + yield* events.project(SessionEvent.RetryScheduled, (event) => run(db, event)) yield* events.project(SessionEvent.Compaction.Ended, (event) => run(db, event)) yield* events.project(SessionEvent.RevertEvent.Staged, (event) => db diff --git a/packages/core/src/session/run-coordinator.ts b/packages/core/src/session/run-coordinator.ts index 8524b1c1f8..1550280164 100644 --- a/packages/core/src/session/run-coordinator.ts +++ b/packages/core/src/session/run-coordinator.ts @@ -3,7 +3,7 @@ export * as SessionRunCoordinator from "./run-coordinator" import { Deferred, Effect, Exit, Fiber, FiberSet, Scope } from "effect" /** Serializes execution for each key while allowing different keys to run concurrently. */ -export interface Coordinator { +export interface Coordinator { /** Snapshots keys with an execution owned by this coordinator. */ readonly active: Effect.Effect> /** Starts an execution while idle, or joins the active execution and returns its exit. */ @@ -11,7 +11,7 @@ export interface Coordinator { /** Rings the doorbell: an idle key starts an execution; an active one drains again before settling. */ readonly wake: (key: Key) => Effect.Effect /** Stops the active execution, clears its doorbell, and waits for cleanup. No-op when idle. */ - readonly interrupt: (key: Key) => Effect.Effect + readonly interrupt: (key: Key, reason?: Reason) => Effect.Effect /** Resolves once no execution is active for the key. Returns immediately when already idle and never starts work. */ readonly awaitIdle: (key: Key) => Effect.Effect } @@ -23,11 +23,13 @@ export interface Coordinator { * closes the gap between a drain's last eligibility check and the idle transition, since * those cannot be one atomic step. `done` resolves joiners with this execution's exit. */ -type Execution = { +type Execution = { readonly done: Deferred.Deferred owner?: Fiber.Fiber pendingWake: boolean stopping: boolean + settling: boolean + interruptionReason?: Reason } /** @@ -41,19 +43,21 @@ type Execution = { * waiters get this exit * ``` */ -export const make = (options: { +export const make = (options: { readonly drain: (key: Key, force: boolean) => Effect.Effect + /** Runs once when a process-local busy period begins, before its first drain. */ + readonly started?: (key: Key) => Effect.Effect /** * Runs in the execution fiber for every exit, including interruption, after the final * drain and before the execution settles (waiters resolve after it completes). */ - readonly settled?: (key: Key, exit: Exit.Exit) => Effect.Effect -}): Effect.Effect, never, Scope.Scope> => + readonly settled?: (key: Key, exit: Exit.Exit, reason?: Reason) => Effect.Effect +}): Effect.Effect, never, Scope.Scope> => Effect.gen(function* () { - const executions = new Map>() + const executions = new Map>() const fork = yield* FiberSet.makeRuntime() - const loop = (key: Key, execution: Execution, force: boolean): Effect.Effect => + const loop = (key: Key, execution: Execution, force: boolean): Effect.Effect => Effect.suspend(() => options.drain(key, force)).pipe( Effect.flatMap(() => Effect.suspend(() => { @@ -66,15 +70,25 @@ export const make = (options: { ) const start = (key: Key, force: boolean) => { - const execution: Execution = { done: Deferred.makeUnsafe(), pendingWake: false, stopping: false } + const execution: Execution = { + done: Deferred.makeUnsafe(), + pendingWake: false, + stopping: false, + settling: false, + } executions.set(key, execution) // The leading yield lets `owner` be assigned before the drain can settle, and keeps // failing self-waking executions from growing the stack across successor starts. // Drains start one tick after wake; callers observe progress through events or run. execution.owner = fork( Effect.yieldNow.pipe( + Effect.andThen(Effect.uninterruptible(options.started?.(key) ?? Effect.void)), Effect.andThen(loop(key, execution, force)), - Effect.onExit((exit) => options.settled?.(key, exit) ?? Effect.void), + Effect.onExit((exit) => + Effect.sync(() => { + execution.settling = true + }).pipe(Effect.andThen(options.settled?.(key, exit, execution.interruptionReason) ?? Effect.void)), + ), Effect.onExit((exit) => Effect.sync(() => settle(key, execution, exit))), Effect.exit, Effect.asVoid, @@ -85,7 +99,7 @@ export const make = (options: { // A doorbell that survives the execution loop (rung after the loop decided to end, or // during failure or interruption cleanup) starts a fresh execution for the remaining work. - const settle = (key: Key, execution: Execution, exit: Exit.Exit) => { + const settle = (key: Key, execution: Execution, exit: Exit.Exit) => { if (execution.pendingWake) start(key, false) else executions.delete(key) Deferred.doneUnsafe(execution.done, exit) @@ -112,12 +126,13 @@ export const make = (options: { start(key, false) }) - const interrupt = (key: Key): Effect.Effect => + const interrupt = (key: Key, reason?: Reason): Effect.Effect => Effect.suspend(() => { const execution = executions.get(key) - if (execution?.owner === undefined) return Effect.void + if (execution?.owner === undefined || execution.stopping || execution.settling) return Effect.void execution.stopping = true execution.pendingWake = false + execution.interruptionReason = reason return Fiber.interrupt(execution.owner) }) diff --git a/packages/core/src/session/runner/index.ts b/packages/core/src/session/runner/index.ts index 7c2a6f463d..c612c9b598 100644 --- a/packages/core/src/session/runner/index.ts +++ b/packages/core/src/session/runner/index.ts @@ -3,13 +3,19 @@ export * as SessionRunner from "./index" import type { LLMError } from "@opencode-ai/llm" import { Context, Effect } from "effect" import { SessionSchema } from "../schema" -import type { MessageDecodeError } from "../error" +import type { MessageDecodeError, StepFailedError, UserInterruptedError } from "../error" import { SessionRunnerModel } from "./model" import type { SystemContext } from "../../system-context/index" import type { ToolOutputStore } from "../../tool-output-store" export type RunError = - LLMError | SessionRunnerModel.Error | MessageDecodeError | SystemContext.InitializationBlocked | ToolOutputStore.Error + | LLMError + | SessionRunnerModel.Error + | MessageDecodeError + | StepFailedError + | UserInterruptedError + | SystemContext.InitializationBlocked + | ToolOutputStore.Error /** Runs one local continuation from already-recorded Session history. */ export interface Interface { diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 6213ef872c..3e6475a947 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -10,7 +10,8 @@ import { isContextOverflowFailure, type ProviderErrorEvent, } from "@opencode-ai/llm" -import { Cause, DateTime, Effect, Exit, FiberSet, Layer, Option, Semaphore, Stream } from "effect" +import { SessionError } from "@opencode-ai/schema/session-error" +import { Cause, Effect, Exit, FiberSet, Layer, Option, Semaphore, Stream } from "effect" import { AgentV2 } from "../../agent" import { Config } from "../../config" import { Database } from "../../database/database" @@ -31,6 +32,7 @@ import { SessionCompaction } from "../compaction" import { SessionEvent } from "../event" import { SessionHistory } from "../history" import { SessionInput } from "../input" +import { SessionMessage } from "../message" import { SessionSchema } from "../schema" import { SessionStore } from "../store" import { SessionTitle } from "../title" @@ -43,6 +45,9 @@ import { SessionRunnerSystemPrompt } from "./system-prompt" import { Snapshot } from "../../snapshot" import { makeLocationNode } from "../../effect/app-node" import { llmClient } from "../../effect/app-node-platform" +import { StepFailedError, UserInterruptedError } from "../error" +import { toSessionError } from "../to-session-error" +import { SessionRunnerRetry } from "./retry" /** * Runs one durable coding-agent Session until it settles. @@ -53,10 +58,10 @@ import { llmClient } from "../../effect/app-node-platform" * - Session ownership and controls * - [x] Coordinate one local active drain per Session; explicit resumes join and prompt wakeups coalesce. * - [ ] Replace local ownership with durable multi-node ownership when clustered. - * - [ ] Mark busy, retrying, idle, interrupted, or terminal-failure status durably. + * - [x] Publish durable historical execution lifecycle and bounded retry observations. * - [ ] Honor interruption and reject stale work after runtime attachment replacement. * - [x] Honor optional agent step limits. - * - [ ] Bound provider retries and repeated identical tool calls. + * - [ ] Bound repeated identical tool calls (provider retries are bounded). * * - Runtime context assembly * - Track V1 runtime-context parity canonically in `specs/v2/session.md`. @@ -65,7 +70,7 @@ import { llmClient } from "../../effect/app-node-platform" * - [x] Translate every projected V2 Session message variant into canonical * `@opencode-ai/llm` messages. * - [ ] Resolve policy-filtered built-in, MCP, plugin, and structured-output tool definitions. - * - [x] Stream exactly one `llm.stream(request)` physical attempt. + * - [x] Stream exactly one `llm.stream(request)` call per attempt. * - [x] Persist assistant text and usage events incrementally as they arrive. * - [ ] Persist snapshots, patches, and retry notices incrementally as they arrive. * - [x] Persist reasoning, provider errors, and tool-call events incrementally as they arrive. @@ -86,7 +91,7 @@ import { llmClient } from "../../effect/app-node-platform" * - [ ] Coalesce streamed deltas and add covering projected-history indexes. * - [ ] Update title, summaries, compaction state, and cleanup in bounded background work. * - * Use `llm.stream(request)` for each physical attempt. Keep tool execution and continuation here. + * Use `llm.stream(request)` for each attempt. Keep tool execution and continuation here. * Durable continuation recovery remains a separate future slice with an explicit retry policy. * * The current slice loads V2 history, translates it, resolves a model through a core service, and persists one @@ -136,14 +141,14 @@ const layer = Layer.effect( sessionID, assistantMessageID: message.id, callID: tool.id, - error: { type: "unknown", message: "Tool execution interrupted" }, + error: { type: "tool.stale", message: "Tool execution interrupted", name: tool.name }, executed: tool.executed === true, }) } } }) - const awaitToolFibers = (fibers: FiberSet.FiberSet) => + const awaitToolFibers = (fibers: FiberSet.FiberSet) => Effect.raceFirst(FiberSet.join(fibers), FiberSet.awaitEmpty(fibers)) // Match V1: dismissing a question halts the loop instead of becoming model-facing tool output. @@ -168,6 +173,7 @@ const layer = Layer.effect( promotion: SessionInput.Delivery | undefined, step: number, recoverOverflow?: typeof compaction.compactAfterOverflow, + assistantMessageID?: SessionMessage.ID, ) { const session = yield* getSession(sessionID) if (session.location.directory !== location.directory || session.location.workspaceID !== location.workspaceID) @@ -181,7 +187,7 @@ const layer = Layer.effect( loadSystemContext(agent, session.id), session.id, ) - const toolFibers = yield* FiberSet.make() + const toolFibers = yield* FiberSet.make() let needsContinuation = false let currentStep = step if (promotion) { @@ -227,20 +233,21 @@ const layer = Layer.effect( model: resolved.ref, provider: model.provider, snapshot: startSnapshot, + assistantMessageID, }) const publication = Semaphore.makeUnsafe(1) // Durable publishes are serialized so tool fibers and step settlement never interleave // mid-event. const serialized = (effect: Effect.Effect) => publication.withPermit(effect) - const publish = (event: LLMEvent, outputPaths: ReadonlyArray = []) => - serialized(publisher.publish(event, outputPaths)) + const publish = (event: LLMEvent, outputPaths: ReadonlyArray = [], error?: SessionError.Error) => + serialized(publisher.publish(event, outputPaths, error)) let overflowFailure: ProviderErrorEvent | undefined const providerStream = llm.stream(request).pipe( Stream.runForEach((event) => Effect.gen(function* () { if (overflowFailure || publisher.hasProviderError()) return if (LLMEvent.is.providerError(event)) { - if (isContextOverflowFailure(event) && !publisher.hasAssistantStarted()) { + if (isContextOverflowFailure(event) && !publisher.hasRetryEvidence()) { overflowFailure = event return } @@ -248,7 +255,12 @@ const layer = Layer.effect( yield* publish(event) if (event.type !== "tool-call" || event.providerExecuted) return if (!toolMaterialization) { - yield* serialized(publisher.failUnsettledTools("Tools are disabled after the maximum agent steps")) + yield* serialized( + publisher.failUnsettledTools({ + type: "tool.execution", + message: "Tools are disabled after the maximum agent steps", + }), + ) return } needsContinuation = true @@ -271,6 +283,15 @@ const layer = Layer.effect( output: settlement.output, }), settlement.outputPaths ?? [], + settlement.error, + ).pipe( + Effect.andThen( + settlement.error?.type === "permission.rejected" + ? serialized(publisher.failAssistant(settlement.error)).pipe( + Effect.andThen(Effect.fail(new UserInterruptedError())), + ) + : Effect.void, + ), ), ), ), @@ -317,7 +338,7 @@ const layer = Layer.effect( // restart the step instead of surfacing the provider error. if ( recoverOverflow && - !publisher.hasAssistantStarted() && + !publisher.hasRetryEvidence() && isContextOverflowFailure(overflowFailure ?? streamFailure) && (yield* restore(recoverOverflow({ sessionID: session.id, messages: context, request }))) ) @@ -329,8 +350,26 @@ const layer = Layer.effect( if (overflowFailure) yield* publish(overflowFailure) const llmFailure = streamFailure instanceof LLMError ? streamFailure : undefined if (llmFailure && !publisher.hasProviderError()) { - yield* serialized(publisher.failUnsettledTools("Provider did not return a tool result", true)) - yield* serialized(publisher.failAssistant(llmFailure.reason.message)) + const error = toSessionError(llmFailure) + if ( + SessionRunnerRetry.isRetryable(llmFailure) && + !publisher.hasRetryEvidence() && + (agent.info?.steps === undefined || currentStep < agent.info.steps) + ) { + return yield* new SessionRunnerRetry.RetryableFailure({ + cause: llmFailure, + assistantMessageID: yield* publisher.startAssistant(), + error, + step: currentStep, + }) + } + yield* serialized( + publisher.failUnsettledTools( + { type: "tool.result-missing", message: "Provider did not return a tool result" }, + true, + ), + ) + yield* serialized(publisher.failAssistant(error)) } // Provider error events only arrive from the stream, so the flag is final here. const providerFailed = publisher.hasProviderError() @@ -340,27 +379,30 @@ const layer = Layer.effect( const settled = yield* restore(awaitToolFibers(toolFibers)).pipe(Effect.exit) const toolsInterrupted = settled._tag === "Failure" && Cause.hasInterrupts(settled.cause) const questionDismissed = settled._tag === "Failure" && isQuestionRejected(settled.cause) + const settledError = + settled._tag === "Failure" ? Option.getOrUndefined(Cause.findErrorOption(settled.cause)) : undefined + const permissionRejected = settledError instanceof UserInterruptedError - if (questionDismissed || streamInterrupted || toolsInterrupted) { + if (questionDismissed || permissionRejected || streamInterrupted || toolsInterrupted) { yield* FiberSet.clear(toolFibers) - yield* serialized(publisher.failUnsettledTools("Tool execution interrupted")) - yield* serialized(publisher.failAssistant("Step interrupted")) + yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" })) + yield* serialized(publisher.failAssistant({ type: "aborted", message: "Step interrupted" })) // Match V1: dismissing a question halts the loop like an interruption. - if (questionDismissed) return yield* Effect.interrupt + if (questionDismissed || permissionRejected) return yield* new UserInterruptedError() } // A settled tool fiber failure is one of two things. A defect from a tool // implementation becomes a failed tool call the model can read, and the step still // settles so the model may recover. A typed infrastructure failure (tool output // could not be persisted) also fails the assistant and then fails the drain. - const settledFailure = settled._tag === "Failure" && !toolsInterrupted ? settled.cause : undefined + const settledFailure = + settled._tag === "Failure" && !toolsInterrupted && !permissionRejected ? settled.cause : undefined const infraError = settledFailure === undefined ? undefined : Option.getOrUndefined(Cause.findErrorOption(settledFailure)) if (settledFailure !== undefined) { const failure = infraError ?? Cause.squash(settledFailure) - const message = failure instanceof Error ? failure.message : String(failure) - yield* serialized(publisher.failUnsettledTools(`Tool execution failed: ${message}`)) - if (infraError !== undefined) - yield* serialized(publisher.failAssistant(`Tool execution failed: ${message}`)) + const error = toSessionError(failure) + yield* serialized(publisher.failUnsettledTools(error)) + if (infraError !== undefined) yield* serialized(publisher.failAssistant(error)) } const stepSettlement = publisher.stepSettlement() @@ -369,13 +411,21 @@ const layer = Layer.effect( if (stepSettlement && stepEndedCleanly) yield* publishStepEnd(stepSettlement) // A provider error orphans recorded local calls; a clean stream can still leave // hosted calls without results. - if (providerFailed) yield* serialized(publisher.failUnsettledTools("Tool execution interrupted")) + if (providerFailed) + yield* serialized(publisher.failUnsettledTools({ type: "aborted", message: "Tool execution interrupted" })) if (stream._tag === "Success" && !providerFailed) - yield* serialized(publisher.failUnsettledTools("Provider did not return a tool result", true)) + yield* serialized( + publisher.failUnsettledTools( + { type: "tool.result-missing", message: "Provider did not return a tool result" }, + true, + ), + ) if (stream._tag === "Failure") return yield* Effect.failCause(stream.cause) if (settled._tag === "Failure" && (toolsInterrupted || infraError !== undefined)) return yield* Effect.failCause(settled.cause) + const stepFailure = publisher.stepFailure() + if (stepFailure) return yield* new StepFailedError({ error: stepFailure }) return { _tag: "Completed", needsContinuation: !providerFailed && needsContinuation, @@ -396,8 +446,31 @@ const layer = Layer.effect( let recoverOverflow: typeof compaction.compactAfterOverflow | undefined = compaction.compactAfterOverflow let currentPromotion = promotion let currentStep = step + let assistantMessageID: SessionMessage.ID | undefined while (true) { - const attempt = yield* attemptStep(sessionID, currentPromotion, currentStep, recoverOverflow) + const attempt = yield* Effect.suspend(() => + attemptStep(sessionID, currentPromotion, currentStep, recoverOverflow, assistantMessageID), + ).pipe( + Effect.tapError((error) => + error instanceof SessionRunnerRetry.RetryableFailure + ? Effect.sync(() => { + currentStep = error.step + 1 + assistantMessageID = error.assistantMessageID + currentPromotion = undefined + }) + : Effect.void, + ), + Effect.retryOrElse(SessionRunnerRetry.schedule(events, sessionID), (error) => { + if (!(error instanceof SessionRunnerRetry.RetryableFailure)) return Effect.fail(error) + return events + .publish(SessionEvent.Step.Failed, { + sessionID, + assistantMessageID: error.assistantMessageID, + error: error.error, + }) + .pipe(Effect.andThen(Effect.fail(error.cause))) + }), + ) if (attempt._tag === "Completed") return { needsContinuation: attempt.needsContinuation, step: attempt.step } if (attempt._tag === "RestartAfterOverflowCompaction") recoverOverflow = undefined yield* Effect.yieldNow @@ -406,8 +479,7 @@ const layer = Layer.effect( } }) - // ExecutionSettled is published per execution (busy period) by SessionExecution, not per - // drain here. + // Execution lifecycle is published per busy period by SessionExecution, not per drain here. const drain = Effect.fn("SessionRunner.drain")(function* (input: { readonly sessionID: SessionSchema.ID readonly force: boolean diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index c16e4713f8..c158c1954f 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -5,6 +5,7 @@ import { ModelV2 } from "../../model" import { SessionEvent } from "../event" import { SessionMessage } from "../message" import { SessionSchema } from "../schema" +import { SessionError } from "@opencode-ai/schema/session-error" type Input = { readonly sessionID: SessionSchema.ID @@ -12,6 +13,7 @@ type Input = { readonly model: ModelV2.Ref readonly provider: string readonly snapshot?: string + readonly assistantMessageID?: SessionMessage.ID } const safe = (value: number | undefined) => Math.max(0, Number.isFinite(value) ? (value ?? 0) : 0) @@ -42,10 +44,10 @@ const message = (value: unknown) => { type SettledOutput = | { readonly structured: Record; readonly content: ToolOutput["content"] } - | { readonly error: { readonly type: "unknown"; readonly message: string } } + | { readonly error: SessionError.Error } const settledOutput = (value: ToolOutput | undefined, result: ToolResultValue): SettledOutput => { - if (result.type === "error") return { error: { type: "unknown", message: message(result.value) } } + if (result.type === "error") return { error: { type: "tool.execution", message: message(result.value) } } const settled = value ?? ToolOutput.fromResultValue(result) if (!settled) throw new Error(`Unsupported tool result: ${message(result)}`) return { structured: record(settled.structured), content: settled.content } @@ -64,16 +66,23 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) providerExecuted: boolean } >() - let assistantMessageID: SessionMessage.ID | undefined - let assistantActive = false + let assistantMessageID = input.assistantMessageID + let stepStarted = false let assistantFailed = false let providerFailed = false - let stepSettlement: { readonly finish: string; readonly tokens: ReturnType } | undefined + let retryEvidence = false + let stepFailure: SessionError.Error | undefined + let stepSettlement: + | { + readonly finish: Extract["reason"] + readonly tokens: ReturnType + } + | undefined const startAssistant = Effect.fnUntraced(function* () { - if (assistantMessageID !== undefined) return assistantMessageID - assistantMessageID = SessionMessage.ID.create() - assistantActive = true + if (stepStarted && assistantMessageID !== undefined) return assistantMessageID + assistantMessageID ??= SessionMessage.ID.create() + stepStarted = true yield* events.publish(SessionEvent.Step.Started, { ...input, assistantMessageID, @@ -197,21 +206,21 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) yield* flushFragments() }) - const failAssistant = Effect.fnUntraced(function* (message: string) { + const failAssistant = Effect.fnUntraced(function* (error: SessionError.Error) { if (assistantFailed) return yield* flush() const assistantMessageID = yield* startAssistant() - assistantActive = false assistantFailed = true + stepFailure = error yield* events.publish(SessionEvent.Step.Failed, { sessionID: input.sessionID, assistantMessageID, - error: { type: "unknown", message }, + error, }) }) const failUnsettledTools = Effect.fn("SessionRunner.failUnsettledTools")(function* ( - message: string, + error: SessionError.Error, hostedOnly = false, ) { for (const [callID, tool] of tools) { @@ -221,7 +230,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) sessionID: input.sessionID, assistantMessageID: tool.assistantMessageID, callID, - error: { type: "unknown", message }, + error, executed: tool.providerExecuted, }) } @@ -235,11 +244,14 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) const publish = Effect.fn("SessionRunner.publishLLMEvent")(function* ( event: LLMEvent, outputPaths: ReadonlyArray = [], + error?: SessionError.Error, ) { switch (event.type) { case "step-start": + yield* startAssistant() return case "text-start": + retryEvidence = true yield* text.start(event.id) yield* events.publish(SessionEvent.Text.Started, { sessionID: input.sessionID, @@ -258,6 +270,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) yield* text.end(event.id) return case "reasoning-start": + retryEvidence = true yield* reasoning.start(event.id) yield* events.publish(SessionEvent.Reasoning.Started, { sessionID: input.sessionID, @@ -277,6 +290,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) yield* reasoning.end(event.id, providerState(event.providerMetadata)) return case "tool-input-start": + retryEvidence = true yield* startToolInput(event) return case "tool-input-delta": { @@ -298,6 +312,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) yield* endToolInput(event) return case "tool-call": { + retryEvidence = true if (!tools.has(event.id)) yield* startToolInput(event) const tool = tools.get(event.id)! if (!tool.inputEnded) yield* endToolInput(event) @@ -318,6 +333,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) return } case "tool-result": { + retryEvidence = true const tool = tools.get(event.id) if (!tool?.called) return yield* Effect.die(new Error(`Tool result before call: ${event.id}`)) if (tool.name !== event.name) @@ -327,7 +343,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) return yield* Effect.die(new Error(`Duplicate tool result: ${event.id}`)) } tool.settled = true - const result = settledOutput(event.output, event.result) + const result = error ? { error } : settledOutput(event.output, event.result) const executed = event.providerExecuted === true || tool.providerExecuted const resultState = providerState(event.providerMetadata) if ("error" in result) { @@ -355,6 +371,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) return } case "tool-error": { + retryEvidence = true const tool = tools.get(event.id) if (!tool?.called) return yield* Effect.die(new Error(`Tool error before call: ${event.id}`)) if (tool.name !== event.name) @@ -365,7 +382,10 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) sessionID: input.sessionID, assistantMessageID: tool.assistantMessageID, callID: event.id, - error: { type: "unknown", message: event.message }, + error: + event.message === `Unknown tool: ${event.name}` + ? { type: "tool.unknown", message: event.message, name: event.name } + : { type: "tool.execution", message: event.message }, executed: tool.providerExecuted, resultState: providerState(event.providerMetadata), }) @@ -373,15 +393,19 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) } case "step-finish": yield* flush() - assistantActive = false if (stepSettlement) return yield* Effect.die(new Error("Duplicate step finish")) + if (event.reason === "content-filter") { + providerFailed = true + yield* failAssistant({ type: "provider.content-filter", message: "Provider blocked the response" }) + return + } stepSettlement = { finish: event.reason, tokens: tokens(event.usage) } return case "finish": return case "provider-error": providerFailed = true - yield* failAssistant(event.message) + yield* failAssistant({ type: "provider.unknown", message: event.message }) return } }) @@ -391,9 +415,9 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input) flush, failAssistant, failUnsettledTools, - hasActiveAssistant: () => assistantActive, - hasAssistantStarted: () => assistantMessageID !== undefined, hasProviderError: () => providerFailed, + hasRetryEvidence: () => retryEvidence, + stepFailure: () => stepFailure, stepSettlement: () => stepSettlement, startAssistant, assistantMessageID: assistantMessageIDForTool, diff --git a/packages/core/src/session/runner/retry.ts b/packages/core/src/session/runner/retry.ts new file mode 100644 index 0000000000..3ada83b4e4 --- /dev/null +++ b/packages/core/src/session/runner/retry.ts @@ -0,0 +1,67 @@ +export * as SessionRunnerRetry from "./retry" + +import { LLMError } from "@opencode-ai/llm" +import { SessionError } from "@opencode-ai/schema/session-error" +import { Data, Duration, Effect, Schedule } from "effect" +import { EventV2 } from "../../event" +import { SessionEvent } from "../event" +import { SessionMessage } from "../message" +import { SessionSchema } from "../schema" +import type { SessionRunner } from "./index" + +export class RetryableFailure extends Data.TaggedError("SessionRunner.RetryableFailure")<{ + readonly cause: LLMError + readonly assistantMessageID: SessionMessage.ID + readonly error: SessionError.Error + readonly step: number +}> {} + +export function isRetryable(error: LLMError) { + switch (error.reason._tag) { + case "RateLimit": + case "ProviderInternal": + case "Transport": + return true + case "Authentication": + case "QuotaExceeded": + case "ContentPolicy": + case "InvalidProviderOutput": + case "InvalidRequest": + case "NoRoute": + case "UnknownProvider": + return false + default: { + const exhaustive: never = error.reason + return exhaustive + } + } +} + +const retryAfter = (failure: RetryableFailure) => { + if (failure.cause.reason._tag === "RateLimit" || failure.cause.reason._tag === "ProviderInternal") + return failure.cause.reason.retryAfterMs + return undefined +} + +export const schedule = (events: EventV2.Interface, sessionID: SessionSchema.ID) => + Schedule.exponential("2 seconds").pipe( + Schedule.take(4), + Schedule.setInputType(), + Schedule.passthrough, + Schedule.while(({ input }) => input instanceof RetryableFailure), + Schedule.modifyDelay((failure, delay) => { + const minimum = failure instanceof RetryableFailure ? retryAfter(failure) : undefined + return Effect.succeed(minimum === undefined ? delay : Duration.max(delay, Duration.millis(minimum))) + }), + Schedule.tap((metadata) => + metadata.input instanceof RetryableFailure + ? events.publish(SessionEvent.RetryScheduled, { + sessionID, + assistantMessageID: metadata.input.assistantMessageID, + attempt: metadata.attempt + 1, + at: metadata.now + Duration.toMillis(metadata.duration), + error: metadata.input.error, + }) + : Effect.void, + ), + ) diff --git a/packages/core/src/session/to-session-error.ts b/packages/core/src/session/to-session-error.ts new file mode 100644 index 0000000000..db42d1a0a9 --- /dev/null +++ b/packages/core/src/session/to-session-error.ts @@ -0,0 +1,65 @@ +import { LLMError, ToolFailure } from "@opencode-ai/llm" +import { SessionError } from "@opencode-ai/schema/session-error" +import { PermissionV2 } from "../permission" +import { QuestionV2 } from "../question" +import { Integration } from "../integration" +import { ToolOutputStore } from "../tool-output-store" +import { StepFailedError, UserInterruptedError } from "./error" +import { SessionRunnerModel } from "./runner/model" + +export function toSessionError(cause: unknown): SessionError.Error { + if (cause instanceof LLMError) { + switch (cause.reason._tag) { + case "RateLimit": + return { + type: "provider.rate-limit", + message: cause.reason.message, + retryAfterMs: cause.reason.retryAfterMs, + } + case "Authentication": + return { type: "provider.auth", message: cause.reason.message } + case "QuotaExceeded": + return { type: "provider.quota", message: cause.reason.message } + case "ContentPolicy": + return { type: "provider.content-filter", message: cause.reason.message } + case "Transport": + return { type: "provider.transport", message: cause.reason.message } + case "ProviderInternal": + return { type: "provider.internal", message: cause.reason.message } + case "InvalidProviderOutput": + return { type: "provider.invalid-output", message: cause.reason.message } + case "InvalidRequest": + return { type: "provider.invalid-request", message: cause.reason.message } + case "NoRoute": + return { type: "provider.no-route", message: cause.reason.message } + case "UnknownProvider": + return { type: "provider.unknown", message: cause.reason.message } + default: { + const exhaustive: never = cause.reason + return exhaustive + } + } + } + if (cause instanceof PermissionV2.RejectedError) + return { + type: "permission.rejected", + message: cause.message, + permission: cause.permission, + resources: [...cause.resources], + } + if (cause instanceof QuestionV2.RejectedError) return { type: "aborted", message: cause.message, reason: "user" } + if (cause instanceof ToolFailure) + return cause.error === undefined ? { type: "tool.execution", message: cause.message } : toSessionError(cause.error) + if (cause instanceof StepFailedError) return cause.error + if (cause instanceof UserInterruptedError) return { type: "aborted", message: cause.message, reason: "user" } + if ( + cause instanceof SessionRunnerModel.ModelNotSelectedError || + cause instanceof SessionRunnerModel.ModelUnavailableError || + cause instanceof SessionRunnerModel.VariantUnavailableError || + cause instanceof SessionRunnerModel.UnsupportedApiError + ) + return { type: "provider.no-route", message: cause.message } + if (cause instanceof Integration.AuthorizationError) return { type: "provider.auth", message: cause.message } + if (cause instanceof ToolOutputStore.StorageError) return { type: "unknown", message: cause.message } + return { type: "unknown", message: cause instanceof Error ? cause.message : String(cause) } +} diff --git a/packages/core/src/tool/apply-patch.ts b/packages/core/src/tool/apply-patch.ts index 2f481d8a0d..88cc0eccab 100644 --- a/packages/core/src/tool/apply-patch.ts +++ b/packages/core/src/tool/apply-patch.ts @@ -73,12 +73,12 @@ export const Plugin = { toModelOutput: ({ output }) => [{ type: "text", text: toModelOutput(output) }], execute: (input, context) => { const applied: Array = [] - const fail = (path: string) => { + const fail = (path: string, error?: unknown) => { const prefix = applied.length === 0 ? `Unable to apply patch at ${path}` : `Patch partially applied before failing at ${path}. Applied: ${applied.map((item) => item.resource).join(", ")}` - return new ToolFailure({ message: prefix }) + return new ToolFailure({ message: prefix, error }) } return Effect.gen(function* () { const source = { @@ -150,7 +150,7 @@ export const Plugin = { before, after: update.content, }) - }).pipe(Effect.mapError(() => fail(hunk.path))) + }).pipe(Effect.mapError((error) => fail(hunk.path, error))) } const patchFiles = prepared.map(patchFile) @@ -180,11 +180,11 @@ export const Plugin = { content: change.content, }) applied.push({ type: change.type, resource: result.resource, target: result.target }) - }).pipe(Effect.mapError(() => fail(change.path))), + }).pipe(Effect.mapError((error) => fail(change.path, error))), { discard: true }, ) return { applied, files: patchFiles } - }).pipe(Effect.mapError((error) => (error instanceof ToolFailure ? error : fail("patch")))) + }).pipe(Effect.mapError((error) => (error instanceof ToolFailure ? error : fail("patch", error)))) }, }), "edit", diff --git a/packages/core/src/tool/edit.ts b/packages/core/src/tool/edit.ts index 17ff28cfc1..bce16fa428 100644 --- a/packages/core/src/tool/edit.ts +++ b/packages/core/src/tool/edit.ts @@ -111,8 +111,9 @@ export const Plugin = { error instanceof FileMutation.StaleContentError ? new ToolFailure({ message: "File changed after permission approval. Read it again before editing.", + error, }) - : new ToolFailure({ message: `Unable to edit ${input.path}` }), + : new ToolFailure({ message: `Unable to edit ${input.path}`, error }), ), ) diff --git a/packages/core/src/tool/glob.ts b/packages/core/src/tool/glob.ts index dcea52120f..c19f2bad0a 100644 --- a/packages/core/src/tool/glob.ts +++ b/packages/core/src/tool/glob.ts @@ -100,7 +100,7 @@ export const Plugin = { Effect.mapError((error) => error instanceof ToolFailure ? error - : new ToolFailure({ message: `Unable to find files matching ${input.pattern}` }), + : new ToolFailure({ message: `Unable to find files matching ${input.pattern}`, error }), ), ), }), diff --git a/packages/core/src/tool/grep.ts b/packages/core/src/tool/grep.ts index 35ac537c91..f1de1468d4 100644 --- a/packages/core/src/tool/grep.ts +++ b/packages/core/src/tool/grep.ts @@ -131,7 +131,7 @@ export const Plugin = { Effect.mapError((error) => error instanceof ToolFailure ? error - : new ToolFailure({ message: `Unable to grep for ${input.pattern}` }), + : new ToolFailure({ message: `Unable to grep for ${input.pattern}`, error }), ), ), }), diff --git a/packages/core/src/tool/question.ts b/packages/core/src/tool/question.ts index 218edb57e9..a5644ecea9 100644 --- a/packages/core/src/tool/question.ts +++ b/packages/core/src/tool/question.ts @@ -67,7 +67,7 @@ export const Plugin = { source: { type: "tool", messageID: context.assistantMessageID, callID: context.toolCallID }, }) .pipe( - Effect.mapError(() => new ToolFailure({ message: "Permission denied: question" })), + Effect.mapError((error) => new ToolFailure({ message: "Permission denied: question", error })), Effect.andThen( question .ask({ diff --git a/packages/core/src/tool/read.ts b/packages/core/src/tool/read.ts index 16e229d97d..919a477dc9 100644 --- a/packages/core/src/tool/read.ts +++ b/packages/core/src/tool/read.ts @@ -106,7 +106,9 @@ export const Plugin = { start: type === "directory" ? resolved : dirname(resolved), stop: root, }) - const candidates = (yield* Effect.forEach(discovered, fs.resolve)).filter((file) => dirname(file) !== root) + const candidates = (yield* Effect.forEach(discovered, fs.resolve)).filter( + (file) => dirname(file) !== root, + ) if (candidates.length === 0) return yield* sessionInstructions.load({ sessionID: context.sessionID, paths: candidates }) }).pipe( @@ -130,7 +132,7 @@ export const Plugin = { error instanceof Image.SizeError ? error.message : `Unable to read ${input.path}` - return new ToolFailure({ message }) + return new ToolFailure({ message, error }) }), ) }, diff --git a/packages/core/src/tool/registry.ts b/packages/core/src/tool/registry.ts index 1dc7478c11..1a6b42731d 100644 --- a/packages/core/src/tool/registry.ts +++ b/packages/core/src/tool/registry.ts @@ -12,6 +12,8 @@ import { definition, permission, registrationEntries, settle, type AnyTool, type import { Tools } from "./tools" import { ToolHooks } from "./hooks" import { makeLocationNode } from "../effect/app-node" +import { SessionError } from "@opencode-ai/schema/session-error" +import { toSessionError } from "../session/to-session-error" export type ExecuteInput = { readonly sessionID: SessionSchema.ID @@ -40,6 +42,7 @@ export interface Settlement { readonly result: ToolResultValue readonly output?: ToolOutput readonly outputPaths?: ReadonlyArray + readonly error?: SessionError.Error } export class Service extends Context.Service()("@opencode/v2/ToolRegistry") {} @@ -60,9 +63,15 @@ const registryLayer = Layer.effect( type: "error" as const, value: advertised ? `Stale tool call: ${input.call.name}` : `Unknown tool: ${input.call.name}`, }, + error: advertised + ? ({ type: "tool.stale", message: `Stale tool call: ${input.call.name}`, name: input.call.name } as const) + : ({ type: "tool.unknown", message: `Unknown tool: ${input.call.name}`, name: input.call.name } as const), } if (advertised && registration.identity !== advertised) - return { result: { type: "error" as const, value: `Stale tool call: ${input.call.name}` } } + return { + result: { type: "error" as const, value: `Stale tool call: ${input.call.name}` }, + error: { type: "tool.stale" as const, message: `Stale tool call: ${input.call.name}`, name: input.call.name }, + } // Hooks fire only for hosted/local tools; provider-executed calls never reach settleWith. const beforeEvent: ToolHooks.BeforeEvent = { tool: input.call.name, @@ -73,22 +82,33 @@ const registryLayer = Layer.effect( input: input.call.input, } yield* toolHooks.runBefore(beforeEvent) - const pending = yield* settle(registration.tool, { ...input.call, input: beforeEvent.input }, { - sessionID: input.sessionID, - agent: input.agent, - assistantMessageID: input.assistantMessageID, - toolCallID: input.call.id, - }).pipe( + const pending = yield* settle( + registration.tool, + { ...input.call, input: beforeEvent.input }, + { + sessionID: input.sessionID, + agent: input.agent, + assistantMessageID: input.assistantMessageID, + toolCallID: input.call.id, + }, + ).pipe( Effect.map((output) => ({ output })), Effect.catchTag("LLM.ToolFailure", (failure) => - Effect.succeed({ result: { type: "error" as const, value: failure.message } }), + Effect.succeed({ + result: { type: "error" as const, value: failure.message }, + error: toSessionError(failure), + }), ), ) let settlement: Settlement if ("result" in pending) { settlement = pending } else { - const bounded = yield* resources.bound({ sessionID: input.sessionID, toolCallID: input.call.id, output: pending.output }) + const bounded = yield* resources.bound({ + sessionID: input.sessionID, + toolCallID: input.call.id, + output: pending.output, + }) const result = ToolOutput.toResultValue(bounded.output) settlement = result.type === "error" @@ -115,6 +135,7 @@ const registryLayer = Layer.effect( result: afterEvent.result, ...(afterEvent.output !== undefined ? { output: afterEvent.output } : {}), ...(afterEvent.outputPaths !== undefined ? { outputPaths: afterEvent.outputPaths } : {}), + ...(settlement.error !== undefined ? { error: settlement.error } : {}), } }) @@ -157,7 +178,10 @@ const registryLayer = Layer.effect( settle: (input) => { const registration = registrations.get(input.call.name) if (registration) return settleWith(input, registration.identity) - return Effect.succeed({ result: { type: "error", value: `Unknown tool: ${input.call.name}` } }) + return Effect.succeed({ + result: { type: "error", value: `Unknown tool: ${input.call.name}` }, + error: { type: "tool.unknown", message: `Unknown tool: ${input.call.name}`, name: input.call.name }, + }) }, } }), diff --git a/packages/core/src/tool/shell.ts b/packages/core/src/tool/shell.ts index c3644e4810..31be44abf3 100644 --- a/packages/core/src/tool/shell.ts +++ b/packages/core/src/tool/shell.ts @@ -247,9 +247,9 @@ export const Plugin = { } } - const result = yield* runtime.job.block({ id: job.id, sessionID: context.sessionID }).pipe( - Effect.onInterrupt(() => runtime.job.cancel(job.id).pipe(Effect.ignore)), - ) + const result = yield* runtime.job + .block({ id: job.id, sessionID: context.sessionID }) + .pipe(Effect.onInterrupt(() => runtime.job.cancel(job.id).pipe(Effect.ignore))) if (result?.type === "backgrounded") { yield* notifyWhenDone(context.sessionID, context.toolCallID, input.command) return { @@ -260,14 +260,19 @@ export const Plugin = { ...(warnings.length ? { warnings } : {}), } } - if (result?.info.status === "error") return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) + if (result?.info.status === "error") + return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) if (result?.info.status === "cancelled") return yield* Effect.fail(new Error("Command cancelled")) return { ...(yield* settleShell()), ...(warnings.length ? { warnings } : {}), } - }).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to execute command: ${input.command}` }))), + }).pipe( + Effect.mapError( + (error) => new ToolFailure({ message: `Unable to execute command: ${input.command}`, error }), + ), + ), }), }) .pipe(Effect.orDie) diff --git a/packages/core/src/tool/subagent.ts b/packages/core/src/tool/subagent.ts index 517d1ce65f..e1916c9942 100644 --- a/packages/core/src/tool/subagent.ts +++ b/packages/core/src/tool/subagent.ts @@ -104,7 +104,9 @@ export const Plugin = { const parent = yield* runtime.session .get(context.sessionID) .pipe( - Effect.mapError(() => new ToolFailure({ message: `Parent session not found: ${context.sessionID}` })), + Effect.mapError( + (error) => new ToolFailure({ message: `Parent session not found: ${context.sessionID}`, error }), + ), ) const agent = yield* agents.resolve(input.agent) if (agent === undefined) return yield* new ToolFailure({ message: `Unknown agent: ${input.agent}` }) @@ -123,7 +125,9 @@ export const Plugin = { // session (V1 deriveSubagentSessionPermission). MVP uses the agent's own permissions. }) .pipe( - Effect.mapError(() => new ToolFailure({ message: `Parent session not found: ${context.sessionID}` })), + Effect.mapError( + (error) => new ToolFailure({ message: `Parent session not found: ${context.sessionID}`, error }), + ), ) const background = input.background === true diff --git a/packages/core/src/tool/todowrite.ts b/packages/core/src/tool/todowrite.ts index 5f34ebd5aa..b900f9ace2 100644 --- a/packages/core/src/tool/todowrite.ts +++ b/packages/core/src/tool/todowrite.ts @@ -46,7 +46,7 @@ export const Plugin = { }) yield* todos.update({ sessionID: context.sessionID, todos: input.todos }) return { todos: input.todos } - }).pipe(Effect.mapError(() => new ToolFailure({ message: "Unable to update todos" }))), + }).pipe(Effect.mapError((error) => new ToolFailure({ message: "Unable to update todos", error }))), }), }) .pipe(Effect.orDie) diff --git a/packages/core/src/tool/webfetch.ts b/packages/core/src/tool/webfetch.ts index efac4a2a75..9b10b775b6 100644 --- a/packages/core/src/tool/webfetch.ts +++ b/packages/core/src/tool/webfetch.ts @@ -170,7 +170,7 @@ export const Plugin = { format: input.format, output, } - }).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to fetch ${input.url}` }))), + }).pipe(Effect.mapError((error) => new ToolFailure({ message: `Unable to fetch ${input.url}`, error }))), }), }) .pipe(Effect.orDie) diff --git a/packages/core/src/tool/websearch.ts b/packages/core/src/tool/websearch.ts index 80a2422cee..4aead0364d 100644 --- a/packages/core/src/tool/websearch.ts +++ b/packages/core/src/tool/websearch.ts @@ -243,7 +243,11 @@ export const Plugin = { provider, text: text ?? NO_RESULTS, } - }).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to search the web for ${input.query}` }))) + }).pipe( + Effect.mapError( + (error) => new ToolFailure({ message: `Unable to search the web for ${input.query}`, error }), + ), + ) }, }), }) diff --git a/packages/core/src/tool/write.ts b/packages/core/src/tool/write.ts index 73885b2187..8c94c3d1cf 100644 --- a/packages/core/src/tool/write.ts +++ b/packages/core/src/tool/write.ts @@ -83,7 +83,7 @@ export const Plugin = { source, }) return yield* files.writeTextPreservingBom({ target, content: input.content }) - }).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to write ${input.path}` }))), + }).pipe(Effect.mapError((error) => new ToolFailure({ message: `Unable to write ${input.path}`, error }))), }), "edit", ), diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index b36de1abac..1954b29add 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -15,6 +15,7 @@ import eventSourcedSessionInputMigration from "@opencode-ai/core/database/migrat import contextEpochAgentMigration from "@opencode-ai/core/database/migration/20260605042240_add_context_epoch_agent" import simplifyIntegrationCredentialsMigration from "@opencode-ai/core/database/migration/20260611192811_lush_chimera" import simplifySessionInputMigration from "@opencode-ai/core/database/migration/20260622202450_simplify_session_input" +import resetExecutionErrorsMigration from "@opencode-ai/core/database/migration/20260703210000_reset_v2_execution_errors" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { EventV2 } from "@opencode-ai/core/event" @@ -38,6 +39,29 @@ const run = (effect: Effect.Effect) => const makeDb = EffectDrizzleSqlite.makeWithDefaults() describe("DatabaseMigration", () => { + test("resets incompatible V2 execution and error history", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE session_input (id text PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE session_message (id text PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE event (id text PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id text PRIMARY KEY, seq integer NOT NULL)`) + yield* db.run(sql`INSERT INTO session_input (id) VALUES ('input')`) + yield* db.run(sql`INSERT INTO session_message (id) VALUES ('message')`) + yield* db.run(sql`INSERT INTO event (id) VALUES ('event')`) + yield* db.run(sql`INSERT INTO event_sequence (aggregate_id, seq) VALUES ('session', 1)`) + + yield* DatabaseMigration.applyOnly(db, [resetExecutionErrorsMigration]) + + expect(yield* db.get(sql`SELECT id FROM session_input`)).toBeUndefined() + expect(yield* db.get(sql`SELECT id FROM session_message`)).toBeUndefined() + expect(yield* db.get(sql`SELECT id FROM event`)).toBeUndefined() + expect(yield* db.get(sql`SELECT aggregate_id FROM event_sequence`)).toBeUndefined() + }), + ) + }) + test("serializes concurrent embedded initialization for one database path", async () => { await using tmp = await tmpdir() const filename = path.join(tmp.path, "embedded.sqlite") diff --git a/packages/core/test/session-error.test.ts b/packages/core/test/session-error.test.ts new file mode 100644 index 0000000000..2eabd7ce8d --- /dev/null +++ b/packages/core/test/session-error.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test" +import { + AuthenticationReason, + ContentPolicyReason, + InvalidProviderOutputReason, + InvalidRequestReason, + LLMError, + NoRouteReason, + ModelID, + ProviderID, + ProviderInternalReason, + QuotaExceededReason, + RateLimitReason, + TransportReason, + UnknownProviderReason, + ToolFailure, +} from "@opencode-ai/llm" +import { PermissionV2 } from "@opencode-ai/core/permission" +import { toSessionError } from "@opencode-ai/core/session/to-session-error" +import { SessionRunnerRetry } from "@opencode-ai/core/session/runner/retry" + +const llm = (reason: LLMError["reason"]) => new LLMError({ module: "test", method: "stream", reason }) + +describe("toSessionError", () => { + test("maps every LLM reason to the closed wire type", () => { + expect(toSessionError(llm(new RateLimitReason({ message: "rate", retryAfterMs: 123 })))).toEqual({ + type: "provider.rate-limit", + message: "rate", + retryAfterMs: 123, + }) + expect(toSessionError(llm(new AuthenticationReason({ message: "auth", kind: "invalid" }))).type).toBe( + "provider.auth", + ) + expect(toSessionError(llm(new QuotaExceededReason({ message: "quota" }))).type).toBe("provider.quota") + expect(toSessionError(llm(new ContentPolicyReason({ message: "blocked" }))).type).toBe("provider.content-filter") + expect(toSessionError(llm(new TransportReason({ message: "transport" }))).type).toBe("provider.transport") + expect(toSessionError(llm(new ProviderInternalReason({ message: "internal", status: 500 }))).type).toBe( + "provider.internal", + ) + expect(toSessionError(llm(new InvalidProviderOutputReason({ message: "output" }))).type).toBe( + "provider.invalid-output", + ) + expect(toSessionError(llm(new InvalidRequestReason({ message: "request" }))).type).toBe("provider.invalid-request") + expect( + toSessionError( + llm( + new NoRouteReason({ + route: "route", + provider: ProviderID.make("provider"), + model: ModelID.make("model"), + }), + ), + ).type, + ).toBe("provider.no-route") + expect(toSessionError(llm(new UnknownProviderReason({ message: "unknown" }))).type).toBe("provider.unknown") + }) + + test("preserves structured permission rejection data without inventing resources", () => { + const rejected = new PermissionV2.RejectedError({ permission: "external_directory", resources: [] }) + expect(toSessionError(rejected)).toEqual({ + type: "permission.rejected", + message: "Permission rejected: external_directory", + permission: "external_directory", + resources: [], + }) + expect(toSessionError(new ToolFailure({ message: rejected.message, error: rejected }))).toEqual({ + type: "permission.rejected", + message: "Permission rejected: external_directory", + permission: "external_directory", + resources: [], + }) + }) + + test("retries only rate limits, provider-internal failures, and transport failures", () => { + const eligible = [ + llm(new RateLimitReason({ message: "rate" })), + llm(new ProviderInternalReason({ message: "internal", status: 500 })), + llm(new TransportReason({ message: "transport" })), + ] + const ineligible = [ + llm(new AuthenticationReason({ message: "auth", kind: "invalid" })), + llm(new QuotaExceededReason({ message: "quota" })), + llm(new ContentPolicyReason({ message: "blocked" })), + llm(new InvalidProviderOutputReason({ message: "output" })), + llm(new InvalidRequestReason({ message: "request" })), + llm(new NoRouteReason({ route: "route", provider: ProviderID.make("provider"), model: ModelID.make("model") })), + llm(new UnknownProviderReason({ message: "unknown" })), + ] + + expect(eligible.map(SessionRunnerRetry.isRetryable)).toEqual([true, true, true]) + expect(ineligible.map(SessionRunnerRetry.isRetryable)).toEqual([false, false, false, false, false, false, false]) + }) +}) diff --git a/packages/core/test/session-execution-local.test.ts b/packages/core/test/session-execution-local.test.ts new file mode 100644 index 0000000000..be1c4aebea --- /dev/null +++ b/packages/core/test/session-execution-local.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test" +import { LLMError, TransportReason } from "@opencode-ai/llm" +import { terminal } from "@opencode-ai/core/session/execution/local" +import { UserInterruptedError } from "@opencode-ai/core/session/error" +import { ToolOutputStore } from "@opencode-ai/core/tool-output-store" +import { Effect, Exit } from "effect" + +describe("SessionExecutionLocal lifecycle", () => { + test("classifies success and typed failure terminals", () => { + expect(terminal(Exit.succeed(undefined))).toEqual({ type: "succeeded" }) + expect( + terminal( + Exit.fail( + new LLMError({ + module: "test", + method: "stream", + reason: new TransportReason({ message: "Disconnected" }), + }), + ), + ), + ).toEqual({ type: "failed", error: { type: "provider.transport", message: "Disconnected" } }) + const storage = new ToolOutputStore.StorageError({ operation: "encode", cause: new Error("invalid output") }) + expect(terminal(Exit.fail(storage))).toEqual({ + type: "failed", + error: { type: "unknown", message: storage.message }, + }) + }) + + test("defaults owner-scope interruption to shutdown and preserves explicit reasons", () => { + const interrupted = Effect.runSyncExit(Effect.interrupt) + expect(terminal(interrupted)).toEqual({ type: "interrupted", reason: "shutdown" }) + expect(terminal(interrupted, "user")).toEqual({ type: "interrupted", reason: "user" }) + expect(terminal(interrupted, "superseded")).toEqual({ type: "interrupted", reason: "superseded" }) + expect(terminal(Exit.fail(new UserInterruptedError()))).toEqual({ type: "interrupted", reason: "user" }) + }) +}) diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index 08fbf64e2b..3cdebdd0c0 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -432,6 +432,73 @@ describe("SessionProjector", () => { }), ) + it.effect("projects retry state and clears it at the next step or execution terminal", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "test", + directory: "/project", + title: "test", + version: "test", + }) + .run() + .pipe(Effect.orDie) + const events = yield* EventV2.Service + const first = SessionMessage.ID.make("msg_retry_first") + const second = SessionMessage.ID.make("msg_retry_second") + yield* events.publish(SessionEvent.Step.Started, { sessionID, assistantMessageID: first, agent: "build", model }) + yield* events.publish(SessionEvent.RetryScheduled, { + sessionID, + assistantMessageID: first, + attempt: 2, + at: 2_000, + error: { type: "provider.transport", message: "Disconnected" }, + }) + + const decode = (row: typeof SessionMessageTable.$inferSelect) => + Schema.decodeUnknownSync(SessionMessage.Message)({ ...row.data, id: row.id, type: row.type }) + const firstRow = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, first)) + .get() + .pipe(Effect.orDie) + const projected = firstRow ?? (yield* Effect.die(new Error("Missing retry projection"))) + expect(decode(projected)).toMatchObject({ + retry: { attempt: 2, at: DateTime.makeUnsafe(2_000), error: { type: "provider.transport" } }, + }) + + yield* events.publish(SessionEvent.Step.Started, { sessionID, assistantMessageID: second, agent: "build", model }) + yield* events.publish(SessionEvent.RetryScheduled, { + sessionID, + assistantMessageID: second, + attempt: 3, + at: 6_000, + error: { type: "provider.internal", message: "Unavailable" }, + }) + yield* events.publish(SessionEvent.Execution.Interrupted, { sessionID, reason: "shutdown" }) + + const rows = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.session_id, sessionID)) + .orderBy(asc(SessionMessageTable.seq)) + .all() + .pipe(Effect.orDie) + expect(decode(rows[0])).not.toHaveProperty("retry") + expect(decode(rows[1])).not.toHaveProperty("retry") + }), + ) + it.effect("updates only the newest incomplete assistant projection", () => Effect.gen(function* () { const { db } = yield* Database.Service diff --git a/packages/core/test/session-run-coordinator.test.ts b/packages/core/test/session-run-coordinator.test.ts index ecc4e613ac..c566c3f33c 100644 --- a/packages/core/test/session-run-coordinator.test.ts +++ b/packages/core/test/session-run-coordinator.test.ts @@ -104,8 +104,10 @@ describe("SessionRunCoordinator", () => { Effect.gen(function* () { const failure = new Error("failed") const defect = new Error("defect") + const settled: Exit.Exit[] = [] const coordinator = yield* SessionRunCoordinator.make({ drain: (key: string) => (key === "failure" ? Effect.fail(failure) : Effect.die(defect)), + settled: (_key, exit) => Effect.sync(() => void settled.push(exit)), }) const failed = yield* coordinator.run("failure").pipe(Effect.exit) @@ -115,6 +117,25 @@ describe("SessionRunCoordinator", () => { const died = yield* coordinator.run("defect").pipe(Effect.exit) expect(Exit.isFailure(died) && Cause.hasDies(died.cause)).toBeTrue() expect(Array.from(yield* coordinator.active)).toEqual([]) + expect(settled).toHaveLength(2) + }), + ), + ) + + it.effect("preserves settlement hook defects while releasing ownership", () => + Effect.scoped( + Effect.gen(function* () { + const defect = new Error("terminal publication failed") + const coordinator = yield* SessionRunCoordinator.make({ + drain: () => Effect.void, + settled: () => Effect.die(defect), + }) + + const exit = yield* coordinator.run("session").pipe(Effect.exit) + + expect(Exit.isFailure(exit) && Cause.hasDies(exit.cause)).toBe(true) + if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBe(defect) + expect(yield* coordinator.active).toEqual(new Set()) }), ), ) @@ -209,8 +230,41 @@ describe("SessionRunCoordinator", () => { it.effect("does nothing when interrupted while idle", () => Effect.scoped( Effect.gen(function* () { - const coordinator = yield* SessionRunCoordinator.make({ drain: () => Effect.void }) - yield* coordinator.interrupt("session") + const reasons: Array = [] + const coordinator = yield* SessionRunCoordinator.make({ + drain: () => Effect.void, + settled: (_key, _exit, reason) => Effect.sync(() => void reasons.push(reason)), + }) + yield* coordinator.interrupt("session", "user") + yield* coordinator.run("session") + expect(reasons).toEqual([undefined]) + }), + ), + ) + + it.effect("does not attach a late interrupt reason after terminal settlement starts", () => + Effect.scoped( + Effect.gen(function* () { + const settling = yield* Deferred.make() + const release = yield* Deferred.make() + const reasons: Array = [] + const coordinator = yield* SessionRunCoordinator.make({ + drain: () => Effect.void, + settled: (_key, _exit, reason) => + Deferred.succeed(settling, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(Effect.sync(() => void reasons.push(reason))), + ), + }) + + const run = yield* coordinator.run("session").pipe(Effect.forkChild) + yield* Deferred.await(settling) + yield* coordinator.interrupt("session", "user") + yield* Deferred.succeed(release, undefined) + yield* Fiber.join(run) + yield* coordinator.run("session") + + expect(reasons).toEqual([undefined, undefined]) }), ), ) @@ -221,25 +275,28 @@ describe("SessionRunCoordinator", () => { const started = yield* Deferred.make() const interrupted = yield* Deferred.make() let runs = 0 - const coordinator = yield* SessionRunCoordinator.make({ + const reasons: Array = [] + const coordinator = yield* SessionRunCoordinator.make({ drain: () => Effect.sync(() => ++runs).pipe( Effect.andThen(Deferred.succeed(started, undefined)), Effect.andThen(Effect.never), Effect.onInterrupt(() => Deferred.succeed(interrupted, undefined)), ), + settled: (_key, _exit, reason) => Effect.sync(() => void reasons.push(reason)), }) const resumed = yield* coordinator.run("session").pipe(Effect.forkChild) yield* Deferred.await(started) yield* coordinator.wake("session") - yield* coordinator.interrupt("session") + yield* coordinator.interrupt("session", "user") yield* Deferred.await(interrupted) const exit = yield* Fiber.await(resumed) expect(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)).toBeTrue() expect(Array.from(yield* coordinator.active)).toEqual([]) expect(runs).toBe(1) + expect(reasons).toEqual(["user"]) }), ), ) @@ -252,6 +309,7 @@ describe("SessionRunCoordinator", () => { const cleanupGate = yield* Deferred.make() const secondStarted = yield* Deferred.make() let runs = 0 + let starts = 0 const coordinator = yield* SessionRunCoordinator.make({ drain: () => Effect.sync(() => ++runs).pipe( @@ -266,6 +324,7 @@ describe("SessionRunCoordinator", () => { : Deferred.succeed(secondStarted, undefined), ), ), + started: () => Effect.sync(() => starts++).pipe(Effect.asVoid), }) yield* coordinator.wake("session") @@ -278,6 +337,7 @@ describe("SessionRunCoordinator", () => { yield* Deferred.await(secondStarted) expect(runs).toBe(2) + expect(starts).toBe(2) }), ), ) @@ -399,6 +459,7 @@ describe("SessionRunCoordinator", () => { const gate = yield* Deferred.make() const idle = yield* Deferred.make() let drains = 0 + let starts = 0 const settled: Exit.Exit[] = [] const coordinator = yield* SessionRunCoordinator.make({ drain: () => @@ -410,6 +471,7 @@ describe("SessionRunCoordinator", () => { ), Effect.asVoid, ), + started: () => Effect.sync(() => starts++).pipe(Effect.asVoid), settled: (_key, exit) => Effect.sync(() => void settled.push(exit)).pipe( Effect.andThen(Deferred.succeed(idle, undefined)), @@ -424,6 +486,7 @@ describe("SessionRunCoordinator", () => { yield* Deferred.await(idle) expect(drains).toBe(2) + expect(starts).toBe(1) expect(settled).toHaveLength(1) expect(Exit.isSuccess(settled[0]!)).toBe(true) }), diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index ac14921734..e69379bb93 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -149,3 +149,37 @@ test("step finish records settlement without publishing step ended", async () => expect(published.some((event) => event.type === "step.ended.2")).toBe(false) expect(publisher.stepSettlement()).toMatchObject({ finish: "stop" }) }) + +test("content-filter finish fails a contentless step", async () => { + const { published, publisher } = capture() + await Effect.runPromise(publisher.publish(LLMEvent.stepStart({ index: 0 }))) + await Effect.runPromise(publisher.publish(LLMEvent.stepFinish({ index: 0, reason: "content-filter" }))) + + expect(published.map((event) => event.type)).toEqual(["session.step.started.1", "session.step.failed.1"]) + expect(published.at(-1)?.data).toMatchObject({ + error: { type: "provider.content-filter", message: "Provider blocked the response" }, + }) + expect(publisher.stepSettlement()).toBeUndefined() +}) + +test("content-filter finish preserves partial streamed text and never ends the step successfully", async () => { + const { published, publisher } = capture() + await Effect.runPromise( + Effect.forEach( + [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "text" }), + LLMEvent.textDelta({ id: "text", text: "Partial" }), + LLMEvent.stepFinish({ index: 0, reason: "content-filter" }), + ], + (event) => publisher.publish(event), + { discard: true }, + ), + ) + + expect(published.some((event) => event.type === "session.step.ended.1")).toBe(false) + expect(published.find((event) => event.type === "session.text.ended.1")?.data).toMatchObject({ text: "Partial" }) + expect(published.find((event) => event.type === "session.step.failed.1")?.data).toMatchObject({ + error: { type: "provider.content-filter" }, + }) +}) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index c9320730a7..7af006bb37 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -4,8 +4,10 @@ import { LLMError, LLMEvent, Model, + ToolFailure, TransportReason, InvalidRequestReason, + RateLimitReason, type LLMClientShape, type LLMRequest, } from "@opencode-ai/llm" @@ -26,6 +28,7 @@ import { AbsolutePath } from "@opencode-ai/core/schema" import { SessionV2 } from "@opencode-ai/core/session" import { Snapshot } from "@opencode-ai/core/snapshot" import { SessionEvent } from "@opencode-ai/core/session/event" +import { UserInterruptedError } from "@opencode-ai/core/session/error" import { SessionCompaction } from "@opencode-ai/core/session/compaction" import { SessionTitle } from "@opencode-ai/core/session/title" import { SessionInput } from "@opencode-ai/core/session/input" @@ -61,7 +64,8 @@ import { McpGuidance } from "@opencode-ai/core/mcp/guidance" import { ModelV2 } from "@opencode-ai/core/model" import { Location } from "@opencode-ai/core/location" import { ProviderV2 } from "@opencode-ai/core/provider" -import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" +import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Option, Schema, Stream } from "effect" +import { TestClock } from "effect/testing" import { asc, eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -368,6 +372,20 @@ const providerUnavailable = () => reason: new TransportReason({ message: "Provider unavailable" }), }) +const invalidRequest = () => + new LLMError({ + module: "test", + method: "stream", + reason: new InvalidRequestReason({ message: "Invalid request" }), + }) + +const rateLimited = (retryAfterMs?: number) => + new LLMError({ + module: "test", + method: "stream", + reason: new RateLimitReason({ message: "Rate limited", retryAfterMs }), + }) + const setupOverflowRecovery = Effect.gen(function* () { yield* setup const session = yield* SessionV2.Service @@ -510,6 +528,7 @@ const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string const verifyEphemeralDeltas = (kind: FragmentKind) => Effect.gen(function* () { yield* setup + requests.length = 0 const session = yield* SessionV2.Service const prompt = `Stream ${kind}` const chunks = Array.from({ length: 32 }, (_, index) => `${index},`) @@ -542,6 +561,7 @@ const verifyEphemeralDeltas = (kind: FragmentKind) => const verifyPartialFlushOnFailure = (kind: FragmentKind) => Effect.gen(function* () { yield* setup + requests.length = 0 const session = yield* SessionV2.Service const prompt = `Fail after ${kind}` const fixture = fragmentFixture(kind, fragmentID(kind, "partial"), ["Partial"]) @@ -555,10 +575,11 @@ const verifyPartialFlushOnFailure = (kind: FragmentKind) => { type: "assistant", finish: "error", - error: { type: "unknown", message: "Provider unavailable" }, + error: { type: "provider.transport", message: "Provider unavailable" }, content: [fixture.expectedContent], }, ]) + expect(requests).toHaveLength(1) }) const verifyPartialFlushOnInterruption = (kind: FragmentKind) => @@ -583,7 +604,7 @@ const verifyPartialFlushOnInterruption = (kind: FragmentKind) => { type: "assistant", finish: "error", - error: { type: "unknown", message: "Step interrupted" }, + error: { type: "aborted", message: "Step interrupted" }, content: [ kind === "tool input" ? { type: "tool", id: fragmentID(kind, "interrupted"), state: { status: "error" } } @@ -1369,7 +1390,7 @@ describe("SessionRunnerLLM", () => { overflow(), ] yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Continue" }), resume: false }) - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long") expect(requests).toHaveLength(3) expect(yield* session.context(sessionID)).toMatchObject([ @@ -1415,7 +1436,7 @@ describe("SessionRunnerLLM", () => { [LLMEvent.providerError({ message: "summary unavailable" })], ] yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Continue" }), resume: false }) - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long") expect(requests).toHaveLength(2) const context = yield* session.context(sessionID) @@ -1560,7 +1581,7 @@ describe("SessionRunnerLLM", () => { state: { status: "error", input: { path: "README.md" }, - error: { type: "unknown", message: "Denied" }, + error: { type: "tool.execution", message: "Denied" }, }, }, { @@ -2363,7 +2384,7 @@ describe("SessionRunnerLLM", () => { requests.length = 0 responses = undefined response = [] - streamFailure = providerUnavailable() + streamFailure = invalidRequest() streamGate = yield* Deferred.make() streamStarted = yield* Deferred.make() @@ -2430,7 +2451,10 @@ describe("SessionRunnerLLM", () => { { type: "tool", id: "call-interrupted", - state: { status: "error", error: { type: "unknown", message: "Tool execution interrupted" } }, + state: { + status: "error", + error: { type: "tool.stale", message: "Tool execution interrupted", name: "echo" }, + }, }, ], }, @@ -2673,7 +2697,7 @@ describe("SessionRunnerLLM", () => { requests.length = 0 responses = undefined response = [] - streamFailure = providerUnavailable() + streamFailure = invalidRequest() streamGate = yield* Deferred.make() streamStarted = yield* Deferred.make() @@ -2779,7 +2803,7 @@ describe("SessionRunnerLLM", () => { id: "call-defect", state: { status: "error", - error: { type: "unknown", message: "Tool execution failed: unexpected tool defect" }, + error: { type: "unknown", message: "unexpected tool defect" }, }, }, ], @@ -2822,13 +2846,80 @@ describe("SessionRunnerLLM", () => { status: "error", error: { type: "unknown", - message: expect.stringContaining("Tool execution failed: Failed to encode tool output"), + message: expect.stringContaining("Failed to encode tool output"), + }, + }, + }, + ], + finish: "error", + error: { type: "unknown", message: expect.stringContaining("Failed to encode tool output") }, + }, + ]) + }), + ) + + it.effect("preserves permission rejection and stops before continuation", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + const registry = yield* ToolRegistry.Service + yield* registry.register({ + permissionfail: Tool.make({ + description: "Reject a permission", + input: Schema.Struct({}), + output: Schema.Struct({}), + execute: () => + new ToolFailure({ + message: "Permission rejected: edit", + error: new PermissionV2.RejectedError({ permission: "edit", resources: ["src/index.ts"] }), + }), + }), + }) + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Reject permission" }), resume: false }) + requests.length = 0 + responses = [ + [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolCall({ id: "call-permission", name: "permissionfail", input: {} }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ], + [LLMEvent.stepStart({ index: 0 }), LLMEvent.stepFinish({ index: 0, reason: "stop" })], + ] + + const exit = yield* session.resume(sessionID).pipe(Effect.exit) + + expect(exit._tag).toBe("Failure") + expect(requests).toHaveLength(1) + expect(yield* session.context(sessionID)).toMatchObject([ + { type: "user" }, + { + type: "assistant", + finish: "error", + error: { + type: "permission.rejected", + message: "Permission rejected: edit", + permission: "edit", + resources: ["src/index.ts"], + }, + content: [ + { + type: "tool", + id: "call-permission", + state: { + status: "error", + error: { + type: "permission.rejected", + message: "Permission rejected: edit", + permission: "edit", + resources: ["src/index.ts"], }, }, }, ], }, ]) + expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.ended.1") }), ) @@ -2870,7 +2961,8 @@ describe("SessionRunnerLLM", () => { const exit = yield* Fiber.join(run) expect(exit._tag).toBe("Failure") - if (exit._tag === "Failure") expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true) + if (exit._tag === "Failure") + expect(Option.getOrUndefined(Cause.findErrorOption(exit.cause))).toBeInstanceOf(UserInterruptedError) expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Ask then stop" }, @@ -2880,7 +2972,7 @@ describe("SessionRunnerLLM", () => { { type: "tool", id: "call-question", - state: { status: "error", error: { type: "unknown", message: "Tool execution interrupted" } }, + state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, }, ], }, @@ -2952,7 +3044,7 @@ describe("SessionRunnerLLM", () => { { type: "tool", id: "call-before-interrupt", - state: { status: "error", error: { type: "unknown", message: "Tool execution interrupted" } }, + state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, }, ], }, @@ -2993,7 +3085,7 @@ describe("SessionRunnerLLM", () => { expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Interrupt provider" }, - { type: "assistant", finish: "error", error: { type: "unknown", message: "Step interrupted" } }, + { type: "assistant", finish: "error", error: { type: "aborted", message: "Step interrupted" } }, ]) expect(yield* recordedEventTypes(sessionID)).toContain("session.step.failed.1") yield* session.interrupt(sessionID) @@ -3028,12 +3120,12 @@ describe("SessionRunnerLLM", () => { { type: "assistant", finish: "error", - error: { type: "unknown", message: "Step interrupted" }, + error: { type: "aborted", message: "Step interrupted" }, content: [ { type: "tool", id: "call-await-interrupt", - state: { status: "error", error: { type: "unknown", message: "Tool execution interrupted" } }, + state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, }, ], }, @@ -3156,12 +3248,12 @@ describe("SessionRunnerLLM", () => { streamStarted = undefined response = [LLMEvent.stepStart({ index: 0 }), LLMEvent.providerError({ message: "Provider unavailable" })] - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable") expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Fail durably" }, - { type: "assistant", finish: "error", error: { type: "unknown", message: "Provider unavailable" } }, + { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } }, ]) }), ) @@ -3175,16 +3267,43 @@ describe("SessionRunnerLLM", () => { requests.length = 0 response = [LLMEvent.providerError({ message: "Provider unavailable" })] - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable") expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Fail before step" }, - { type: "assistant", finish: "error", error: { type: "unknown", message: "Provider unavailable" } }, + { type: "assistant", finish: "error", error: { type: "provider.unknown", message: "Provider unavailable" } }, ]) }), ) + it.effect("projects content-filter finishes as visible terminal failures", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Blocked response" }), resume: false }) + response = [ + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "partial" }), + LLMEvent.textDelta({ id: "partial", text: "Partial" }), + LLMEvent.stepFinish({ index: 0, reason: "content-filter" }), + LLMEvent.finish({ reason: "content-filter" }), + ] + + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider blocked the response") + expect(yield* session.context(sessionID)).toMatchObject([ + { type: "user" }, + { + type: "assistant", + finish: "error", + error: { type: "provider.content-filter" }, + content: [{ type: "text", text: "Partial" }], + }, + ]) + expect(yield* recordedEventTypes(sessionID)).not.toContain("session.step.ended.1") + }), + ) + it.effect("does not recover context overflow after durable assistant output", () => Effect.gen(function* () { yield* setup @@ -3199,7 +3318,7 @@ describe("SessionRunnerLLM", () => { LLMEvent.textEnd({ id: "text-partial" }), LLMEvent.providerError({ message: "prompt too long", classification: "context-overflow" }), ] - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("prompt too long") expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ @@ -3219,18 +3338,146 @@ describe("SessionRunnerLLM", () => { yield* setup const session = yield* SessionV2.Service yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Fail raw stream durably" }), resume: false }) - const failure = providerUnavailable() + const failure = invalidRequest() responseStream = Stream.fail(failure) expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure) yield* replaySessionProjection(sessionID) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Fail raw stream durably" }, - { type: "assistant", finish: "error", error: { type: "unknown", message: "Provider unavailable" } }, + { type: "assistant", finish: "error", error: { type: "provider.invalid-request", message: "Invalid request" } }, ]) }), ) + it.effect("retries eligible pre-output failures after exponential backoff", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Retry transport" }), resume: false }) + requests.length = 0 + responseStream = Stream.fail(providerUnavailable()) + response = fragmentFixture("text", "retry-success", ["Recovered"]).completeEvents + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + while (requests.length < 1) yield* Effect.yieldNow + yield* TestClock.adjust("1999 millis") + expect(requests).toHaveLength(1) + yield* TestClock.adjust("1 millis") + yield* Fiber.join(run) + + expect(requests).toHaveLength(2) + const eventTypes = yield* recordedEventTypes(sessionID) + expect(eventTypes).toContain("session.retry.scheduled.1") + expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2) + expect(yield* session.context(sessionID)).toMatchObject([ + { type: "user" }, + { type: "assistant", finish: "stop", content: [{ type: "text", text: "Recovered" }] }, + ]) + yield* replaySessionProjection(sessionID) + expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1) + }), + ) + + it.effect("uses a larger provider retry-after delay", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Retry rate limit" }), resume: false }) + requests.length = 0 + responseStream = Stream.fail(rateLimited(5_000)) + response = fragmentFixture("text", "retry-after-success", ["Recovered"]).completeEvents + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + while (requests.length < 1) yield* Effect.yieldNow + yield* TestClock.adjust("4999 millis") + expect(requests).toHaveLength(1) + yield* TestClock.adjust("1 millis") + yield* Fiber.join(run) + expect(requests).toHaveLength(2) + }), + ) + + it.effect("stops after five total retry attempts", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Exhaust retries" }), resume: false }) + requests.length = 0 + streamFailure = providerUnavailable() + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + while (requests.length < 1) yield* Effect.yieldNow + for (const [index, delay] of [2_000, 4_000, 8_000, 16_000].entries()) { + yield* TestClock.adjust(delay) + while (requests.length < index + 2) yield* Effect.yieldNow + } + expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(streamFailure) + expect(requests).toHaveLength(5) + + const database = (yield* Database.Service).db + const retries = yield* database + .select({ data: EventTable.data }) + .from(EventTable) + .where(eq(EventTable.type, "session.retry.scheduled.1")) + .orderBy(asc(EventTable.seq)) + .all() + .pipe(Effect.orDie) + expect(retries.map((event) => event.data)).toMatchObject([ + { attempt: 2, at: 2_000 }, + { attempt: 3, at: 6_000 }, + { attempt: 4, at: 14_000 }, + { attempt: 5, at: 30_000 }, + ]) + expect((yield* recordedEventTypes(sessionID)).filter((type) => type === "session.step.started.1")).toHaveLength(5) + expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1) + }), + ) + + it.effect("counts retry attempts against the agent step allowance", () => + Effect.gen(function* () { + yield* setup + const agents = yield* AgentV2.Service + yield* agents.transform((editor) => + editor.update(AgentV2.ID.make("build"), (agent) => { + agent.steps = 2 + }), + ) + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Bound retries by steps" }), resume: false }) + requests.length = 0 + const failure = providerUnavailable() + responseStream = Stream.fail(failure) + streamFailure = failure + + const run = yield* session.resume(sessionID).pipe(Effect.forkChild) + while (requests.length < 1) yield* Effect.yieldNow + yield* TestClock.adjust("2 seconds") + expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure) + + expect(requests).toHaveLength(2) + const eventTypes = yield* recordedEventTypes(sessionID) + expect(eventTypes.filter((type) => type === "session.step.started.1")).toHaveLength(2) + expect(eventTypes.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(1) + expect((yield* session.context(sessionID)).filter((message) => message.type === "assistant")).toHaveLength(1) + }), + ) + + it.effect("does not retry non-eligible provider failures", () => + Effect.gen(function* () { + yield* setup + const session = yield* SessionV2.Service + yield* session.prompt({ sessionID, prompt: Prompt.make({ text: "Do not retry" }), resume: false }) + requests.length = 0 + const failure = invalidRequest() + streamFailure = failure + + expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure) + expect(requests).toHaveLength(1) + expect(yield* recordedEventTypes(sessionID)).not.toContain("session.retry.scheduled.1") + }), + ) + it.effect("does not continue automatically after a provider error follows a local tool call", () => Effect.gen(function* () { yield* setup @@ -3249,7 +3496,7 @@ describe("SessionRunnerLLM", () => { LLMEvent.providerError({ message: "Provider unavailable" }), ] - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable") expect(requests).toHaveLength(1) expect(executions.slice(executionCount)).toEqual(["settled"]) @@ -3274,7 +3521,7 @@ describe("SessionRunnerLLM", () => { LLMEvent.providerError({ message: "Provider unavailable" }), ] - yield* session.resume(sessionID) + expect((yield* session.resume(sessionID).pipe(Effect.flip)).message).toBe("Provider unavailable") expect(requests).toHaveLength(1) expect(yield* session.context(sessionID)).toMatchObject([ @@ -3321,6 +3568,7 @@ describe("SessionRunnerLLM", () => { prompt: Prompt.make({ text: "Fail hosted tool on raw failure" }), resume: false, }) + requests.length = 0 const failure = providerUnavailable() responseStream = Stream.concat( Stream.fromIterable([ @@ -3336,13 +3584,14 @@ describe("SessionRunnerLLM", () => { ) expect(yield* session.resume(sessionID).pipe(Effect.flip)).toBe(failure) + expect(requests).toHaveLength(1) yield* replaySessionProjection(sessionID) expect(yield* session.context(sessionID)).toMatchObject([ { type: "user", text: "Fail hosted tool on raw failure" }, { type: "assistant", finish: "error", - error: { type: "unknown", message: "Provider unavailable" }, + error: { type: "provider.transport", message: "Provider unavailable" }, content: [{ type: "tool", id: "call-hosted-raw-failure", state: { status: "error" } }], }, ]) diff --git a/packages/core/test/shared-schema.test.ts b/packages/core/test/shared-schema.test.ts index eb45e49106..51bdad9887 100644 --- a/packages/core/test/shared-schema.test.ts +++ b/packages/core/test/shared-schema.test.ts @@ -107,6 +107,7 @@ test("Core reuses the canonical shared schemas", async () => { [coreIntegration.Ref, Integration.Ref], [coreLocation.Ref, Location.Ref], [coreLLM.ProviderMetadata, LLM.ProviderMetadata], + [coreLLM.FinishReason, LLM.FinishReason], [coreLLM.ToolTextContent, LLM.ToolTextContent], [coreLLM.ToolFileContent, LLM.ToolFileContent], [coreLLM.ToolContent, LLM.ToolContent], @@ -147,7 +148,7 @@ test("Core reuses the canonical shared schemas", async () => { [coreSessionInput.Delivery, SessionInput.Delivery], [coreSessionInput.Admitted, SessionInput.Admitted], [coreSessionMessage.ID, SessionMessage.ID], - [coreSessionMessage.UnknownError, SessionMessage.UnknownError], + [coreSessionMessage.AssistantRetry, SessionMessage.AssistantRetry], [coreSessionMessage.AgentSelected, SessionMessage.AgentSelected], [coreSessionMessage.ModelSelected, SessionMessage.ModelSelected], [coreSessionMessage.User, SessionMessage.User], diff --git a/packages/llm/src/schema/ids.ts b/packages/llm/src/schema/ids.ts index 279a3097e1..4775caf2ef 100644 --- a/packages/llm/src/schema/ids.ts +++ b/packages/llm/src/schema/ids.ts @@ -1,5 +1,5 @@ import { Schema } from "effect" -import { ProviderMetadata } from "@opencode-ai/schema/llm" +import { LLM, ProviderMetadata } from "@opencode-ai/schema/llm" export { ProviderMetadata } @@ -36,7 +36,7 @@ export type TextVerbosity = Schema.Schema.Type export const MessageRole = Schema.Literals(["system", "user", "assistant", "tool"]) export type MessageRole = Schema.Schema.Type -export const FinishReason = Schema.Literals(["stop", "length", "tool-calls", "content-filter", "error", "unknown"]) +export const FinishReason = LLM.FinishReason export type FinishReason = Schema.Schema.Type export const JsonSchema = Schema.Record(Schema.String, Schema.Unknown) diff --git a/packages/opencode/src/cli/cmd/run/noninteractive.ts b/packages/opencode/src/cli/cmd/run/noninteractive.ts index e0b0356b7b..3df7788e14 100644 --- a/packages/opencode/src/cli/cmd/run/noninteractive.ts +++ b/packages/opencode/src/cli/cmd/run/noninteractive.ts @@ -165,8 +165,8 @@ export async function runNonInteractivePrompt(input: Input) { } } if ( - event.type === "session.execution.settled" && - event.data.outcome === "interrupted" && + event.type === "session.execution.interrupted" && + event.data.reason === "user" && (interrupted || permissionRejected || questionRejected || formCancelled) ) { return @@ -351,16 +351,25 @@ export async function runNonInteractivePrompt(input: Input) { if (!emit("error", time, { error: event.data.error })) UI.error(event.data.error.message) continue } - if (event.type === "session.execution.settled") { - if (event.data.outcome === "failure" && !emittedError && !questionRejected && !formCancelled) { + if (event.type === "session.execution.failed") { + if (!emittedError && !questionRejected && !formCancelled) { emittedError = true process.exitCode = 1 - const error = event.data.error ?? { type: "unknown", message: "Session execution failed" } - if (!emit("error", time, { error })) UI.error(error.message) + if (!emit("error", time, { error: event.data.error })) UI.error(event.data.error.message) } - if (event.data.outcome === "interrupted" && interrupted) process.exitCode = 130 return } + if (event.type === "session.execution.interrupted") { + if (event.data.reason === "user" && interrupted) process.exitCode = 130 + if (event.data.reason !== "user" && !emittedError) { + emittedError = true + process.exitCode = 1 + const error = { type: "aborted" as const, message: `Session interrupted: ${event.data.reason}` } + if (!emit("error", time, { error })) UI.error(error.message) + } + return + } + if (event.type === "session.execution.succeeded") return } } diff --git a/packages/opencode/src/cli/cmd/run/stream-v2.subagent.ts b/packages/opencode/src/cli/cmd/run/stream-v2.subagent.ts index 31ee32c083..d5cd520459 100644 --- a/packages/opencode/src/cli/cmd/run/stream-v2.subagent.ts +++ b/packages/opencode/src/cli/cmd/run/stream-v2.subagent.ts @@ -645,9 +645,23 @@ export function createSubagentTracker(input: SubagentTrackerInput): SubagentTrac notifyDetail(child) return } - if (event.type === "session.execution.settled") { + if (event.type === "session.execution.started") { + child.status = "running" + touch(child, event.created) + input.emit() + return + } + if ( + event.type === "session.execution.succeeded" || + event.type === "session.execution.failed" || + event.type === "session.execution.interrupted" + ) { child.status = - event.data.outcome === "success" ? "completed" : event.data.outcome === "interrupted" ? "cancelled" : "error" + event.type === "session.execution.succeeded" + ? "completed" + : event.type === "session.execution.interrupted" + ? "cancelled" + : "error" touch(child, event.created) input.emit() } diff --git a/packages/opencode/src/cli/cmd/run/stream-v2.transport.ts b/packages/opencode/src/cli/cmd/run/stream-v2.transport.ts index 9e6cff8117..94e7f22896 100644 --- a/packages/opencode/src/cli/cmd/run/stream-v2.transport.ts +++ b/packages/opencode/src/cli/cmd/run/stream-v2.transport.ts @@ -785,7 +785,6 @@ export async function createSessionTransport(input: StreamInput): Promise 0 ? total.toLocaleString() : "" write([], { - phase: event.data.finish === "tool-calls" ? "running" : "idle", usage: event.data.cost ? `${usage} · ${money.format(event.data.cost)}` : usage, }) return @@ -798,21 +797,33 @@ export async function createSessionTransport(input: StreamInput): Promise, event: unknown, thin }) } +function shell(status: "running" | "exited" = "running") { + return { + id: "sh_1", + status, + command: "pwd", + cwd: "/tmp", + shell: "/bin/sh", + file: "/tmp/shell-output", + metadata: {}, + time: { started: 1, completed: status === "exited" ? 2 : undefined }, + } +} + function assistant(id: string, extra: Record = {}) { return { type: "message.updated", @@ -332,9 +345,7 @@ describe("run session data", () => { type: "session.shell.started", properties: { sessionID: "session-1", - timestamp: 1, - callID: "call-1", - command: "pwd", + shell: shell(), }, }) @@ -342,10 +353,10 @@ describe("run session data", () => { expect.objectContaining({ kind: "tool", phase: "start", - partID: "shell:call-1", + partID: "shell:sh_1", tool: "bash", shell: { - callID: "call-1", + callID: "sh_1", command: "pwd", }, }), @@ -356,9 +367,8 @@ describe("run session data", () => { type: "session.shell.ended", properties: { sessionID: "session-1", - timestamp: 2, - callID: "call-1", - output: "/tmp/demo\n", + shell: shell("exited"), + output: { output: "/tmp/demo\n", cursor: 10, size: 10, truncated: false }, }, }) @@ -366,12 +376,12 @@ describe("run session data", () => { expect.objectContaining({ kind: "tool", phase: "progress", - partID: "shell:call-1", + partID: "shell:sh_1", tool: "bash", text: "/tmp/demo\n", toolState: "completed", shell: { - callID: "call-1", + callID: "sh_1", command: "pwd", }, }), @@ -383,9 +393,7 @@ describe("run session data", () => { type: "session.shell.started", properties: { sessionID: "session-1", - timestamp: 1, - callID: "call-1", - command: "pwd", + shell: shell(), }, }).data @@ -395,7 +403,7 @@ describe("run session data", () => { tool({ id: "tool-1", messageID: "msg-1", - callID: "call-1", + callID: "sh_1", tool: "bash", state: { status: "running", @@ -412,9 +420,8 @@ describe("run session data", () => { type: "session.shell.ended", properties: { sessionID: "session-1", - timestamp: 2, - callID: "call-1", - output: "/tmp/demo\n", + shell: shell("exited"), + output: { output: "/tmp/demo\n", cursor: 10, size: 10, truncated: false }, }, }).data @@ -424,7 +431,7 @@ describe("run session data", () => { tool({ id: "tool-1", messageID: "msg-1", - callID: "call-1", + callID: "sh_1", tool: "bash", state: { status: "completed", @@ -449,7 +456,7 @@ describe("run session data", () => { tool({ id: "tool-1", messageID: "msg-1", - callID: "call-1", + callID: "sh_1", tool: "bash", state: { status: "running", @@ -466,9 +473,7 @@ describe("run session data", () => { type: "session.shell.started", properties: { sessionID: "session-1", - timestamp: 1, - callID: "call-1", - command: "pwd", + shell: shell(), }, }).commits, ).toEqual([]) @@ -478,7 +483,7 @@ describe("run session data", () => { tool({ id: "tool-1", messageID: "msg-1", - callID: "call-1", + callID: "sh_1", tool: "bash", state: { status: "completed", @@ -500,9 +505,8 @@ describe("run session data", () => { type: "session.shell.ended", properties: { sessionID: "session-1", - timestamp: 2, - callID: "call-1", - output: "/tmp/demo\n", + shell: shell("exited"), + output: { output: "/tmp/demo\n", cursor: 10, size: 10, truncated: false }, }, }).commits, ).toEqual([]) diff --git a/packages/opencode/test/cli/run/stream-v2.transport.test.ts b/packages/opencode/test/cli/run/stream-v2.transport.test.ts index 2f5bd68a25..d83c44661d 100644 --- a/packages/opencode/test/cli/run/stream-v2.transport.test.ts +++ b/packages/opencode/test/cli/run/stream-v2.transport.test.ts @@ -220,8 +220,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) await turn @@ -268,8 +269,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) }) return ok({ @@ -362,8 +364,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) }) return ok({ @@ -459,8 +462,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) }) return ok({ @@ -908,8 +912,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.interrupted", + durable: durable("ses_1"), + data: { sessionID: "ses_1", reason: "user" }, }) await turn @@ -974,8 +979,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) await turn @@ -1036,8 +1042,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.interrupted", + durable: durable("ses_1"), + data: { sessionID: "ses_1", reason: "user" }, }) await turn @@ -1262,8 +1269,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) }) return ok({ @@ -1339,8 +1347,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) }) return ok(undefined) as never @@ -1409,8 +1418,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_unrelated_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) await Bun.sleep(0) await Bun.sleep(0) @@ -1430,8 +1440,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_skill_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_1", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_1"), + data: { sessionID: "ses_1" }, }) await turn @@ -1561,8 +1572,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_child_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_child", outcome: "success" }, + type: "session.execution.succeeded", + durable: durable("ses_child"), + data: { sessionID: "ses_child" }, }) while (!states().some((state) => state.tabs.some((tab) => tab.status === "completed"))) await Bun.sleep(0) await transport.close() @@ -1618,8 +1630,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_child_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_child", outcome: "interrupted" }, + type: "session.execution.interrupted", + durable: durable("ses_child"), + data: { sessionID: "ses_child", reason: "user" }, }) await Bun.sleep(0) resolveGet?.() @@ -1718,8 +1731,9 @@ describe("V2 mini transport", () => { events.push({ id: "evt_child_settled", created: 0, - type: "session.execution.settled", - data: { sessionID: "ses_child", outcome: "interrupted" }, + type: "session.execution.interrupted", + durable: durable("ses_child"), + data: { sessionID: "ses_child", reason: "shutdown" }, }) while (!states().some((state) => state.tabs.some((tab) => tab.status === "cancelled"))) await Bun.sleep(0) diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index 1454fab1cf..8628edfaa3 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -20,6 +20,7 @@ export { Revert } from "./revert.js" export { Session } from "./session.js" export { Vcs } from "./vcs.js" export { SessionInput } from "./session-input.js" +export { SessionError } from "./session-error.js" export { SessionMessage } from "./session-message.js" export { Shell } from "./shell.js" export { Skill } from "./skill.js" diff --git a/packages/schema/src/llm.ts b/packages/schema/src/llm.ts index 5e19ecc6d3..2bc1d93bd8 100644 --- a/packages/schema/src/llm.ts +++ b/packages/schema/src/llm.ts @@ -8,6 +8,9 @@ export const ProviderMetadata = Schema.Record(Schema.String, Schema.Record(Schem }) export type ProviderMetadata = Schema.Schema.Type +export const FinishReason = Schema.Literals(["stop", "length", "tool-calls", "content-filter", "error", "unknown"]) +export type FinishReason = typeof FinishReason.Type + export interface ToolTextContent extends Schema.Schema.Type {} export const ToolTextContent = Schema.Struct({ type: Schema.Literal("text"), diff --git a/packages/schema/src/session-error.ts b/packages/schema/src/session-error.ts new file mode 100644 index 0000000000..0dc41eceba --- /dev/null +++ b/packages/schema/src/session-error.ts @@ -0,0 +1,72 @@ +export * as SessionError from "./session-error.js" + +import { Schema } from "effect" +import { optional } from "./schema.js" + +const Message = { message: Schema.String } + +const ProviderRateLimit = Schema.Struct({ + type: Schema.Literal("provider.rate-limit"), + ...Message, + retryAfterMs: Schema.Finite.pipe(optional), +}) +const ProviderAuth = Schema.Struct({ type: Schema.Literal("provider.auth"), ...Message }) +const ProviderQuota = Schema.Struct({ type: Schema.Literal("provider.quota"), ...Message }) +const ProviderContentFilter = Schema.Struct({ type: Schema.Literal("provider.content-filter"), ...Message }) +const ProviderTransport = Schema.Struct({ type: Schema.Literal("provider.transport"), ...Message }) +const ProviderInternal = Schema.Struct({ type: Schema.Literal("provider.internal"), ...Message }) +const ProviderInvalidOutput = Schema.Struct({ type: Schema.Literal("provider.invalid-output"), ...Message }) +const ProviderInvalidRequest = Schema.Struct({ type: Schema.Literal("provider.invalid-request"), ...Message }) +const ProviderNoRoute = Schema.Struct({ type: Schema.Literal("provider.no-route"), ...Message }) +const ProviderUnknown = Schema.Struct({ type: Schema.Literal("provider.unknown"), ...Message }) +const PermissionRejected = Schema.Struct({ + type: Schema.Literal("permission.rejected"), + ...Message, + permission: Schema.String, + resources: Schema.Array(Schema.String), +}) +const ToolUnknown = Schema.Struct({ type: Schema.Literal("tool.unknown"), ...Message, name: Schema.String }) +const ToolStale = Schema.Struct({ + type: Schema.Literal("tool.stale"), + ...Message, + name: Schema.String.pipe(optional), +}) +const ToolExecution = Schema.Struct({ type: Schema.Literal("tool.execution"), ...Message }) +const ToolResultMissing = Schema.Struct({ + type: Schema.Literal("tool.result-missing"), + ...Message, + callID: Schema.String.pipe(optional), +}) +const Aborted = Schema.Struct({ + type: Schema.Literal("aborted"), + ...Message, + reason: Schema.Literals(["user", "shutdown", "timeout"]).pipe(optional), +}) +const Unknown = Schema.Struct({ + type: Schema.Literal("unknown"), + ...Message, + agent: Schema.String.pipe(optional), +}) + +export const Error = Schema.Union([ + ProviderRateLimit, + ProviderAuth, + ProviderQuota, + ProviderContentFilter, + ProviderTransport, + ProviderInternal, + ProviderInvalidOutput, + ProviderInvalidRequest, + ProviderNoRoute, + ProviderUnknown, + PermissionRejected, + ToolUnknown, + ToolStale, + ToolExecution, + ToolResultMissing, + Aborted, + Unknown, +]) + .pipe(Schema.toTaggedUnion("type")) + .annotate({ identifier: "Session.StructuredError" }) +export type Error = typeof Error.Type diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index d14f8b7a8d..485f57e7e7 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -4,15 +4,17 @@ import { Schema } from "effect" import { optional } from "./schema.js" import { Event } from "./event.js" import { ToolContent } from "./llm.js" +import { FinishReason } from "./llm.js" import { Delivery } from "./session-delivery.js" import { Model } from "./model.js" -import { NonNegativeInt, RelativePath } from "./schema.js" +import { NonNegativeInt, PositiveInt, RelativePath } from "./schema.js" import { FileAttachment, Prompt } from "./prompt.js" import { SessionID } from "./session-id.js" import { Location } from "./location.js" import { SessionMessage } from "./session-message.js" import { Revert } from "./revert.js" import { Shell as ShellSchema } from "./shell.js" +import { SessionError } from "./session-error.js" export { FileAttachment } @@ -41,16 +43,6 @@ const options = { version: 1, }, } as const -const stepSettlementOptions = { - durable: { - aggregate: "sessionID", - version: 1, - }, -} as const - -export const UnknownError = SessionMessage.UnknownError -export type UnknownError = SessionMessage.UnknownError - export const AgentSelected = Event.durable({ type: "session.agent.selected", ...options, @@ -120,15 +112,27 @@ export const PromptAdmitted = Event.durable({ }) export type PromptAdmitted = typeof PromptAdmitted.Type -export const ExecutionSettled = Event.ephemeral({ - type: "session.execution.settled", - schema: { - ...Base, - outcome: Schema.Literals(["success", "failure", "interrupted"]), - error: UnknownError.pipe(optional), - }, -}) -export type ExecutionSettled = typeof ExecutionSettled.Type +export namespace Execution { + export const Started = Event.durable({ type: "session.execution.started", ...options, schema: Base }) + export type Started = typeof Started.Type + + export const Succeeded = Event.durable({ type: "session.execution.succeeded", ...options, schema: Base }) + export type Succeeded = typeof Succeeded.Type + + export const Failed = Event.durable({ + type: "session.execution.failed", + ...options, + schema: { ...Base, error: SessionError.Error }, + }) + export type Failed = typeof Failed.Type + + export const Interrupted = Event.durable({ + type: "session.execution.interrupted", + ...options, + schema: { ...Base, reason: Schema.Literals(["user", "shutdown", "superseded"]) }, + }) + export type Interrupted = typeof Interrupted.Type +} export const ContextUpdated = Event.durable({ type: "session.context.updated", @@ -204,11 +208,11 @@ export namespace Step { export const Ended = Event.durable({ type: "session.step.ended", - ...stepSettlementOptions, + ...options, schema: { ...Base, assistantMessageID: SessionMessage.ID, - finish: Schema.String, + finish: FinishReason, cost: Schema.Finite, tokens: Schema.Struct({ input: Schema.Finite, @@ -227,11 +231,11 @@ export namespace Step { export const Failed = Event.durable({ type: "session.step.failed", - ...stepSettlementOptions, + ...options, schema: { ...Base, assistantMessageID: SessionMessage.ID, - error: UnknownError, + error: SessionError.Error, }, }) export type Failed = typeof Failed.Type @@ -393,7 +397,7 @@ export namespace Tool { ...options, schema: { ...ToolBase, - error: UnknownError, + error: SessionError.Error, result: Schema.Unknown.pipe(optional), executed: Schema.Boolean, resultState: SessionMessage.ProviderState.pipe(optional), @@ -402,28 +406,18 @@ export namespace Tool { export type Failed = typeof Failed.Type } -export const RetryError = Schema.Struct({ - message: Schema.String, - statusCode: Schema.Finite.pipe(optional), - isRetryable: Schema.Boolean, - responseHeaders: Schema.Record(Schema.String, Schema.String).pipe(optional), - responseBody: Schema.String.pipe(optional), - metadata: Schema.Record(Schema.String, Schema.String).pipe(optional), -}).annotate({ - identifier: "session.retry.error", -}) -export interface RetryError extends Schema.Schema.Type {} - -export const Retried = Event.durable({ - type: "session.retried", +export const RetryScheduled = Event.durable({ + type: "session.retry.scheduled", ...options, schema: { ...Base, - attempt: Schema.Finite, - error: RetryError, + assistantMessageID: SessionMessage.ID, + attempt: PositiveInt, + at: NonNegativeInt, + error: SessionError.Error, }, }) -export type Retried = typeof Retried.Type +export type RetryScheduled = typeof RetryScheduled.Type export namespace Compaction { export const Started = Event.durable({ @@ -480,7 +474,10 @@ export const Definitions = Event.inventory( Forked, PromptPromoted, PromptAdmitted, - ExecutionSettled, + Execution.Started, + Execution.Succeeded, + Execution.Failed, + Execution.Interrupted, ContextUpdated, Synthetic, Skill.Activated, @@ -502,7 +499,7 @@ export const Definitions = Event.inventory( Tool.Progress, Tool.Success, Tool.Failed, - Retried, + RetryScheduled, Compaction.Started, Compaction.Delta, Compaction.Ended, diff --git a/packages/schema/src/session-message.ts b/packages/schema/src/session-message.ts index f6246007a8..aafef1dba2 100644 --- a/packages/schema/src/session-message.ts +++ b/packages/schema/src/session-message.ts @@ -5,11 +5,13 @@ import { optional } from "./schema.js" import { ToolContent } from "./llm.js" import { Model } from "./model.js" import { FileAttachment, Prompt } from "./prompt.js" -import { DateTimeUtcFromMillis, RelativePath, statics } from "./schema.js" +import { DateTimeUtcFromMillis, PositiveInt, RelativePath, statics } from "./schema.js" import { SessionID } from "./session-id.js" import { ascending } from "./identifier.js" import { Event } from "./event.js" import { Shell as ShellSchema } from "./shell.js" +import { FinishReason } from "./llm.js" +import { SessionError } from "./session-error.js" export const ID = Schema.String.check(Schema.isStartsWith("msg_")).pipe( Schema.brand("Session.Message.ID"), @@ -20,12 +22,6 @@ export const ID = Schema.String.check(Schema.isStartsWith("msg_")).pipe( ) export type ID = typeof ID.Type -export interface UnknownError extends Schema.Schema.Type {} -export const UnknownError = Schema.Struct({ - type: Schema.Literal("unknown"), - message: Schema.String, -}).annotate({ identifier: "Session.Error.Unknown" }) - const Base = { id: ID, metadata: Schema.Record(Schema.String, Schema.Unknown).pipe(optional), @@ -128,7 +124,7 @@ export const ToolStateError = Schema.Struct({ input: Schema.Record(Schema.String, Schema.Unknown), content: ToolContent.pipe(Schema.Array), structured: Schema.Record(Schema.String, Schema.Unknown), - error: UnknownError, + error: SessionError.Error, result: Schema.Unknown.pipe(optional), }).annotate({ identifier: "Session.Message.ToolState.Error" }) @@ -176,6 +172,13 @@ export const AssistantContent = Schema.Union([AssistantText, AssistantReasoning, ) export type AssistantContent = AssistantText | AssistantReasoning | AssistantTool +export interface AssistantRetry extends Schema.Schema.Type {} +export const AssistantRetry = Schema.Struct({ + attempt: PositiveInt, + at: DateTimeUtcFromMillis, + error: SessionError.Error, +}).annotate({ identifier: "Session.Message.Assistant.Retry" }) + export interface Assistant extends Schema.Schema.Type {} export const Assistant = Schema.Struct({ ...Base, @@ -188,7 +191,7 @@ export const Assistant = Schema.Struct({ end: Schema.String.pipe(optional), files: Schema.Array(RelativePath).pipe(optional), }).pipe(optional), - finish: Schema.String.pipe(optional), + finish: FinishReason.pipe(optional), cost: Schema.Finite.pipe(optional), tokens: Schema.Struct({ input: Schema.Finite, @@ -196,7 +199,8 @@ export const Assistant = Schema.Struct({ reasoning: Schema.Finite, cache: Schema.Struct({ read: Schema.Finite, write: Schema.Finite }), }).pipe(optional), - error: UnknownError.pipe(optional), + error: SessionError.Error.pipe(optional), + retry: AssistantRetry.pipe(optional), time: Schema.Struct({ created: DateTimeUtcFromMillis, completed: DateTimeUtcFromMillis.pipe(optional), diff --git a/packages/schema/test/event-manifest.test.ts b/packages/schema/test/event-manifest.test.ts index 82b695ca8c..451e44b26a 100644 --- a/packages/schema/test/event-manifest.test.ts +++ b/packages/schema/test/event-manifest.test.ts @@ -101,6 +101,10 @@ describe("public event manifest", () => { "session.forked.1", "session.prompt.promoted.1", "session.prompt.admitted.1", + "session.execution.started.1", + "session.execution.succeeded.1", + "session.execution.failed.1", + "session.execution.interrupted.1", "session.context.updated.1", "session.synthetic.1", "session.skill.activated.1", @@ -119,7 +123,7 @@ describe("public event manifest", () => { "session.tool.failed.1", "session.reasoning.started.1", "session.reasoning.ended.1", - "session.retried.1", + "session.retry.scheduled.1", "session.compaction.started.1", "session.compaction.ended.1", "session.revert.staged.1", diff --git a/packages/schema/test/session-error.test.ts b/packages/schema/test/session-error.test.ts new file mode 100644 index 0000000000..6a15ad3d85 --- /dev/null +++ b/packages/schema/test/session-error.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test" +import { Schema } from "effect" +import { LLM, SessionError } from "../src/index.js" + +describe("SessionError", () => { + test("exports one identified closed union", () => { + expect(SessionError.Error.ast.annotations?.identifier).toBe("Session.StructuredError") + expect(Object.keys(SessionError).filter((key) => key !== "SessionError")).toEqual(["Error"]) + }) + + test("round trips every closed error type through JSON", () => { + const values: SessionError.Error[] = [ + { type: "provider.rate-limit", message: "Slow down", retryAfterMs: 2_500 }, + { type: "provider.auth", message: "Authentication failed" }, + { type: "provider.quota", message: "Quota exhausted" }, + { type: "provider.content-filter", message: "Response blocked" }, + { type: "provider.transport", message: "Connection failed" }, + { type: "provider.internal", message: "Provider failed" }, + { type: "provider.invalid-output", message: "Malformed response" }, + { type: "provider.invalid-request", message: "Invalid request" }, + { type: "provider.no-route", message: "No route" }, + { type: "provider.unknown", message: "Unknown provider failure" }, + { type: "permission.rejected", message: "Permission rejected", permission: "read", resources: ["a"] }, + { type: "tool.unknown", message: "Unknown tool", name: "missing" }, + { type: "tool.stale", message: "Stale tool", name: "old" }, + { type: "tool.execution", message: "Tool failed" }, + { type: "tool.result-missing", message: "Missing result", callID: "call_1" }, + { type: "aborted", message: "Interrupted", reason: "user" }, + { type: "unknown", message: "Unexpected", agent: "build" }, + ] + const codec = Schema.fromJsonString(SessionError.Error) + + for (const value of values) { + const encoded = Schema.encodeSync(codec)(value) + expect(Schema.decodeUnknownSync(codec)(encoded)).toEqual(value) + } + }) + + test("rejects unknown types and missing messages", () => { + expect(() => + Schema.decodeUnknownSync(SessionError.Error)({ type: "provider.timeout", message: "Timeout" }), + ).toThrow() + expect(() => Schema.decodeUnknownSync(SessionError.Error)({ type: "provider.auth" })).toThrow() + }) +}) + +test("FinishReason is the closed browser-safe provider set", () => { + const reasons = ["stop", "length", "tool-calls", "content-filter", "error", "unknown"] as const + expect(reasons.map((reason) => Schema.decodeUnknownSync(LLM.FinishReason)(reason))).toEqual([...reasons]) + expect(() => Schema.decodeUnknownSync(LLM.FinishReason)("other")).toThrow() +}) diff --git a/packages/sdk/js/script/build.ts b/packages/sdk/js/script/build.ts index 220bf43da3..b7ee7d6135 100755 --- a/packages/sdk/js/script/build.ts +++ b/packages/sdk/js/script/build.ts @@ -60,7 +60,7 @@ if (schemas) { visit({ ...document, components: { ...document.components, schemas: undefined } }) for (const name of Object.keys(schemas)) { if ( - /^(SessionAgentSelected|SessionModelSelected|SessionMoved|SessionRenamed|SessionForked|SessionPromptPromoted|SessionPromptAdmitted|SessionExecutionSettled|SessionContextUpdated|SessionSynthetic|SessionSkillActivated|SessionShellStarted|SessionShellEnded|SessionStepStarted|SessionStepEnded|SessionStepFailed|SessionTextStarted|SessionTextDelta|SessionTextEnded|SessionReasoningStarted|SessionReasoningDelta|SessionReasoningEnded|SessionToolInputStarted|SessionToolInputDelta|SessionToolInputEnded|SessionToolCalled|SessionToolProgress|SessionToolSuccess|SessionToolFailed|SessionRetried|SessionCompactionStarted|SessionCompactionDelta|SessionCompactionEnded|SessionRevertStaged|SessionRevertCleared|SessionRevertCommitted)1$/.test( + /^(SessionAgentSelected|SessionModelSelected|SessionMoved|SessionRenamed|SessionForked|SessionPromptPromoted|SessionPromptAdmitted|SessionExecutionStarted|SessionExecutionSucceeded|SessionExecutionFailed|SessionExecutionInterrupted|SessionContextUpdated|SessionSynthetic|SessionSkillActivated|SessionShellStarted|SessionShellEnded|SessionStepStarted|SessionStepEnded|SessionStepFailed|SessionTextStarted|SessionTextDelta|SessionTextEnded|SessionReasoningStarted|SessionReasoningDelta|SessionReasoningEnded|SessionToolInputStarted|SessionToolInputDelta|SessionToolInputEnded|SessionToolCalled|SessionToolProgress|SessionToolSuccess|SessionToolFailed|SessionRetryScheduled|SessionCompactionStarted|SessionCompactionDelta|SessionCompactionEnded|SessionRevertStaged|SessionRevertCleared|SessionRevertCommitted)1$/.test( name, ) && !reachable.has(name) @@ -100,17 +100,23 @@ await createClient({ const generatedTypesPath = "./src/v2/gen/types.gen.ts" const generatedTypes = await Bun.file(generatedTypesPath).text() if ( - /export type (SessionAgentSelected|SessionModelSelected|SessionMoved|SessionRenamed|SessionForked|SessionPromptPromoted|SessionPromptAdmitted|SessionExecutionSettled|SessionContextUpdated|SessionSynthetic|SessionSkillActivated|SessionShellStarted|SessionShellEnded|SessionStepStarted|SessionStepEnded|SessionStepFailed|SessionTextStarted|SessionTextDelta|SessionTextEnded|SessionReasoningStarted|SessionReasoningDelta|SessionReasoningEnded|SessionToolInputStarted|SessionToolInputDelta|SessionToolInputEnded|SessionToolCalled|SessionToolProgress|SessionToolSuccess|SessionToolFailed|SessionRetried|SessionCompactionStarted|SessionCompactionDelta|SessionCompactionEnded|SessionRevertStaged|SessionRevertCleared|SessionRevertCommitted)1 =/.test( + /export type (SessionAgentSelected|SessionModelSelected|SessionMoved|SessionRenamed|SessionForked|SessionPromptPromoted|SessionPromptAdmitted|SessionExecutionStarted|SessionExecutionSucceeded|SessionExecutionFailed|SessionExecutionInterrupted|SessionContextUpdated|SessionSynthetic|SessionSkillActivated|SessionShellStarted|SessionShellEnded|SessionStepStarted|SessionStepEnded|SessionStepFailed|SessionTextStarted|SessionTextDelta|SessionTextEnded|SessionReasoningStarted|SessionReasoningDelta|SessionReasoningEnded|SessionToolInputStarted|SessionToolInputDelta|SessionToolInputEnded|SessionToolCalled|SessionToolProgress|SessionToolSuccess|SessionToolFailed|SessionRetryScheduled|SessionCompactionStarted|SessionCompactionDelta|SessionCompactionEnded|SessionRevertStaged|SessionRevertCleared|SessionRevertCommitted)1 =/.test( generatedTypes, ) ) { throw new Error("Session history generated duplicate Session event variants") } -const logTypesPatched = generatedTypes.replace( - /(export type V2SessionLogData = \{[\s\S]*?query\?: \{\s*after\?: )string/, - "$1number", -) -if (logTypesPatched === generatedTypes) { +const duplicateSessionErrorStart = generatedTypes.indexOf("export type SessionStructuredError2 =") +const duplicateSessionErrorEnd = generatedTypes.indexOf("\n\nexport type ", duplicateSessionErrorStart + 1) +if (duplicateSessionErrorStart === -1 || duplicateSessionErrorEnd === -1) { + throw new Error("Session structured error duplicate prune did not apply") +} +const sessionErrorTypesPatched = + generatedTypes.slice(0, duplicateSessionErrorStart) + generatedTypes.slice(duplicateSessionErrorEnd + 2) +const logTypesPatched = sessionErrorTypesPatched + .replaceAll("SessionStructuredError2", "SessionStructuredError") + .replace(/(export type V2SessionLogData = \{[\s\S]*?query\?: \{\s*after\?: )string/, "$1number") +if (logTypesPatched === sessionErrorTypesPatched) { throw new Error("Session log numeric query patch did not apply") } const sessionListTypesPatched = logTypesPatched.replace( @@ -128,12 +134,15 @@ if (sessionMessagesTypesPatched === sessionListTypesPatched) { throw new Error("Session messages numeric query patch did not apply") } const eventSubscribeTypesPatched = sessionMessagesTypesPatched.replace( - /(export type V2EventSubscribeResponses = \{\s*\/\*\*[\s\S]*?\*\/\s*200: )\{\s*id: string \| null;?\s*event: string;?\s*data: V2EventStreamV2;?\s*\};?/, + /(export type V2EventSubscribeResponses = \{\s*\/\*\*[\s\S]*?\*\/\s*200: )\{\s*id: string \| null;?\s*event: string;?\s*data: V2EventStream(?:V2)?;?\s*\};?/, "$1V2Event", ) if (eventSubscribeTypesPatched === sessionMessagesTypesPatched) { throw new Error("Event subscribe response patch did not apply") } +if (/SessionStructuredError\d/.test(eventSubscribeTypesPatched)) { + throw new Error("Session structured error generated a name-mangled duplicate") +} await Bun.write(generatedTypesPath, eventSubscribeTypesPatched) const querySerializerPath = "./src/v2/gen/client/utils.gen.ts" diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 4ffc01c30e..5455bee56e 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -24,7 +24,10 @@ export type Event = | EventSessionForked | EventSessionPromptPromoted | EventSessionPromptAdmitted - | EventSessionExecutionSettled + | EventSessionExecutionStarted + | EventSessionExecutionSucceeded + | EventSessionExecutionFailed + | EventSessionExecutionInterrupted | EventSessionContextUpdated | EventSessionSynthetic | EventSessionSkillActivated @@ -46,7 +49,7 @@ export type Event = | EventSessionToolProgress | EventSessionToolSuccess | EventSessionToolFailed - | EventSessionRetried + | EventSessionRetryScheduled | EventSessionCompactionStarted | EventSessionCompactionDelta | EventSessionCompactionEnded @@ -919,11 +922,32 @@ export type GlobalEvent = { } | { id: string - type: "session.execution.settled" + type: "session.execution.started" properties: { sessionID: string - outcome: "success" | "failure" | "interrupted" - error?: SessionErrorUnknown + } + } + | { + id: string + type: "session.execution.succeeded" + properties: { + sessionID: string + } + } + | { + id: string + type: "session.execution.failed" + properties: { + sessionID: string + error: SessionStructuredError + } + } + | { + id: string + type: "session.execution.interrupted" + properties: { + sessionID: string + reason: "user" | "shutdown" | "superseded" } } | { @@ -994,7 +1018,7 @@ export type GlobalEvent = { properties: { sessionID: string assistantMessageID: string - finish: string + finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost: number tokens: { input: number @@ -1015,7 +1039,7 @@ export type GlobalEvent = { properties: { sessionID: string assistantMessageID: string - error: SessionErrorUnknown + error: SessionStructuredError } } | { @@ -1153,7 +1177,7 @@ export type GlobalEvent = { sessionID: string assistantMessageID: string callID: string - error: SessionErrorUnknown + error: SessionStructuredError result?: unknown executed: boolean resultState?: SessionMessageProviderState @@ -1161,11 +1185,13 @@ export type GlobalEvent = { } | { id: string - type: "session.retried" + type: "session.retry.scheduled" properties: { sessionID: string + assistantMessageID: string attempt: number - error: SessionRetryError + at: number + error: SessionStructuredError } } | { @@ -1710,6 +1736,10 @@ export type GlobalEvent = { | SyncEventSessionForked | SyncEventSessionPromptPromoted | SyncEventSessionPromptAdmitted + | SyncEventSessionExecutionStarted + | SyncEventSessionExecutionSucceeded + | SyncEventSessionExecutionFailed + | SyncEventSessionExecutionInterrupted | SyncEventSessionContextUpdated | SyncEventSessionSynthetic | SyncEventSessionSkillActivated @@ -1728,7 +1758,7 @@ export type GlobalEvent = { | SyncEventSessionToolProgress | SyncEventSessionToolSuccess | SyncEventSessionToolFailed - | SyncEventSessionRetried + | SyncEventSessionRetryScheduled | SyncEventSessionCompactionStarted | SyncEventSessionCompactionEnded | SyncEventSessionRevertStaged @@ -2878,6 +2908,10 @@ export type SessionDurableEvent = | SessionForked | SessionPromptPromoted | SessionPromptAdmitted + | SessionExecutionStarted + | SessionExecutionSucceeded + | SessionExecutionFailed + | SessionExecutionInterrupted | SessionContextUpdated | SessionSynthetic | SessionSkillActivated @@ -2896,7 +2930,7 @@ export type SessionDurableEvent = | SessionToolProgress | SessionToolSuccess | SessionToolFailed - | SessionRetried + | SessionRetryScheduled | SessionCompactionStarted | SessionCompactionEnded | SessionRevertStaged @@ -3019,7 +3053,10 @@ export type V2Event = | SessionForked | SessionPromptPromoted | SessionPromptAdmitted - | SessionExecutionSettled + | SessionExecutionStarted + | SessionExecutionSucceeded + | SessionExecutionFailed + | SessionExecutionInterrupted | SessionContextUpdated | SessionSynthetic | SessionSkillActivated @@ -3041,7 +3078,7 @@ export type V2Event = | SessionToolProgress | SessionToolSuccess | SessionToolFailed - | SessionRetried + | SessionRetryScheduled | SessionCompactionStarted | SessionCompactionDelta | SessionCompactionEnded @@ -3258,10 +3295,83 @@ export type PromptAgentAttachment = { source?: PromptSource } -export type SessionErrorUnknown = { - type: "unknown" - message: string -} +export type SessionStructuredError = + | { + type: "provider.rate-limit" + message: string + retryAfterMs?: number + } + | { + type: "provider.auth" + message: string + } + | { + type: "provider.quota" + message: string + } + | { + type: "provider.content-filter" + message: string + } + | { + type: "provider.transport" + message: string + } + | { + type: "provider.internal" + message: string + } + | { + type: "provider.invalid-output" + message: string + } + | { + type: "provider.invalid-request" + message: string + } + | { + type: "provider.no-route" + message: string + } + | { + type: "provider.unknown" + message: string + } + | { + type: "permission.rejected" + message: string + permission: string + resources: Array + } + | { + type: "tool.unknown" + message: string + name: string + } + | { + type: "tool.stale" + message: string + name?: string + } + | { + type: "tool.execution" + message: string + } + | { + type: "tool.result-missing" + message: string + callID?: string + } + | { + type: "aborted" + message: string + reason?: "user" | "shutdown" | "timeout" + } + | { + type: "unknown" + message: string + agent?: string + } export type SessionMessageProviderState = { [key: string]: unknown @@ -3281,19 +3391,6 @@ export type ToolFileContent = { export type LlmToolContent = ToolTextContent | ToolFileContent -export type SessionRetryError = { - message: string - statusCode?: number - isRetryable: boolean - responseHeaders?: { - [key: string]: string - } - responseBody?: string - metadata?: { - [key: string]: string - } -} - export type FileDiff = { path: string status: "added" | "modified" | "deleted" @@ -3713,6 +3810,64 @@ export type SyncEventSessionPromptAdmitted = { } } +export type SyncEventSessionExecutionStarted = { + type: "sync" + id: string + syncEvent: { + type: "session.execution.started.1" + id: string + seq: number + aggregateID: string + data: { + sessionID: string + } + } +} + +export type SyncEventSessionExecutionSucceeded = { + type: "sync" + id: string + syncEvent: { + type: "session.execution.succeeded.1" + id: string + seq: number + aggregateID: string + data: { + sessionID: string + } + } +} + +export type SyncEventSessionExecutionFailed = { + type: "sync" + id: string + syncEvent: { + type: "session.execution.failed.1" + id: string + seq: number + aggregateID: string + data: { + sessionID: string + error: SessionStructuredError + } + } +} + +export type SyncEventSessionExecutionInterrupted = { + type: "sync" + id: string + syncEvent: { + type: "session.execution.interrupted.1" + id: string + seq: number + aggregateID: string + data: { + sessionID: string + reason: "user" | "shutdown" | "superseded" + } + } +} + export type SyncEventSessionContextUpdated = { type: "sync" id: string @@ -3828,7 +3983,7 @@ export type SyncEventSessionStepEnded = { data: { sessionID: string assistantMessageID: string - finish: string + finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost: number tokens: { input: number @@ -3856,7 +4011,7 @@ export type SyncEventSessionStepFailed = { data: { sessionID: string assistantMessageID: string - error: SessionErrorUnknown + error: SessionStructuredError } } } @@ -4036,7 +4191,7 @@ export type SyncEventSessionToolFailed = { sessionID: string assistantMessageID: string callID: string - error: SessionErrorUnknown + error: SessionStructuredError result?: unknown executed: boolean resultState?: SessionMessageProviderState @@ -4044,18 +4199,20 @@ export type SyncEventSessionToolFailed = { } } -export type SyncEventSessionRetried = { +export type SyncEventSessionRetryScheduled = { type: "sync" id: string syncEvent: { - type: "session.retried.1" + type: "session.retry.scheduled.1" id: string seq: number aggregateID: string data: { sessionID: string + assistantMessageID: string attempt: number - error: SessionRetryError + at: number + error: SessionStructuredError } } } @@ -4413,7 +4570,7 @@ export type SessionMessageToolStateError = { structured: { [key: string]: unknown } - error: SessionErrorUnknown + error: SessionStructuredError result?: unknown } @@ -4437,6 +4594,12 @@ export type SessionMessageAssistantTool = { } } +export type SessionMessageAssistantRetry = { + attempt: number + at: number + error: SessionStructuredError +} + export type SessionMessageAssistant = { id: string metadata?: { @@ -4455,7 +4618,7 @@ export type SessionMessageAssistant = { end?: string files?: Array } - finish?: string + finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost?: number tokens?: { input: number @@ -4466,7 +4629,8 @@ export type SessionMessageAssistant = { write: number } } - error?: SessionErrorUnknown + error?: SessionStructuredError + retry?: SessionMessageAssistantRetry } export type SessionMessageCompaction = { @@ -4638,6 +4802,80 @@ export type SessionPromptAdmitted = { } } +export type SessionExecutionStarted = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.started" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef + data: { + sessionID: string + } +} + +export type SessionExecutionSucceeded = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.succeeded" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef + data: { + sessionID: string + } +} + +export type SessionExecutionFailed = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.failed" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef + data: { + sessionID: string + error: SessionStructuredError + } +} + +export type SessionExecutionInterrupted = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.interrupted" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef + data: { + sessionID: string + reason: "user" | "shutdown" | "superseded" + } +} + export type SessionContextUpdated = { id: string created: number @@ -4782,7 +5020,7 @@ export type SessionStepEnded = { data: { sessionID: string assistantMessageID: string - finish: string + finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost: number tokens: { input: number @@ -4814,7 +5052,7 @@ export type SessionStepFailed = { data: { sessionID: string assistantMessageID: string - error: SessionErrorUnknown + error: SessionStructuredError } } @@ -5034,20 +5272,20 @@ export type SessionToolFailed = { sessionID: string assistantMessageID: string callID: string - error: SessionErrorUnknown + error: SessionStructuredError result?: unknown executed: boolean resultState?: SessionMessageProviderState } } -export type SessionRetried = { +export type SessionRetryScheduled = { id: string created: number metadata?: { [key: string]: unknown } - type: "session.retried" + type: "session.retry.scheduled" durable: { aggregateID: string seq: number @@ -5056,8 +5294,10 @@ export type SessionRetried = { location?: LocationRef data: { sessionID: string + assistantMessageID: string attempt: number - error: SessionRetryError + at: number + error: SessionStructuredError } } @@ -5684,21 +5924,6 @@ export type MessagePartRemoved = { } } -export type SessionExecutionSettled = { - id: string - created: number - metadata?: { - [key: string]: unknown - } - type: "session.execution.settled" - location?: LocationRef - data: { - sessionID: string - outcome: "success" | "failure" | "interrupted" - error?: SessionErrorUnknown - } -} - export type SessionTextDelta = { id: string created: number @@ -6784,13 +7009,37 @@ export type EventSessionPromptAdmitted = { } } -export type EventSessionExecutionSettled = { +export type EventSessionExecutionStarted = { id: string - type: "session.execution.settled" + type: "session.execution.started" properties: { sessionID: string - outcome: "success" | "failure" | "interrupted" - error?: SessionErrorUnknown + } +} + +export type EventSessionExecutionSucceeded = { + id: string + type: "session.execution.succeeded" + properties: { + sessionID: string + } +} + +export type EventSessionExecutionFailed = { + id: string + type: "session.execution.failed" + properties: { + sessionID: string + error: SessionStructuredError + } +} + +export type EventSessionExecutionInterrupted = { + id: string + type: "session.execution.interrupted" + properties: { + sessionID: string + reason: "user" | "shutdown" | "superseded" } } @@ -6868,7 +7117,7 @@ export type EventSessionStepEnded = { properties: { sessionID: string assistantMessageID: string - finish: string + finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost: number tokens: { input: number @@ -6890,7 +7139,7 @@ export type EventSessionStepFailed = { properties: { sessionID: string assistantMessageID: string - error: SessionErrorUnknown + error: SessionStructuredError } } @@ -7041,20 +7290,22 @@ export type EventSessionToolFailed = { sessionID: string assistantMessageID: string callID: string - error: SessionErrorUnknown + error: SessionStructuredError result?: unknown executed: boolean resultState?: SessionMessageProviderState } } -export type EventSessionRetried = { +export type EventSessionRetryScheduled = { id: string - type: "session.retried" + type: "session.retry.scheduled" properties: { sessionID: string + assistantMessageID: string attempt: number - error: SessionRetryError + at: number + error: SessionStructuredError } } @@ -8086,11 +8337,6 @@ export type SessionMessageToolStateCompleted2 = { result?: unknown } -export type SessionErrorUnknown2 = { - type: "unknown" - message: string -} - export type SessionMessageToolStateError2 = { status: "error" input: { @@ -8100,7 +8346,7 @@ export type SessionMessageToolStateError2 = { structured: { [key: string]: unknown } - error: SessionErrorUnknown2 + error: SessionStructuredError result?: unknown } @@ -8124,6 +8370,12 @@ export type SessionMessageAssistantTool2 = { } } +export type SessionMessageAssistantRetry2 = { + attempt: number + at: number + error: SessionStructuredError +} + export type SessionMessageAssistant2 = { id: string metadata?: { @@ -8142,7 +8394,7 @@ export type SessionMessageAssistant2 = { end?: string files?: Array } - finish?: string + finish?: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost?: number tokens?: { input: number @@ -8153,7 +8405,8 @@ export type SessionMessageAssistant2 = { write: number } } - error?: SessionErrorUnknown2 + error?: SessionStructuredError + retry?: SessionMessageAssistantRetry2 } export type SessionMessageCompaction2 = { @@ -8328,6 +8581,80 @@ export type SessionPromptAdmitted2 = { } } +export type SessionExecutionStarted2 = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.started" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef2 + data: { + sessionID: string + } +} + +export type SessionExecutionSucceeded2 = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.succeeded" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef2 + data: { + sessionID: string + } +} + +export type SessionExecutionFailed2 = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.failed" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef2 + data: { + sessionID: string + error: SessionStructuredError + } +} + +export type SessionExecutionInterrupted2 = { + id: string + created: number + metadata?: { + [key: string]: unknown + } + type: "session.execution.interrupted" + durable: { + aggregateID: string + seq: number + version: number + } + location?: LocationRef2 + data: { + sessionID: string + reason: "user" | "shutdown" | "superseded" + } +} + export type SessionContextUpdated2 = { id: string created: number @@ -8490,7 +8817,7 @@ export type SessionStepEnded2 = { data: { sessionID: string assistantMessageID: string - finish: string + finish: "stop" | "length" | "tool-calls" | "content-filter" | "error" | "unknown" cost: number tokens: { input: number @@ -8522,7 +8849,7 @@ export type SessionStepFailed2 = { data: { sessionID: string assistantMessageID: string - error: SessionErrorUnknown2 + error: SessionStructuredError } } @@ -8762,33 +9089,20 @@ export type SessionToolFailed2 = { sessionID: string assistantMessageID: string callID: string - error: SessionErrorUnknown2 + error: SessionStructuredError result?: unknown executed: boolean resultState?: SessionMessageProviderState7 } } -export type SessionRetryError2 = { - message: string - statusCode?: number - isRetryable: boolean - responseHeaders?: { - [key: string]: string - } - responseBody?: string - metadata?: { - [key: string]: string - } -} - -export type SessionRetried2 = { +export type SessionRetryScheduled2 = { id: string created: number metadata?: { [key: string]: unknown } - type: "session.retried" + type: "session.retry.scheduled" durable: { aggregateID: string seq: number @@ -8797,8 +9111,10 @@ export type SessionRetried2 = { location?: LocationRef2 data: { sessionID: string + assistantMessageID: string attempt: number - error: SessionRetryError2 + at: number + error: SessionStructuredError } } @@ -8906,6 +9222,10 @@ export type SessionDurableEventV2 = | SessionForked2 | SessionPromptPromoted2 | SessionPromptAdmitted2 + | SessionExecutionStarted2 + | SessionExecutionSucceeded2 + | SessionExecutionFailed2 + | SessionExecutionInterrupted2 | SessionContextUpdated2 | SessionSynthetic2 | SessionSkillActivated2 @@ -8924,7 +9244,7 @@ export type SessionDurableEventV2 = | SessionToolProgress2 | SessionToolSuccess2 | SessionToolFailed2 - | SessionRetried2 + | SessionRetryScheduled2 | SessionCompactionStarted2 | SessionCompactionEnded2 | SessionRevertStaged2 @@ -10142,21 +10462,6 @@ export type MessagePartRemoved2 = { } } -export type SessionExecutionSettled2 = { - id: string - created: number - metadata?: { - [key: string]: unknown - } - type: "session.execution.settled" - location?: LocationRef2 - data: { - sessionID: string - outcome: "success" | "failure" | "interrupted" - error?: SessionErrorUnknown2 - } -} - export type SessionTextDelta2 = { id: string created: number @@ -11073,7 +11378,10 @@ export type V2EventV2 = | SessionForked2 | SessionPromptPromoted2 | SessionPromptAdmitted2 - | SessionExecutionSettled2 + | SessionExecutionStarted2 + | SessionExecutionSucceeded2 + | SessionExecutionFailed2 + | SessionExecutionInterrupted2 | SessionContextUpdated2 | SessionSynthetic2 | SessionSkillActivated2 @@ -11095,7 +11403,7 @@ export type V2EventV2 = | SessionToolProgress2 | SessionToolSuccess2 | SessionToolFailed2 - | SessionRetried2 + | SessionRetryScheduled2 | SessionCompactionStarted2 | SessionCompactionDelta2 | SessionCompactionEnded2 diff --git a/packages/tui/src/context/data.tsx b/packages/tui/src/context/data.tsx index 77c00d9ab5..878c1fcd15 100644 --- a/packages/tui/src/context/data.tsx +++ b/packages/tui/src/context/data.tsx @@ -267,7 +267,6 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ setStore("session", "info", event.data.sessionID, "title", event.data.title) break case "session.prompt.promoted": { - setSessionStatus(event.data.sessionID, "running") message.update(event.data.sessionID, (draft, index) => { const position = index.get(event.data.inputID) if (position === undefined) return @@ -321,7 +320,6 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ }) break case "session.shell.started": - setSessionStatus(event.data.sessionID, "running") message.update(event.data.sessionID, (draft, index) => { message.append(draft, index, { id: messageIDFromEvent(event.id), @@ -332,7 +330,6 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ }) break case "session.shell.ended": - setSessionStatus(event.data.sessionID, "idle") message.update(event.data.sessionID, (draft) => { const match = message.shell(draft, event.data.shell.id) if (!match) return @@ -342,11 +339,24 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ }) break case "session.step.started": - setSessionStatus(event.data.sessionID, "running") message.update(event.data.sessionID, (draft, index) => { - if (index.has(event.data.assistantMessageID)) return + const position = index.get(event.data.assistantMessageID) + const existing = position === undefined ? undefined : draft[position] + if (existing?.type === "assistant") { + existing.agent = event.data.agent + existing.model = event.data.model + existing.retry = undefined + existing.error = undefined + existing.finish = undefined + existing.time.completed = undefined + if (event.data.snapshot) existing.snapshot = { ...existing.snapshot, start: event.data.snapshot } + return + } const currentAssistant = message.activeAssistant(draft) - if (currentAssistant) currentAssistant.time.completed = event.created + if (currentAssistant) { + currentAssistant.retry = undefined + currentAssistant.time.completed = event.created + } message.append(draft, index, { id: event.data.assistantMessageID, type: "assistant", @@ -359,7 +369,6 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ }) break case "session.step.ended": - setSessionStatus(event.data.sessionID, "running") message.update(event.data.sessionID, (draft, index) => { const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID) if (!currentAssistant) return @@ -378,6 +387,7 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ currentAssistant.time.completed = event.created currentAssistant.finish = "error" currentAssistant.error = event.data.error + currentAssistant.retry = undefined }) break case "session.text.started": @@ -518,17 +528,33 @@ export const { use: useData, provider: DataProvider } = createSimpleContext({ } }) break - case "session.compaction.started": + case "session.retry.scheduled": + message.update(event.data.sessionID, (draft, index) => { + const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID) + if (!currentAssistant) return + currentAssistant.retry = { + attempt: event.data.attempt, + at: event.data.at, + error: event.data.error, + } + }) + break + case "session.execution.started": setSessionStatus(event.data.sessionID, "running") + break + case "session.compaction.started": setStore("session", "compaction", event.data.sessionID, "") break - case "session.retried": - setSessionStatus(event.data.sessionID, "running") - break - case "session.execution.settled": + case "session.execution.succeeded": + case "session.execution.failed": + case "session.execution.interrupted": setSessionStatus(event.data.sessionID, "idle") if (store.session.compaction[event.data.sessionID] !== undefined) setStore("session", "compaction", event.data.sessionID, undefined) + message.update(event.data.sessionID, (draft) => { + const currentAssistant = message.activeAssistant(draft) + if (currentAssistant) currentAssistant.retry = undefined + }) break case "session.revert.staged": if (store.session.info[event.data.sessionID]) diff --git a/packages/tui/src/feature-plugins/system/notifications.ts b/packages/tui/src/feature-plugins/system/notifications.ts index 416ba8c466..056b8455eb 100644 --- a/packages/tui/src/feature-plugins/system/notifications.ts +++ b/packages/tui/src/feature-plugins/system/notifications.ts @@ -27,8 +27,8 @@ function sessionErrorMessage(error: SessionError) { } const tui: TuiPlugin = async (api) => { - const active = new Set() const errored = new Set() + const terminal = new Set() const questions = new Set() const permissions = new Set() @@ -57,14 +57,13 @@ const tui: TuiPlugin = async (api) => { }) const started = (sessionID: string) => { - active.add(sessionID) errored.delete(sessionID) + terminal.delete(sessionID) } const ended = (sessionID: string) => { - if (!active.has(sessionID)) return - active.delete(sessionID) - + if (terminal.has(sessionID)) return + terminal.add(sessionID) if (errored.has(sessionID)) { errored.delete(sessionID) return @@ -74,28 +73,25 @@ const tui: TuiPlugin = async (api) => { notify(api, sessionID, "Session done", session?.parentID ? "subagent_done" : "done") } - api.event.on("session.prompt.promoted", (event) => started(event.data.sessionID)) - api.event.on("session.shell.started", (event) => started(event.data.sessionID)) - api.event.on("session.step.started", (event) => started(event.data.sessionID)) - api.event.on("session.retried", (event) => started(event.data.sessionID)) - api.event.on("session.compaction.started", (event) => started(event.data.sessionID)) - api.event.on("session.shell.ended", (event) => ended(event.data.sessionID)) - api.event.on("session.step.ended", (event) => { - if (event.data.finish === "tool-calls") return - ended(event.data.sessionID) - }) - api.event.on("session.step.failed", (event) => { + api.event.on("session.execution.started", (event) => started(event.data.sessionID)) + api.event.on("session.execution.succeeded", (event) => ended(event.data.sessionID)) + api.event.on("session.execution.interrupted", (event) => ended(event.data.sessionID)) + api.event.on("session.execution.failed", (event) => { const sessionID = event.data.sessionID - if (!active.has(sessionID)) return + if (errored.has(sessionID)) { + ended(sessionID) + return + } errored.add(sessionID) - notify(api, sessionID, "Session error", "error") + notify(api, sessionID, event.data.error.message, "error") ended(sessionID) }) api.event.on("session.error", (event) => { const sessionID = event.data.sessionID if (!sessionID) return - if (!active.has(sessionID)) return + if (api.state.session.status(sessionID)?.type !== "busy") return + if (errored.has(sessionID)) return errored.add(sessionID) notify(api, sessionID, sessionErrorMessage(event.data.error), "error") }) diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index 7b19ebfdd1..d0be3592b5 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -1214,6 +1214,15 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) { {errorMessage(props.message.error)} + + {(retry) => ( + + + Retry attempt {retry().attempt} scheduled: {retry().error.message} [{retry().error.type}] + + + )} + @@ -1553,6 +1562,15 @@ function AssistantMessage(props: { message: SessionMessageAssistant; last: boole {errorMessage(props.message.error)} + + {(retry) => ( + + + Retry attempt {retry().attempt} scheduled: {retry().error.message} [{retry().error.type}] + + + )} + diff --git a/packages/tui/src/routes/session/rows.ts b/packages/tui/src/routes/session/rows.ts index 61dd6e7fac..5b7652d22f 100644 --- a/packages/tui/src/routes/session/rows.ts +++ b/packages/tui/src/routes/session/rows.ts @@ -131,6 +131,14 @@ export function createSessionRows(sessionID: Accessor) { }), ) + const removeFooter = (messageID: string) => + setRows( + produce((draft) => { + const index = draft.findIndex((row) => row.type === "assistant-footer" && row.messageID === messageID) + if (index !== -1) draft.splice(index, 1) + }), + ) + const latestFragmentRef = (messageID: string, kind: "text" | "reasoning") => { const message = data.session.message.get(sessionID(), messageID) const ordinal = message?.type === "assistant" ? message.content.filter((part) => part.type === kind).length - 1 : 0 @@ -183,6 +191,12 @@ export function createSessionRows(sessionID: Accessor) { if (event.data.sessionID === sessionID()) appendPart({ messageID: event.data.assistantMessageID, partID: event.data.callID }, event.data.name) }), + data.on("session.retry.scheduled", (event) => { + if (event.data.sessionID === sessionID()) appendFooter(event.data.assistantMessageID) + }), + data.on("session.step.started", (event) => { + if (event.data.sessionID === sessionID()) removeFooter(event.data.assistantMessageID) + }), data.on("session.step.ended", (event) => { if (event.data.sessionID !== sessionID() || ["tool-calls", "unknown"].includes(event.data.finish)) return appendFooter(event.data.assistantMessageID) @@ -212,7 +226,7 @@ export function reduceSessionRows(messages: SessionMessage[]) { if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return append(rows, { messageID: message.id, partID }, part) }) - if ((message.finish && !["tool-calls", "unknown"].includes(message.finish)) || message.error) { + if ((message.finish && !["tool-calls", "unknown"].includes(message.finish)) || message.error || message.retry) { completePrevious(rows) rows.push({ type: "assistant-footer", messageID: message.id }) } diff --git a/packages/tui/test/cli/cmd/tui/notifications.test.ts b/packages/tui/test/cli/cmd/tui/notifications.test.ts index 44cd98b133..8e1a5a2257 100644 --- a/packages/tui/test/cli/cmd/tui/notifications.test.ts +++ b/packages/tui/test/cli/cmd/tui/notifications.test.ts @@ -49,6 +49,7 @@ async function setup() { state: { session: { get: (sessionID: string) => sessions[sessionID], + status: () => ({ type: "busy" }), }, }, }), @@ -87,46 +88,34 @@ function durable(sessionID: string) { return { aggregateID: sessionID, seq: 0, version: 1 } } -function stepStarted(id: string, sessionID = "session"): V2Event { +function executionStarted(id: string, sessionID = "session"): V2Event { return { id, created: 0, - type: "session.step.started", + type: "session.execution.started", durable: durable(sessionID), - data: { - sessionID, - assistantMessageID: `msg_${id}`, - agent: "build", - model: { id: "model", providerID: "provider" }, - }, + data: { sessionID }, } } -function stepEnded(id: string, sessionID = "session", finish = "stop"): V2Event { +function executionSucceeded(id: string, sessionID = "session"): V2Event { return { id, created: 0, - type: "session.step.ended", + type: "session.execution.succeeded", durable: durable(sessionID), - data: { - sessionID, - assistantMessageID: `msg_${id}`, - finish, - cost: 0, - tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, - }, + data: { sessionID }, } } -function stepFailed(id: string, sessionID = "session"): V2Event { +function executionFailed(id: string, sessionID = "session"): V2Event { return { id, created: 0, - type: "session.step.failed", + type: "session.execution.failed", durable: durable(sessionID), data: { sessionID, - assistantMessageID: `msg_${id}`, error: { type: "unknown", message: "boom" }, }, } @@ -187,12 +176,12 @@ describe("internal notifications TUI plugin", () => { ]) }) - test("notifies when an active session becomes idle and suppresses no-op idle", async () => { + test("notifies for terminal lifecycle events even when attached after execution started", async () => { const harness = await setup() - harness.emit(stepEnded("event-1")) - harness.emit(stepStarted("event-2")) - harness.emit(stepEnded("event-3")) + harness.emit(executionSucceeded("event-1")) + harness.emit(executionStarted("event-2")) + harness.emit(executionSucceeded("event-3")) expect(harness.notifications).toEqual([ { @@ -201,6 +190,12 @@ describe("internal notifications TUI plugin", () => { notification: { when: "blurred" }, sound: { name: "done", when: "always" }, }, + { + title: "Demo session", + message: "Session done", + notification: { when: "blurred" }, + sound: { name: "done", when: "always" }, + }, ]) }) @@ -208,8 +203,8 @@ describe("internal notifications TUI plugin", () => { const harness = await setup() harness.emit({ id: "event-1", created: 0, type: "question.asked", data: question("question-1", "subagent") }) - harness.emit(stepStarted("event-2", "subagent")) - harness.emit(stepEnded("event-3", "subagent")) + harness.emit(executionStarted("event-2", "subagent")) + harness.emit(executionSucceeded("event-3", "subagent")) expect(harness.notifications).toEqual([ { @@ -230,14 +225,14 @@ describe("internal notifications TUI plugin", () => { test("notifies session errors once and suppresses the following idle done notification", async () => { const harness = await setup() - harness.emit(stepStarted("event-1")) - harness.emit(stepFailed("event-2")) - harness.emit(stepEnded("event-3")) + harness.emit(executionStarted("event-1")) + harness.emit(executionFailed("event-2")) + harness.emit(executionSucceeded("event-3")) expect(harness.notifications).toEqual([ { title: "Demo session", - message: "Session error", + message: "boom", notification: { when: "blurred" }, sound: { name: "error", when: "always" }, }, @@ -247,20 +242,21 @@ describe("internal notifications TUI plugin", () => { test("special-cases aborts and model response timeouts", async () => { const harness = await setup() - harness.emit(stepStarted("event-1", "abort")) + harness.emit(executionStarted("event-1", "abort")) harness.emit({ id: "event-2", created: 0, type: "session.error", data: { sessionID: "abort", error: { name: "MessageAbortedError", data: { message: "Aborted" } } }, }) - harness.emit(stepStarted("event-3", "timeout")) + harness.emit(executionStarted("event-3", "timeout")) harness.emit({ id: "event-4", created: 0, type: "session.error", data: { sessionID: "timeout", error: { name: "UnknownError", data: { message: "SSE read timed out" } } }, }) + harness.emit(executionFailed("event-5", "timeout")) expect(harness.notifications).toEqual([ { diff --git a/packages/tui/test/cli/tui/data.test.tsx b/packages/tui/test/cli/tui/data.test.tsx index 0fce5202ba..b354908091 100644 --- a/packages/tui/test/cli/tui/data.test.tsx +++ b/packages/tui/test/cli/tui/data.test.tsx @@ -8,7 +8,7 @@ import { onMount } from "solid-js" import { ProjectProvider } from "../../../src/context/project" import { SDKProvider } from "../../../src/context/sdk" import { DataProvider, useData } from "../../../src/context/data" -import { createSessionRows } from "../../../src/routes/session/rows" +import { createSessionRows, type SessionRow } from "../../../src/routes/session/rows" import { createApi, createClient, createEventStream, createFetch, directory, json } from "../../fixture/tui-sdk" import { TestTuiContexts } from "../../fixture/tui-environment" @@ -177,16 +177,11 @@ test("reconnects the event stream and bootstraps fresh data", async () => { await wait(() => requests.active === 2 && data.connection.status() === "connected", 4000) emitEvent(events, { - id: "evt_step_started_after_reconnect", + id: "evt_execution_started_after_reconnect", created: 1, - type: "session.step.started", + type: "session.execution.started", durable: durable("session-new"), - data: { - sessionID: "session-new", - assistantMessageID: "message-new", - agent: "build", - model: { id: "model", providerID: "provider" }, - }, + data: { sessionID: "session-new" }, }) await wait(() => data.session.status("session-new") === "running") resolveActive(json({ data: {}, watermarks: {} })) @@ -353,9 +348,11 @@ test("tracks session status from active sessions and execution events", async () return json({ data: { "session-active": { type: "running" } }, watermarks: {} }) }, events) let data!: ReturnType + let rows!: SessionRow[] function Probe() { data = useData() + rows = createSessionRows(() => "session-retry") return } @@ -375,6 +372,15 @@ test("tracks session status from active sessions and execution events", async () await wait(() => data.session.status("session-active") === "running") expect(data.session.status("session-idle")).toBe("idle") + emitEvent(events, { + id: "evt_execution_started", + created: 0, + type: "session.execution.started", + durable: durable("session-live"), + data: { sessionID: "session-live" }, + }) + await wait(() => data.session.status("session-live") === "running") + emitEvent(events, { id: "evt_step_started", created: 0, @@ -387,8 +393,6 @@ test("tracks session status from active sessions and execution events", async () model: { id: "model", providerID: "provider" }, }, }) - await wait(() => data.session.status("session-live") === "running") - emitEvent(events, { id: "evt_step_ended", created: 0, @@ -409,16 +413,23 @@ test("tracks session status from active sessions and execution events", async () expect(data.session.status("session-live")).toBe("running") emitEvent(events, { - id: "evt_execution_settled", + id: "evt_execution_succeeded", created: 0, - type: "session.execution.settled", - data: { - sessionID: "session-live", - outcome: "success", - }, + type: "session.execution.succeeded", + durable: durable("session-live", 1, 3), + data: { sessionID: "session-live" }, }) await wait(() => data.session.status("session-live") === "idle") + emitEvent(events, { + id: "evt_failed_execution_started", + created: 0, + type: "session.execution.started", + durable: durable("session-failed"), + data: { sessionID: "session-failed" }, + }) + await wait(() => data.session.status("session-failed") === "running") + emitEvent(events, { id: "evt_failed_step_started", created: 0, @@ -431,8 +442,6 @@ test("tracks session status from active sessions and execution events", async () model: { id: "model", providerID: "provider" }, }, }) - await wait(() => data.session.status("session-failed") === "running") - emitEvent(events, { id: "evt_step_failed", created: 0, @@ -441,27 +450,113 @@ test("tracks session status from active sessions and execution events", async () data: { sessionID: "session-failed", assistantMessageID: "message-failed", - error: { type: "unknown", message: "Provider unavailable" }, + error: { type: "provider.content-filter", message: "Provider blocked the response" }, }, }) await wait(() => { const assistant = data.session.message.get("session-failed", "message-failed") - return assistant?.type === "assistant" && assistant.finish === "error" + return ( + assistant?.type === "assistant" && + assistant.finish === "error" && + assistant.error?.type === "provider.content-filter" + ) }) expect(data.session.status("session-failed")).toBe("running") emitEvent(events, { - id: "evt_failed_execution_settled", + id: "evt_failed_execution_failed", created: 0, - type: "session.execution.settled", + type: "session.execution.failed", + durable: durable("session-failed", 1, 3), data: { sessionID: "session-failed", - outcome: "failure", - error: { type: "unknown", message: "Provider unavailable" }, + error: { type: "provider.content-filter", message: "Provider blocked the response" }, }, }) await wait(() => data.session.status("session-failed") === "idle") + emitEvent(events, { + id: "evt_retry_execution_started", + created: 0, + type: "session.execution.started", + durable: durable("session-retry"), + data: { sessionID: "session-retry" }, + }) + emitEvent(events, { + id: "evt_retry_step_started", + created: 0, + type: "session.step.started", + durable: durable("session-retry", 1, 2), + data: { + sessionID: "session-retry", + assistantMessageID: "message-retry", + agent: "build", + model: { id: "model", providerID: "provider" }, + }, + }) + emitEvent(events, { + id: "evt_retry_scheduled", + created: 0, + type: "session.retry.scheduled", + durable: durable("session-retry", 1, 3), + data: { + sessionID: "session-retry", + assistantMessageID: "message-retry", + attempt: 2, + at: 2_000, + error: { type: "provider.transport", message: "Disconnected" }, + }, + }) + await wait(() => { + const assistant = data.session.message.get("session-retry", "message-retry") + return assistant?.type === "assistant" && assistant.retry?.attempt === 2 + }) + await wait(() => rows.some((row) => row.type === "assistant-footer" && row.messageID === "message-retry")) + emitEvent(events, { + id: "evt_retry_next_step", + created: 2_000, + type: "session.step.started", + durable: durable("session-retry", 1, 4), + data: { + sessionID: "session-retry", + assistantMessageID: "message-retry", + agent: "build", + model: { id: "model", providerID: "provider" }, + }, + }) + await wait(() => { + const assistant = data.session.message.get("session-retry", "message-retry") + return assistant?.type === "assistant" && assistant.retry === undefined + }) + await wait(() => !rows.some((row) => row.type === "assistant-footer" && row.messageID === "message-retry")) + expect(data.session.message.list("session-retry").filter((message) => message.type === "assistant")).toHaveLength(1) + emitEvent(events, { + id: "evt_retry_scheduled_again", + created: 2_000, + type: "session.retry.scheduled", + durable: durable("session-retry", 1, 5), + data: { + sessionID: "session-retry", + assistantMessageID: "message-retry", + attempt: 3, + at: 6_000, + error: { type: "provider.transport", message: "Disconnected again" }, + }, + }) + await wait(() => { + const assistant = data.session.message.get("session-retry", "message-retry") + return assistant?.type === "assistant" && assistant.retry?.attempt === 3 + }) + emitEvent(events, { + id: "evt_retry_interrupted", + created: 2_000, + type: "session.execution.interrupted", + durable: durable("session-retry", 1, 6), + data: { sessionID: "session-retry", reason: "shutdown" }, + }) + await wait(() => data.session.status("session-retry") === "idle") + expect(data.session.message.get("session-retry", "message-retry")).not.toHaveProperty("retry") + emitEvent(events, { id: "evt_compaction_started", created: 0, diff --git a/packages/tui/test/cli/tui/session-rows.test.ts b/packages/tui/test/cli/tui/session-rows.test.ts index 46b1f91a37..e55e42231b 100644 --- a/packages/tui/test/cli/tui/session-rows.test.ts +++ b/packages/tui/test/cli/tui/session-rows.test.ts @@ -198,6 +198,17 @@ test("renders synthetic messages with descriptions", () => { ]) }) +test("renders a footer for a pre-output retry assistant after replay", () => { + const message = assistant("assistant-retry", []) + message.retry = { + attempt: 2, + at: 2_000, + error: { type: "provider.transport", message: "Disconnected" }, + } + + expect(reduceSessionRows([message])).toEqual([{ type: "assistant-footer", messageID: "assistant-retry" }]) +}) + function assistant(id: string, content: SessionMessageAssistant["content"]): SessionMessageAssistant { return { type: "assistant", diff --git a/specs/v2/schema-changelog.md b/specs/v2/schema-changelog.md index 4e5d3edba6..6856116b6e 100644 --- a/specs/v2/schema-changelog.md +++ b/specs/v2/schema-changelog.md @@ -1,5 +1,18 @@ # V2 Schema Changelog +## 2026-07-03: Add Execution Lifecycle, Retry, And Structured Session Errors + +- Replace live-only `session.execution.settled` and unused `session.retried` with durable v1 `session.execution.started`, `session.execution.succeeded`, `session.execution.failed`, `session.execution.interrupted`, and `session.retry.scheduled` events. +- Add the closed, dot-cased `SessionError` wire union and browser-safe `FinishReason` contract. +- Project retry state onto the current assistant and classify content-filter finishes as failed steps. +- Reuse one projected assistant across pre-output retry steps; each provider call remains a distinct step and consumes agent allowance. + +Compatibility: + +- Experimental V2 event, sequence, input, and message-projection rows are reset. Durable event contracts restart at v1. +- Execution lifecycle events are historical observations of one process-local coordinator busy period. Unmatched starts never establish current liveness or recovery work; `/api/session/active` remains the current-process liveness authority. +- Scheduled retries are historical UI state after a crash and never trigger provider recovery. + ## 2026-07-03: Require Durable Envelope On Durable Events - Make the wire `durable` envelope required on durable event definitions. diff --git a/specs/v2/session.md b/specs/v2/session.md index 14fd632a67..4d072e9ce4 100644 --- a/specs/v2/session.md +++ b/specs/v2/session.md @@ -21,7 +21,7 @@ sessions.prompt({ id?, sessionID, prompt, delivery?, resume? }) sessions.interrupt(sessionID) -> interrupts active execution on this process - -> waits for runner cleanup and settlement + -> waits for runner cleanup and a terminal lifecycle observation -> clears a coalesced follow-up wake already registered with this coordinator -> preserves durable inbox rows for a later wake or resume -> idle or missing Session is a no-op @@ -49,6 +49,12 @@ SessionExecution.resume(sessionID) The local runner issues one explicit `llm.stream(request)` per step, projects each complete local tool call durably before eagerly starting its structured child execution, awaits every started tool fiber after provider-stream closure, and reloads projected history once before continuation. Promoting any new user input resets the selected agent's configured step allowance; multiple steers promoted at one boundary reset it once. Tool settlement events carry the owning assistant message ID because provider-local call IDs may repeat across steps. Before assembling a provider request, the runner durably fails any local tool still projected as `running` from a previous process with `Tool execution interrupted`; abandoned side effects are never silently replayed. +`session.execution.started.1` and exactly one of `session.execution.succeeded.1`, `session.execution.failed.1`, or `session.execution.interrupted.1` observe one process-local coordinator busy period, including coalesced drains and joined resumes. These durable rows are history, not a durable execution identity: replay must never infer current liveness, recovery, grouping, or resumability from an unmatched start. A drain has no durable identity or transcript boundary. `/api/session/active` is the authority for current process-local liveness, and is empty after restart. User interruption records `reason: "user"`; owner-scope interruption defaults to `"shutdown"`; `"superseded"` is reserved for explicit replacement. + +Core retries only typed rate-limit, provider-internal, and transport failures before durable assistant text, reasoning, tool-call, tool-output, or tool-execution evidence. The initial call plus at most four retries use two-second exponential backoff, raised when a provider's `retryAfterMs` is larger. Every retry attempt remains a distinct step and consumes the selected agent's step allowance, while all pre-output attempts reuse one assistant message ID so retry state never creates empty transcript messages. Repeated `session.step.started.1` facts reopen that assistant projection idempotently. `session.retry.scheduled.1` is committed before each delay with the upcoming one-based attempt and absolute epoch-millisecond time, then projects onto `Assistant.retry`. The next `session.step.started.1` or terminal failure/interruption clears it. A scheduled retry surviving a crash is historical UI state only and never triggers recovery. + +A normalized `step-finish` with `content-filter` publishes `session.step.failed.1` with `provider.content-filter`, never `session.step.ended.1`. Any partial streamed content remains visible; a contentless filtered response still has a failed assistant projection. + Projected hosted tools preserve call-side and settlement-side provider metadata separately so settlement and interruption recovery cannot erase continuation identifiers. Provider-native reasoning and provider metadata replay only while the historical assistant model matches the selected continuation model; after a model switch, visible reasoning text remains ordinary assistant text and provider-native metadata is omitted. ## Context Epochs @@ -154,7 +160,7 @@ Status: `complete` is usable in the native V2 path, `partial` covers only part o | Prompt/reference expansion | Configured-reference expansion | missing | Resolve aliases and emit durable model-visible reference context or failures. | | Prompt/reference expansion | Native synthetic expansion replay | partial | V2 replays synthetic messages but only the V1 compatibility path creates them. | -Provider timeout, retry, and watchdog policy is intentionally deferred. The runner does not impose a universal provider-stream inactivity or absolute timeout. A future slice should design configurable policy around provider behavior, durable failure reporting, and local drain-chain release rather than hardcoding one default for every provider. +Provider timeout and watchdog policy is intentionally deferred. Retry tuning beyond the narrow safe policy above remains separate work; the runner does not impose a universal provider-stream inactivity or absolute timeout. Inbox delivery is explicit: @@ -168,7 +174,7 @@ Execution has two entry points: Post-crash continuation recovery is intentionally deferred. A wake does not infer that ambiguous provider work is safe to retry after an input has already been promoted. Explicit `run` may deliberately continue from durable projected history. A future recovery slice should model provider-dispatch ambiguity, required continuation, queued-input promotion, retry policy, and visible recovery status together. It must not assume an enclosing durable execution identity that the Session model does not otherwise need. -A process-global `SessionRunCoordinator` serializes execution for each local Session while allowing different Sessions to run concurrently. Resumes join active execution, overlapping wakes coalesce into one follow-up, and interruption stops current process-local execution without deleting durable inbox work. The runner enters the Session's current Location when execution starts and fences each new step against that Location. +A process-global `SessionRunCoordinator` serializes execution for each local Session while allowing different Sessions to run concurrently. Resumes join active execution, overlapping wakes coalesce into one follow-up, and interruption stops current process-local execution without deleting durable inbox work. The runner enters the Session's current Location when execution starts and fences each new step against that Location. Its durable lifecycle events are historical observations only; they do not replace the coordinator's process-local active registry. The coordinator's active registry is also the source for `sessions.active()`. It represents only foreground Session drains owned by the current process; background subagents and tasks do not add parent Sessions to this registry. The snapshot is runtime state and is empty after a process restart. diff --git a/specs/v2/todo.md b/specs/v2/todo.md index 6f09684a1b..d93520b626 100644 --- a/specs/v2/todo.md +++ b/specs/v2/todo.md @@ -55,7 +55,7 @@ Next reviewed slices: ### Deferred durable continuation recovery -Do not infer that ambiguous provider work is safe to retry from an advisory wake. +Do not infer that ambiguous provider work is safe to retry from an advisory wake, an unmatched historical `session.execution.started` event, or a surviving `session.retry.scheduled` projection. The first inbox-driven runner intentionally omits outer physical-attempt markers until they have a concrete consumer and a complete recovery policy.