From 5a0ba34d6435810d46808fffce4ef3f75a507614 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:58:58 -0500 Subject: [PATCH] fix(core): expire stale shell output (#43554) --- packages/core/src/file-retention.ts | 23 ++++++++++ packages/core/src/shell.ts | 51 ++++++++++++++++++++--- packages/core/src/tool-output.ts | 19 +++------ packages/core/test/shell-cleanup.test.ts | 53 ++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 18 deletions(-) create mode 100644 packages/core/src/file-retention.ts create mode 100644 packages/core/test/shell-cleanup.test.ts diff --git a/packages/core/src/file-retention.ts b/packages/core/src/file-retention.ts new file mode 100644 index 0000000000..aa94d050c6 --- /dev/null +++ b/packages/core/src/file-retention.ts @@ -0,0 +1,23 @@ +export * as FileRetention from "./file-retention.js" + +import { Duration, Effect, Option } from "effect" +import { FSUtil } from "@opencode-ai/util/fs-util" + +export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( + fs: FSUtil.Interface, + files: ReadonlyArray, + retention: Duration.Input, +) { + const cutoff = Date.now() - Duration.toMillis(retention) + yield* Effect.forEach( + files, + (file) => + Effect.gen(function* () { + const info = yield* fs.stat(file).pipe(Effect.catch(() => Effect.succeed(undefined))) + const mtime = info && Option.getOrUndefined(info.mtime) + if (!mtime || mtime.getTime() >= cutoff) return + yield* fs.remove(file).pipe(Effect.catch(() => Effect.void)) + }), + { concurrency: 8, discard: true }, + ) +}) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 0800804396..51598e7b33 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,14 +1,16 @@ export * as Shell from "./shell.js" import path from "path" -import { Context, Deferred, Duration, Effect, Fiber, Layer, Schema, Stream } from "effect" +import { Context, Deferred, Duration, Effect, Fiber, Layer, Schema, Schedule, Stream } from "effect" import { ChildProcess } from "effect/unstable/process" import { produce } from "immer" import { Shell } from "@opencode-ai/schema/shell" import { AppProcess } from "@opencode-ai/util/process" -import { makeLocationNode } from "@opencode-ai/util/effect/app-node" +import { makeGlobalNode, makeLocationNode } from "@opencode-ai/util/effect/app-node" +import { FSUtil } from "@opencode-ai/util/fs-util" import { Bus } from "./bus.js" import { Environment } from "./environment/index.js" +import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { Global } from "@opencode-ai/util/global" import { ShellSelect } from "./shell/select.js" @@ -21,9 +23,11 @@ export class NotFoundError extends Schema.TaggedError()("Shell.No id: Shell.ID, }) {} -// Exited processes stay observable (status, exit code, retained output) until removed explicitly. -// Cap retention so abandoned commands do not accumulate unbounded state and output files. +// Keep recent exited processes observable in memory, including their file-backed output. +// The process-local cap complements the time-based sweep, which also cleans files left by restarts. const EXITED_LIMIT = 25 +export const RETENTION = Duration.days(7) +export const DIRECTORY = "shell" type Info = Shell.Info @@ -67,6 +71,42 @@ export interface Interface { export class Service extends Context.Service()("@opencode/Shell") {} +export const cleanup = Effect.fn("Shell.cleanup")(function* () { + const fs = yield* FSUtil.Service + const global = yield* Global.Service + const directory = path.join(global.data, DIRECTORY) + const projects = yield* fs.readDirectoryEntries(directory).pipe( + Effect.map((entries) => entries.filter((entry) => entry.type === "directory")), + Effect.catch(() => Effect.succeed([])), + ) + const files = yield* Effect.forEach( + projects, + (project) => + fs.readDirectoryEntries(path.join(directory, project.name)).pipe( + Effect.map((entries) => + entries.flatMap((entry) => + entry.type === "file" && /^sh_[0-9a-f]{12}.*\.out$/.test(entry.name) + ? [path.join(directory, project.name, entry.name)] + : [], + ), + ), + Effect.catch(() => Effect.succeed([])), + ), + { concurrency: 8 }, + ) + yield* FileRetention.cleanup(fs, files.flat(), RETENTION) +}) + +const cleanupLayer = Layer.effectDiscard( + cleanup().pipe(Effect.repeat(Schedule.spaced(Duration.hours(1))), Effect.forkScoped), +) + +const cleanupNode = makeGlobalNode({ + name: "shell-output-cleanup", + layer: cleanupLayer, + deps: [FSUtil.node, Global.node], +}) + const layer = () => Layer.effect( Service, @@ -83,7 +123,7 @@ const layer = () => const sessions = new Map() const exitOrder: string[] = [] - const outputDir = path.join(global.data, "shell", location.project.id) + const outputDir = path.join(global.data, DIRECTORY, location.project.id) const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) @@ -358,5 +398,6 @@ export const node = makeLocationNode({ Environment.node, PluginHooks.node, SessionEnvironment.node, + cleanupNode, ], }) diff --git a/packages/core/src/tool-output.ts b/packages/core/src/tool-output.ts index 17fe252dff..b2a29c2431 100644 --- a/packages/core/src/tool-output.ts +++ b/packages/core/src/tool-output.ts @@ -2,10 +2,11 @@ export * as ToolOutput from "./tool-output.js" import path from "path" import type { Tool } from "@opencode-ai/schema/tool" -import { Context, Duration, Effect, Layer, Option, Schedule } from "effect" +import { Context, Duration, Effect, Layer, Schedule } from "effect" import { makeGlobalNode, makeLocationNode } from "@opencode-ai/util/effect/app-node" import { FSUtil } from "@opencode-ai/util/fs-util" import { Global } from "@opencode-ai/util/global" +import { FileRetention } from "./file-retention.js" import { Identifier } from "./id/id.js" import { State } from "./state.js" @@ -33,22 +34,14 @@ export interface Interface extends State.Transformable { export class Service extends Context.Service()("@opencode/ToolOutput") {} const cleanup = Effect.fn("ToolOutput.cleanup")(function* (fs: FSUtil.Interface, directory: string) { - const cutoff = Date.now() - Duration.toMillis(RETENTION) const entries = yield* fs.readDirectory(directory).pipe( Effect.map((entries) => entries.filter((entry) => /^tool_[0-9a-f]{12}/.test(entry))), Effect.catch(() => Effect.succeed([])), ) - yield* Effect.forEach( - entries, - (entry) => - Effect.gen(function* () { - const file = path.join(directory, entry) - const info = yield* fs.stat(file).pipe(Effect.catch(() => Effect.succeed(undefined))) - const mtime = info && Option.getOrUndefined(info.mtime) - if (!mtime || mtime.getTime() >= cutoff) return - yield* fs.remove(file).pipe(Effect.catch(() => Effect.void)) - }), - { concurrency: 8, discard: true }, + yield* FileRetention.cleanup( + fs, + entries.map((entry) => path.join(directory, entry)), + RETENTION, ) }) diff --git a/packages/core/test/shell-cleanup.test.ts b/packages/core/test/shell-cleanup.test.ts new file mode 100644 index 0000000000..41804a25e0 --- /dev/null +++ b/packages/core/test/shell-cleanup.test.ts @@ -0,0 +1,53 @@ +import { describe, expect } from "bun:test" +import path from "path" +import { Effect } from "effect" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { Shell } from "@opencode-ai/core/shell" +import { LayerNode } from "@opencode-ai/util/effect/layer-node" +import { FSUtil } from "@opencode-ai/util/fs-util" +import { Global } from "@opencode-ai/util/global" +import { tmpdir } from "./fixture/tmpdir" +import { it } from "./lib/effect" + +const withStore = (body: (fs: FSUtil.Interface, root: string) => Effect.Effect) => + Effect.acquireUseRelease( + Effect.promise(() => tmpdir()), + (tmp) => { + const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [ + [Global.node, Global.layerWith({ data: tmp.path })], + ]) + return Effect.gen(function* () { + const fs = yield* FSUtil.Service + return yield* body(fs, tmp.path) + }).pipe(Effect.provide(layer)) + }, + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + +describe("Shell cleanup", () => { + it.live("removes expired output files across projects", () => + withStore((fs, root) => + Effect.gen(function* () { + const first = path.join(root, Shell.DIRECTORY, "first") + const second = path.join(root, Shell.DIRECTORY, "second") + const old = path.join(first, "sh_0123456789abABCDEFGHIJKLMN.out") + const recent = path.join(second, "sh_0123456789abNOPQRSTUVWXYZ0.out") + const unrelated = path.join(first, "notes.out") + yield* fs.ensureDir(first) + yield* fs.ensureDir(second) + yield* fs.writeFileString(old, "old") + yield* fs.writeFileString(recent, "recent") + yield* fs.writeFileString(unrelated, "unrelated") + const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) + yield* fs.utimes(old, new Date(), expired) + yield* fs.utimes(unrelated, new Date(), expired) + + yield* Shell.cleanup() + + expect(yield* fs.exists(old)).toBe(false) + expect(yield* fs.exists(recent)).toBe(true) + expect(yield* fs.exists(unrelated)).toBe(true) + }), + ), + ) +})