core: migrate from custom JSON storage to standard Drizzle migrations to improve database reliability and performance

This replaces the previous manual JSON file system with standard Drizzle migrations, enabling:
- Proper database schema migrations with timestamp-based versioning
- Batched migration for faster migration of large datasets
- Better data integrity with proper table schemas instead of JSON blobs
- Easier database upgrades and rollback capabilities

Migration changes:
- Todo table now uses individual columns with composite PK instead of JSON blob
- Share table removes unused download share data
- Session diff table moved from database table to file storage
- All migrations now use proper Drizzle format with per-folder layout

Users will see a one-time migration on next run that migrates existing JSON data to the new SQLite database.
This commit is contained in:
Dax Raad
2026-01-27 12:36:05 -05:00
parent 5e1639de2b
commit a48a5a3462
26 changed files with 1499 additions and 1407 deletions
+7 -21
View File
@@ -11,8 +11,8 @@ import { Identifier } from "../id/id"
import { Installation } from "../installation"
import { Database, NotFoundError, eq } from "../storage/db"
import { SessionTable, MessageTable, PartTable, SessionDiffTable } from "./session.sql"
import { ShareTable } from "../share/share.sql"
import { SessionTable, MessageTable, PartTable } from "./session.sql"
import { Storage } from "@/storage/storage"
import { Log } from "../util/log"
import { MessageV2 } from "./message-v2"
import { Instance } from "../project/instance"
@@ -153,16 +153,6 @@ export namespace Session {
})
export type Info = z.output<typeof Info>
export const ShareInfo = z
.object({
secret: z.string(),
url: z.string(),
})
.meta({
ref: "SessionShare",
})
export type ShareInfo = z.output<typeof ShareInfo>
export const Event = {
Created: BusEvent.define(
"session.created",
@@ -323,11 +313,6 @@ export namespace Session {
return fromRow(row)
})
export const getShare = fn(Identifier.schema("session"), async (id) => {
const row = Database.use((db) => db.select().from(ShareTable).where(eq(ShareTable.session_id, id)).get())
return row?.data
})
export const share = fn(Identifier.schema("session"), async (id) => {
const cfg = await Config.get()
if (cfg.share === "disabled") {
@@ -498,10 +483,11 @@ export namespace Session {
)
export const diff = fn(Identifier.schema("session"), async (sessionID) => {
const row = Database.use((db) =>
db.select().from(SessionDiffTable).where(eq(SessionDiffTable.session_id, sessionID)).get(),
)
return row?.data ?? []
try {
return await Storage.read<Snapshot.FileDiff[]>(["session_diff", sessionID])
} catch {
return []
}
})
export const messages = fn(
+3 -8
View File
@@ -5,7 +5,8 @@ import { MessageV2 } from "./message-v2"
import { Session } from "."
import { Log } from "../util/log"
import { Database, eq } from "../storage/db"
import { SessionDiffTable, MessageTable, PartTable } from "./session.sql"
import { MessageTable, PartTable } from "./session.sql"
import { Storage } from "@/storage/storage"
import { Bus } from "../bus"
import { SessionPrompt } from "./prompt"
import { SessionSummary } from "./summary"
@@ -60,13 +61,7 @@ export namespace SessionRevert {
if (revert.snapshot) revert.diff = await Snapshot.diff(revert.snapshot)
const rangeMessages = all.filter((msg) => msg.info.id >= revert!.messageID)
const diffs = await SessionSummary.computeDiff({ messages: rangeMessages })
Database.use((db) =>
db
.insert(SessionDiffTable)
.values({ session_id: input.sessionID, data: diffs })
.onConflictDoUpdate({ target: SessionDiffTable.session_id, set: { data: diffs } })
.run(),
)
await Storage.write(["session_diff", input.sessionID], diffs)
Bus.publish(Session.Event.Diff, {
sessionID: input.sessionID,
diff: diffs,
+15 -15
View File
@@ -1,8 +1,7 @@
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core"
import { sqliteTable, text, integer, index, primaryKey } from "drizzle-orm/sqlite-core"
import { ProjectTable } from "../project/project.sql"
import type { MessageV2 } from "./message-v2"
import type { Snapshot } from "@/snapshot"
import type { Todo } from "./todo"
import type { PermissionNext } from "@/permission/next"
export const SessionTable = sqliteTable(
@@ -61,19 +60,20 @@ export const PartTable = sqliteTable(
(table) => [index("part_message_idx").on(table.message_id), index("part_session_idx").on(table.session_id)],
)
export const SessionDiffTable = sqliteTable("session_diff", {
session_id: text()
.primaryKey()
.references(() => SessionTable.id, { onDelete: "cascade" }),
data: text({ mode: "json" }).notNull().$type<Snapshot.FileDiff[]>(),
})
export const TodoTable = sqliteTable("todo", {
session_id: text()
.primaryKey()
.references(() => SessionTable.id, { onDelete: "cascade" }),
data: text({ mode: "json" }).notNull().$type<Todo.Info[]>(),
})
export const TodoTable = sqliteTable(
"todo",
{
session_id: text()
.notNull()
.references(() => SessionTable.id, { onDelete: "cascade" }),
id: text().notNull(),
content: text().notNull(),
status: text().notNull(),
priority: text().notNull(),
position: integer().notNull(),
},
(table) => [primaryKey({ columns: [table.session_id, table.id] }), index("todo_session_idx").on(table.session_id)],
)
export const PermissionTable = sqliteTable("permission", {
project_id: text()
+7 -13
View File
@@ -11,8 +11,7 @@ import { Snapshot } from "@/snapshot"
import { Log } from "@/util/log"
import path from "path"
import { Instance } from "@/project/instance"
import { Database, eq } from "@/storage/db"
import { SessionDiffTable } from "./session.sql"
import { Storage } from "@/storage/storage"
import { Bus } from "@/bus"
import { LLM } from "./llm"
@@ -56,13 +55,7 @@ export namespace SessionSummary {
files: diffs.length,
},
})
Database.use((db) =>
db
.insert(SessionDiffTable)
.values({ session_id: input.sessionID, data: diffs })
.onConflictDoUpdate({ target: SessionDiffTable.session_id, set: { data: diffs } })
.run(),
)
await Storage.write(["session_diff", input.sessionID], diffs)
Bus.publish(Session.Event.Diff, {
sessionID: input.sessionID,
diff: diffs,
@@ -124,10 +117,11 @@ export namespace SessionSummary {
messageID: Identifier.schema("message").optional(),
}),
async (input) => {
const row = Database.use((db) =>
db.select().from(SessionDiffTable).where(eq(SessionDiffTable.session_id, input.sessionID)).get(),
)
return row?.data ?? []
try {
return await Storage.read<Snapshot.FileDiff[]>(["session_diff", input.sessionID])
} catch {
return []
}
},
)
+26 -10
View File
@@ -1,7 +1,7 @@
import { BusEvent } from "@/bus/bus-event"
import { Bus } from "@/bus"
import z from "zod"
import { Database, eq } from "../storage/db"
import { Database, eq, asc } from "../storage/db"
import { TodoTable } from "./session.sql"
export namespace Todo {
@@ -26,18 +26,34 @@ export namespace Todo {
}
export function update(input: { sessionID: string; todos: Info[] }) {
Database.use((db) =>
db
.insert(TodoTable)
.values({ session_id: input.sessionID, data: input.todos })
.onConflictDoUpdate({ target: TodoTable.session_id, set: { data: input.todos } })
.run(),
)
Database.transaction((db) => {
db.delete(TodoTable).where(eq(TodoTable.session_id, input.sessionID)).run()
if (input.todos.length === 0) return
db.insert(TodoTable)
.values(
input.todos.map((todo, position) => ({
session_id: input.sessionID,
id: todo.id,
content: todo.content,
status: todo.status,
priority: todo.priority,
position,
})),
)
.run()
})
Bus.publish(Event.Updated, input)
}
export function get(sessionID: string) {
const row = Database.use((db) => db.select().from(TodoTable).where(eq(TodoTable.session_id, sessionID)).get())
return row?.data ?? []
const rows = Database.use((db) =>
db.select().from(TodoTable).where(eq(TodoTable.session_id, sessionID)).orderBy(asc(TodoTable.position)).all(),
)
return rows.map((row) => ({
id: row.id,
content: row.content,
status: row.status,
priority: row.priority,
}))
}
}