core: migrate from custom JSON storage to standard Drizzle migrations to improve database reliability and performance
This replaces the previous manual JSON file system with standard Drizzle migrations, enabling: - Proper database schema migrations with timestamp-based versioning - Batched migration for faster migration of large datasets - Better data integrity with proper table schemas instead of JSON blobs - Easier database upgrades and rollback capabilities Migration changes: - Todo table now uses individual columns with composite PK instead of JSON blob - Share table removes unused download share data - Session diff table moved from database table to file storage - All migrations now use proper Drizzle format with per-folder layout Users will see a one-time migration on next run that migrates existing JSON data to the new SQLite database.
This commit is contained in:
@@ -11,8 +11,8 @@ import { Identifier } from "../id/id"
|
||||
import { Installation } from "../installation"
|
||||
|
||||
import { Database, NotFoundError, eq } from "../storage/db"
|
||||
import { SessionTable, MessageTable, PartTable, SessionDiffTable } from "./session.sql"
|
||||
import { ShareTable } from "../share/share.sql"
|
||||
import { SessionTable, MessageTable, PartTable } from "./session.sql"
|
||||
import { Storage } from "@/storage/storage"
|
||||
import { Log } from "../util/log"
|
||||
import { MessageV2 } from "./message-v2"
|
||||
import { Instance } from "../project/instance"
|
||||
@@ -153,16 +153,6 @@ export namespace Session {
|
||||
})
|
||||
export type Info = z.output<typeof Info>
|
||||
|
||||
export const ShareInfo = z
|
||||
.object({
|
||||
secret: z.string(),
|
||||
url: z.string(),
|
||||
})
|
||||
.meta({
|
||||
ref: "SessionShare",
|
||||
})
|
||||
export type ShareInfo = z.output<typeof ShareInfo>
|
||||
|
||||
export const Event = {
|
||||
Created: BusEvent.define(
|
||||
"session.created",
|
||||
@@ -323,11 +313,6 @@ export namespace Session {
|
||||
return fromRow(row)
|
||||
})
|
||||
|
||||
export const getShare = fn(Identifier.schema("session"), async (id) => {
|
||||
const row = Database.use((db) => db.select().from(ShareTable).where(eq(ShareTable.session_id, id)).get())
|
||||
return row?.data
|
||||
})
|
||||
|
||||
export const share = fn(Identifier.schema("session"), async (id) => {
|
||||
const cfg = await Config.get()
|
||||
if (cfg.share === "disabled") {
|
||||
@@ -498,10 +483,11 @@ export namespace Session {
|
||||
)
|
||||
|
||||
export const diff = fn(Identifier.schema("session"), async (sessionID) => {
|
||||
const row = Database.use((db) =>
|
||||
db.select().from(SessionDiffTable).where(eq(SessionDiffTable.session_id, sessionID)).get(),
|
||||
)
|
||||
return row?.data ?? []
|
||||
try {
|
||||
return await Storage.read<Snapshot.FileDiff[]>(["session_diff", sessionID])
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
})
|
||||
|
||||
export const messages = fn(
|
||||
|
||||
@@ -5,7 +5,8 @@ import { MessageV2 } from "./message-v2"
|
||||
import { Session } from "."
|
||||
import { Log } from "../util/log"
|
||||
import { Database, eq } from "../storage/db"
|
||||
import { SessionDiffTable, MessageTable, PartTable } from "./session.sql"
|
||||
import { MessageTable, PartTable } from "./session.sql"
|
||||
import { Storage } from "@/storage/storage"
|
||||
import { Bus } from "../bus"
|
||||
import { SessionPrompt } from "./prompt"
|
||||
import { SessionSummary } from "./summary"
|
||||
@@ -60,13 +61,7 @@ export namespace SessionRevert {
|
||||
if (revert.snapshot) revert.diff = await Snapshot.diff(revert.snapshot)
|
||||
const rangeMessages = all.filter((msg) => msg.info.id >= revert!.messageID)
|
||||
const diffs = await SessionSummary.computeDiff({ messages: rangeMessages })
|
||||
Database.use((db) =>
|
||||
db
|
||||
.insert(SessionDiffTable)
|
||||
.values({ session_id: input.sessionID, data: diffs })
|
||||
.onConflictDoUpdate({ target: SessionDiffTable.session_id, set: { data: diffs } })
|
||||
.run(),
|
||||
)
|
||||
await Storage.write(["session_diff", input.sessionID], diffs)
|
||||
Bus.publish(Session.Event.Diff, {
|
||||
sessionID: input.sessionID,
|
||||
diff: diffs,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core"
|
||||
import { sqliteTable, text, integer, index, primaryKey } from "drizzle-orm/sqlite-core"
|
||||
import { ProjectTable } from "../project/project.sql"
|
||||
import type { MessageV2 } from "./message-v2"
|
||||
import type { Snapshot } from "@/snapshot"
|
||||
import type { Todo } from "./todo"
|
||||
import type { PermissionNext } from "@/permission/next"
|
||||
|
||||
export const SessionTable = sqliteTable(
|
||||
@@ -61,19 +60,20 @@ export const PartTable = sqliteTable(
|
||||
(table) => [index("part_message_idx").on(table.message_id), index("part_session_idx").on(table.session_id)],
|
||||
)
|
||||
|
||||
export const SessionDiffTable = sqliteTable("session_diff", {
|
||||
session_id: text()
|
||||
.primaryKey()
|
||||
.references(() => SessionTable.id, { onDelete: "cascade" }),
|
||||
data: text({ mode: "json" }).notNull().$type<Snapshot.FileDiff[]>(),
|
||||
})
|
||||
|
||||
export const TodoTable = sqliteTable("todo", {
|
||||
session_id: text()
|
||||
.primaryKey()
|
||||
.references(() => SessionTable.id, { onDelete: "cascade" }),
|
||||
data: text({ mode: "json" }).notNull().$type<Todo.Info[]>(),
|
||||
})
|
||||
export const TodoTable = sqliteTable(
|
||||
"todo",
|
||||
{
|
||||
session_id: text()
|
||||
.notNull()
|
||||
.references(() => SessionTable.id, { onDelete: "cascade" }),
|
||||
id: text().notNull(),
|
||||
content: text().notNull(),
|
||||
status: text().notNull(),
|
||||
priority: text().notNull(),
|
||||
position: integer().notNull(),
|
||||
},
|
||||
(table) => [primaryKey({ columns: [table.session_id, table.id] }), index("todo_session_idx").on(table.session_id)],
|
||||
)
|
||||
|
||||
export const PermissionTable = sqliteTable("permission", {
|
||||
project_id: text()
|
||||
|
||||
@@ -11,8 +11,7 @@ import { Snapshot } from "@/snapshot"
|
||||
import { Log } from "@/util/log"
|
||||
import path from "path"
|
||||
import { Instance } from "@/project/instance"
|
||||
import { Database, eq } from "@/storage/db"
|
||||
import { SessionDiffTable } from "./session.sql"
|
||||
import { Storage } from "@/storage/storage"
|
||||
import { Bus } from "@/bus"
|
||||
|
||||
import { LLM } from "./llm"
|
||||
@@ -56,13 +55,7 @@ export namespace SessionSummary {
|
||||
files: diffs.length,
|
||||
},
|
||||
})
|
||||
Database.use((db) =>
|
||||
db
|
||||
.insert(SessionDiffTable)
|
||||
.values({ session_id: input.sessionID, data: diffs })
|
||||
.onConflictDoUpdate({ target: SessionDiffTable.session_id, set: { data: diffs } })
|
||||
.run(),
|
||||
)
|
||||
await Storage.write(["session_diff", input.sessionID], diffs)
|
||||
Bus.publish(Session.Event.Diff, {
|
||||
sessionID: input.sessionID,
|
||||
diff: diffs,
|
||||
@@ -124,10 +117,11 @@ export namespace SessionSummary {
|
||||
messageID: Identifier.schema("message").optional(),
|
||||
}),
|
||||
async (input) => {
|
||||
const row = Database.use((db) =>
|
||||
db.select().from(SessionDiffTable).where(eq(SessionDiffTable.session_id, input.sessionID)).get(),
|
||||
)
|
||||
return row?.data ?? []
|
||||
try {
|
||||
return await Storage.read<Snapshot.FileDiff[]>(["session_diff", input.sessionID])
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { BusEvent } from "@/bus/bus-event"
|
||||
import { Bus } from "@/bus"
|
||||
import z from "zod"
|
||||
import { Database, eq } from "../storage/db"
|
||||
import { Database, eq, asc } from "../storage/db"
|
||||
import { TodoTable } from "./session.sql"
|
||||
|
||||
export namespace Todo {
|
||||
@@ -26,18 +26,34 @@ export namespace Todo {
|
||||
}
|
||||
|
||||
export function update(input: { sessionID: string; todos: Info[] }) {
|
||||
Database.use((db) =>
|
||||
db
|
||||
.insert(TodoTable)
|
||||
.values({ session_id: input.sessionID, data: input.todos })
|
||||
.onConflictDoUpdate({ target: TodoTable.session_id, set: { data: input.todos } })
|
||||
.run(),
|
||||
)
|
||||
Database.transaction((db) => {
|
||||
db.delete(TodoTable).where(eq(TodoTable.session_id, input.sessionID)).run()
|
||||
if (input.todos.length === 0) return
|
||||
db.insert(TodoTable)
|
||||
.values(
|
||||
input.todos.map((todo, position) => ({
|
||||
session_id: input.sessionID,
|
||||
id: todo.id,
|
||||
content: todo.content,
|
||||
status: todo.status,
|
||||
priority: todo.priority,
|
||||
position,
|
||||
})),
|
||||
)
|
||||
.run()
|
||||
})
|
||||
Bus.publish(Event.Updated, input)
|
||||
}
|
||||
|
||||
export function get(sessionID: string) {
|
||||
const row = Database.use((db) => db.select().from(TodoTable).where(eq(TodoTable.session_id, sessionID)).get())
|
||||
return row?.data ?? []
|
||||
const rows = Database.use((db) =>
|
||||
db.select().from(TodoTable).where(eq(TodoTable.session_id, sessionID)).orderBy(asc(TodoTable.position)).all(),
|
||||
)
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
content: row.content,
|
||||
status: row.status,
|
||||
priority: row.priority,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user