Compare commits

...

3 Commits

Author SHA1 Message Date
starptech 54a2dd9235 fix(core): guard model-data migration on valid JSON 2026-06-03 16:50:43 +02:00
starptech 0104216d17 chore: Remove local install script 2026-06-03 16:27:51 +02:00
starptech fe28f3b1dd fix(core): reduce memory peaks in long sessions 2026-06-03 16:26:06 +02:00
16 changed files with 2229 additions and 54 deletions
@@ -0,0 +1,9 @@
ALTER TABLE `part` ADD `data_model` text;
--> statement-breakpoint
UPDATE part
SET data_model = json_remove(data, '$.state.metadata')
WHERE json_valid(data)
AND length(CAST(data AS BLOB)) > 65536
AND json_extract(data, '$.type') = 'tool'
AND json_extract(data, '$.state.status') = 'completed'
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536;
@@ -0,0 +1,1755 @@
{
"version": "7",
"dialect": "sqlite",
"id": "274148ba-cf62-4392-83e5-0235d1c4f043",
"prevIds": [
"80f2378a-ed35-45cb-9d3b-9f4837fac801"
],
"ddl": [
{
"name": "workspace",
"entityType": "tables"
},
{
"name": "data_migration",
"entityType": "tables"
},
{
"name": "account_state",
"entityType": "tables"
},
{
"name": "account",
"entityType": "tables"
},
{
"name": "control_account",
"entityType": "tables"
},
{
"name": "event_sequence",
"entityType": "tables"
},
{
"name": "event",
"entityType": "tables"
},
{
"name": "permission",
"entityType": "tables"
},
{
"name": "project_directory",
"entityType": "tables"
},
{
"name": "project",
"entityType": "tables"
},
{
"name": "message",
"entityType": "tables"
},
{
"name": "part",
"entityType": "tables"
},
{
"name": "session_message",
"entityType": "tables"
},
{
"name": "session",
"entityType": "tables"
},
{
"name": "todo",
"entityType": "tables"
},
{
"name": "session_share",
"entityType": "tables"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "type",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": "''",
"generated": null,
"name": "name",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "branch",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "directory",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "extra",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "project_id",
"entityType": "columns",
"table": "workspace"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_used",
"entityType": "columns",
"table": "workspace"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "name",
"entityType": "columns",
"table": "data_migration"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_completed",
"entityType": "columns",
"table": "data_migration"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "account_state"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "active_account_id",
"entityType": "columns",
"table": "account_state"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "active_org_id",
"entityType": "columns",
"table": "account_state"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "email",
"entityType": "columns",
"table": "account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "url",
"entityType": "columns",
"table": "account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "access_token",
"entityType": "columns",
"table": "account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "refresh_token",
"entityType": "columns",
"table": "account"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "token_expiry",
"entityType": "columns",
"table": "account"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "account"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "email",
"entityType": "columns",
"table": "control_account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "url",
"entityType": "columns",
"table": "control_account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "access_token",
"entityType": "columns",
"table": "control_account"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "refresh_token",
"entityType": "columns",
"table": "control_account"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "token_expiry",
"entityType": "columns",
"table": "control_account"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "active",
"entityType": "columns",
"table": "control_account"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "control_account"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "control_account"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "aggregate_id",
"entityType": "columns",
"table": "event_sequence"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "seq",
"entityType": "columns",
"table": "event_sequence"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "owner_id",
"entityType": "columns",
"table": "event_sequence"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "event"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "aggregate_id",
"entityType": "columns",
"table": "event"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "seq",
"entityType": "columns",
"table": "event"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "type",
"entityType": "columns",
"table": "event"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data",
"entityType": "columns",
"table": "event"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "permission"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "project_id",
"entityType": "columns",
"table": "permission"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "action",
"entityType": "columns",
"table": "permission"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "resource",
"entityType": "columns",
"table": "permission"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "permission"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "permission"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "project_id",
"entityType": "columns",
"table": "project_directory"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "directory",
"entityType": "columns",
"table": "project_directory"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "type",
"entityType": "columns",
"table": "project_directory"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "project_directory"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "worktree",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "vcs",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "name",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "icon_url",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "icon_url_override",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "icon_color",
"entityType": "columns",
"table": "project"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "project"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "project"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_initialized",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "sandboxes",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "commands",
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "message"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "message"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "message"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "message"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data",
"entityType": "columns",
"table": "message"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "part"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "message_id",
"entityType": "columns",
"table": "part"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "part"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "part"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "part"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data",
"entityType": "columns",
"table": "part"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data_model",
"entityType": "columns",
"table": "part"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "session_message"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "session_message"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "type",
"entityType": "columns",
"table": "session_message"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "session_message"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "session_message"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data",
"entityType": "columns",
"table": "session_message"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "project_id",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "workspace_id",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "parent_id",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "slug",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "directory",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "path",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "title",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "version",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "share_url",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "summary_additions",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "summary_deletions",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "summary_files",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "summary_diffs",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "metadata",
"entityType": "columns",
"table": "session"
},
{
"type": "real",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "cost",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "tokens_input",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "tokens_output",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "tokens_reasoning",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "tokens_cache_read",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": "0",
"generated": null,
"name": "tokens_cache_write",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "revert",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "permission",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "agent",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "model",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_compacting",
"entityType": "columns",
"table": "session"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_archived",
"entityType": "columns",
"table": "session"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "todo"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "content",
"entityType": "columns",
"table": "todo"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "status",
"entityType": "columns",
"table": "todo"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "priority",
"entityType": "columns",
"table": "todo"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "position",
"entityType": "columns",
"table": "todo"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "todo"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "todo"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "session_share"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "session_share"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "secret",
"entityType": "columns",
"table": "session_share"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "url",
"entityType": "columns",
"table": "session_share"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "session_share"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_updated",
"entityType": "columns",
"table": "session_share"
},
{
"columns": [
"project_id"
],
"tableTo": "project",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_workspace_project_id_project_id_fk",
"entityType": "fks",
"table": "workspace"
},
{
"columns": [
"active_account_id"
],
"tableTo": "account",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "SET NULL",
"nameExplicit": false,
"name": "fk_account_state_active_account_id_account_id_fk",
"entityType": "fks",
"table": "account_state"
},
{
"columns": [
"aggregate_id"
],
"tableTo": "event_sequence",
"columnsTo": [
"aggregate_id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_event_aggregate_id_event_sequence_aggregate_id_fk",
"entityType": "fks",
"table": "event"
},
{
"columns": [
"project_id"
],
"tableTo": "project",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_permission_project_id_project_id_fk",
"entityType": "fks",
"table": "permission"
},
{
"columns": [
"project_id"
],
"tableTo": "project",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_project_directory_project_id_project_id_fk",
"entityType": "fks",
"table": "project_directory"
},
{
"columns": [
"session_id"
],
"tableTo": "session",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_message_session_id_session_id_fk",
"entityType": "fks",
"table": "message"
},
{
"columns": [
"message_id"
],
"tableTo": "message",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_part_message_id_message_id_fk",
"entityType": "fks",
"table": "part"
},
{
"columns": [
"session_id"
],
"tableTo": "session",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_session_message_session_id_session_id_fk",
"entityType": "fks",
"table": "session_message"
},
{
"columns": [
"project_id"
],
"tableTo": "project",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_session_project_id_project_id_fk",
"entityType": "fks",
"table": "session"
},
{
"columns": [
"session_id"
],
"tableTo": "session",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_todo_session_id_session_id_fk",
"entityType": "fks",
"table": "todo"
},
{
"columns": [
"session_id"
],
"tableTo": "session",
"columnsTo": [
"id"
],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_session_share_session_id_session_id_fk",
"entityType": "fks",
"table": "session_share"
},
{
"columns": [
"email",
"url"
],
"nameExplicit": false,
"name": "control_account_pk",
"entityType": "pks",
"table": "control_account"
},
{
"columns": [
"project_id",
"directory"
],
"nameExplicit": false,
"name": "project_directory_pk",
"entityType": "pks",
"table": "project_directory"
},
{
"columns": [
"session_id",
"position"
],
"nameExplicit": false,
"name": "todo_pk",
"entityType": "pks",
"table": "todo"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "workspace_pk",
"table": "workspace",
"entityType": "pks"
},
{
"columns": [
"name"
],
"nameExplicit": false,
"name": "data_migration_pk",
"table": "data_migration",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "account_state_pk",
"table": "account_state",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "account_pk",
"table": "account",
"entityType": "pks"
},
{
"columns": [
"aggregate_id"
],
"nameExplicit": false,
"name": "event_sequence_pk",
"table": "event_sequence",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "event_pk",
"table": "event",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "permission_pk",
"table": "permission",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "project_pk",
"table": "project",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "message_pk",
"table": "message",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "part_pk",
"table": "part",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "session_message_pk",
"table": "session_message",
"entityType": "pks"
},
{
"columns": [
"id"
],
"nameExplicit": false,
"name": "session_pk",
"table": "session",
"entityType": "pks"
},
{
"columns": [
"session_id"
],
"nameExplicit": false,
"name": "session_share_pk",
"table": "session_share",
"entityType": "pks"
},
{
"columns": [
{
"value": "project_id",
"isExpression": false
},
{
"value": "action",
"isExpression": false
},
{
"value": "resource",
"isExpression": false
}
],
"isUnique": true,
"where": null,
"origin": "manual",
"name": "permission_project_action_resource_idx",
"entityType": "indexes",
"table": "permission"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
},
{
"value": "time_created",
"isExpression": false
},
{
"value": "id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "message_session_time_created_id_idx",
"entityType": "indexes",
"table": "message"
},
{
"columns": [
{
"value": "message_id",
"isExpression": false
},
{
"value": "id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "part_message_id_id_idx",
"entityType": "indexes",
"table": "part"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "part_session_idx",
"entityType": "indexes",
"table": "part"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_message_session_idx",
"entityType": "indexes",
"table": "session_message"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
},
{
"value": "type",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_message_session_type_idx",
"entityType": "indexes",
"table": "session_message"
},
{
"columns": [
{
"value": "time_created",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_message_time_created_idx",
"entityType": "indexes",
"table": "session_message"
},
{
"columns": [
{
"value": "project_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_project_idx",
"entityType": "indexes",
"table": "session"
},
{
"columns": [
{
"value": "workspace_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_workspace_idx",
"entityType": "indexes",
"table": "session"
},
{
"columns": [
{
"value": "parent_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_parent_idx",
"entityType": "indexes",
"table": "session"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "todo_session_idx",
"entityType": "indexes",
"table": "todo"
}
],
"renames": []
}
+5 -1
View File
@@ -8,12 +8,15 @@ import { Flag } from "../flag/flag"
import { isAbsolute, join } from "path"
import { DatabaseMigration } from "./migration"
import { InstallationChannel } from "../installation/version"
import { Sqlite } from "./sqlite"
const makeDatabase = EffectDrizzleSqlite.makeWithDefaults()
type DatabaseShape = Effect.Success<typeof makeDatabase>
export interface Interface {
db: DatabaseShape
// Lazy property getters cannot yield an Effect.
sync: Sqlite.DrizzleClient
}
export class Service extends Context.Service<Service, Interface>()("@opencode/v2/storage/Database") {}
@@ -22,6 +25,7 @@ export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const db = yield* makeDatabase
const sync = yield* Sqlite.Drizzle
yield* db.run("PRAGMA journal_mode = WAL")
yield* db.run("PRAGMA synchronous = NORMAL")
@@ -31,7 +35,7 @@ export const layer = Layer.effect(
yield* db.run("PRAGMA wal_checkpoint(PASSIVE)")
yield* DatabaseMigration.apply(db)
return { db }
return { db, sync }
}).pipe(Effect.orDie),
)
+1
View File
@@ -27,5 +27,6 @@ export const migrations = (
import("./migration/20260601202201_amazing_prowler"),
import("./migration/20260602002951_lowly_union_jack"),
import("./migration/20260602182828_add_project_directories"),
import("./migration/20260603120017_warm_guardsmen"),
])
).map((module) => module.default) satisfies DatabaseMigration.Migration[]
@@ -0,0 +1,22 @@
import { Effect } from "effect"
import type { DatabaseMigration } from "../migration"
export default {
id: "20260603120017_warm_guardsmen",
up(tx) {
return Effect.gen(function* () {
yield* tx.run(`ALTER TABLE \`part\` ADD \`data_model\` text;`)
// Keep canonical history intact while avoiding prompt-time decoding. This
// one-time transactional backfill may briefly grow the WAL on large stores.
yield* tx.run(`
UPDATE part
SET data_model = json_remove(data, '$.state.metadata')
WHERE json_valid(data)
AND length(CAST(data AS BLOB)) > 65536
AND json_extract(data, '$.type') = 'tool'
AND json_extract(data, '$.state.status') = 'completed'
AND length(CAST(json_extract(data, '$.state.metadata') AS BLOB)) > 65536
`)
})
},
} satisfies DatabaseMigration.Migration
+26
View File
@@ -0,0 +1,26 @@
export * as SessionPartModelData from "./model-data"
import type { SessionV1 } from "../v1/session"
type V1PartData<Data extends SessionV1.Part = SessionV1.Part> = Data extends SessionV1.Part
? Omit<Data, "id" | "sessionID" | "messageID">
: never
export type ModelData = Omit<V1PartData<SessionV1.ToolPart>, "state"> & {
state: Omit<SessionV1.ToolStateCompleted, "metadata">
}
export const THRESHOLD = 64 * 1024
// Strip UI-only metadata only when the stored prompt projection benefits.
export function create(data: unknown): ModelData | null {
if (!data || typeof data !== "object") return null
if (!("type" in data) || data.type !== "tool") return null
if (!("state" in data) || !data.state || typeof data.state !== "object") return null
if (!("status" in data.state) || data.state.status !== "completed") return null
if (!("metadata" in data.state)) return null
const metadata = JSON.stringify(data.state.metadata)
if (!metadata || Buffer.byteLength(metadata) <= THRESHOLD) return null
const { metadata: _, ...state } = data.state
return { ...data, state } as ModelData
}
+12 -5
View File
@@ -11,6 +11,7 @@ import { SessionMessage } from "./message"
import { SessionMessageUpdater } from "./message-updater"
import { MessageTable, PartTable, SessionMessageTable, SessionTable } from "./sql"
import type { DeepMutable } from "../schema"
import { SessionPartModelData } from "./model-data"
type DatabaseService = Database.Interface["db"]
@@ -309,7 +310,7 @@ export const layer = Layer.effectDiscard(
yield* events.project(SessionV1.Event.MessageRemoved, (event) =>
Effect.gen(function* () {
const rows = yield* db
.select()
.select({ session_id: PartTable.session_id, data: PartTable.data })
.from(PartTable)
.where(and(eq(PartTable.message_id, event.data.messageID), eq(PartTable.session_id, event.data.sessionID)))
.all()
@@ -328,7 +329,7 @@ export const layer = Layer.effectDiscard(
yield* events.project(SessionV1.Event.PartRemoved, (event) =>
Effect.gen(function* () {
const row = yield* db
.select()
.select({ session_id: PartTable.session_id, data: PartTable.data })
.from(PartTable)
.where(and(eq(PartTable.id, event.data.partID), eq(PartTable.session_id, event.data.sessionID)))
.get()
@@ -348,11 +349,17 @@ export const layer = Layer.effectDiscard(
const messageID = event.data.part.messageID
const sessionID = event.data.part.sessionID
const data = partData(event.data.part)
const row = yield* db.select().from(PartTable).where(eq(PartTable.id, id)).get().pipe(Effect.orDie)
const data_model = SessionPartModelData.create(data)
const row = yield* db
.select({ session_id: PartTable.session_id, data: PartTable.data })
.from(PartTable)
.where(eq(PartTable.id, id))
.get()
.pipe(Effect.orDie)
yield* db
.insert(PartTable)
.values({ id, message_id: messageID, session_id: sessionID, time_created: event.data.time, data })
.onConflictDoUpdate({ target: PartTable.id, set: { data } })
.values({ id, message_id: messageID, session_id: sessionID, time_created: event.data.time, data, data_model })
.onConflictDoUpdate({ target: PartTable.id, set: { data, data_model } })
.run()
.pipe(Effect.orDie)
const previous = row && usage(row.data)
+3
View File
@@ -9,6 +9,7 @@ import type { SessionSchema } from "./schema"
import type { MessageID, PartID, SessionV1 } from "../v1/session"
import { WorkspaceV2 } from "../workspace"
import { Timestamps } from "../database/schema.sql"
import type { ModelData } from "./model-data"
type SessionMessageData = Omit<(typeof SessionMessage.Message)["Encoded"], "type" | "id">
type V1MessageData = Omit<SessionV1.Info, "id" | "sessionID">
@@ -85,6 +86,8 @@ export const PartTable = sqliteTable(
session_id: text().$type<SessionSchema.ID>().notNull(),
...Timestamps,
data: text({ mode: "json" }).notNull().$type<V1PartData>(),
// Derived prompt projection; data remains canonical.
data_model: text({ mode: "json" }).$type<ModelData>(),
},
(table) => [
index("part_message_id_id_idx").on(table.message_id, table.id),
+28 -1
View File
@@ -14,6 +14,7 @@ import { AbsolutePath } from "@opencode-ai/core/schema"
import { SessionSchema } from "@opencode-ai/core/session/schema"
import { SessionTable } from "@opencode-ai/core/session/sql"
import sessionMetadataMigration from "@opencode-ai/core/database/migration/20260511173437_session-metadata"
import partModelDataMigration from "@opencode-ai/core/database/migration/20260603120017_warm_guardsmen"
import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient"
const run = <A, E>(effect: Effect.Effect<A, E, SqlClientService>) =>
@@ -43,7 +44,7 @@ describe("DatabaseMigration", () => {
expect(yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session'`)).toEqual({
name: "session",
})
expect(yield* db.get(sql`SELECT count(*) as count FROM migration`)).toEqual({ count: 25 })
expect(yield* db.get(sql`SELECT count(*) as count FROM migration`)).toEqual({ count: 26 })
}),
)
})
@@ -77,6 +78,32 @@ describe("DatabaseMigration", () => {
)
})
test("backfills lightweight model data for oversized completed tool metadata", async () => {
await run(
Effect.gen(function* () {
const db = yield* makeDb
yield* db.run(sql`CREATE TABLE part (id text PRIMARY KEY, data text NOT NULL)`)
const large = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "x".repeat(70_000) } } })
const unicode = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "😀".repeat(20_000) } } })
const small = JSON.stringify({ type: "tool", state: { status: "completed", metadata: { diff: "small" } } })
const malformed = "{" + "x".repeat(70_000)
yield* db.run(sql`INSERT INTO part (id, data) VALUES (${"large"}, ${large}), (${"unicode"}, ${unicode}), (${"small"}, ${small}), (${"malformed"}, ${malformed})`)
yield* DatabaseMigration.applyOnly(db, [partModelDataMigration])
expect(yield* db.get(sql`SELECT data, data_model FROM part WHERE id = ${"large"}`)).toEqual({
data: large,
data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }),
})
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"small"}`)).toEqual({ data_model: null })
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"unicode"}`)).toEqual({
data_model: JSON.stringify({ type: "tool", state: { status: "completed" } }),
})
expect(yield* db.get(sql`SELECT data_model FROM part WHERE id = ${"malformed"}`)).toEqual({ data_model: null })
}),
)
})
test("normalizes Windows storage paths and leaves POSIX paths untouched", async () => {
await run(
Effect.gen(function* () {
+3
View File
@@ -12,6 +12,7 @@ import path from "path"
import { FSUtil } from "@opencode-ai/core/fs-util"
import { Effect, Schema } from "effect"
import type { InstanceContext } from "@/project/instance-context"
import { SessionPartModelData } from "@opencode-ai/core/session/model-data"
const decodeMessageInfo = Schema.decodeUnknownSync(SessionV1.Info)
const decodePart = Schema.decodeUnknownSync(SessionV1.Part)
@@ -212,6 +213,8 @@ const runImport = Effect.fn("Cli.import.body")(function* (file: string, ctx: Ins
message_id: messageID,
session_id: row.id,
data: partData,
// Keep imported history on the same prompt path as live updates.
data_model: SessionPartModelData.create(partData),
})
.onConflictDoNothing()
.run()
+1 -1
View File
@@ -402,7 +402,7 @@ export const layer = Layer.effect(
{ context: [], prompt: undefined },
)
const nextPrompt = compacting.prompt ?? buildPrompt({ previousSummary, context: compacting.context })
const msgs = structuredClone(selected.head)
const msgs = MessageV2.cloneForTransform(selected.head)
yield* plugin.trigger("experimental.chat.messages.transform", {}, { messages: msgs })
const modelMessages = yield* MessageV2.toModelMessagesEffect(msgs, model, {
stripMedia: true,
+177 -35
View File
@@ -29,6 +29,7 @@ import { eq } from "drizzle-orm"
import { inArray } from "drizzle-orm"
import { lt } from "drizzle-orm"
import { or } from "drizzle-orm"
import { sql } from "drizzle-orm"
import { MessageTable, PartTable, SessionTable } from "@opencode-ai/core/session/sql"
import { ProviderError } from "@/provider/error"
import { iife } from "@/util/iife"
@@ -47,6 +48,7 @@ interface FetchDecompressionError extends Error {
}
export const SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:"
const COMPACTED_TOOL_OUTPUT = "[Old tool result content cleared]"
export { isMedia }
function truncateToolOutput(text: string, maxChars?: number) {
@@ -96,7 +98,7 @@ const info = (row: typeof MessageTable.$inferSelect) =>
sessionID: row.session_id,
}) as Info
const part = (row: typeof PartTable.$inferSelect) =>
const part = (row: Pick<typeof PartTable.$inferSelect, "id" | "session_id" | "message_id" | "data">) =>
({
...row.data,
id: row.id,
@@ -107,20 +109,41 @@ const part = (row: typeof PartTable.$inferSelect) =>
const older = (row: Cursor) =>
or(lt(MessageTable.time_created, row.time), and(eq(MessageTable.time_created, row.time), lt(MessageTable.id, row.id)))
function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$inferSelect)[]) {
function hydrate(
db: Database.Interface["db"],
rows: (typeof MessageTable.$inferSelect)[],
sync?: Database.Interface["sync"],
) {
const ids = rows.map((row) => row.id)
const partByMessage = new Map<string, Part[]>()
return Effect.gen(function* () {
if (ids.length > 0) {
const partRows = yield* db
.select()
.from(PartTable)
.where(inArray(PartTable.message_id, ids))
.orderBy(PartTable.message_id, PartTable.id)
.all()
.pipe(Effect.orDie)
const partRows = sync
? yield* db
.select({
id: PartTable.id,
message_id: PartTable.message_id,
session_id: PartTable.session_id,
// Keep oversized UI metadata available to extensions without decoding it for every prompt.
data: sql`coalesce(${PartTable.data_model}, ${PartTable.data})`
.mapWith(PartTable.data)
.as("data"),
lazy_metadata: sql<number>`${PartTable.data_model} IS NOT NULL`.as("lazy_metadata"),
})
.from(PartTable)
.where(inArray(PartTable.message_id, ids))
.orderBy(PartTable.message_id, PartTable.id)
.all()
.pipe(Effect.orDie)
: yield* db
.select({ id: PartTable.id, message_id: PartTable.message_id, session_id: PartTable.session_id, data: PartTable.data })
.from(PartTable)
.where(inArray(PartTable.message_id, ids))
.orderBy(PartTable.message_id, PartTable.id)
.all()
.pipe(Effect.orDie)
for (const row of partRows) {
const next = part(row)
const next = "lazy_metadata" in row && row.lazy_metadata && sync ? lazyMetadata(sync, part(row)) : part(row)
const list = partByMessage.get(row.message_id)
if (list) list.push(next)
else partByMessage.set(row.message_id, [next])
@@ -134,6 +157,67 @@ function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$infer
})
}
function lazyMetadata(db: Database.Interface["sync"], part: Part) {
if (part.type !== "tool" || part.state.status !== "completed") return part
defineLazyMetadata(part.state, () => {
// Prompt history is short-lived. Resolve against canonical storage only when
// an extension explicitly reads metadata instead of decoding it every turn.
const row = db
.select({ metadata: sql<string>`json_extract(${PartTable.data}, '$.state.metadata')` })
.from(PartTable)
.where(eq(PartTable.id, part.id))
.get()
return row?.metadata ? JSON.parse(row.metadata) : {}
})
return part
}
function defineLazyMetadata(state: SessionV1.ToolStateCompleted, load: () => Record<string, any>) {
let metadata: Record<string, any> | undefined
Object.defineProperty(state, "metadata", {
configurable: true,
enumerable: true,
get() {
metadata ??= load()
return metadata
},
set(value: Record<string, any>) {
metadata = value
},
})
}
export function cloneForTransform(input: WithParts[]) {
// structuredClone resolves getters, so mask and restore lazy metadata.
const lazy = new Map<string, () => Record<string, any>>()
const masked = input.map((msg) => ({
...msg,
parts: msg.parts.map((part) => {
if (part.type !== "tool" || part.state.status !== "completed") return part
const load = Object.getOwnPropertyDescriptor(part.state, "metadata")?.get
if (!load) return part
lazy.set(part.id, () => structuredClone(load.call(part.state)))
return {
...part,
state: Object.fromEntries(
Object.keys(part.state)
.filter((key) => key !== "metadata")
.map((key) => [key, Reflect.get(part.state, key)]),
),
} as Part
}),
}))
const result = structuredClone(masked) as WithParts[]
for (const msg of result) {
for (const part of msg.parts) {
if (part.type !== "tool" || part.state.status !== "completed") continue
const load = lazy.get(part.id)
if (load) defineLazyMetadata(part.state, load)
}
}
return result
}
function providerMeta(metadata: Record<string, any> | undefined) {
if (!metadata) return undefined
const { providerExecuted: _, ...rest } = metadata
@@ -302,7 +386,7 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
toolNames.add(part.tool)
if (part.state.status === "completed") {
const outputText = part.state.time.compacted
? "[Old tool result content cleared]"
? COMPACTED_TOOL_OUTPUT
: truncateToolOutput(part.state.output, options?.toolOutputMaxChars)
const attachments = part.state.time.compacted || options?.stripMedia ? [] : (part.state.attachments ?? [])
@@ -433,12 +517,17 @@ export function toModelMessages(
return Effect.runPromise(toModelMessagesEffect(input, model, options).pipe(Effect.provide(EffectLogger.layer)))
}
export const page = Effect.fn("MessageV2.page")(function* (input: {
type PageInput = {
sessionID: SessionID
limit: number
before?: string
}) {
const { db } = yield* Database.Service
}
const pageWithOptions = Effect.fnUntraced(function* (
input: PageInput & { lazyCompletedToolMetadata?: boolean },
) {
const database = yield* Database.Service
const db = database.db
const before = input.before ? cursor.decode(input.before) : undefined
const where = before
? and(eq(MessageTable.session_id, input.sessionID), older(before))
@@ -467,7 +556,7 @@ export const page = Effect.fn("MessageV2.page")(function* (input: {
const more = rows.length > input.limit
const slice = more ? rows.slice(0, input.limit) : rows
const items = yield* hydrate(db, slice)
const items = yield* hydrate(db, slice, input.lazyCompletedToolMetadata ? database.sync : undefined)
items.reverse()
const tail = slice.at(-1)
return {
@@ -477,6 +566,10 @@ export const page = Effect.fn("MessageV2.page")(function* (input: {
}
})
export const page = Effect.fn("MessageV2.page")(function* (input: PageInput) {
return yield* pageWithOptions(input)
})
export function stream(sessionID: SessionID) {
const size = 50
return Effect.gen(function* () {
@@ -504,7 +597,7 @@ export function parts(messageID: MessageID) {
return Effect.gen(function* () {
const { db } = yield* Database.Service
const rows = yield* db
.select()
.select({ id: PartTable.id, message_id: PartTable.message_id, session_id: PartTable.session_id, data: PartTable.data })
.from(PartTable)
.where(eq(PartTable.message_id, messageID))
.orderBy(PartTable.id)
@@ -529,29 +622,57 @@ export const get = Effect.fn("MessageV2.get")(function* (input: { sessionID: Ses
}
})
export const related = Effect.fn("MessageV2.related")(function* (input: { sessionID: SessionID; messageID: MessageID }) {
const { db } = yield* Database.Service
return yield* db
.select()
.from(MessageTable)
.where(
and(
eq(MessageTable.session_id, input.sessionID),
or(
eq(MessageTable.id, input.messageID),
and(
sql`json_extract(${MessageTable.data}, '$.role') = 'assistant'`,
sql`json_extract(${MessageTable.data}, '$.parentID') = ${input.messageID}`,
),
),
),
)
.orderBy(MessageTable.time_created, MessageTable.id)
.all()
.pipe(Effect.flatMap((rows) => hydrate(db, rows)), Effect.orDie)
})
export function filterCompacted(msgs: Iterable<WithParts>) {
const result = [] as WithParts[]
const completed = new Set<string>()
let retain: MessageID | undefined
const state = compactedState()
for (const msg of msgs) {
result.push(msg)
if (retain) {
if (msg.info.id === retain) break
continue
}
if (msg.info.role === "user" && completed.has(msg.info.id)) {
const part = msg.parts.find((item): item is CompactionPart => item.type === "compaction")
if (!part) continue
if (!part.tail_start_id) break
retain = part.tail_start_id
if (msg.info.id === retain) break
continue
}
if (msg.info.role === "user" && completed.has(msg.info.id) && msg.parts.some((part) => part.type === "compaction"))
break
if (msg.info.role === "assistant" && msg.info.summary && msg.info.finish && !msg.info.error)
completed.add(msg.info.parentID)
if (reachedCompactedBoundary(state, msg)) break
}
return reorderCompacted(result)
}
function compactedState() {
return { completed: new Set<string>(), retain: undefined as MessageID | undefined }
}
function reachedCompactedBoundary(state: ReturnType<typeof compactedState>, msg: WithParts) {
if (state.retain) return msg.info.id === state.retain
if (msg.info.role === "user" && state.completed.has(msg.info.id)) {
const part = msg.parts.find((item): item is CompactionPart => item.type === "compaction")
if (!part) return false
if (!part.tail_start_id) return true
state.retain = part.tail_start_id
return msg.info.id === state.retain
}
if (msg.info.role === "assistant" && msg.info.summary && msg.info.finish && !msg.info.error)
state.completed.add(msg.info.parentID)
return false
}
function reorderCompacted(result: WithParts[]) {
result.reverse()
const compactionIndex = result.findLastIndex(
(msg) =>
@@ -583,7 +704,28 @@ export function filterCompacted(msgs: Iterable<WithParts>) {
}
export const filterCompactedEffect = Effect.fnUntraced(function* (sessionID: SessionID) {
return filterCompacted(yield* stream(sessionID))
// Stop paging once older compacted history would be discarded anyway.
const size = 50
const result = [] as WithParts[]
const state = compactedState()
let before: string | undefined
while (true) {
const next = yield* pageWithOptions({ sessionID, limit: size, before, lazyCompletedToolMetadata: true }).pipe(
Effect.catchIf(NotFoundError.isInstance, () =>
Effect.succeed({ items: [] as WithParts[], more: false, cursor: undefined }),
),
)
if (next.items.length === 0) break
for (let i = next.items.length - 1; i >= 0; i--) {
const item = next.items[i]
if (!item) continue
result.push(item)
if (reachedCompactedBoundary(state, item)) return reorderCompacted(result)
}
if (!next.more || !next.cursor) break
before = next.cursor
}
return reorderCompacted(result)
})
// filterCompacted reorders messages for model consumption
+1 -1
View File
@@ -720,7 +720,7 @@ export const layer: Layer.Layer<
const getPart: Interface["getPart"] = Effect.fn("Session.getPart")(function* (input) {
const row = yield* db
.select()
.select({ id: PartTable.id, session_id: PartTable.session_id, message_id: PartTable.message_id, data: PartTable.data })
.from(PartTable)
.where(
and(
+11 -8
View File
@@ -5,6 +5,9 @@ import { Snapshot } from "@/snapshot"
import { Session } from "./session"
import { SessionID, MessageID } from "./schema"
import { Config } from "@/config/config"
import { MessageV2 } from "./message-v2"
import { Database } from "@opencode-ai/core/database/database"
import { NotFoundError } from "@/storage/storage"
function unquoteGitPath(input: string) {
if (!input.startsWith('"')) return input
@@ -77,6 +80,7 @@ export const layer = Layer.effect(
const snapshot = yield* Snapshot.Service
const events = yield* EventV2Bridge.Service
const config = yield* Config.Service
const database = yield* Database.Service
const computeDiff = Effect.fn("SessionSummary.computeDiff")(function* (input: { messages: SessionV1.WithParts[] }) {
let from: string | undefined
@@ -112,12 +116,7 @@ export const layer = Layer.effect(
})
yield* events.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: [] })
if ((yield* config.get()).snapshot === false) return
const all = yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie)
if (!all.length) return
const messages = all.filter(
(m) => m.info.id === input.messageID || (m.info.role === "assistant" && m.info.parentID === input.messageID),
)
const messages = yield* MessageV2.related(input).pipe(Effect.provideService(Database.Service, database))
const target = messages.find((m) => m.info.id === input.messageID)
if (!target || target.info.role !== "user") return
const msgDiffs = yield* computeDiff({ messages })
@@ -127,8 +126,11 @@ export const layer = Layer.effect(
const diff = Effect.fn("SessionSummary.diff")(function* (input: { sessionID: SessionID; messageID?: MessageID }) {
if (!input.messageID) return []
const message = (yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie)).find(
(item) => item.info.id === input.messageID,
const message = yield* MessageV2.get({ sessionID: input.sessionID, messageID: input.messageID }).pipe(
Effect.provideService(Database.Service, database),
Effect.catchIf(NotFoundError.isInstance, () =>
sessions.get(input.sessionID).pipe(Effect.orDie, Effect.as(undefined)),
),
)
if (!message || message.info.role !== "user") return []
const diffs = message.info.summary?.diffs ?? []
@@ -150,6 +152,7 @@ export const defaultLayer = Layer.suspend(() =>
Layer.provide(Snapshot.defaultLayer),
Layer.provide(EventV2Bridge.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Database.defaultLayer),
),
)
@@ -382,6 +382,20 @@ function autocontinue(enabled: boolean) {
})
}
function messagesTransform(inspect: (messages: SessionV1.WithParts[]) => void) {
return Layer.mock(Plugin.Service)({
trigger: <Name extends string, Input, Output>(name: Name, _input: Input, output: Output) => {
if (name !== "experimental.chat.messages.transform") return Effect.succeed(output)
return Effect.sync(() => {
inspect((output as { messages: SessionV1.WithParts[] }).messages)
return output
})
},
list: () => Effect.succeed([]),
init: () => Effect.void,
})
}
describe("session.compaction.isOverflow", () => {
it.live(
"returns true when token count exceeds usable context",
@@ -682,8 +696,18 @@ describe("session.compaction.prune", () => {
status: "completed",
input: {},
output: "x".repeat(200_000),
attachments: [
{
id: PartID.ascending(),
messageID: b.id,
sessionID: info.id,
type: "file",
mime: "text/plain",
url: "data:text/plain;base64,eA==",
},
],
title: "done",
metadata: {},
metadata: { output: "x".repeat(200_000), description: "done" },
time: { start: Date.now(), end: Date.now() },
},
})
@@ -713,9 +737,45 @@ describe("session.compaction.prune", () => {
expect(part?.state.status).toBe("completed")
if (part?.type === "tool" && part.state.status === "completed") {
expect(part.state.time.compacted).toBeNumber()
expect(part.state.output).toHaveLength(200_000)
expect(part.state.metadata.output).toHaveLength(200_000)
expect(part.state.attachments).toHaveLength(1)
const compacted = (yield* MessageV2.filterCompactedEffect(info.id))
.flatMap((msg) => msg.parts)
.find((part) => part.type === "tool")
expect(compacted?.type).toBe("tool")
if (compacted?.type === "tool" && compacted.state.status === "completed") {
expect(Object.getOwnPropertyDescriptor(compacted.state, "metadata")?.get).toBeFunction()
expect(compacted.state.output).toHaveLength(200_000)
expect(compacted.state.metadata.output).toHaveLength(200_000)
expect(compacted.state.attachments).toHaveLength(1)
expect(JSON.parse(JSON.stringify(compacted.state)).metadata.output).toHaveLength(200_000)
part.state.metadata = { description: "small" }
yield* ssn.updatePart(part)
const small = (yield* MessageV2.filterCompactedEffect(info.id))
.flatMap((msg) => msg.parts)
.find((item) => item.id === part.id)
expect(small?.type).toBe("tool")
if (small?.type === "tool" && small.state.status === "completed") {
expect(Object.getOwnPropertyDescriptor(small.state, "metadata")?.get).toBeUndefined()
expect(small.state.metadata).toEqual({ description: "small" })
}
part.state.metadata = { output: "x".repeat(200_000), description: "large again" }
yield* ssn.updatePart(part)
const largeAgain = (yield* MessageV2.filterCompactedEffect(info.id))
.flatMap((msg) => msg.parts)
.find((item) => item.id === part.id)
expect(largeAgain?.type).toBe("tool")
if (largeAgain?.type === "tool" && largeAgain.state.status === "completed") {
expect(Object.getOwnPropertyDescriptor(largeAgain.state, "metadata")?.get).toBeFunction()
expect(largeAgain.state.metadata.output).toHaveLength(200_000)
}
}
}
}),
{
config: {
compaction: { prune: true },
@@ -815,6 +875,52 @@ describe("session.compaction.prune", () => {
})
describe("session.compaction.process", () => {
itCompaction.instance(
"keeps oversized tool metadata lazy through the plugin transform clone",
() => {
let lazy = false
return Effect.gen(function* () {
const test = yield* TestInstance
const ssn = yield* SessionNs.Service
const session = yield* ssn.create({})
const first = yield* createUserMessage(session.id, "first")
const assistant = yield* createAssistantMessage(session.id, first.id, test.directory)
yield* ssn.updatePart({
id: PartID.ascending(),
messageID: assistant.id,
sessionID: session.id,
type: "tool",
tool: "apply_patch",
callID: "call_test",
state: {
status: "completed",
input: {},
output: "done",
title: "done",
metadata: { diff: "x".repeat(70_000) },
time: { start: Date.now(), end: Date.now() },
},
})
const parent = yield* createUserMessage(session.id, "compact")
const msgs = yield* MessageV2.filterCompactedEffect(session.id)
yield* SessionCompaction.use.process({ parentID: parent.id, messages: msgs, sessionID: session.id, auto: false })
expect(lazy).toBe(true)
}).pipe(
withCompaction({
config: cfg({ tail_turns: 0 }),
plugin: messagesTransform((messages) => {
const part = messages.flatMap((msg) => msg.parts).find((part) => part.type === "tool")
if (part?.type === "tool" && part.state.status === "completed") {
lazy = Object.getOwnPropertyDescriptor(part.state, "metadata")?.get !== undefined
}
}),
}),
)
},
)
it.instance(
"throws when parent is not a user message",
Effect.gen(function* () {
@@ -553,6 +553,32 @@ describe("MessageV2.get", () => {
)
})
describe("MessageV2.related", () => {
it.instance("returns one user turn and its assistant replies", () =>
withSession(({ session, sessionID }) =>
Effect.gen(function* () {
yield* addUser(sessionID, "older")
const user = yield* addUser(sessionID, "target")
const first = yield* addAssistant(sessionID, user)
const second = yield* addAssistant(sessionID, user)
yield* session.updatePart({
id: PartID.ascending(),
sessionID,
messageID: second,
type: "text",
text: "response",
})
yield* addUser(sessionID, "newer")
const result = yield* MessageV2.related({ sessionID, messageID: user })
expect(result.map((item) => item.info.id)).toEqual([user, first, second])
expect(result[2].parts).toHaveLength(1)
}),
),
)
})
describe("Session.messages", () => {
it.instance("returns all messages in chronological order across pages", () =>
withSession(({ session, sessionID }) =>
@@ -645,6 +671,31 @@ describe("MessageV2.filterCompacted", () => {
),
)
it.instance("effect stops at compaction boundary beyond the first page", () =>
withSession(({ session, sessionID }) =>
Effect.gen(function* () {
yield* fill(sessionID, 55, (index: number) => index)
const compact = yield* addUser(sessionID)
yield* addCompactionPart(sessionID, compact)
const summary = yield* addAssistant(sessionID, compact, { summary: true, finish: "end_turn" })
yield* session.updatePart({
id: PartID.ascending(),
sessionID,
messageID: summary,
type: "text",
text: "summary",
})
yield* fill(sessionID, 55)
const expected = MessageV2.filterCompacted(yield* MessageV2.stream(sessionID))
const result = yield* MessageV2.filterCompactedEffect(sessionID)
expect(result).toEqual(expected)
expect(result).toHaveLength(57)
}),
),
)
it.live("handles empty iterable", () =>
Effect.sync(() => {
const result = MessageV2.filterCompacted([])
@@ -665,6 +716,22 @@ describe("MessageV2.filterCompacted", () => {
),
)
it.instance("does not break on summary without matching compaction part", () =>
withSession(({ sessionID }) =>
Effect.gen(function* () {
const user = yield* addUser(sessionID, "hello")
yield* addAssistant(sessionID, user, { summary: true, finish: "end_turn" })
yield* addUser(sessionID, "world")
const expected = MessageV2.filterCompacted(yield* MessageV2.stream(sessionID))
const result = yield* MessageV2.filterCompactedEffect(sessionID)
expect(result).toEqual(expected)
expect(result).toHaveLength(3)
}),
),
)
it.instance("skips assistant with error even if marked as summary", () =>
withSession(({ sessionID }) =>
Effect.gen(function* () {