resolve merge conflicts
This commit is contained in:
Johnny Amancio
@@ -62,8 +62,8 @@
|
||||
"light": "frappeText"
|
||||
},
|
||||
"textMuted": {
|
||||
"dark": "frappeSubtext1",
|
||||
"light": "frappeSubtext1"
|
||||
"dark": "frappeOverlay2",
|
||||
"light": "frappeOverlay2"
|
||||
},
|
||||
"background": {
|
||||
"dark": "frappeBase",
|
||||
|
||||
@@ -62,8 +62,8 @@
|
||||
"light": "macText"
|
||||
},
|
||||
"textMuted": {
|
||||
"dark": "macSubtext1",
|
||||
"light": "macSubtext1"
|
||||
"dark": "macOverlay2",
|
||||
"light": "macOverlay2"
|
||||
},
|
||||
"background": {
|
||||
"dark": "macBase",
|
||||
|
||||
@@ -63,7 +63,7 @@
|
||||
"success": { "dark": "darkGreen", "light": "lightGreen" },
|
||||
"info": { "dark": "darkTeal", "light": "lightTeal" },
|
||||
"text": { "dark": "darkText", "light": "lightText" },
|
||||
"textMuted": { "dark": "darkSubtext1", "light": "lightSubtext1" },
|
||||
"textMuted": { "dark": "darkOverlay2", "light": "lightOverlay2" },
|
||||
"background": { "dark": "darkBase", "light": "lightBase" },
|
||||
"backgroundPanel": { "dark": "darkMantle", "light": "lightMantle" },
|
||||
"backgroundElement": { "dark": "darkCrust", "light": "lightCrust" },
|
||||
|
||||
@@ -169,7 +169,9 @@ export function DialogProvider(props: ParentProps) {
|
||||
evt.preventDefault()
|
||||
evt.stopPropagation()
|
||||
}}
|
||||
onMouseUp={!Flag.KILO_EXPERIMENTAL_DISABLE_COPY_ON_SELECT ? () => Selection.copy(renderer, toast) : undefined}
|
||||
onMouseUp={
|
||||
!Flag.KILO_EXPERIMENTAL_DISABLE_COPY_ON_SELECT ? () => Selection.copy(renderer, toast) : undefined
|
||||
}
|
||||
>
|
||||
<Show when={value.stack.length}>
|
||||
<Dialog onClose={() => value.clear()} size={value.size}>
|
||||
|
||||
@@ -31,9 +31,12 @@ export namespace Flag {
|
||||
export const KILO_DISABLE_AUTOCOMPACT = truthy("KILO_DISABLE_AUTOCOMPACT")
|
||||
export const KILO_DISABLE_MODELS_FETCH = truthy("KILO_DISABLE_MODELS_FETCH")
|
||||
export const KILO_DISABLE_CLAUDE_CODE = truthy("KILO_DISABLE_CLAUDE_CODE")
|
||||
export const KILO_DISABLE_CLAUDE_CODE_PROMPT = KILO_DISABLE_CLAUDE_CODE || truthy("KILO_DISABLE_CLAUDE_CODE_PROMPT")
|
||||
export const KILO_DISABLE_CLAUDE_CODE_SKILLS = KILO_DISABLE_CLAUDE_CODE || truthy("KILO_DISABLE_CLAUDE_CODE_SKILLS")
|
||||
export const KILO_DISABLE_EXTERNAL_SKILLS = KILO_DISABLE_CLAUDE_CODE_SKILLS || truthy("KILO_DISABLE_EXTERNAL_SKILLS")
|
||||
export const KILO_DISABLE_CLAUDE_CODE_PROMPT =
|
||||
KILO_DISABLE_CLAUDE_CODE || truthy("KILO_DISABLE_CLAUDE_CODE_PROMPT")
|
||||
export const KILO_DISABLE_CLAUDE_CODE_SKILLS =
|
||||
KILO_DISABLE_CLAUDE_CODE || truthy("KILO_DISABLE_CLAUDE_CODE_SKILLS")
|
||||
export const KILO_DISABLE_EXTERNAL_SKILLS =
|
||||
KILO_DISABLE_CLAUDE_CODE_SKILLS || truthy("KILO_DISABLE_EXTERNAL_SKILLS")
|
||||
export declare const KILO_DISABLE_PROJECT_CONFIG: boolean
|
||||
export const KILO_FAKE_VCS = process.env["KILO_FAKE_VCS"]
|
||||
export declare const KILO_CLIENT: string
|
||||
@@ -46,15 +49,17 @@ export namespace Flag {
|
||||
export const KILO_EXPERIMENTAL_FILEWATCHER = Config.boolean("KILO_EXPERIMENTAL_FILEWATCHER").pipe(
|
||||
Config.withDefault(false),
|
||||
)
|
||||
export const KILO_EXPERIMENTAL_DISABLE_FILEWATCHER = Config.boolean("KILO_EXPERIMENTAL_DISABLE_FILEWATCHER").pipe(
|
||||
Config.withDefault(false),
|
||||
)
|
||||
export const KILO_EXPERIMENTAL_ICON_DISCOVERY = KILO_EXPERIMENTAL || truthy("KILO_EXPERIMENTAL_ICON_DISCOVERY")
|
||||
export const KILO_EXPERIMENTAL_DISABLE_FILEWATCHER = Config.boolean(
|
||||
"KILO_EXPERIMENTAL_DISABLE_FILEWATCHER",
|
||||
).pipe(Config.withDefault(false))
|
||||
export const KILO_EXPERIMENTAL_ICON_DISCOVERY =
|
||||
KILO_EXPERIMENTAL || truthy("KILO_EXPERIMENTAL_ICON_DISCOVERY")
|
||||
|
||||
const copy = process.env["KILO_EXPERIMENTAL_DISABLE_COPY_ON_SELECT"]
|
||||
export const KILO_EXPERIMENTAL_DISABLE_COPY_ON_SELECT =
|
||||
copy === undefined ? process.platform === "win32" : truthy("KILO_EXPERIMENTAL_DISABLE_COPY_ON_SELECT")
|
||||
export const KILO_ENABLE_EXA = truthy("KILO_ENABLE_EXA") || KILO_EXPERIMENTAL || truthy("KILO_EXPERIMENTAL_EXA")
|
||||
export const KILO_ENABLE_EXA =
|
||||
truthy("KILO_ENABLE_EXA") || KILO_EXPERIMENTAL || truthy("KILO_EXPERIMENTAL_EXA")
|
||||
export const KILO_EXPERIMENTAL_BASH_DEFAULT_TIMEOUT_MS = number("KILO_EXPERIMENTAL_BASH_DEFAULT_TIMEOUT_MS")
|
||||
export const KILO_EXPERIMENTAL_OUTPUT_TOKEN_MAX = number("KILO_EXPERIMENTAL_OUTPUT_TOKEN_MAX")
|
||||
export const KILO_EXPERIMENTAL_OXFMT = KILO_EXPERIMENTAL || truthy("KILO_EXPERIMENTAL_OXFMT")
|
||||
|
||||
@@ -112,7 +112,7 @@ let cli = yargs(hideBin(process.argv)) // kilocode_change
|
||||
})
|
||||
|
||||
process.env.AGENT = "1"
|
||||
process.env.KILO = "1"
|
||||
process.env.OPENCODE = "1"
|
||||
process.env.KILO_PID = String(process.pid)
|
||||
|
||||
Log.Default.info("opencode", {
|
||||
|
||||
@@ -19,6 +19,9 @@ import { iife } from "@/util/iife"
|
||||
import { Global } from "../global"
|
||||
import path from "path"
|
||||
import { Filesystem } from "../util/filesystem"
|
||||
import { Effect, Layer, ServiceMap } from "effect"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { makeRuntime } from "@/effect/run-service"
|
||||
|
||||
// Direct imports for bundled providers
|
||||
import { createAmazonBedrock, type AmazonBedrockProviderSettings } from "@ai-sdk/amazon-bedrock"
|
||||
@@ -893,6 +896,29 @@ export namespace Provider {
|
||||
})
|
||||
export type Info = z.infer<typeof Info>
|
||||
|
||||
export interface Interface {
|
||||
readonly list: () => Effect.Effect<Record<ProviderID, Info>>
|
||||
readonly getProvider: (providerID: ProviderID) => Effect.Effect<Info>
|
||||
readonly getModel: (providerID: ProviderID, modelID: ModelID) => Effect.Effect<Model>
|
||||
readonly getLanguage: (model: Model) => Effect.Effect<LanguageModelV3>
|
||||
readonly closest: (
|
||||
providerID: ProviderID,
|
||||
query: string[],
|
||||
) => Effect.Effect<{ providerID: ProviderID; modelID: string } | undefined>
|
||||
readonly getSmallModel: (providerID: ProviderID) => Effect.Effect<Model | undefined>
|
||||
readonly defaultModel: () => Effect.Effect<{ providerID: ProviderID; modelID: ModelID }>
|
||||
}
|
||||
|
||||
interface State {
|
||||
models: Map<string, LanguageModelV3>
|
||||
providers: Record<ProviderID, Info>
|
||||
sdk: Map<string, BundledSDK>
|
||||
modelLoaders: Record<string, CustomModelLoader>
|
||||
varsLoaders: Record<string, CustomVarsLoader>
|
||||
}
|
||||
|
||||
export class Service extends ServiceMap.Service<Service, Interface>()("@opencode/Provider") {}
|
||||
|
||||
function fromModelsDevModel(provider: ModelsDev.Provider, model: ModelsDev.Model): Model {
|
||||
const m: Model = {
|
||||
id: ModelID.make(model.id),
|
||||
@@ -978,581 +1004,646 @@ export namespace Provider {
|
||||
}
|
||||
}
|
||||
|
||||
const state = Instance.state(async () => {
|
||||
using _ = log.time("state")
|
||||
const config = await Config.get()
|
||||
const modelsDev = await ModelsDev.get()
|
||||
const database = mapValues(modelsDev, fromModelsDevProvider)
|
||||
const layer: Layer.Layer<Service, never, Config.Service | Auth.Service> = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const config = yield* Config.Service
|
||||
const auth = yield* Auth.Service
|
||||
|
||||
const disabled = new Set(config.disabled_providers ?? [])
|
||||
const enabled = config.enabled_providers ? new Set(config.enabled_providers) : null
|
||||
const cache = yield* InstanceState.make<State>(() =>
|
||||
Effect.gen(function* () {
|
||||
using _ = log.time("state")
|
||||
const cfg = yield* config.get()
|
||||
const modelsDev = yield* Effect.promise(() => ModelsDev.get())
|
||||
const database = mapValues(modelsDev, fromModelsDevProvider)
|
||||
|
||||
function isProviderAllowed(providerID: ProviderID): boolean {
|
||||
if (enabled && !enabled.has(providerID)) return false
|
||||
if (disabled.has(providerID)) return false
|
||||
return true
|
||||
}
|
||||
const disabled = new Set(cfg.disabled_providers ?? [])
|
||||
const enabled = cfg.enabled_providers ? new Set(cfg.enabled_providers) : null
|
||||
|
||||
const providers: Record<ProviderID, Info> = {} as Record<ProviderID, Info>
|
||||
const languages = new Map<string, LanguageModelV3>()
|
||||
const modelLoaders: {
|
||||
[providerID: string]: CustomModelLoader
|
||||
} = {}
|
||||
const varsLoaders: {
|
||||
[providerID: string]: CustomVarsLoader
|
||||
} = {}
|
||||
const sdk = new Map<string, BundledSDK>()
|
||||
const discoveryLoaders: {
|
||||
[providerID: string]: CustomDiscoverModels
|
||||
} = {}
|
||||
|
||||
log.info("init")
|
||||
|
||||
const configProviders = Object.entries(config.provider ?? {})
|
||||
|
||||
function mergeProvider(providerID: ProviderID, provider: Partial<Info>) {
|
||||
const existing = providers[providerID]
|
||||
if (existing) {
|
||||
// @ts-expect-error
|
||||
providers[providerID] = mergeDeep(existing, provider)
|
||||
return
|
||||
}
|
||||
const match = database[providerID]
|
||||
if (!match) return
|
||||
// @ts-expect-error
|
||||
providers[providerID] = mergeDeep(match, provider)
|
||||
}
|
||||
|
||||
// extend database from config
|
||||
for (const [providerID, provider] of configProviders) {
|
||||
const existing = database[providerID]
|
||||
const parsed: Info = {
|
||||
id: ProviderID.make(providerID),
|
||||
name: provider.name ?? existing?.name ?? providerID,
|
||||
env: provider.env ?? existing?.env ?? [],
|
||||
options: mergeDeep(existing?.options ?? {}, provider.options ?? {}),
|
||||
source: "config",
|
||||
models: existing?.models ?? {},
|
||||
}
|
||||
|
||||
for (const [modelID, model] of Object.entries(provider.models ?? {})) {
|
||||
const existingModel = parsed.models[model.id ?? modelID]
|
||||
const name = iife(() => {
|
||||
if (model.name) return model.name
|
||||
if (model.id && model.id !== modelID) return modelID
|
||||
return existingModel?.name ?? modelID
|
||||
})
|
||||
const parsedModel: Model = {
|
||||
id: ModelID.make(modelID),
|
||||
api: {
|
||||
id: model.id ?? existingModel?.api.id ?? modelID,
|
||||
npm:
|
||||
model.provider?.npm ??
|
||||
provider.npm ??
|
||||
existingModel?.api.npm ??
|
||||
modelsDev[providerID]?.npm ??
|
||||
"@ai-sdk/openai-compatible",
|
||||
url: model.provider?.api ?? provider?.api ?? existingModel?.api.url ?? modelsDev[providerID]?.api,
|
||||
},
|
||||
status: model.status ?? existingModel?.status ?? "active",
|
||||
name,
|
||||
providerID: ProviderID.make(providerID),
|
||||
capabilities: {
|
||||
temperature: model.temperature ?? existingModel?.capabilities.temperature ?? false,
|
||||
reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? false,
|
||||
attachment: model.attachment ?? existingModel?.capabilities.attachment ?? false,
|
||||
toolcall: model.tool_call ?? existingModel?.capabilities.toolcall ?? true,
|
||||
input: {
|
||||
text: model.modalities?.input?.includes("text") ?? existingModel?.capabilities.input.text ?? true,
|
||||
audio: model.modalities?.input?.includes("audio") ?? existingModel?.capabilities.input.audio ?? false,
|
||||
image: model.modalities?.input?.includes("image") ?? existingModel?.capabilities.input.image ?? false,
|
||||
video: model.modalities?.input?.includes("video") ?? existingModel?.capabilities.input.video ?? false,
|
||||
pdf: model.modalities?.input?.includes("pdf") ?? existingModel?.capabilities.input.pdf ?? false,
|
||||
},
|
||||
output: {
|
||||
text: model.modalities?.output?.includes("text") ?? existingModel?.capabilities.output.text ?? true,
|
||||
audio: model.modalities?.output?.includes("audio") ?? existingModel?.capabilities.output.audio ?? false,
|
||||
image: model.modalities?.output?.includes("image") ?? existingModel?.capabilities.output.image ?? false,
|
||||
video: model.modalities?.output?.includes("video") ?? existingModel?.capabilities.output.video ?? false,
|
||||
pdf: model.modalities?.output?.includes("pdf") ?? existingModel?.capabilities.output.pdf ?? false,
|
||||
},
|
||||
interleaved: model.interleaved ?? false,
|
||||
},
|
||||
cost: {
|
||||
input: model?.cost?.input ?? existingModel?.cost?.input ?? 0,
|
||||
output: model?.cost?.output ?? existingModel?.cost?.output ?? 0,
|
||||
cache: {
|
||||
read: model?.cost?.cache_read ?? existingModel?.cost?.cache.read ?? 0,
|
||||
write: model?.cost?.cache_write ?? existingModel?.cost?.cache.write ?? 0,
|
||||
},
|
||||
},
|
||||
options: mergeDeep(existingModel?.options ?? {}, model.options ?? {}),
|
||||
limit: {
|
||||
context: model.limit?.context ?? existingModel?.limit?.context ?? 0,
|
||||
output: model.limit?.output ?? existingModel?.limit?.output ?? 0,
|
||||
},
|
||||
headers: mergeDeep(existingModel?.headers ?? {}, model.headers ?? {}),
|
||||
family: model.family ?? existingModel?.family ?? "",
|
||||
release_date: model.release_date ?? existingModel?.release_date ?? "",
|
||||
variants: {},
|
||||
|
||||
// kilocode_change start
|
||||
recommendedIndex: model.recommendedIndex ?? existingModel?.recommendedIndex,
|
||||
prompt: model.prompt ?? existingModel?.prompt,
|
||||
isFree: model.isFree ?? existingModel?.isFree,
|
||||
ai_sdk_provider: model.ai_sdk_provider ?? existingModel?.ai_sdk_provider,
|
||||
// kilocode_change end
|
||||
}
|
||||
const merged = mergeDeep(ProviderTransform.variants(parsedModel), model.variants ?? {})
|
||||
parsedModel.variants = mapValues(
|
||||
pickBy(merged, (v) => !v.disabled),
|
||||
(v) => omit(v, ["disabled"]),
|
||||
)
|
||||
parsed.models[modelID] = parsedModel
|
||||
}
|
||||
database[providerID] = parsed
|
||||
}
|
||||
|
||||
// load env
|
||||
const env = Env.all()
|
||||
for (const [id, provider] of Object.entries(database)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
const apiKey = provider.env.map((item) => env[item]).find(Boolean)
|
||||
if (!apiKey) continue
|
||||
mergeProvider(providerID, {
|
||||
source: "env",
|
||||
key: provider.env.length === 1 ? apiKey : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
// load apikeys
|
||||
for (const [id, provider] of Object.entries(await Auth.all())) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
if (provider.type === "api") {
|
||||
mergeProvider(providerID, {
|
||||
source: "api",
|
||||
key: provider.key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const plugin of await Plugin.list()) {
|
||||
if (!plugin.auth) continue
|
||||
const providerID = ProviderID.make(plugin.auth.provider)
|
||||
if (disabled.has(providerID)) continue
|
||||
|
||||
const auth = await Auth.get(providerID)
|
||||
if (!auth) continue
|
||||
if (!plugin.auth.loader) continue
|
||||
|
||||
if (auth) {
|
||||
const options = await plugin.auth.loader(() => Auth.get(providerID) as any, database[plugin.auth.provider])
|
||||
const opts = options ?? {}
|
||||
const patch: Partial<Info> = providers[providerID] ? { options: opts } : { source: "custom", options: opts }
|
||||
mergeProvider(providerID, patch)
|
||||
}
|
||||
}
|
||||
|
||||
for (const [id, fn] of Object.entries(CUSTOM_LOADERS)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
const data = database[providerID]
|
||||
if (!data) {
|
||||
log.error("Provider does not exist in model list " + providerID)
|
||||
continue
|
||||
}
|
||||
const result = await fn(data)
|
||||
if (result && (result.autoload || providers[providerID])) {
|
||||
if (result.getModel) modelLoaders[providerID] = result.getModel
|
||||
if (result.vars) varsLoaders[providerID] = result.vars
|
||||
if (result.discoverModels) discoveryLoaders[providerID] = result.discoverModels
|
||||
const opts = result.options ?? {}
|
||||
const patch: Partial<Info> = providers[providerID] ? { options: opts } : { source: "custom", options: opts }
|
||||
mergeProvider(providerID, patch)
|
||||
}
|
||||
}
|
||||
|
||||
// load config
|
||||
for (const [id, provider] of configProviders) {
|
||||
const providerID = ProviderID.make(id)
|
||||
const partial: Partial<Info> = { source: "config" }
|
||||
if (provider.env) partial.env = provider.env
|
||||
if (provider.name) partial.name = provider.name
|
||||
if (provider.options) partial.options = provider.options
|
||||
mergeProvider(providerID, partial)
|
||||
}
|
||||
|
||||
for (const [id, provider] of Object.entries(providers)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (!isProviderAllowed(providerID)) {
|
||||
delete providers[providerID]
|
||||
continue
|
||||
}
|
||||
|
||||
const configProvider = config.provider?.[providerID]
|
||||
|
||||
for (const [modelID, model] of Object.entries(provider.models)) {
|
||||
model.api.id = model.api.id ?? model.id ?? modelID
|
||||
if (
|
||||
modelID === "gpt-5-chat-latest" ||
|
||||
(providerID === ProviderID.openrouter && modelID === "openai/gpt-5-chat")
|
||||
)
|
||||
delete provider.models[modelID]
|
||||
if (model.status === "alpha" && !Flag.KILO_ENABLE_EXPERIMENTAL_MODELS) delete provider.models[modelID]
|
||||
if (model.status === "deprecated") delete provider.models[modelID]
|
||||
if (
|
||||
(configProvider?.blacklist && configProvider.blacklist.includes(modelID)) ||
|
||||
(configProvider?.whitelist && !configProvider.whitelist.includes(modelID))
|
||||
)
|
||||
delete provider.models[modelID]
|
||||
|
||||
model.variants = mapValues(ProviderTransform.variants(model), (v) => v)
|
||||
|
||||
// Filter out disabled variants from config
|
||||
const configVariants = configProvider?.models?.[modelID]?.variants
|
||||
if (configVariants && model.variants) {
|
||||
const merged = mergeDeep(model.variants, configVariants)
|
||||
model.variants = mapValues(
|
||||
pickBy(merged, (v) => !v.disabled),
|
||||
(v) => omit(v, ["disabled"]),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(provider.models).length === 0) {
|
||||
delete providers[providerID]
|
||||
continue
|
||||
}
|
||||
|
||||
log.info("found", { providerID })
|
||||
}
|
||||
|
||||
const gitlab = ProviderID.make("gitlab")
|
||||
if (discoveryLoaders[gitlab] && providers[gitlab]) {
|
||||
await (async () => {
|
||||
const discovered = await discoveryLoaders[gitlab]()
|
||||
for (const [modelID, model] of Object.entries(discovered)) {
|
||||
if (!providers[gitlab].models[modelID]) {
|
||||
providers[gitlab].models[modelID] = model
|
||||
function isProviderAllowed(providerID: ProviderID): boolean {
|
||||
if (enabled && !enabled.has(providerID)) return false
|
||||
if (disabled.has(providerID)) return false
|
||||
return true
|
||||
}
|
||||
}
|
||||
})().catch((e) => log.warn("state discovery error", { id: "gitlab", error: e }))
|
||||
}
|
||||
|
||||
return {
|
||||
models: languages,
|
||||
providers,
|
||||
sdk,
|
||||
modelLoaders,
|
||||
varsLoaders,
|
||||
}
|
||||
})
|
||||
const providers: Record<ProviderID, Info> = {} as Record<ProviderID, Info>
|
||||
const languages = new Map<string, LanguageModelV3>()
|
||||
const modelLoaders: {
|
||||
[providerID: string]: CustomModelLoader
|
||||
} = {}
|
||||
const varsLoaders: {
|
||||
[providerID: string]: CustomVarsLoader
|
||||
} = {}
|
||||
const sdk = new Map<string, BundledSDK>()
|
||||
const discoveryLoaders: {
|
||||
[providerID: string]: CustomDiscoverModels
|
||||
} = {}
|
||||
|
||||
export async function list() {
|
||||
return state().then((state) => state.providers)
|
||||
}
|
||||
log.info("init")
|
||||
|
||||
async function getSDK(model: Model) {
|
||||
try {
|
||||
using _ = log.time("getSDK", {
|
||||
providerID: model.providerID,
|
||||
})
|
||||
const s = await state()
|
||||
const provider = s.providers[model.providerID]
|
||||
const options = { ...provider.options }
|
||||
const configProviders = Object.entries(cfg.provider ?? {})
|
||||
|
||||
if (model.providerID === "google-vertex" && !model.api.npm.includes("@ai-sdk/openai-compatible")) {
|
||||
delete options.fetch
|
||||
}
|
||||
|
||||
if (model.api.npm.includes("@ai-sdk/openai-compatible") && options["includeUsage"] !== false) {
|
||||
options["includeUsage"] = true
|
||||
}
|
||||
|
||||
const baseURL = iife(() => {
|
||||
let url =
|
||||
typeof options["baseURL"] === "string" && options["baseURL"] !== "" ? options["baseURL"] : model.api.url
|
||||
if (!url) return
|
||||
|
||||
// some models/providers have variable urls, ex: "https://${AZURE_RESOURCE_NAME}.services.ai.azure.com/anthropic/v1"
|
||||
// We track this in models.dev, and then when we are resolving the baseURL
|
||||
// we need to string replace that literal: "${AZURE_RESOURCE_NAME}"
|
||||
const loader = s.varsLoaders[model.providerID]
|
||||
if (loader) {
|
||||
const vars = loader(options)
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
const field = "${" + key + "}"
|
||||
url = url.replaceAll(field, value)
|
||||
function mergeProvider(providerID: ProviderID, provider: Partial<Info>) {
|
||||
const existing = providers[providerID]
|
||||
if (existing) {
|
||||
// @ts-expect-error
|
||||
providers[providerID] = mergeDeep(existing, provider)
|
||||
return
|
||||
}
|
||||
const match = database[providerID]
|
||||
if (!match) return
|
||||
// @ts-expect-error
|
||||
providers[providerID] = mergeDeep(match, provider)
|
||||
}
|
||||
}
|
||||
|
||||
url = url.replace(/\$\{([^}]+)\}/g, (item, key) => {
|
||||
const val = Env.get(String(key))
|
||||
return val ?? item
|
||||
})
|
||||
return url
|
||||
})
|
||||
// extend database from config
|
||||
for (const [providerID, provider] of configProviders) {
|
||||
const existing = database[providerID]
|
||||
const parsed: Info = {
|
||||
id: ProviderID.make(providerID),
|
||||
name: provider.name ?? existing?.name ?? providerID,
|
||||
env: provider.env ?? existing?.env ?? [],
|
||||
options: mergeDeep(existing?.options ?? {}, provider.options ?? {}),
|
||||
source: "config",
|
||||
models: existing?.models ?? {},
|
||||
}
|
||||
|
||||
if (baseURL !== undefined) options["baseURL"] = baseURL
|
||||
if (options["apiKey"] === undefined && provider.key) options["apiKey"] = provider.key
|
||||
if (model.headers)
|
||||
options["headers"] = {
|
||||
...options["headers"],
|
||||
...model.headers,
|
||||
}
|
||||
for (const [modelID, model] of Object.entries(provider.models ?? {})) {
|
||||
const existingModel = parsed.models[model.id ?? modelID]
|
||||
const name = iife(() => {
|
||||
if (model.name) return model.name
|
||||
if (model.id && model.id !== modelID) return modelID
|
||||
return existingModel?.name ?? modelID
|
||||
})
|
||||
const parsedModel: Model = {
|
||||
id: ModelID.make(modelID),
|
||||
api: {
|
||||
id: model.id ?? existingModel?.api.id ?? modelID,
|
||||
npm:
|
||||
model.provider?.npm ??
|
||||
provider.npm ??
|
||||
existingModel?.api.npm ??
|
||||
modelsDev[providerID]?.npm ??
|
||||
"@ai-sdk/openai-compatible",
|
||||
url: model.provider?.api ?? provider?.api ?? existingModel?.api.url ?? modelsDev[providerID]?.api,
|
||||
},
|
||||
status: model.status ?? existingModel?.status ?? "active",
|
||||
name,
|
||||
providerID: ProviderID.make(providerID),
|
||||
capabilities: {
|
||||
temperature: model.temperature ?? existingModel?.capabilities.temperature ?? false,
|
||||
reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? false,
|
||||
attachment: model.attachment ?? existingModel?.capabilities.attachment ?? false,
|
||||
toolcall: model.tool_call ?? existingModel?.capabilities.toolcall ?? true,
|
||||
input: {
|
||||
text: model.modalities?.input?.includes("text") ?? existingModel?.capabilities.input.text ?? true,
|
||||
audio:
|
||||
model.modalities?.input?.includes("audio") ?? existingModel?.capabilities.input.audio ?? false,
|
||||
image:
|
||||
model.modalities?.input?.includes("image") ?? existingModel?.capabilities.input.image ?? false,
|
||||
video:
|
||||
model.modalities?.input?.includes("video") ?? existingModel?.capabilities.input.video ?? false,
|
||||
pdf: model.modalities?.input?.includes("pdf") ?? existingModel?.capabilities.input.pdf ?? false,
|
||||
},
|
||||
output: {
|
||||
text: model.modalities?.output?.includes("text") ?? existingModel?.capabilities.output.text ?? true,
|
||||
audio:
|
||||
model.modalities?.output?.includes("audio") ?? existingModel?.capabilities.output.audio ?? false,
|
||||
image:
|
||||
model.modalities?.output?.includes("image") ?? existingModel?.capabilities.output.image ?? false,
|
||||
video:
|
||||
model.modalities?.output?.includes("video") ?? existingModel?.capabilities.output.video ?? false,
|
||||
pdf: model.modalities?.output?.includes("pdf") ?? existingModel?.capabilities.output.pdf ?? false,
|
||||
},
|
||||
interleaved: model.interleaved ?? false,
|
||||
},
|
||||
cost: {
|
||||
input: model?.cost?.input ?? existingModel?.cost?.input ?? 0,
|
||||
output: model?.cost?.output ?? existingModel?.cost?.output ?? 0,
|
||||
cache: {
|
||||
read: model?.cost?.cache_read ?? existingModel?.cost?.cache.read ?? 0,
|
||||
write: model?.cost?.cache_write ?? existingModel?.cost?.cache.write ?? 0,
|
||||
},
|
||||
},
|
||||
options: mergeDeep(existingModel?.options ?? {}, model.options ?? {}),
|
||||
limit: {
|
||||
context: model.limit?.context ?? existingModel?.limit?.context ?? 0,
|
||||
output: model.limit?.output ?? existingModel?.limit?.output ?? 0,
|
||||
},
|
||||
headers: mergeDeep(existingModel?.headers ?? {}, model.headers ?? {}),
|
||||
family: model.family ?? existingModel?.family ?? "",
|
||||
release_date: model.release_date ?? existingModel?.release_date ?? "",
|
||||
variants: {},
|
||||
}
|
||||
const merged = mergeDeep(ProviderTransform.variants(parsedModel), model.variants ?? {})
|
||||
parsedModel.variants = mapValues(
|
||||
pickBy(merged, (v) => !v.disabled),
|
||||
(v) => omit(v, ["disabled"]),
|
||||
)
|
||||
parsed.models[modelID] = parsedModel
|
||||
}
|
||||
database[providerID] = parsed
|
||||
}
|
||||
|
||||
const key = Hash.fast(
|
||||
JSON.stringify({
|
||||
providerID: model.providerID,
|
||||
npm: model.api.npm,
|
||||
options,
|
||||
}),
|
||||
)
|
||||
const existing = s.sdk.get(key)
|
||||
if (existing) return existing
|
||||
// load env
|
||||
const env = Env.all()
|
||||
for (const [id, provider] of Object.entries(database)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
const apiKey = provider.env.map((item) => env[item]).find(Boolean)
|
||||
if (!apiKey) continue
|
||||
mergeProvider(providerID, {
|
||||
source: "env",
|
||||
key: provider.env.length === 1 ? apiKey : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
const customFetch = options["fetch"]
|
||||
const chunkTimeout = options["chunkTimeout"]
|
||||
delete options["chunkTimeout"]
|
||||
// load apikeys
|
||||
const auths = yield* auth.all().pipe(Effect.orDie)
|
||||
for (const [id, provider] of Object.entries(auths)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
if (provider.type === "api") {
|
||||
mergeProvider(providerID, {
|
||||
source: "api",
|
||||
key: provider.key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
|
||||
// Preserve custom fetch if it exists, wrap it with timeout logic
|
||||
const fetchFn = customFetch ?? fetch
|
||||
const opts = init ?? {}
|
||||
const chunkAbortCtl = typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
|
||||
const signals: AbortSignal[] = []
|
||||
const plugins = yield* Effect.promise(() => Plugin.list())
|
||||
for (const plugin of plugins) {
|
||||
if (!plugin.auth) continue
|
||||
const providerID = ProviderID.make(plugin.auth.provider)
|
||||
if (disabled.has(providerID)) continue
|
||||
|
||||
// kilocode_change start - apply connection-phase timeout only
|
||||
// Use an AbortController so we can cancel the timer once headers arrive,
|
||||
// preventing healthy streaming responses from being aborted mid-stream.
|
||||
const ms = options["timeout"] ?? REQUEST_TIMEOUT_MS
|
||||
const controller = ms !== false ? new AbortController() : undefined
|
||||
if (controller) {
|
||||
const signals: AbortSignal[] = [controller.signal]
|
||||
if (opts.signal) signals.push(opts.signal)
|
||||
opts.signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0]
|
||||
}
|
||||
const timer =
|
||||
controller && typeof ms === "number"
|
||||
? setTimeout(() => controller.abort(new DOMException("The operation timed out.", "TimeoutError")), ms)
|
||||
: undefined
|
||||
// kilocode_change end
|
||||
const pluginAuth = yield* auth.get(providerID).pipe(Effect.orDie)
|
||||
if (!pluginAuth) continue
|
||||
if (!plugin.auth.loader) continue
|
||||
|
||||
// Strip openai itemId metadata following what codex does
|
||||
// Codex uses #[serde(skip_serializing)] on id fields for all item types:
|
||||
// Message, Reasoning, FunctionCall, LocalShellCall, CustomToolCall, WebSearchCall
|
||||
// IDs are only re-attached for Azure with store=true
|
||||
if (model.api.npm === "@ai-sdk/openai" && opts.body && opts.method === "POST") {
|
||||
const body = JSON.parse(opts.body as string)
|
||||
const isAzure = model.providerID.includes("azure")
|
||||
const keepIds = isAzure && body.store === true
|
||||
if (!keepIds && Array.isArray(body.input)) {
|
||||
for (const item of body.input) {
|
||||
if ("id" in item) {
|
||||
delete item.id
|
||||
const options = yield* Effect.promise(() =>
|
||||
plugin.auth!.loader!(() => Auth.get(providerID) as any, database[plugin.auth!.provider]),
|
||||
)
|
||||
const opts = options ?? {}
|
||||
const patch: Partial<Info> = providers[providerID] ? { options: opts } : { source: "custom", options: opts }
|
||||
mergeProvider(providerID, patch)
|
||||
}
|
||||
|
||||
for (const [id, fn] of Object.entries(CUSTOM_LOADERS)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (disabled.has(providerID)) continue
|
||||
const data = database[providerID]
|
||||
if (!data) {
|
||||
log.error("Provider does not exist in model list " + providerID)
|
||||
continue
|
||||
}
|
||||
const result = yield* Effect.promise(() => fn(data))
|
||||
if (result && (result.autoload || providers[providerID])) {
|
||||
if (result.getModel) modelLoaders[providerID] = result.getModel
|
||||
if (result.vars) varsLoaders[providerID] = result.vars
|
||||
if (result.discoverModels) discoveryLoaders[providerID] = result.discoverModels
|
||||
const opts = result.options ?? {}
|
||||
const patch: Partial<Info> = providers[providerID]
|
||||
? { options: opts }
|
||||
: { source: "custom", options: opts }
|
||||
mergeProvider(providerID, patch)
|
||||
}
|
||||
}
|
||||
|
||||
// load config
|
||||
for (const [id, provider] of configProviders) {
|
||||
const providerID = ProviderID.make(id)
|
||||
const partial: Partial<Info> = { source: "config" }
|
||||
if (provider.env) partial.env = provider.env
|
||||
if (provider.name) partial.name = provider.name
|
||||
if (provider.options) partial.options = provider.options
|
||||
mergeProvider(providerID, partial)
|
||||
}
|
||||
|
||||
for (const [id, provider] of Object.entries(providers)) {
|
||||
const providerID = ProviderID.make(id)
|
||||
if (!isProviderAllowed(providerID)) {
|
||||
delete providers[providerID]
|
||||
continue
|
||||
}
|
||||
|
||||
const configProvider = cfg.provider?.[providerID]
|
||||
|
||||
for (const [modelID, model] of Object.entries(provider.models)) {
|
||||
model.api.id = model.api.id ?? model.id ?? modelID
|
||||
if (
|
||||
modelID === "gpt-5-chat-latest" ||
|
||||
(providerID === ProviderID.openrouter && modelID === "openai/gpt-5-chat")
|
||||
)
|
||||
delete provider.models[modelID]
|
||||
if (model.status === "alpha" && !Flag.KILO_ENABLE_EXPERIMENTAL_MODELS) delete provider.models[modelID]
|
||||
if (model.status === "deprecated") delete provider.models[modelID]
|
||||
if (
|
||||
(configProvider?.blacklist && configProvider.blacklist.includes(modelID)) ||
|
||||
(configProvider?.whitelist && !configProvider.whitelist.includes(modelID))
|
||||
)
|
||||
delete provider.models[modelID]
|
||||
|
||||
model.variants = mapValues(ProviderTransform.variants(model), (v) => v)
|
||||
|
||||
const configVariants = configProvider?.models?.[modelID]?.variants
|
||||
if (configVariants && model.variants) {
|
||||
const merged = mergeDeep(model.variants, configVariants)
|
||||
model.variants = mapValues(
|
||||
pickBy(merged, (v) => !v.disabled),
|
||||
(v) => omit(v, ["disabled"]),
|
||||
)
|
||||
}
|
||||
}
|
||||
opts.body = JSON.stringify(body)
|
||||
|
||||
if (Object.keys(provider.models).length === 0) {
|
||||
delete providers[providerID]
|
||||
continue
|
||||
}
|
||||
|
||||
log.info("found", { providerID })
|
||||
}
|
||||
|
||||
const gitlab = ProviderID.make("gitlab")
|
||||
if (discoveryLoaders[gitlab] && providers[gitlab]) {
|
||||
yield* Effect.promise(async () => {
|
||||
try {
|
||||
const discovered = await discoveryLoaders[gitlab]()
|
||||
for (const [modelID, model] of Object.entries(discovered)) {
|
||||
if (!providers[gitlab].models[modelID]) {
|
||||
providers[gitlab].models[modelID] = model
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn("state discovery error", { id: "gitlab", error: e })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
models: languages,
|
||||
providers,
|
||||
sdk,
|
||||
modelLoaders,
|
||||
varsLoaders,
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
const list = Effect.fn("Provider.list")(() => InstanceState.use(cache, (s) => s.providers))
|
||||
|
||||
async function resolveSDK(model: Model, s: State) {
|
||||
try {
|
||||
using _ = log.time("getSDK", {
|
||||
providerID: model.providerID,
|
||||
})
|
||||
const provider = s.providers[model.providerID]
|
||||
const options = { ...provider.options }
|
||||
|
||||
if (model.providerID === "google-vertex" && !model.api.npm.includes("@ai-sdk/openai-compatible")) {
|
||||
delete options.fetch
|
||||
}
|
||||
|
||||
if (model.api.npm.includes("@ai-sdk/openai-compatible") && options["includeUsage"] !== false) {
|
||||
options["includeUsage"] = true
|
||||
}
|
||||
|
||||
const baseURL = iife(() => {
|
||||
let url =
|
||||
typeof options["baseURL"] === "string" && options["baseURL"] !== "" ? options["baseURL"] : model.api.url
|
||||
if (!url) return
|
||||
|
||||
const loader = s.varsLoaders[model.providerID]
|
||||
if (loader) {
|
||||
const vars = loader(options)
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
const field = "${" + key + "}"
|
||||
url = url.replaceAll(field, value)
|
||||
}
|
||||
}
|
||||
|
||||
url = url.replace(/\$\{([^}]+)\}/g, (item, key) => {
|
||||
const val = Env.get(String(key))
|
||||
return val ?? item
|
||||
})
|
||||
return url
|
||||
})
|
||||
|
||||
if (baseURL !== undefined) options["baseURL"] = baseURL
|
||||
if (options["apiKey"] === undefined && provider.key) options["apiKey"] = provider.key
|
||||
if (model.headers)
|
||||
options["headers"] = {
|
||||
...options["headers"],
|
||||
...model.headers,
|
||||
}
|
||||
|
||||
const key = Hash.fast(
|
||||
JSON.stringify({
|
||||
providerID: model.providerID,
|
||||
npm: model.api.npm,
|
||||
options,
|
||||
}),
|
||||
)
|
||||
const existing = s.sdk.get(key)
|
||||
if (existing) return existing
|
||||
|
||||
const customFetch = options["fetch"]
|
||||
const chunkTimeout = options["chunkTimeout"]
|
||||
delete options["chunkTimeout"]
|
||||
|
||||
options["fetch"] = async (input: any, init?: BunFetchRequestInit) => {
|
||||
const fetchFn = customFetch ?? fetch
|
||||
const opts = init ?? {}
|
||||
const chunkAbortCtl =
|
||||
typeof chunkTimeout === "number" && chunkTimeout > 0 ? new AbortController() : undefined
|
||||
// kilocode_change start - apply connection-phase timeout via AbortController so the timer
|
||||
// can be cancelled once headers arrive, preventing healthy streaming responses from being
|
||||
// aborted mid-stream.
|
||||
const ms = options["timeout"] ?? REQUEST_TIMEOUT_MS
|
||||
const controller = ms !== false ? new AbortController() : undefined
|
||||
const signals: AbortSignal[] = []
|
||||
|
||||
if (opts.signal) signals.push(opts.signal)
|
||||
if (chunkAbortCtl) signals.push(chunkAbortCtl.signal)
|
||||
if (controller) signals.push(controller.signal)
|
||||
|
||||
const combined = signals.length === 0 ? null : signals.length === 1 ? signals[0] : AbortSignal.any(signals)
|
||||
if (combined) opts.signal = combined
|
||||
|
||||
const timer =
|
||||
controller && typeof ms === "number"
|
||||
? setTimeout(() => controller.abort(new DOMException("The operation timed out.", "TimeoutError")), ms)
|
||||
: undefined
|
||||
// kilocode_change end
|
||||
|
||||
// Strip openai itemId metadata following what codex does
|
||||
if (model.api.npm === "@ai-sdk/openai" && opts.body && opts.method === "POST") {
|
||||
const body = JSON.parse(opts.body as string)
|
||||
const isAzure = model.providerID.includes("azure")
|
||||
const keepIds = isAzure && body.store === true
|
||||
if (!keepIds && Array.isArray(body.input)) {
|
||||
for (const item of body.input) {
|
||||
if ("id" in item) {
|
||||
delete item.id
|
||||
}
|
||||
}
|
||||
opts.body = JSON.stringify(body)
|
||||
}
|
||||
}
|
||||
|
||||
// kilocode_change start - clear connection-phase timeout once headers arrive
|
||||
try {
|
||||
const res = await fetchFn(input, {
|
||||
...opts,
|
||||
// @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682
|
||||
timeout: false,
|
||||
})
|
||||
if (timer !== undefined) clearTimeout(timer)
|
||||
if (!chunkAbortCtl) return res
|
||||
return wrapSSE(res, chunkTimeout, chunkAbortCtl)
|
||||
} catch (err) {
|
||||
if (timer !== undefined) clearTimeout(timer)
|
||||
throw err
|
||||
}
|
||||
// kilocode_change end
|
||||
}
|
||||
|
||||
const bundledFn = BUNDLED_PROVIDERS[model.api.npm]
|
||||
if (bundledFn) {
|
||||
log.info("using bundled provider", {
|
||||
providerID: model.providerID,
|
||||
pkg: model.api.npm,
|
||||
})
|
||||
const loaded = bundledFn({
|
||||
name: model.providerID,
|
||||
...options,
|
||||
})
|
||||
s.sdk.set(key, loaded)
|
||||
return loaded as SDK
|
||||
}
|
||||
|
||||
let installedPath: string
|
||||
if (!model.api.npm.startsWith("file://")) {
|
||||
installedPath = await BunProc.install(model.api.npm, "latest")
|
||||
} else {
|
||||
log.info("loading local provider", { pkg: model.api.npm })
|
||||
installedPath = model.api.npm
|
||||
}
|
||||
|
||||
const mod = await import(installedPath)
|
||||
|
||||
const fn = mod[Object.keys(mod).find((key) => key.startsWith("create"))!]
|
||||
const loaded = fn({
|
||||
name: model.providerID,
|
||||
...options,
|
||||
})
|
||||
s.sdk.set(key, loaded)
|
||||
return loaded as SDK
|
||||
} catch (e) {
|
||||
throw new InitError({ providerID: model.providerID }, { cause: e })
|
||||
}
|
||||
}
|
||||
|
||||
const getProvider = Effect.fn("Provider.getProvider")((providerID: ProviderID) =>
|
||||
InstanceState.use(cache, (s) => s.providers[providerID]),
|
||||
)
|
||||
|
||||
const getModel = Effect.fn("Provider.getModel")(function* (providerID: ProviderID, modelID: ModelID) {
|
||||
const s = yield* InstanceState.get(cache)
|
||||
const provider = s.providers[providerID]
|
||||
if (!provider) {
|
||||
const available = Object.keys(s.providers)
|
||||
const matches = fuzzysort.go(providerID, available, { limit: 3, threshold: -10000 })
|
||||
throw new ModelNotFoundError({ providerID, modelID, suggestions: matches.map((m) => m.target) })
|
||||
}
|
||||
|
||||
const info = provider.models[modelID]
|
||||
if (!info) {
|
||||
const available = Object.keys(provider.models)
|
||||
const matches = fuzzysort.go(modelID, available, { limit: 3, threshold: -10000 })
|
||||
throw new ModelNotFoundError({ providerID, modelID, suggestions: matches.map((m) => m.target) })
|
||||
}
|
||||
return info
|
||||
})
|
||||
|
||||
const getLanguage = Effect.fn("Provider.getLanguage")(function* (model: Model) {
|
||||
const s = yield* InstanceState.get(cache)
|
||||
const key = `${model.providerID}/${model.id}`
|
||||
if (s.models.has(key)) return s.models.get(key)!
|
||||
|
||||
return yield* Effect.promise(async () => {
|
||||
const provider = s.providers[model.providerID]
|
||||
const sdk = await resolveSDK(model, s)
|
||||
|
||||
try {
|
||||
const language = s.modelLoaders[model.providerID]
|
||||
? await s.modelLoaders[model.providerID](sdk, model.api.id, {
|
||||
...provider.options,
|
||||
...model.options,
|
||||
})
|
||||
: sdk.languageModel(model.api.id)
|
||||
s.models.set(key, language)
|
||||
return language
|
||||
} catch (e) {
|
||||
if (e instanceof NoSuchModelError)
|
||||
throw new ModelNotFoundError(
|
||||
{
|
||||
modelID: model.id,
|
||||
providerID: model.providerID,
|
||||
},
|
||||
{ cause: e },
|
||||
)
|
||||
throw e
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const closest = Effect.fn("Provider.closest")(function* (providerID: ProviderID, query: string[]) {
|
||||
const s = yield* InstanceState.get(cache)
|
||||
const provider = s.providers[providerID]
|
||||
if (!provider) return undefined
|
||||
for (const item of query) {
|
||||
for (const modelID of Object.keys(provider.models)) {
|
||||
if (modelID.includes(item)) return { providerID, modelID }
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
|
||||
const getSmallModel = Effect.fn("Provider.getSmallModel")(function* (providerID: ProviderID) {
|
||||
const cfg = yield* config.get()
|
||||
|
||||
if (cfg.small_model) {
|
||||
const parsed = parseModel(cfg.small_model)
|
||||
return yield* getModel(parsed.providerID, parsed.modelID)
|
||||
}
|
||||
|
||||
const s = yield* InstanceState.get(cache)
|
||||
const provider = s.providers[providerID]
|
||||
if (!provider) return undefined
|
||||
|
||||
let priority = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-haiku-4.5",
|
||||
"3-5-haiku",
|
||||
"3.5-haiku",
|
||||
"gemini-3-flash",
|
||||
"gemini-2.5-flash",
|
||||
"gpt-5-nano",
|
||||
]
|
||||
if (providerID.startsWith("opencode")) {
|
||||
priority = ["gpt-5-nano"]
|
||||
}
|
||||
if (providerID.startsWith("github-copilot")) {
|
||||
priority = ["gpt-5-mini", "claude-haiku-4.5", ...priority]
|
||||
}
|
||||
// kilocode_change start
|
||||
if (providerID.startsWith("kilo")) {
|
||||
priority = ["kilo-auto/small"]
|
||||
}
|
||||
// kilocode_change end
|
||||
for (const item of priority) {
|
||||
if (providerID === ProviderID.amazonBedrock) {
|
||||
const crossRegionPrefixes = ["global.", "us.", "eu."]
|
||||
const candidates = Object.keys(provider.models).filter((m) => m.includes(item))
|
||||
|
||||
const globalMatch = candidates.find((m) => m.startsWith("global."))
|
||||
if (globalMatch) return yield* getModel(providerID, ModelID.make(globalMatch))
|
||||
|
||||
const region = provider.options?.region
|
||||
if (region) {
|
||||
const regionPrefix = region.split("-")[0]
|
||||
if (regionPrefix === "us" || regionPrefix === "eu") {
|
||||
const regionalMatch = candidates.find((m) => m.startsWith(`${regionPrefix}.`))
|
||||
if (regionalMatch) return yield* getModel(providerID, ModelID.make(regionalMatch))
|
||||
}
|
||||
}
|
||||
|
||||
const unprefixed = candidates.find((m) => !crossRegionPrefixes.some((p) => m.startsWith(p)))
|
||||
if (unprefixed) return yield* getModel(providerID, ModelID.make(unprefixed))
|
||||
} else {
|
||||
for (const model of Object.keys(provider.models)) {
|
||||
if (model.includes(item)) return yield* getModel(providerID, ModelID.make(model))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kilocode_change start - clear timeout once headers arrive
|
||||
try {
|
||||
const response = await fetchFn(input, {
|
||||
...opts,
|
||||
// @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682
|
||||
timeout: false,
|
||||
})
|
||||
if (timer !== undefined) clearTimeout(timer)
|
||||
return response
|
||||
} catch (err) {
|
||||
if (timer !== undefined) clearTimeout(timer)
|
||||
throw err
|
||||
// kilocode_change start - fall back to kilo's auto small model if the requested provider has no match
|
||||
const kiloProvider = s.providers[ProviderID.kilo]
|
||||
if (kiloProvider && kiloProvider.models["kilo-auto/small"]) {
|
||||
return yield* getModel(ProviderID.make("kilo"), ModelID.make("kilo-auto/small"))
|
||||
}
|
||||
// kilocode_change end
|
||||
}
|
||||
|
||||
const bundledFn = BUNDLED_PROVIDERS[model.api.npm]
|
||||
if (bundledFn) {
|
||||
log.info("using bundled provider", {
|
||||
providerID: model.providerID,
|
||||
pkg: model.api.npm,
|
||||
})
|
||||
const loaded = bundledFn({
|
||||
name: model.providerID,
|
||||
...options,
|
||||
})
|
||||
s.sdk.set(key, loaded)
|
||||
return loaded as SDK
|
||||
}
|
||||
|
||||
let installedPath: string
|
||||
if (!model.api.npm.startsWith("file://")) {
|
||||
installedPath = await BunProc.install(model.api.npm, "latest")
|
||||
} else {
|
||||
log.info("loading local provider", { pkg: model.api.npm })
|
||||
installedPath = model.api.npm
|
||||
}
|
||||
|
||||
const mod = await import(installedPath)
|
||||
|
||||
const fn = mod[Object.keys(mod).find((key) => key.startsWith("create"))!]
|
||||
const loaded = fn({
|
||||
name: model.providerID,
|
||||
...options,
|
||||
return undefined
|
||||
})
|
||||
s.sdk.set(key, loaded)
|
||||
return loaded as SDK
|
||||
} catch (e) {
|
||||
throw new InitError({ providerID: model.providerID }, { cause: e })
|
||||
}
|
||||
|
||||
const defaultModel = Effect.fn("Provider.defaultModel")(function* () {
|
||||
const cfg = yield* config.get()
|
||||
if (cfg.model) return parseModel(cfg.model)
|
||||
|
||||
const s = yield* InstanceState.get(cache)
|
||||
const recent = yield* Effect.promise(() =>
|
||||
Filesystem.readJson<{
|
||||
recent?: { providerID: ProviderID; modelID: ModelID }[]
|
||||
}>(path.join(Global.Path.state, "model.json"))
|
||||
.then((x): { providerID: ProviderID; modelID: ModelID }[] => (Array.isArray(x.recent) ? x.recent : []))
|
||||
.catch((): { providerID: ProviderID; modelID: ModelID }[] => []),
|
||||
)
|
||||
for (const entry of recent) {
|
||||
const provider = s.providers[entry.providerID]
|
||||
if (!provider) continue
|
||||
if (!provider.models[entry.modelID]) continue
|
||||
return { providerID: entry.providerID, modelID: entry.modelID }
|
||||
}
|
||||
|
||||
const provider = Object.values(s.providers).find(
|
||||
(p) => !cfg.provider || Object.keys(cfg.provider).includes(p.id),
|
||||
)
|
||||
if (!provider) throw new Error("no providers found")
|
||||
const [model] = sort(Object.values(provider.models))
|
||||
if (!model) throw new Error("no models found")
|
||||
return {
|
||||
providerID: provider.id,
|
||||
modelID: model.id,
|
||||
}
|
||||
})
|
||||
|
||||
return Service.of({ list, getProvider, getModel, getLanguage, closest, getSmallModel, defaultModel })
|
||||
}),
|
||||
)
|
||||
|
||||
const { runPromise } = makeRuntime(
|
||||
Service,
|
||||
layer.pipe(Layer.provide(Config.defaultLayer), Layer.provide(Auth.defaultLayer)),
|
||||
)
|
||||
|
||||
export async function list() {
|
||||
return runPromise((svc) => svc.list())
|
||||
}
|
||||
|
||||
export async function getProvider(providerID: ProviderID) {
|
||||
return state().then((s) => s.providers[providerID])
|
||||
return runPromise((svc) => svc.getProvider(providerID))
|
||||
}
|
||||
|
||||
export async function getModel(providerID: ProviderID, modelID: ModelID) {
|
||||
const s = await state()
|
||||
const provider = s.providers[providerID]
|
||||
if (!provider) {
|
||||
const availableProviders = Object.keys(s.providers)
|
||||
const matches = fuzzysort.go(providerID, availableProviders, {
|
||||
limit: 3,
|
||||
threshold: -10000,
|
||||
})
|
||||
const suggestions = matches.map((m) => m.target)
|
||||
throw new ModelNotFoundError({ providerID, modelID, suggestions })
|
||||
}
|
||||
|
||||
const info = provider.models[modelID]
|
||||
if (!info) {
|
||||
const availableModels = Object.keys(provider.models)
|
||||
const matches = fuzzysort.go(modelID, availableModels, {
|
||||
limit: 3,
|
||||
threshold: -10000,
|
||||
})
|
||||
const suggestions = matches.map((m) => m.target)
|
||||
throw new ModelNotFoundError({ providerID, modelID, suggestions })
|
||||
}
|
||||
return info
|
||||
return runPromise((svc) => svc.getModel(providerID, modelID))
|
||||
}
|
||||
|
||||
export async function getLanguage(model: Model): Promise<LanguageModelV3> {
|
||||
const s = await state()
|
||||
const key = `${model.providerID}/${model.id}`
|
||||
if (s.models.has(key)) return s.models.get(key)!
|
||||
|
||||
const provider = s.providers[model.providerID]
|
||||
const sdk = await getSDK(model)
|
||||
|
||||
try {
|
||||
const language = s.modelLoaders[model.providerID]
|
||||
? await s.modelLoaders[model.providerID](sdk, model.api.id, {
|
||||
...provider.options,
|
||||
...model.options,
|
||||
})
|
||||
: sdk.languageModel(model.api.id)
|
||||
s.models.set(key, language)
|
||||
return language
|
||||
} catch (e) {
|
||||
if (e instanceof NoSuchModelError)
|
||||
throw new ModelNotFoundError(
|
||||
{
|
||||
modelID: model.id,
|
||||
providerID: model.providerID,
|
||||
},
|
||||
{ cause: e },
|
||||
)
|
||||
throw e
|
||||
}
|
||||
export async function getLanguage(model: Model) {
|
||||
return runPromise((svc) => svc.getLanguage(model))
|
||||
}
|
||||
|
||||
export async function closest(providerID: ProviderID, query: string[]) {
|
||||
const s = await state()
|
||||
const provider = s.providers[providerID]
|
||||
if (!provider) return undefined
|
||||
for (const item of query) {
|
||||
for (const modelID of Object.keys(provider.models)) {
|
||||
if (modelID.includes(item))
|
||||
return {
|
||||
providerID,
|
||||
modelID,
|
||||
}
|
||||
}
|
||||
}
|
||||
return runPromise((svc) => svc.closest(providerID, query))
|
||||
}
|
||||
|
||||
export async function getSmallModel(providerID: ProviderID) {
|
||||
const cfg = await Config.get()
|
||||
return runPromise((svc) => svc.getSmallModel(providerID))
|
||||
}
|
||||
|
||||
if (cfg.small_model) {
|
||||
const parsed = parseModel(cfg.small_model)
|
||||
return getModel(parsed.providerID, parsed.modelID)
|
||||
}
|
||||
|
||||
const provider = await state().then((state) => state.providers[providerID])
|
||||
if (provider) {
|
||||
let priority = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-haiku-4.5",
|
||||
"3-5-haiku",
|
||||
"3.5-haiku",
|
||||
"gemini-3-flash",
|
||||
"gemini-2.5-flash",
|
||||
"gpt-5-nano",
|
||||
]
|
||||
// kilocode_change start
|
||||
if (providerID.startsWith("kilo")) {
|
||||
priority = ["kilo-auto/small"]
|
||||
}
|
||||
// kilocode_change end
|
||||
if (providerID.startsWith("github-copilot")) {
|
||||
// prioritize free models for github copilot
|
||||
priority = ["gpt-5-mini", "claude-haiku-4.5", ...priority]
|
||||
}
|
||||
for (const item of priority) {
|
||||
if (providerID === ProviderID.amazonBedrock) {
|
||||
const crossRegionPrefixes = ["global.", "us.", "eu."]
|
||||
const candidates = Object.keys(provider.models).filter((m) => m.includes(item))
|
||||
|
||||
// Model selection priority:
|
||||
// 1. global. prefix (works everywhere)
|
||||
// 2. User's region prefix (us., eu.)
|
||||
// 3. Unprefixed model
|
||||
const globalMatch = candidates.find((m) => m.startsWith("global."))
|
||||
if (globalMatch) return getModel(providerID, ModelID.make(globalMatch))
|
||||
|
||||
const region = provider.options?.region
|
||||
if (region) {
|
||||
const regionPrefix = region.split("-")[0]
|
||||
if (regionPrefix === "us" || regionPrefix === "eu") {
|
||||
const regionalMatch = candidates.find((m) => m.startsWith(`${regionPrefix}.`))
|
||||
if (regionalMatch) return getModel(providerID, ModelID.make(regionalMatch))
|
||||
}
|
||||
}
|
||||
|
||||
const unprefixed = candidates.find((m) => !crossRegionPrefixes.some((p) => m.startsWith(p)))
|
||||
if (unprefixed) return getModel(providerID, ModelID.make(unprefixed))
|
||||
} else {
|
||||
for (const model of Object.keys(provider.models)) {
|
||||
if (model.includes(item)) return getModel(providerID, ModelID.make(model))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kilocode_change start
|
||||
// Check if kilo provider is available before using it
|
||||
const kiloProvider = await state().then((state) => state.providers[ProviderID.kilo])
|
||||
if (kiloProvider && kiloProvider.models["kilo-auto/small"]) {
|
||||
return getModel(ProviderID.make("kilo"), ModelID.make("kilo-auto/small"))
|
||||
}
|
||||
// kilocode_change end
|
||||
|
||||
return undefined
|
||||
export async function defaultModel() {
|
||||
return runPromise((svc) => svc.defaultModel())
|
||||
}
|
||||
|
||||
const priority = ["gpt-5", "claude-sonnet-4", "big-pickle", "gemini-3-pro"]
|
||||
@@ -1565,33 +1656,6 @@ export namespace Provider {
|
||||
)
|
||||
}
|
||||
|
||||
export async function defaultModel() {
|
||||
const cfg = await Config.get()
|
||||
if (cfg.model) return parseModel(cfg.model)
|
||||
|
||||
const providers = await list()
|
||||
const recent = (await Filesystem.readJson<{
|
||||
recent?: { providerID: ProviderID; modelID: ModelID }[]
|
||||
}>(path.join(Global.Path.state, "model.json"))
|
||||
.then((x) => (Array.isArray(x.recent) ? x.recent : []))
|
||||
.catch(() => [])) as { providerID: ProviderID; modelID: ModelID }[]
|
||||
for (const entry of recent) {
|
||||
const provider = providers[entry.providerID]
|
||||
if (!provider) continue
|
||||
if (!provider.models[entry.modelID]) continue
|
||||
return { providerID: entry.providerID, modelID: entry.modelID }
|
||||
}
|
||||
|
||||
const provider = Object.values(providers).find((p) => !cfg.provider || Object.keys(cfg.provider).includes(p.id))
|
||||
if (!provider) throw new Error("no providers found")
|
||||
const [model] = sort(Object.values(provider.models))
|
||||
if (!model) throw new Error("no models found")
|
||||
return {
|
||||
providerID: provider.id,
|
||||
modelID: model.id,
|
||||
}
|
||||
}
|
||||
|
||||
export function parseModel(model: string) {
|
||||
const [providerID, ...rest] = model.split("/")
|
||||
return {
|
||||
|
||||
@@ -34,7 +34,9 @@ async function resolveRelative(instruction: string): Promise<string[]> {
|
||||
return Filesystem.globUp(instruction, Instance.directory, Instance.worktree).catch(() => [])
|
||||
}
|
||||
if (!Flag.KILO_CONFIG_DIR) {
|
||||
log.warn(`Skipping relative instruction "${instruction}" - no KILO_CONFIG_DIR set while project config is disabled`)
|
||||
log.warn(
|
||||
`Skipping relative instruction "${instruction}" - no KILO_CONFIG_DIR set while project config is disabled`,
|
||||
)
|
||||
return []
|
||||
}
|
||||
return Filesystem.globUp(instruction, Flag.KILO_CONFIG_DIR, Flag.KILO_CONFIG_DIR).catch(() => [])
|
||||
|
||||
@@ -404,7 +404,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const match = input.processor.partFromToolCall(options.toolCallId)
|
||||
if (!match || match.state.status !== "running") return
|
||||
if (!match || !["running", "pending"].includes(match.state.status)) return
|
||||
yield* sessions.updatePart({
|
||||
...match,
|
||||
state: {
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
import { Log } from "../util/log"
|
||||
import path from "path"
|
||||
import fs from "fs/promises"
|
||||
import { Global } from "../global"
|
||||
import { Filesystem } from "../util/filesystem"
|
||||
import { lazy } from "../util/lazy"
|
||||
import { Lock } from "../util/lock"
|
||||
import { NamedError } from "@opencode-ai/util/error"
|
||||
import z from "zod"
|
||||
import { Glob } from "../util/glob"
|
||||
import { git } from "@/util/git"
|
||||
import { AppFileSystem } from "@/filesystem"
|
||||
import { makeRuntime } from "@/effect/run-service"
|
||||
import { Effect, Exit, Layer, Option, RcMap, Schema, ServiceMap, TxReentrantLock } from "effect"
|
||||
|
||||
export namespace Storage {
|
||||
const log = Log.create({ service: "storage" })
|
||||
|
||||
type Migration = (dir: string) => Promise<void>
|
||||
type Migration = (dir: string, fs: AppFileSystem.Interface) => Effect.Effect<void, AppFileSystem.Error>
|
||||
|
||||
export const NotFoundError = NamedError.create(
|
||||
"NotFoundError",
|
||||
@@ -22,36 +20,101 @@ export namespace Storage {
|
||||
}),
|
||||
)
|
||||
|
||||
export type Error = AppFileSystem.Error | InstanceType<typeof NotFoundError>
|
||||
|
||||
const RootFile = Schema.Struct({
|
||||
path: Schema.optional(
|
||||
Schema.Struct({
|
||||
root: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
const SessionFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
})
|
||||
|
||||
const MessageFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
})
|
||||
|
||||
const DiffFile = Schema.Struct({
|
||||
additions: Schema.Number,
|
||||
deletions: Schema.Number,
|
||||
})
|
||||
|
||||
const SummaryFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
projectID: Schema.String,
|
||||
summary: Schema.Struct({ diffs: Schema.Array(DiffFile) }),
|
||||
})
|
||||
|
||||
const decodeRoot = Schema.decodeUnknownOption(RootFile)
|
||||
const decodeSession = Schema.decodeUnknownOption(SessionFile)
|
||||
const decodeMessage = Schema.decodeUnknownOption(MessageFile)
|
||||
const decodeSummary = Schema.decodeUnknownOption(SummaryFile)
|
||||
|
||||
export interface Interface {
|
||||
readonly remove: (key: string[]) => Effect.Effect<void, AppFileSystem.Error>
|
||||
readonly read: <T>(key: string[]) => Effect.Effect<T, Error>
|
||||
readonly update: <T>(key: string[], fn: (draft: T) => void) => Effect.Effect<T, Error>
|
||||
readonly write: <T>(key: string[], content: T) => Effect.Effect<void, AppFileSystem.Error>
|
||||
readonly list: (prefix: string[]) => Effect.Effect<string[][], AppFileSystem.Error>
|
||||
}
|
||||
|
||||
export class Service extends ServiceMap.Service<Service, Interface>()("@opencode/Storage") {}
|
||||
|
||||
function file(dir: string, key: string[]) {
|
||||
return path.join(dir, ...key) + ".json"
|
||||
}
|
||||
|
||||
function missing(err: unknown) {
|
||||
if (!err || typeof err !== "object") return false
|
||||
if ("code" in err && err.code === "ENOENT") return true
|
||||
if ("reason" in err && err.reason && typeof err.reason === "object" && "_tag" in err.reason) {
|
||||
return err.reason._tag === "NotFound"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function parseMigration(text: string) {
|
||||
const value = Number.parseInt(text, 10)
|
||||
return Number.isNaN(value) ? 0 : value
|
||||
}
|
||||
|
||||
const MIGRATIONS: Migration[] = [
|
||||
async (dir) => {
|
||||
Effect.fn("Storage.migration.1")(function* (dir: string, fs: AppFileSystem.Interface) {
|
||||
const project = path.resolve(dir, "../project")
|
||||
if (!(await Filesystem.isDir(project))) return
|
||||
const projectDirs = await Glob.scan("*", {
|
||||
if (!(yield* fs.isDir(project))) return
|
||||
const projectDirs = yield* fs.glob("*", {
|
||||
cwd: project,
|
||||
include: "all",
|
||||
})
|
||||
for (const projectDir of projectDirs) {
|
||||
const fullPath = path.join(project, projectDir)
|
||||
if (!(await Filesystem.isDir(fullPath))) continue
|
||||
const full = path.join(project, projectDir)
|
||||
if (!(yield* fs.isDir(full))) continue
|
||||
log.info(`migrating project ${projectDir}`)
|
||||
let projectID = projectDir
|
||||
const fullProjectDir = path.join(project, projectDir)
|
||||
let worktree = "/"
|
||||
|
||||
if (projectID !== "global") {
|
||||
for (const msgFile of await Glob.scan("storage/session/message/*/*.json", {
|
||||
cwd: path.join(project, projectDir),
|
||||
for (const msgFile of yield* fs.glob("storage/session/message/*/*.json", {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const json = await Filesystem.readJson<any>(msgFile)
|
||||
worktree = json.path?.root
|
||||
if (worktree) break
|
||||
const json = decodeRoot(yield* fs.readJson(msgFile), { onExcessProperty: "preserve" })
|
||||
const root = Option.isSome(json) ? json.value.path?.root : undefined
|
||||
if (!root) continue
|
||||
worktree = root
|
||||
break
|
||||
}
|
||||
if (!worktree) continue
|
||||
if (!(await Filesystem.isDir(worktree))) continue
|
||||
const result = await git(["rev-list", "--max-parents=0", "--all"], {
|
||||
cwd: worktree,
|
||||
})
|
||||
if (!(yield* fs.isDir(worktree))) continue
|
||||
const result = yield* Effect.promise(() =>
|
||||
git(["rev-list", "--max-parents=0", "--all"], {
|
||||
cwd: worktree,
|
||||
}),
|
||||
)
|
||||
const [id] = result
|
||||
.text()
|
||||
.split("\n")
|
||||
@@ -61,157 +124,230 @@ export namespace Storage {
|
||||
if (!id) continue
|
||||
projectID = id
|
||||
|
||||
await Filesystem.writeJson(path.join(dir, "project", projectID + ".json"), {
|
||||
id,
|
||||
vcs: "git",
|
||||
worktree,
|
||||
time: {
|
||||
created: Date.now(),
|
||||
initialized: Date.now(),
|
||||
},
|
||||
})
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "project", projectID + ".json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
id,
|
||||
vcs: "git",
|
||||
worktree,
|
||||
time: {
|
||||
created: Date.now(),
|
||||
initialized: Date.now(),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
|
||||
log.info(`migrating sessions for project ${projectID}`)
|
||||
for (const sessionFile of await Glob.scan("storage/session/info/*.json", {
|
||||
cwd: fullProjectDir,
|
||||
for (const sessionFile of yield* fs.glob("storage/session/info/*.json", {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
|
||||
log.info("copying", {
|
||||
sessionFile,
|
||||
dest,
|
||||
})
|
||||
const session = await Filesystem.readJson<any>(sessionFile)
|
||||
await Filesystem.writeJson(dest, session)
|
||||
log.info(`migrating messages for session ${session.id}`)
|
||||
for (const msgFile of await Glob.scan(`storage/session/message/${session.id}/*.json`, {
|
||||
cwd: fullProjectDir,
|
||||
log.info("copying", { sessionFile, dest })
|
||||
const session = yield* fs.readJson(sessionFile)
|
||||
const info = decodeSession(session, { onExcessProperty: "preserve" })
|
||||
yield* fs.writeWithDirs(dest, JSON.stringify(session, null, 2))
|
||||
if (Option.isNone(info)) continue
|
||||
log.info(`migrating messages for session ${info.value.id}`)
|
||||
for (const msgFile of yield* fs.glob(`storage/session/message/${info.value.id}/*.json`, {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const dest = path.join(dir, "message", session.id, path.basename(msgFile))
|
||||
const next = path.join(dir, "message", info.value.id, path.basename(msgFile))
|
||||
log.info("copying", {
|
||||
msgFile,
|
||||
dest,
|
||||
dest: next,
|
||||
})
|
||||
const message = await Filesystem.readJson<any>(msgFile)
|
||||
await Filesystem.writeJson(dest, message)
|
||||
const message = yield* fs.readJson(msgFile)
|
||||
const item = decodeMessage(message, { onExcessProperty: "preserve" })
|
||||
yield* fs.writeWithDirs(next, JSON.stringify(message, null, 2))
|
||||
if (Option.isNone(item)) continue
|
||||
|
||||
log.info(`migrating parts for message ${message.id}`)
|
||||
for (const partFile of await Glob.scan(`storage/session/part/${session.id}/${message.id}/*.json`, {
|
||||
cwd: fullProjectDir,
|
||||
log.info(`migrating parts for message ${item.value.id}`)
|
||||
for (const partFile of yield* fs.glob(`storage/session/part/${info.value.id}/${item.value.id}/*.json`, {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const dest = path.join(dir, "part", message.id, path.basename(partFile))
|
||||
const part = await Filesystem.readJson(partFile)
|
||||
const out = path.join(dir, "part", item.value.id, path.basename(partFile))
|
||||
const part = yield* fs.readJson(partFile)
|
||||
log.info("copying", {
|
||||
partFile,
|
||||
dest,
|
||||
dest: out,
|
||||
})
|
||||
await Filesystem.writeJson(dest, part)
|
||||
yield* fs.writeWithDirs(out, JSON.stringify(part, null, 2))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
async (dir) => {
|
||||
for (const item of await Glob.scan("session/*/*.json", {
|
||||
}),
|
||||
Effect.fn("Storage.migration.2")(function* (dir: string, fs: AppFileSystem.Interface) {
|
||||
for (const item of yield* fs.glob("session/*/*.json", {
|
||||
cwd: dir,
|
||||
absolute: true,
|
||||
})) {
|
||||
const session = await Filesystem.readJson<any>(item)
|
||||
if (!session.projectID) continue
|
||||
if (!session.summary?.diffs) continue
|
||||
const { diffs } = session.summary
|
||||
await Filesystem.write(path.join(dir, "session_diff", session.id + ".json"), JSON.stringify(diffs))
|
||||
await Filesystem.writeJson(path.join(dir, "session", session.projectID, session.id + ".json"), {
|
||||
...session,
|
||||
summary: {
|
||||
additions: diffs.reduce((sum: any, x: any) => sum + x.additions, 0),
|
||||
deletions: diffs.reduce((sum: any, x: any) => sum + x.deletions, 0),
|
||||
},
|
||||
})
|
||||
const raw = yield* fs.readJson(item)
|
||||
const session = decodeSummary(raw, { onExcessProperty: "preserve" })
|
||||
if (Option.isNone(session)) continue
|
||||
const diffs = session.value.summary.diffs
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "session_diff", session.value.id + ".json"),
|
||||
JSON.stringify(diffs, null, 2),
|
||||
)
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "session", session.value.projectID, session.value.id + ".json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
...(raw as Record<string, unknown>),
|
||||
summary: {
|
||||
additions: diffs.reduce((sum, x) => sum + x.additions, 0),
|
||||
deletions: diffs.reduce((sum, x) => sum + x.deletions, 0),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
}
|
||||
},
|
||||
}),
|
||||
]
|
||||
|
||||
const state = lazy(async () => {
|
||||
const dir = path.join(Global.Path.data, "storage")
|
||||
const migration = await Filesystem.readJson<string>(path.join(dir, "migration"))
|
||||
.then((x) => parseInt(x))
|
||||
.catch(() => 0)
|
||||
for (let index = migration; index < MIGRATIONS.length; index++) {
|
||||
log.info("running migration", { index })
|
||||
const migration = MIGRATIONS[index]
|
||||
await migration(dir).catch(() => log.error("failed to run migration", { index }))
|
||||
await Filesystem.write(path.join(dir, "migration"), (index + 1).toString())
|
||||
}
|
||||
return {
|
||||
dir,
|
||||
}
|
||||
})
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* AppFileSystem.Service
|
||||
const locks = yield* RcMap.make({
|
||||
lookup: () => TxReentrantLock.make(),
|
||||
idleTimeToLive: 0,
|
||||
})
|
||||
const state = yield* Effect.cached(
|
||||
Effect.gen(function* () {
|
||||
const dir = path.join(Global.Path.data, "storage")
|
||||
const marker = path.join(dir, "migration")
|
||||
const migration = yield* fs.readFileString(marker).pipe(
|
||||
Effect.map(parseMigration),
|
||||
Effect.catchIf(missing, () => Effect.succeed(0)),
|
||||
Effect.orElseSucceed(() => 0),
|
||||
)
|
||||
for (let i = migration; i < MIGRATIONS.length; i++) {
|
||||
log.info("running migration", { index: i })
|
||||
const step = MIGRATIONS[i]!
|
||||
const exit = yield* Effect.exit(step(dir, fs))
|
||||
if (Exit.isFailure(exit)) {
|
||||
log.error("failed to run migration", { index: i, cause: exit.cause })
|
||||
break
|
||||
}
|
||||
yield* fs.writeWithDirs(marker, String(i + 1))
|
||||
}
|
||||
return { dir }
|
||||
}),
|
||||
)
|
||||
|
||||
const fail = (target: string): Effect.Effect<never, InstanceType<typeof NotFoundError>> =>
|
||||
Effect.fail(new NotFoundError({ message: `Resource not found: ${target}` }))
|
||||
|
||||
const wrap = <A>(target: string, body: Effect.Effect<A, AppFileSystem.Error>) =>
|
||||
body.pipe(Effect.catchIf(missing, () => fail(target)))
|
||||
|
||||
const writeJson = Effect.fnUntraced(function* (target: string, content: unknown) {
|
||||
yield* fs.writeWithDirs(target, JSON.stringify(content, null, 2))
|
||||
})
|
||||
|
||||
const withResolved = <A, E>(
|
||||
key: string[],
|
||||
fn: (target: string, rw: TxReentrantLock.TxReentrantLock) => Effect.Effect<A, E>,
|
||||
): Effect.Effect<A, E | AppFileSystem.Error> =>
|
||||
Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const target = file((yield* state).dir, key)
|
||||
return yield* fn(target, yield* RcMap.get(locks, target))
|
||||
}),
|
||||
)
|
||||
|
||||
const remove: Interface["remove"] = Effect.fn("Storage.remove")(function* (key: string[]) {
|
||||
yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withWriteLock(rw, fs.remove(target).pipe(Effect.catchIf(missing, () => Effect.void))),
|
||||
)
|
||||
})
|
||||
|
||||
const read: Interface["read"] = <T>(key: string[]) =>
|
||||
Effect.gen(function* () {
|
||||
const value = yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withReadLock(rw, wrap(target, fs.readJson(target))),
|
||||
)
|
||||
return value as T
|
||||
})
|
||||
|
||||
const update: Interface["update"] = <T>(key: string[], fn: (draft: T) => void) =>
|
||||
Effect.gen(function* () {
|
||||
const value = yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withWriteLock(
|
||||
rw,
|
||||
Effect.gen(function* () {
|
||||
const content = yield* wrap(target, fs.readJson(target))
|
||||
fn(content as T)
|
||||
yield* writeJson(target, content)
|
||||
return content
|
||||
}),
|
||||
),
|
||||
)
|
||||
return value as T
|
||||
})
|
||||
|
||||
const write: Interface["write"] = (key: string[], content: unknown) =>
|
||||
Effect.gen(function* () {
|
||||
yield* withResolved(key, (target, rw) => TxReentrantLock.withWriteLock(rw, writeJson(target, content)))
|
||||
})
|
||||
|
||||
const list: Interface["list"] = Effect.fn("Storage.list")(function* (prefix: string[]) {
|
||||
const dir = (yield* state).dir
|
||||
const cwd = path.join(dir, ...prefix)
|
||||
const result = yield* fs
|
||||
.glob("**/*", {
|
||||
cwd,
|
||||
include: "file",
|
||||
})
|
||||
.pipe(Effect.catch(() => Effect.succeed<string[]>([])))
|
||||
return result
|
||||
.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)])
|
||||
.toSorted((a, b) => a.join("/").localeCompare(b.join("/")))
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
remove,
|
||||
read,
|
||||
update,
|
||||
write,
|
||||
list,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(Layer.provide(AppFileSystem.defaultLayer))
|
||||
|
||||
const { runPromise } = makeRuntime(Service, defaultLayer)
|
||||
|
||||
export async function remove(key: string[]) {
|
||||
const dir = await state().then((x) => x.dir)
|
||||
const target = path.join(dir, ...key) + ".json"
|
||||
return withErrorHandling(async () => {
|
||||
await fs.unlink(target).catch(() => {})
|
||||
})
|
||||
return runPromise((svc) => svc.remove(key))
|
||||
}
|
||||
|
||||
export async function read<T>(key: string[]) {
|
||||
const dir = await state().then((x) => x.dir)
|
||||
const target = path.join(dir, ...key) + ".json"
|
||||
return withErrorHandling(async () => {
|
||||
using _ = await Lock.read(target)
|
||||
const result = await Filesystem.readJson<T>(target)
|
||||
return result as T
|
||||
})
|
||||
return runPromise((svc) => svc.read<T>(key))
|
||||
}
|
||||
|
||||
export async function update<T>(key: string[], fn: (draft: T) => void) {
|
||||
const dir = await state().then((x) => x.dir)
|
||||
const target = path.join(dir, ...key) + ".json"
|
||||
return withErrorHandling(async () => {
|
||||
using _ = await Lock.write(target)
|
||||
const content = await Filesystem.readJson<T>(target)
|
||||
fn(content as T)
|
||||
await Filesystem.writeJson(target, content)
|
||||
return content
|
||||
})
|
||||
return runPromise((svc) => svc.update<T>(key, fn))
|
||||
}
|
||||
|
||||
export async function write<T>(key: string[], content: T) {
|
||||
const dir = await state().then((x) => x.dir)
|
||||
const target = path.join(dir, ...key) + ".json"
|
||||
return withErrorHandling(async () => {
|
||||
using _ = await Lock.write(target)
|
||||
await Filesystem.writeJson(target, content)
|
||||
})
|
||||
}
|
||||
|
||||
async function withErrorHandling<T>(body: () => Promise<T>) {
|
||||
return body().catch((e) => {
|
||||
if (!(e instanceof Error)) throw e
|
||||
const errnoException = e as NodeJS.ErrnoException
|
||||
if (errnoException.code === "ENOENT") {
|
||||
throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` })
|
||||
}
|
||||
throw e
|
||||
})
|
||||
return runPromise((svc) => svc.write(key, content))
|
||||
}
|
||||
|
||||
export async function list(prefix: string[]) {
|
||||
const dir = await state().then((x) => x.dir)
|
||||
try {
|
||||
const result = await Glob.scan("**/*", {
|
||||
cwd: path.join(dir, ...prefix),
|
||||
include: "file",
|
||||
}).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]))
|
||||
result.sort()
|
||||
return result
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
return runPromise((svc) => svc.list(prefix))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user