Mirror_Repos/Kilo-Org_kilocode
2
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

feat: implement codebase indexing (#6966)

Browse Source 
* feat(core): implement codebase indexing

* feat(core): gate indexing behind experimental flag

* fix: improvements to slow provider startups and indexing init blocking the world

* test(cli): stabilize indexing startup tests

* test(cli): avoid indexing startup hang

* test(cli): avoid CI startup leaks

* test(cli): annotate provider test change

* fix(indexing): tighten dependency surface

* refactor(cli): move indexing logic into kilocode paths

* test(cli): skip flaky shell cancel prompt cases

* fix(cli): tolerate slow Windows worktree cleanup

* fix(vscode): authenticate indexing status fetch

---------

Co-authored-by: Marius <marius@kilocode.ai>
This commit is contained in:
Josh Holmer
2026-04-27 11:43:27 -04:00
committed by GitHub
parent 1ec3885099
commit f74d54c431
221 changed files with 26181 additions and 977 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+1
View File
@@ -15,6 +15,7 @@ Kilo CLI is an open source AI coding agent that generates code from natural lang
- **Typecheck**: `bun turbo typecheck` (uses `tsgo`, not `tsc`)
- **Test**: `bun test` from `packages/opencode/` (NOT from root -- root blocks tests)
- **Single test**: `bun test test/tool/tool.test.ts` from `packages/opencode/`
- **CLI build artifact size check**: after `bun run script/build.ts --single --skip-install` in `packages/opencode/`, use `du -h dist/*/*/bin/kilo` (scoped package output lives under `dist/@kilocode/`)
- **SDK regen**: After changing server endpoints in `packages/opencode/src/server/`, run `./script/generate.ts` from root to regenerate `packages/sdk/js/`
- **Knip** (unused exports): `bun run knip` from `packages/kilo-vscode/`. CI runs this — all exported types/functions must be imported somewhere. Remove or unexport unused exports before pushing.
- **Source links**: After adding or changing URLs in `packages/kilo-vscode/`, `packages/kilo-vscode/webview-ui/`, or `packages/opencode/src/`, run `bun run script/extract-source-links.ts` from the repo root and commit the updated `packages/kilo-docs/source-links.md`. CI runs this check — the build fails if the file is stale.
bun.lock
+143 -7
View File
@@ -246,6 +246,37 @@
"typescript": "catalog:",
},
},
"packages/kilo-indexing": {
"name": "@kilocode/kilo-indexing",
"version": "7.1.3",
"dependencies": {
"@aws-sdk/client-bedrock-runtime": "3.1005.0",
"@aws-sdk/credential-provider-ini": "3.972.31",
"@kilocode/kilo-gateway": "workspace:*",
"@kilocode/plugin": "workspace:*",
"@lancedb/lancedb": "0.26.2",
"@qdrant/js-client-rest": "1.17.0",
"async-mutex": "0.5.0",
"chokidar": "4.0.3",
"glob": "13.0.6",
"hono": "catalog:",
"hono-openapi": "catalog:",
"ignore": "7.0.5",
"minimatch": "10.2.5",
"openai": "6.27.0",
"p-limit": "7.3.0",
"tree-sitter-wasms": "0.1.13",
"uuid": "14.0.0",
"web-tree-sitter": "0.25.10",
"zod": "catalog:",
},
"devDependencies": {
"@tsconfig/node22": "catalog:",
"@types/node": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:",
},
},
"packages/kilo-jetbrains": {
"name": "@kilocode/kilo-jetbrains",
},
@@ -310,6 +341,7 @@
"dependencies": {
"@anthropic-ai/sdk": "^0.39.0",
"@kilocode/kilo-i18n": "workspace:*",
"@kilocode/kilo-indexing": "workspace:*",
"@kilocode/kilo-ui": "workspace:*",
"@kilocode/sdk": "workspace:*",
"@opencode-ai/ui": "workspace:*",
@@ -406,6 +438,7 @@
"@hono/standard-validator": "0.1.5",
"@hono/zod-validator": "catalog:",
"@kilocode/kilo-gateway": "workspace:*",
"@kilocode/kilo-indexing": "workspace:*",
"@kilocode/kilo-telemetry": "workspace:*",
"@kilocode/plugin": "workspace:*",
"@kilocode/sdk": "workspace:*",
@@ -474,6 +507,7 @@
"strip-ansi": "7.1.2",
"tree-sitter-bash": "0.25.0",
"tree-sitter-powershell": "0.25.10",
"tree-sitter-wasms": "^0.1.12",
"turndown": "7.2.0",
"ulid": "catalog:",
"venice-ai-sdk-provider": "2.0.1",
@@ -843,6 +877,8 @@
"@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
"@aws-sdk/client-bedrock-runtime": ["@aws-sdk/client-bedrock-runtime@3.1005.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.19", "@aws-sdk/credential-provider-node": "^3.972.19", "@aws-sdk/eventstream-handler-node": "^3.972.10", "@aws-sdk/middleware-eventstream": "^3.972.7", "@aws-sdk/middleware-host-header": "^3.972.7", "@aws-sdk/middleware-logger": "^3.972.7", "@aws-sdk/middleware-recursion-detection": "^3.972.7", "@aws-sdk/middleware-user-agent": "^3.972.20", "@aws-sdk/middleware-websocket": "^3.972.12", "@aws-sdk/region-config-resolver": "^3.972.7", "@aws-sdk/token-providers": "3.1005.0", "@aws-sdk/types": "^3.973.5", "@aws-sdk/util-endpoints": "^3.996.4", "@aws-sdk/util-user-agent-browser": "^3.972.7", "@aws-sdk/util-user-agent-node": "^3.973.5", "@smithy/config-resolver": "^4.4.10", "@smithy/core": "^3.23.9", "@smithy/eventstream-serde-browser": "^4.2.11", "@smithy/eventstream-serde-config-resolver": "^4.3.11", "@smithy/eventstream-serde-node": "^4.2.11", "@smithy/fetch-http-handler": "^5.3.13", "@smithy/hash-node": "^4.2.11", "@smithy/invalid-dependency": "^4.2.11", "@smithy/middleware-content-length": "^4.2.11", "@smithy/middleware-endpoint": "^4.4.23", "@smithy/middleware-retry": "^4.4.40", "@smithy/middleware-serde": "^4.2.12", "@smithy/middleware-stack": "^4.2.11", "@smithy/node-config-provider": "^4.3.11", "@smithy/node-http-handler": "^4.4.14", "@smithy/protocol-http": "^5.3.11", "@smithy/smithy-client": "^4.12.3", "@smithy/types": "^4.13.0", "@smithy/url-parser": "^4.2.11", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.39", "@smithy/util-defaults-mode-node": "^4.2.42", "@smithy/util-endpoints": "^3.3.2", "@smithy/util-middleware": "^4.2.11", "@smithy/util-retry": "^4.2.11", "@smithy/util-stream": "^4.5.17", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-IV5vZ6H46ZNsTxsFWkbrJkg+sPe6+3m90k7EejgB/AFCb/YQuseH0+I3B57ew+zoOaXJU71KDPBwsIiMSsikVg=="],
"@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.1025.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.26", "@aws-sdk/credential-provider-node": "^3.972.29", "@aws-sdk/middleware-host-header": "^3.972.8", "@aws-sdk/middleware-logger": "^3.972.8", "@aws-sdk/middleware-recursion-detection": "^3.972.9", "@aws-sdk/middleware-user-agent": "^3.972.28", "@aws-sdk/region-config-resolver": "^3.972.10", "@aws-sdk/types": "^3.973.6", "@aws-sdk/util-endpoints": "^3.996.5", "@aws-sdk/util-user-agent-browser": "^3.972.8", "@aws-sdk/util-user-agent-node": "^3.973.14", "@smithy/config-resolver": "^4.4.13", "@smithy/core": "^3.23.13", "@smithy/fetch-http-handler": "^5.3.15", "@smithy/hash-node": "^4.2.12", "@smithy/invalid-dependency": "^4.2.12", "@smithy/middleware-content-length": "^4.2.12", "@smithy/middleware-endpoint": "^4.4.28", "@smithy/middleware-retry": "^4.4.46", "@smithy/middleware-serde": "^4.2.16", "@smithy/middleware-stack": "^4.2.12", "@smithy/node-config-provider": "^4.3.12", "@smithy/node-http-handler": "^4.5.1", "@smithy/protocol-http": "^5.3.12", "@smithy/smithy-client": "^4.12.8", "@smithy/types": "^4.13.1", "@smithy/url-parser": "^4.2.12", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.44", "@smithy/util-defaults-mode-node": "^4.2.48", "@smithy/util-endpoints": "^3.3.3", "@smithy/util-middleware": "^4.2.12", "@smithy/util-retry": "^4.2.13", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-ke7vyS7Dmo3St1a354AlpAjocZpG25Ql52XQ1AXRD3VQ791FBT7vX+EqGCfAEIWLPxpcldaW2Nny6P6klEOkkw=="],
"@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1025.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.26", "@aws-sdk/credential-provider-node": "^3.972.29", "@aws-sdk/middleware-bucket-endpoint": "^3.972.8", "@aws-sdk/middleware-expect-continue": "^3.972.8", "@aws-sdk/middleware-flexible-checksums": "^3.974.6", "@aws-sdk/middleware-host-header": "^3.972.8", "@aws-sdk/middleware-location-constraint": "^3.972.8", "@aws-sdk/middleware-logger": "^3.972.8", "@aws-sdk/middleware-recursion-detection": "^3.972.9", "@aws-sdk/middleware-sdk-s3": "^3.972.27", "@aws-sdk/middleware-ssec": "^3.972.8", "@aws-sdk/middleware-user-agent": "^3.972.28", "@aws-sdk/region-config-resolver": "^3.972.10", "@aws-sdk/signature-v4-multi-region": "^3.996.15", "@aws-sdk/types": "^3.973.6", "@aws-sdk/util-endpoints": "^3.996.5", "@aws-sdk/util-user-agent-browser": "^3.972.8", "@aws-sdk/util-user-agent-node": "^3.973.14", "@smithy/config-resolver": "^4.4.13", "@smithy/core": "^3.23.13", "@smithy/eventstream-serde-browser": "^4.2.12", "@smithy/eventstream-serde-config-resolver": "^4.3.12", "@smithy/eventstream-serde-node": "^4.2.12", "@smithy/fetch-http-handler": "^5.3.15", "@smithy/hash-blob-browser": "^4.2.13", "@smithy/hash-node": "^4.2.12", "@smithy/hash-stream-node": "^4.2.12", "@smithy/invalid-dependency": "^4.2.12", "@smithy/md5-js": "^4.2.12", "@smithy/middleware-content-length": "^4.2.12", "@smithy/middleware-endpoint": "^4.4.28", "@smithy/middleware-retry": "^4.4.46", "@smithy/middleware-serde": "^4.2.16", "@smithy/middleware-stack": "^4.2.12", "@smithy/node-config-provider": "^4.3.12", "@smithy/node-http-handler": "^4.5.1", "@smithy/protocol-http": "^5.3.12", "@smithy/smithy-client": "^4.12.8", "@smithy/types": "^4.13.1", "@smithy/url-parser": "^4.2.12", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.44", "@smithy/util-defaults-mode-node": "^4.2.48", "@smithy/util-endpoints": "^3.3.3", "@smithy/util-middleware": "^4.2.12", "@smithy/util-retry": "^4.2.13", "@smithy/util-stream": "^4.5.21", "@smithy/util-utf8": "^4.2.2", "@smithy/util-waiter": "^4.2.14", "tslib": "^2.6.2" } }, "sha512-9Byz2fPnuGRRL8DTTD5bYPl1Iwm+ysLiCMgptffa3lNkVLCiUZc5e5TAaOjk0MvyeXieq+jn35AmQL6cgN2KHQ=="],
@@ -871,8 +907,12 @@
"@aws-sdk/credential-providers": ["@aws-sdk/credential-providers@3.1025.0", "", { "dependencies": { "@aws-sdk/client-cognito-identity": "3.1025.0", "@aws-sdk/core": "^3.973.26", "@aws-sdk/credential-provider-cognito-identity": "^3.972.21", "@aws-sdk/credential-provider-env": "^3.972.24", "@aws-sdk/credential-provider-http": "^3.972.26", "@aws-sdk/credential-provider-ini": "^3.972.28", "@aws-sdk/credential-provider-login": "^3.972.28", "@aws-sdk/credential-provider-node": "^3.972.29", "@aws-sdk/credential-provider-process": "^3.972.24", "@aws-sdk/credential-provider-sso": "^3.972.28", "@aws-sdk/credential-provider-web-identity": "^3.972.28", "@aws-sdk/nested-clients": "^3.996.18", "@aws-sdk/types": "^3.973.6", "@smithy/config-resolver": "^4.4.13", "@smithy/core": "^3.23.13", "@smithy/credential-provider-imds": "^4.2.12", "@smithy/node-config-provider": "^4.3.12", "@smithy/property-provider": "^4.2.12", "@smithy/types": "^4.13.1", "tslib": "^2.6.2" } }, "sha512-hOMHzYetTwnpvfbLN8emaw+nnQrqlEV0I5rTrgRKTAx1anzEvls/rD1IXwOvX8Z+B9mgbK+yNFqO3wQkBghI1g=="],
"@aws-sdk/eventstream-handler-node": ["@aws-sdk/eventstream-handler-node@3.972.14", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/eventstream-codec": "^4.2.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-m4X56gxG76/CKfxNVbOFuYwnAZcHgS6HOH8lgp15HoGHIAVTcZfZrXvcYzJFOMLEJgVn+JHBu6EiNV+xSNXXFg=="],
"@aws-sdk/middleware-bucket-endpoint": ["@aws-sdk/middleware-bucket-endpoint@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-arn-parser": "^3.972.3", "@smithy/node-config-provider": "^4.3.14", "@smithy/protocol-http": "^5.3.14", "@smithy/types": "^4.14.1", "@smithy/util-config-provider": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA=="],
"@aws-sdk/middleware-eventstream": ["@aws-sdk/middleware-eventstream@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/protocol-http": "^5.3.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-QUqLs7Af1II9X4fCRAu+EGHG3KHyOp4RkuLhRKoA3NuFlh6TL8i+zXBl8w2LUxqm44B/Kom45hgSlwA1SpTsXQ=="],
"@aws-sdk/middleware-expect-continue": ["@aws-sdk/middleware-expect-continue@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/protocol-http": "^5.3.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ=="],
"@aws-sdk/middleware-flexible-checksums": ["@aws-sdk/middleware-flexible-checksums@3.974.9", "", { "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", "@aws-crypto/util": "5.2.0", "@aws-sdk/core": "^3.974.1", "@aws-sdk/crc64-nvme": "^3.972.7", "@aws-sdk/types": "^3.973.8", "@smithy/is-array-buffer": "^4.2.2", "@smithy/node-config-provider": "^4.3.14", "@smithy/protocol-http": "^5.3.14", "@smithy/types": "^4.14.1", "@smithy/util-middleware": "^4.2.14", "@smithy/util-stream": "^4.5.23", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-ye6xVuMEQ5NCT+yQOryGYsuCXnOwu7iGFGzV+qpXZOWtqXIAAaFostapxj6RCubw36rekVwmdB2lcspFuyNfYQ=="],
@@ -891,13 +931,15 @@
"@aws-sdk/middleware-user-agent": ["@aws-sdk/middleware-user-agent@3.972.31", "", { "dependencies": { "@aws-sdk/core": "^3.974.1", "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-endpoints": "^3.996.7", "@smithy/core": "^3.23.15", "@smithy/protocol-http": "^5.3.14", "@smithy/types": "^4.14.1", "@smithy/util-retry": "^4.3.2", "tslib": "^2.6.2" } }, "sha512-L+hXN2HDomlIsWSHW5DVD7ppccCeRnlHXZ5uHG34ePTjF5bm0I1fmrJLbUGiW97xRXWryit5cjdP4Sx2FwiGog=="],
"@aws-sdk/middleware-websocket": ["@aws-sdk/middleware-websocket@3.972.16", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-format-url": "^3.972.10", "@smithy/eventstream-codec": "^4.2.14", "@smithy/eventstream-serde-browser": "^4.2.14", "@smithy/fetch-http-handler": "^5.3.17", "@smithy/protocol-http": "^5.3.14", "@smithy/signature-v4": "^5.3.14", "@smithy/types": "^4.14.1", "@smithy/util-base64": "^4.3.2", "@smithy/util-hex-encoding": "^4.2.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-86+S9oCyRVGzoMRpQhxkArp7kD2K75GPmaNevd9B6EyNhWoNvnCZZ3WbgN4j7ZT+jvtvBCGZvI2XHsWZJ+BRIg=="],
"@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.996.21", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.1", "@aws-sdk/middleware-host-header": "^3.972.10", "@aws-sdk/middleware-logger": "^3.972.10", "@aws-sdk/middleware-recursion-detection": "^3.972.11", "@aws-sdk/middleware-user-agent": "^3.972.31", "@aws-sdk/region-config-resolver": "^3.972.12", "@aws-sdk/types": "^3.973.8", "@aws-sdk/util-endpoints": "^3.996.7", "@aws-sdk/util-user-agent-browser": "^3.972.10", "@aws-sdk/util-user-agent-node": "^3.973.17", "@smithy/config-resolver": "^4.4.16", "@smithy/core": "^3.23.15", "@smithy/fetch-http-handler": "^5.3.17", "@smithy/hash-node": "^4.2.14", "@smithy/invalid-dependency": "^4.2.14", "@smithy/middleware-content-length": "^4.2.14", "@smithy/middleware-endpoint": "^4.4.30", "@smithy/middleware-retry": "^4.5.3", "@smithy/middleware-serde": "^4.2.18", "@smithy/middleware-stack": "^4.2.14", "@smithy/node-config-provider": "^4.3.14", "@smithy/node-http-handler": "^4.5.3", "@smithy/protocol-http": "^5.3.14", "@smithy/smithy-client": "^4.12.11", "@smithy/types": "^4.14.1", "@smithy/url-parser": "^4.2.14", "@smithy/util-base64": "^4.3.2", "@smithy/util-body-length-browser": "^4.2.2", "@smithy/util-body-length-node": "^4.2.3", "@smithy/util-defaults-mode-browser": "^4.3.47", "@smithy/util-defaults-mode-node": "^4.2.52", "@smithy/util-endpoints": "^3.4.1", "@smithy/util-middleware": "^4.2.14", "@smithy/util-retry": "^4.3.2", "@smithy/util-utf8": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-Me3d/ua2lb2G0bQfFmvCeQQp3+nN6GSPqMxDmi/IQlQ8CrlpQ5C0JJHpz2AnOUkEFI0lBNrAL3Vnt29l44ndkA=="],
"@aws-sdk/region-config-resolver": ["@aws-sdk/region-config-resolver@3.972.12", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/config-resolver": "^4.4.16", "@smithy/node-config-provider": "^4.3.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-QQI43Mxd53nBij0pm8HXC+t4IOC6gnhhZfzxE0OATQyO6QfPV4e+aTIRRuAJKA6Nig/cR8eLwPryqYTX9ZrjAQ=="],
"@aws-sdk/signature-v4-multi-region": ["@aws-sdk/signature-v4-multi-region@3.996.18", "", { "dependencies": { "@aws-sdk/middleware-sdk-s3": "^3.972.30", "@aws-sdk/types": "^3.973.8", "@smithy/protocol-http": "^5.3.14", "@smithy/signature-v4": "^5.3.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-4KT8UXRmvNAP5zKq9UI1MIwbnmSChZncBt89RKu/skMqZSSWGkBZTAJsZ+no+txfmF3kVaUFv31CTBZkQ5BJpQ=="],
"@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1032.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.1", "@aws-sdk/nested-clients": "^3.996.21", "@aws-sdk/types": "^3.973.8", "@smithy/property-provider": "^4.2.14", "@smithy/shared-ini-file-loader": "^4.4.9", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-n+PU8Z+gll7p3wDrH+Wo6fkt8sPrVnq30YYM6Ryga95oJlEneNMEbDHj0iqjMX3V7gaGdJo/hJWyPo4lscP+mA=="],
"@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1005.0", "", { "dependencies": { "@aws-sdk/core": "^3.973.19", "@aws-sdk/nested-clients": "^3.996.8", "@aws-sdk/types": "^3.973.5", "@smithy/property-provider": "^4.2.11", "@smithy/shared-ini-file-loader": "^4.4.6", "@smithy/types": "^4.13.0", "tslib": "^2.6.2" } }, "sha512-vMxd+ivKqSxU9bHx5vmAlFKDAkjGotFU56IOkDa5DaTu1WWwbcse0yFHEm9I537oVvodaiwMl3VBwgHfzQ2rvw=="],
"@aws-sdk/types": ["@aws-sdk/types@3.973.8", "", { "dependencies": { "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw=="],
@@ -905,6 +947,8 @@
"@aws-sdk/util-endpoints": ["@aws-sdk/util-endpoints@3.996.7", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/types": "^4.14.1", "@smithy/url-parser": "^4.2.14", "@smithy/util-endpoints": "^3.4.1", "tslib": "^2.6.2" } }, "sha512-ty4LQxN1QC+YhUP28NfEgZDEGXkyqOQy+BDriBozqHsrYO4JMgiPhfizqOGF7P+euBTZ5Ez6SKlLAMCLo8tzmw=="],
"@aws-sdk/util-format-url": ["@aws-sdk/util-format-url@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/querystring-builder": "^4.2.14", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-DEKiHNJVtNxdyTeQspzY+15Po/kHm6sF0Cs4HV9Q2+lplB63+DrvdeiSoOSdWEWAoO2RcY1veoXVDz2tWxWCgQ=="],
"@aws-sdk/util-locate-window": ["@aws-sdk/util-locate-window@3.965.5", "", { "dependencies": { "tslib": "^2.6.2" } }, "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ=="],
"@aws-sdk/util-user-agent-browser": ["@aws-sdk/util-user-agent-browser@3.972.10", "", { "dependencies": { "@aws-sdk/types": "^3.973.8", "@smithy/types": "^4.14.1", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g=="],
@@ -1403,6 +1447,8 @@
"@kilocode/kilo-i18n": ["@kilocode/kilo-i18n@workspace:packages/kilo-i18n"],
"@kilocode/kilo-indexing": ["@kilocode/kilo-indexing@workspace:packages/kilo-indexing"],
"@kilocode/kilo-jetbrains": ["@kilocode/kilo-jetbrains@workspace:packages/kilo-jetbrains"],
"@kilocode/kilo-telemetry": ["@kilocode/kilo-telemetry@workspace:packages/kilo-telemetry"],
@@ -1421,6 +1467,22 @@
"@kwsites/promise-deferred": ["@kwsites/promise-deferred@1.1.1", "", {}, "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="],
"@lancedb/lancedb": ["@lancedb/lancedb@0.26.2", "", { "dependencies": { "reflect-metadata": "^0.2.2" }, "optionalDependencies": { "@lancedb/lancedb-darwin-arm64": "0.26.2", "@lancedb/lancedb-linux-arm64-gnu": "0.26.2", "@lancedb/lancedb-linux-arm64-musl": "0.26.2", "@lancedb/lancedb-linux-x64-gnu": "0.26.2", "@lancedb/lancedb-linux-x64-musl": "0.26.2", "@lancedb/lancedb-win32-arm64-msvc": "0.26.2", "@lancedb/lancedb-win32-x64-msvc": "0.26.2" }, "peerDependencies": { "apache-arrow": ">=15.0.0 <=18.1.0" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ] }, "sha512-umk4WMCTwJntLquwvUbpqE+TXREolcQVL9MHcxr8EhRjsha88+ATJ4QuS/hpyiE1CG3R/XcgrMgJAGkziPC/gA=="],
"@lancedb/lancedb-darwin-arm64": ["@lancedb/lancedb-darwin-arm64@0.26.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-LAZ/v261eTlv44KoEm+AdqGnohS9IbVVVJkH9+8JTqwhe/k4j4Af8X9cD18tsaJAAtrGxxOCyIJ3wZTiBqrkCw=="],
"@lancedb/lancedb-linux-arm64-gnu": ["@lancedb/lancedb-linux-arm64-gnu@0.26.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-guHKm+zvuQB22dgyn6/sYZJvD6IL9lC24cl6ZuzVX/jYgag/gNLHT86HongrcBjgdjI6+YIGmdfD6b/iAKxn3Q=="],
"@lancedb/lancedb-linux-arm64-musl": ["@lancedb/lancedb-linux-arm64-musl@0.26.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-pR6Hs/0iphItrJYYLf/yrqCC+scPcHpCGl6rHqcU2GHxo5RFpzlMzqW1DiXScGiBRuCcD9HIMec+kBsOgXv4GQ=="],
"@lancedb/lancedb-linux-x64-gnu": ["@lancedb/lancedb-linux-x64-gnu@0.26.2", "", { "os": "linux", "cpu": "x64" }, "sha512-u4UUSPwd2YecgGqWjh9W0MHKgsVwB2Ch2ROpF8AY+IA7kpGsbB18R1/t7v2B0q7pahRy20dgsaku5LH1zuzMRQ=="],
"@lancedb/lancedb-linux-x64-musl": ["@lancedb/lancedb-linux-x64-musl@0.26.2", "", { "os": "linux", "cpu": "x64" }, "sha512-XIS4qkVfGlzmsUPqAG2iKt8ykuz28GfemGC0ijXwu04kC1pYiCFzTpB3UIZjm5oM7OTync1aQ3mGTj1oCciSPA=="],
"@lancedb/lancedb-win32-arm64-msvc": ["@lancedb/lancedb-win32-arm64-msvc@0.26.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-//tZDPitm2PxNvalHP+m+Pf6VvFAeQgcht1+HJnutjH4gp6xYW6ynQlWWFDBmz9WRkUT+mXu2O4FUIhbdNaJSQ=="],
"@lancedb/lancedb-win32-x64-msvc": ["@lancedb/lancedb-win32-x64-msvc@0.26.2", "", { "os": "win32", "cpu": "x64" }, "sha512-GH3pfyzicgPGTb84xMXgujlWDaAnBTmUyjooYiCE2tC24BaehX4hgFhXivamzAEsF5U2eVsA/J60Ppif+skAbA=="],
"@leichtgewicht/ip-codec": ["@leichtgewicht/ip-codec@2.0.5", "", {}, "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw=="],
"@lukeed/ms": ["@lukeed/ms@2.0.2", "", {}, "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA=="],
@@ -1797,6 +1859,10 @@
"@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="],
"@qdrant/js-client-rest": ["@qdrant/js-client-rest@1.17.0", "", { "dependencies": { "@qdrant/openapi-typescript-fetch": "1.2.6", "undici": "^6.23.0" }, "peerDependencies": { "typescript": ">=4.7" } }, "sha512-aZFQeirWVqWAa1a8vJ957LMzcXkFHGbsoRhzc8AkGfg6V0jtK8PlG8/eyyc2xhYsR961FDDx1Tx6nyE0K7lS+A=="],
"@qdrant/openapi-typescript-fetch": ["@qdrant/openapi-typescript-fetch@1.2.6", "", {}, "sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA=="],
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.60.2", "", { "os": "android", "cpu": "arm" }, "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw=="],
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.60.2", "", { "os": "android", "cpu": "arm64" }, "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg=="],
@@ -2227,6 +2293,10 @@
"@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="],
"@types/command-line-args": ["@types/command-line-args@5.2.3", "", {}, "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw=="],
"@types/command-line-usage": ["@types/command-line-usage@5.0.4", "", {}, "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg=="],
"@types/cross-spawn": ["@types/cross-spawn@6.0.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-fXRhhUkG4H3TQk5dBhQ7m/JDdSNHKwR2BBia62lhwEIq9xGiQKLxd6LymNhn47SjXhsUEPmxi+PKw2OkW4LLjA=="],
"@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="],
@@ -2551,6 +2621,8 @@
"anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="],
"apache-arrow": ["apache-arrow@18.1.0", "", { "dependencies": { "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", "@types/command-line-usage": "^5.0.4", "@types/node": "^20.13.0", "command-line-args": "^5.2.1", "command-line-usage": "^7.0.1", "flatbuffers": "^24.3.25", "json-bignum": "^0.0.3", "tslib": "^2.6.2" }, "bin": { "arrow2csv": "bin/arrow2csv.js" } }, "sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg=="],
"app-builder-bin": ["app-builder-bin@5.0.0-alpha.12", "", {}, "sha512-j87o0j6LqPL3QRr8yid6c+Tt5gC7xNfYo6uQIQkorAC6MpeayVMZrEDzKmJJ/Hlv7EnOQpaRm53k6ktDYZyB6w=="],
"app-builder-lib": ["app-builder-lib@26.8.1", "", { "dependencies": { "@develar/schema-utils": "~2.6.5", "@electron/asar": "3.4.1", "@electron/fuses": "^1.8.0", "@electron/get": "^3.0.0", "@electron/notarize": "2.5.0", "@electron/osx-sign": "1.3.3", "@electron/rebuild": "^4.0.3", "@electron/universal": "2.0.3", "@malept/flatpak-bundler": "^0.4.0", "@types/fs-extra": "9.0.13", "async-exit-hook": "^2.0.1", "builder-util": "26.8.1", "builder-util-runtime": "9.5.1", "chromium-pickle-js": "^0.2.0", "ci-info": "4.3.1", "debug": "^4.3.4", "dotenv": "^16.4.5", "dotenv-expand": "^11.0.6", "ejs": "^3.1.8", "electron-publish": "26.8.1", "fs-extra": "^10.1.0", "hosted-git-info": "^4.1.0", "isbinaryfile": "^5.0.0", "jiti": "^2.4.2", "js-yaml": "^4.1.0", "json5": "^2.2.3", "lazy-val": "^1.0.5", "minimatch": "^10.0.3", "plist": "3.1.0", "proper-lockfile": "^4.1.2", "resedit": "^1.7.0", "semver": "~7.7.3", "tar": "^7.5.7", "temp-file": "^3.4.0", "tiny-async-pool": "1.3.0", "which": "^5.0.0" }, "peerDependencies": { "dmg-builder": "26.8.1", "electron-builder-squirrel-windows": "26.8.1" } }, "sha512-p0Im/Dx5C4tmz8QEE1Yn4MkuPC8PrnlRneMhWJj7BBXQfNTJUshM/bp3lusdEsDbvvfJZpXWnYesgSLvwtM2Zw=="],
@@ -2565,6 +2637,8 @@
"aria-query": ["aria-query@5.3.2", "", {}, "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw=="],
"array-back": ["array-back@3.1.0", "", {}, "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q=="],
"array-find-index": ["array-find-index@1.0.2", "", {}, "sha512-M1HQyIXcBGtVywBt8WVdim+lrNaK7VHp99Qt5pSNziXznKHViIBbXWtfRTpEFpF/c4FdfxNAsCCwPp5phBYJtw=="],
"array-flatten": ["array-flatten@1.1.1", "", {}, "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg=="],
@@ -2589,6 +2663,8 @@
"async-lock": ["async-lock@1.4.1", "", {}, "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ=="],
"async-mutex": ["async-mutex@0.5.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA=="],
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
"at-least-node": ["at-least-node@1.0.0", "", {}, "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg=="],
@@ -2753,6 +2829,8 @@
"chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
"chalk-template": ["chalk-template@0.4.0", "", { "dependencies": { "chalk": "^4.1.2" } }, "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg=="],
"character-entities-html4": ["character-entities-html4@2.1.0", "", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="],
"character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="],
@@ -2827,6 +2905,10 @@
"comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="],
"command-line-args": ["command-line-args@5.2.1", "", { "dependencies": { "array-back": "^3.1.0", "find-replace": "^3.0.0", "lodash.camelcase": "^4.3.0", "typical": "^4.0.0" } }, "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg=="],
"command-line-usage": ["command-line-usage@7.0.4", "", { "dependencies": { "array-back": "^6.2.2", "chalk-template": "^0.4.0", "table-layout": "^4.1.1", "typical": "^7.3.0" } }, "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg=="],
"commander": ["commander@14.0.2", "", {}, "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ=="],
"common-ancestor-path": ["common-ancestor-path@2.0.0", "", {}, "sha512-dnN3ibLeoRf2HNC+OlCiNc5d2zxbLJXOtiZUudNFSXZrNSydxcCsSpRzXwfu7BBWCIfHPw+xTayeBvJCP/D8Ng=="],
@@ -3289,12 +3371,16 @@
"find-my-way-ts": ["find-my-way-ts@0.1.6", "", {}, "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA=="],
"find-replace": ["find-replace@3.0.0", "", { "dependencies": { "array-back": "^3.0.1" } }, "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ=="],
"find-up": ["find-up@5.0.0", "", { "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" } }, "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng=="],
"flat": ["flat@5.0.2", "", { "bin": { "flat": "cli.js" } }, "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ=="],
"flat-cache": ["flat-cache@4.0.1", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="],
"flatbuffers": ["flatbuffers@24.12.23", "", {}, "sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA=="],
"flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="],
"follow-redirects": ["follow-redirects@1.16.0", "", {}, "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw=="],
@@ -3601,6 +3687,8 @@
"json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],
"json-bignum": ["json-bignum@0.0.3", "", {}, "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg=="],
"json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="],
"json-parse-even-better-errors": ["json-parse-even-better-errors@5.0.0", "", {}, "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ=="],
@@ -3713,6 +3801,8 @@
"lodash-es": ["lodash-es@4.18.1", "", {}, "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A=="],
"lodash.camelcase": ["lodash.camelcase@4.3.0", "", {}, "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="],
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
"lodash.escaperegexp": ["lodash.escaperegexp@4.1.2", "", {}, "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw=="],
@@ -3979,7 +4069,7 @@
"open": ["open@10.1.2", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "is-wsl": "^3.1.0" } }, "sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw=="],
"openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="],
"openai": ["openai@6.27.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-osTKySlrdYrLYTt0zjhY8yp0JUBmWDCN+Q+QxsV4xMQnnoVFpylgKGgxwN8sSdTNw0G4y+WUXs4eCMWpyDNWZQ=="],
"openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="],
@@ -4013,7 +4103,7 @@
"p-filter": ["p-filter@2.1.0", "", { "dependencies": { "p-map": "^2.0.0" } }, "sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw=="],
"p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="],
"p-limit": ["p-limit@7.3.0", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw=="],
"p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="],
@@ -4263,6 +4353,8 @@
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
"reflect-metadata": ["reflect-metadata@0.2.2", "", {}, "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="],
"regex": ["regex@6.1.0", "", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg=="],
"regex-recursion": ["regex-recursion@6.0.2", "", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg=="],
@@ -4567,6 +4659,8 @@
"table": ["table@6.9.0", "", { "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", "slice-ansi": "^4.0.0", "string-width": "^4.2.3", "strip-ansi": "^6.0.1" } }, "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A=="],
"table-layout": ["table-layout@4.1.1", "", { "dependencies": { "array-back": "^6.2.2", "wordwrapjs": "^5.1.0" } }, "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA=="],
"tailwindcss": ["tailwindcss@4.2.2", "", {}, "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q=="],
"tapable": ["tapable@2.3.2", "", {}, "sha512-1MOpMXuhGzGL5TTCZFItxCc0AARf1EZFQkGqMm7ERKj8+Hgr5oLvJOVFcC+lRmR8hCe2S3jC4T5D7Vg/d7/fhA=="],
@@ -4655,6 +4749,8 @@
"tree-sitter-powershell": ["tree-sitter-powershell@0.25.10", "", { "dependencies": { "node-addon-api": "^7.1.0", "node-gyp-build": "^4.8.0" }, "peerDependencies": { "tree-sitter": "^0.25.0" }, "optionalPeers": ["tree-sitter"] }, "sha512-bEt8QoySpGFnU3aa8WedQyNMaN6aTwy/WUbvIVt0JSKF+BbJoSHNHu+wCbhj7xLMsfB0AuffmiJm+B8gzva8Lg=="],
"tree-sitter-wasms": ["tree-sitter-wasms@0.1.13", "", { "dependencies": { "tree-sitter-wasms": "^0.1.11" } }, "sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ=="],
"treeverse": ["treeverse@3.0.0", "", {}, "sha512-gcANaAnd2QDZFmHFEOF4k7uc1J/6a6z3DJMd/QwEyxLoKGiptJRwid582r7QIsFlFMIZ3SnxfS52S4hm2DHkuQ=="],
"trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="],
@@ -4707,6 +4803,8 @@
"typescript-eslint": ["typescript-eslint@8.58.2", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.58.2", "@typescript-eslint/parser": "8.58.2", "@typescript-eslint/typescript-estree": "8.58.2", "@typescript-eslint/utils": "8.58.2" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-V8iSng9mRbdZjl54VJ9NKr6ZB+dW0J3TzRXRGcSbLIej9jV86ZRtlYeTKDR/QLxXykocJ5icNzbsl2+5TzIvcQ=="],
"typical": ["typical@4.0.0", "", {}, "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw=="],
"uc.micro": ["uc.micro@2.1.0", "", {}, "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="],
"ufo": ["ufo@1.6.3", "", {}, "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q=="],
@@ -4769,7 +4867,7 @@
"utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="],
"uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
"uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="],
"v8-to-istanbul": ["v8-to-istanbul@9.3.0", "", { "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" } }, "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA=="],
@@ -4851,6 +4949,8 @@
"word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="],
"wordwrapjs": ["wordwrapjs@5.1.1", "", {}, "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg=="],
"workerpool": ["workerpool@9.3.4", "", {}, "sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg=="],
"wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="],
@@ -4891,7 +4991,7 @@
"yazl": ["yazl@2.5.1", "", { "dependencies": { "buffer-crc32": "~0.2.3" } }, "sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw=="],
"yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
"yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="],
"yoga-layout": ["yoga-layout@3.2.1", "", {}, "sha512-0LPOt3AxKqMdFBZA3HBAt/t/8vIKq7VaQYbuA8WxCgung+p9TVyKRYdpvCb80HcdTN2NkbIKbhNwKUfm3tQywQ=="],
@@ -4957,6 +5057,8 @@
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
"@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1032.0", "", { "dependencies": { "@aws-sdk/core": "^3.974.1", "@aws-sdk/nested-clients": "^3.996.21", "@aws-sdk/types": "^3.973.8", "@smithy/property-provider": "^4.2.14", "@smithy/shared-ini-file-loader": "^4.4.9", "@smithy/types": "^4.14.1", "tslib": "^2.6.2" } }, "sha512-n+PU8Z+gll7p3wDrH+Wo6fkt8sPrVnq30YYM6Ryga95oJlEneNMEbDHj0iqjMX3V7gaGdJo/hJWyPo4lscP+mA=="],
"@aws-sdk/xml-builder/fast-xml-parser": ["fast-xml-parser@5.5.8", "", { "dependencies": { "fast-xml-builder": "^1.1.4", "path-expression-matcher": "^1.2.0", "strnum": "^2.2.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-Z7Fh2nVQSb2d+poDViM063ix2ZGt9jmY1nWhPfHBOK2Hgnb/OW3P4Et3P/81SEej0J7QbWtJqxO05h8QYfK7LQ=="],
"@azure/core-http/@azure/abort-controller": ["@azure/abort-controller@1.1.0", "", { "dependencies": { "tslib": "^2.2.0" } }, "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw=="],
@@ -5093,6 +5195,8 @@
"@kilocode/kilo-gateway/@opentui/solid": ["@opentui/solid@0.1.75", "", { "dependencies": { "@babel/core": "7.28.0", "@babel/preset-typescript": "7.27.1", "@opentui/core": "0.1.75", "babel-plugin-module-resolver": "5.0.2", "babel-preset-solid": "1.9.9", "s-js": "^0.4.9" }, "peerDependencies": { "solid-js": "1.9.9" } }, "sha512-WjKsZIfrm29znfRlcD9w3uUn/+uvoy2MmeoDwTvg1YOa0OjCTCmjZ43L9imp0m9S4HmVU8ma6o2bR4COzcyDdg=="],
"@kilocode/kilo-indexing/glob": ["glob@13.0.6", "", { "dependencies": { "minimatch": "^10.2.2", "minipass": "^7.1.3", "path-scurry": "^2.0.2" } }, "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw=="],
"@malept/flatpak-bundler/fs-extra": ["fs-extra@9.1.0", "", { "dependencies": { "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ=="],
"@manypkg/find-root/find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="],
@@ -5111,6 +5215,8 @@
"@morphllm/morphsdk/ai": ["ai@6.0.158", "", { "dependencies": { "@ai-sdk/gateway": "3.0.95", "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@opentelemetry/api": "1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-gLTp1UXFtMqKUi3XHs33K7UFglbvojkxF/aq337TxnLGOhHIW9+GyP2jwW4hYX87f1es+wId3VQoPRRu9zEStQ=="],
"@morphllm/morphsdk/openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="],
"@octokit/core/@octokit/graphql": ["@octokit/graphql@7.1.1", "", { "dependencies": { "@octokit/request": "^8.4.1", "@octokit/types": "^13.0.0", "universal-user-agent": "^6.0.0" } }, "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g=="],
"@octokit/core/@octokit/types": ["@octokit/types@13.10.0", "", { "dependencies": { "@octokit/openapi-types": "^24.2.0" } }, "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA=="],
@@ -5199,6 +5305,8 @@
"@protobuf-ts/plugin/typescript": ["typescript@3.9.10", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q=="],
"@qdrant/js-client-rest/undici": ["undici@6.25.0", "", {}, "sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg=="],
"@shikijs/engine-javascript/@shikijs/types": ["@shikijs/types@3.20.0", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-lhYAATn10nkZcBQ0BlzSbJA3wcmL5MXUUF8d2Zzon6saZDlToKaiRX60n2+ZaHJCmXEcZRWNzn+k9vplr8Jhsw=="],
"@shikijs/engine-oniguruma/@shikijs/types": ["@shikijs/types@3.20.0", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-lhYAATn10nkZcBQ0BlzSbJA3wcmL5MXUUF8d2Zzon6saZDlToKaiRX60n2+ZaHJCmXEcZRWNzn+k9vplr8Jhsw=="],
@@ -5359,6 +5467,8 @@
"cacheable-request/get-stream": ["get-stream@5.2.0", "", { "dependencies": { "pump": "^3.0.0" } }, "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA=="],
"chalk-template/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
"cheerio/undici": ["undici@7.25.0", "", {}, "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ=="],
"cheerio/whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
@@ -5367,6 +5477,10 @@
"clone-response/mimic-response": ["mimic-response@1.0.1", "", {}, "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="],
"command-line-usage/array-back": ["array-back@6.2.3", "", {}, "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw=="],
"command-line-usage/typical": ["typical@7.3.0", "", {}, "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw=="],
"compress-commons/is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="],
"conf/env-paths": ["env-paths@3.0.0", "", {}, "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A=="],
@@ -5387,6 +5501,8 @@
"dir-compare/minimatch": ["minimatch@3.1.5", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w=="],
"dir-compare/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="],
"dir-glob/path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="],
"dmg-builder/fs-extra": ["fs-extra@10.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ=="],
@@ -5401,6 +5517,8 @@
"effect/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
"effect/uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
"electron-builder/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
"electron-builder/fs-extra": ["fs-extra@10.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ=="],
@@ -5487,6 +5605,8 @@
"keytar/node-addon-api": ["node-addon-api@4.3.0", "", {}, "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="],
"kilo-code/openai": ["openai@4.104.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" }, "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA=="],
"kilo-code/typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"kilo-code/web-tree-sitter": ["web-tree-sitter@0.24.7", "", {}, "sha512-CdC/TqVFbXqR+C51v38hv6wOPatKEUGxa39scAeFSm98wIhZxAYonhRQPSMmfZ2w7JDI0zQDdzdmgtNk06/krQ=="],
@@ -5567,6 +5687,8 @@
"p-filter/p-map": ["p-map@2.1.0", "", {}, "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="],
"p-locate/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="],
"parent-module/callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="],
"parse-semver/semver": ["semver@5.7.2", "", { "bin": { "semver": "bin/semver" } }, "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g=="],
@@ -5661,6 +5783,8 @@
"table/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"table-layout/array-back": ["array-back@6.2.3", "", {}, "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw=="],
"tar/yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="],
"tar-fs/chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="],
@@ -5837,6 +5961,8 @@
"@morphllm/morphsdk/ai/@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.95", "", { "dependencies": { "@ai-sdk/provider": "3.0.8", "@ai-sdk/provider-utils": "4.0.23", "@vercel/oidc": "3.1.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ZmUNNbZl3V42xwQzPaNUi+s8eqR2lnrxf0bvB6YbLXpLjHYv0k2Y78t12cNOfY0bxGeuVVTLyk856uLuQIuXEQ=="],
"@morphllm/morphsdk/openai/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"@octokit/core/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@24.2.0", "", {}, "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg=="],
"@octokit/endpoint/@octokit/types/@octokit/openapi-types": ["@octokit/openapi-types@24.2.0", "", {}, "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg=="],
@@ -5879,6 +6005,8 @@
"@opencode-ai/plugin/effect/toml": ["toml@3.0.0", "", {}, "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w=="],
"@opencode-ai/plugin/effect/uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
"@opencode-ai/storybook/@storybook/addon-docs/@storybook/csf-plugin": ["@storybook/csf-plugin@10.3.5", "", { "dependencies": { "unplugin": "^2.3.5" }, "peerDependencies": { "esbuild": "*", "rollup": "*", "storybook": "^10.3.5", "vite": "*", "webpack": "*" }, "optionalPeers": ["esbuild", "rollup", "vite", "webpack"] }, "sha512-qlEzNKxOjq86pvrbuMwiGD/bylnsXk1dg7ve0j77YFjEEchqtl7qTlrXvFdNaLA89GhW6D/EV6eOCu/eobPDgw=="],
"@opencode-ai/storybook/@storybook/addon-docs/@storybook/react-dom-shim": ["@storybook/react-dom-shim@10.3.5", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.3.5" } }, "sha512-Gw8R7XZm0zSUH0XAuxlQJhmizsLzyD6x00KOlP6l7oW9eQHXGfxg3seNDG3WrSAcW07iP1/P422kuiriQlOv7g=="],
@@ -5901,10 +6029,14 @@
"@standard-community/standard-json/effect/toml": ["toml@3.0.0", "", {}, "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w=="],
"@standard-community/standard-json/effect/uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
"@standard-community/standard-openapi/effect/@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
"@standard-community/standard-openapi/effect/toml": ["toml@3.0.0", "", {}, "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w=="],
"@standard-community/standard-openapi/effect/uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
"@storybook/addon-links/storybook/open": ["open@10.2.0", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "wsl-utils": "^0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="],
"@storybook/addon-onboarding/storybook/open": ["open@10.2.0", "", { "dependencies": { "default-browser": "^5.2.1", "define-lazy-prop": "^3.0.0", "is-inside-container": "^1.0.0", "wsl-utils": "^0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="],
@@ -6047,6 +6179,8 @@
"c8/yargs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
"chalk-template/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
"crc/buffer/ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
"cross-spawn/which/isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
@@ -6057,6 +6191,8 @@
"dir-compare/minimatch/brace-expansion": ["brace-expansion@1.1.14", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g=="],
"dir-compare/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
"dmg-builder/fs-extra/jsonfile": ["jsonfile@6.2.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg=="],
"dmg-builder/fs-extra/universalify": ["universalify@2.0.1", "", {}, "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw=="],
@@ -6269,6 +6405,8 @@
"ora/log-symbols/is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="],
"p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="],
"pkg-conf/find-up/locate-path": ["locate-path@7.2.0", "", { "dependencies": { "p-locate": "^6.0.0" } }, "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA=="],
"pkg-conf/find-up/path-exists": ["path-exists@5.0.0", "", {}, "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ=="],
@@ -6671,8 +6809,6 @@
"mocha/glob/jackspeak/@isaacs/cliui/wrap-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="],
"pkg-conf/find-up/locate-path/p-locate/p-limit/yocto-queue": ["yocto-queue@1.2.2", "", {}, "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ=="],
"qrcode/yargs/find-up/locate-path/p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="],
"test-exclude/glob/jackspeak/@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
packages/kilo-docs/lib/nav/tools.ts
+1 -1
View File
@@ -12,7 +12,7 @@ export const ToolsNav: NavSection[] = [
{ href: "/automate/tools/search-files", children: "search_files" },
{ href: "/automate/tools/list-files", children: "list_files" },
{ href: "/automate/tools/list-code-definition-names", children: "list_code_definition_names" },
{ href: "/automate/tools/codebase-search", children: "codebase_search" },
{ href: "/automate/tools/semantic-search", children: "semantic_search" },
],
},
{
packages/kilo-docs/lychee.toml
+6 -2
View File
@@ -9,8 +9,8 @@ retry_wait_time = 5
exclude_path = ["node_modules"]
accept = [
"200",
"403",
"200",
"403",
"429",
"301",
"307",
@@ -25,6 +25,10 @@ exclude = [
'^https?://(api|console)\.mistral\.ai',
'^https?://(app|console)\.requesty\.ai',
'^https?://(cloud|console)\.google\.ai',
# API endpoint prefixes are valid but return 4xx to plain GET link checks.
'^https?://ai-gateway\.vercel\.sh/v1/?$',
'^https?://api\.voyageai\.com/v1/embeddings/?$',
'^https?://generativelanguage\.googleapis\.com/v1beta/openai/?$',
'https://opencode.ai/pages/config',
'^localhost:3000/',
'https://tbench.ai/',
packages/kilo-docs/pages/automate/tools/codebase-search.md → packages/kilo-docs/pages/automate/tools/semantic-search.md
+28 -28
View File
@@ -1,10 +1,10 @@
# codebase_search
# semantic_search
{% callout type="info" title="Setup Required" %}
The `codebase_search` tool is part of the [Codebase Indexing](/docs/customize/context/codebase-indexing) feature. It requires additional setup including an embedding provider and vector database.
The `semantic_search` tool is part of the [Codebase Indexing](/docs/customize/context/codebase-indexing) feature. It requires additional setup including an embedding provider and vector database.
{% /callout %}
The `codebase_search` tool performs semantic searches across your entire codebase using AI embeddings. Unlike traditional text-based search, it understands the meaning of your queries and finds relevant code even when exact keywords don't match.
The `semantic_search` tool performs semantic searches across your entire codebase using AI embeddings. Unlike traditional text-based search, it understands the meaning of your queries and finds relevant code even when exact keywords don't match.
---
@@ -70,7 +70,7 @@ This tool is only available when the Codebase Indexing feature is properly confi
## How It Works
When the `codebase_search` tool is invoked, it follows this process:
When the `semantic_search` tool is invoked, it follows this process:
1. **Availability Validation**:
- Verifies that the CodeIndexManager is available and initialized
@@ -112,33 +112,33 @@ When the `codebase_search` tool is invoked, it follows this process:
**Good: Conceptual and specific**
```xml
<codebase_search>
<semantic_search>
<query>user authentication and password validation</query>
</codebase_search>
</semantic_search>
```
**Good: Feature-focused**
```xml
<codebase_search>
<semantic_search>
<query>database connection pool setup</query>
</codebase_search>
</semantic_search>
```
**Good: Problem-oriented**
```xml
<codebase_search>
<semantic_search>
<query>error handling for API requests</query>
</codebase_search>
</semantic_search>
```
**Less effective: Too generic**
```xml
<codebase_search>
<semantic_search>
<query>function</query>
</codebase_search>
</semantic_search>
```
### Query Types That Work Well
@@ -157,28 +157,28 @@ Use the optional `path` parameter to focus searches on specific parts of your co
**Search within API modules:**
```xml
<codebase_search>
<semantic_search>
<query>endpoint validation middleware</query>
<path>src/api</path>
</codebase_search>
</semantic_search>
```
**Search in test files:**
```xml
<codebase_search>
<semantic_search>
<query>mock data setup patterns</query>
<path>tests</path>
</codebase_search>
</semantic_search>
```
**Search specific feature directories:**
```xml
<codebase_search>
<semantic_search>
<query>component state management</query>
<path>src/components/auth</path>
</codebase_search>
</semantic_search>
```
---
@@ -217,42 +217,42 @@ Each search result includes:
Searching for authentication-related code across the entire project:
```xml
<codebase_search>
<semantic_search>
<query>user login and authentication logic</query>
</codebase_search>
</semantic_search>
```
Finding database-related code in a specific directory:
```xml
<codebase_search>
<semantic_search>
<query>database connection and query execution</query>
<path>src/data</path>
</codebase_search>
</semantic_search>
```
Looking for error handling patterns in API code:
```xml
<codebase_search>
<semantic_search>
<query>HTTP error responses and exception handling</query>
<path>src/api</path>
</codebase_search>
</semantic_search>
```
Searching for testing utilities and mock setups:
```xml
<codebase_search>
<semantic_search>
<query>test setup and mock data creation</query>
<path>tests</path>
</codebase_search>
</semantic_search>
```
Finding configuration and environment setup code:
```xml
<codebase_search>
<semantic_search>
<query>environment variables and application configuration</query>
</codebase_search>
</semantic_search>
```
packages/kilo-docs/pages/customize/context/codebase-indexing.md
+2 -2
View File
@@ -17,7 +17,7 @@ When enabled, the indexing system:
1. **Parses your code** using Tree-sitter to identify semantic blocks (functions, classes, methods)
2. **Creates embeddings** of each code block using AI models
3. **Stores vectors** in a Qdrant database for fast similarity search
4. **Provides the [`codebase_search`](/docs/automate/tools/codebase-search) tool** to Kilo Code for intelligent code discovery
4. **Provides the [`semantic_search`](/docs/automate/tools/semantic-search) tool** to Kilo Code for intelligent code discovery
This enables natural language queries like "user authentication logic" or "database connection handling" to find relevant code across your entire project.
@@ -202,7 +202,7 @@ If your local embedding server is based on llama.cpp (including Ollama), indexin
## Using the Search Feature
Once indexed, Kilo Code can use the [`codebase_search`](/docs/automate/tools/codebase-search) tool to find relevant code:
Once indexed, Kilo Code can use the [`semantic_search`](/docs/automate/tools/semantic-search) tool to find relevant code:
**Example Queries:**
packages/kilo-docs/previous-docs-redirects.js
+6
View File
@@ -807,6 +807,12 @@ module.exports = [
basePath: false,
permanent: true,
},
{
source: "/docs/automate/tools/codebase-search",
destination: "/docs/automate/tools/semantic-search",
basePath: false,
permanent: true,
},
{
source: "/docs/automate/kiloclaw/:path*",
destination: "/docs/kiloclaw/:path*",
packages/kilo-docs/public/img/screenshot-tests/kilo-vscode/visual-regression/settings/indexing-provider-blur-race-chromium-linux.png
+3
View File
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6edf67819cd2115c4c85a15b16f7b948a2cb48f19c15ee66ea606f3d0e1971aa
size 58183
packages/kilo-indexing/package.json
+60
View File
@@ -0,0 +1,60 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@kilocode/kilo-indexing",
"version": "7.1.3",
"type": "module",
"license": "MIT",
"description": "Standalone indexing engine and host helpers for Kilo Code",
"keywords": [
"kilo",
"kilocode",
"indexing",
"plugin",
"search",
"embeddings"
],
"exports": {
".": "./src/index.ts",
"./config": "./src/config.ts",
"./detect": "./src/detect.ts",
"./engine": "./src/indexing/index.ts",
"./server": "./src/server/routes.ts",
"./status": "./src/status.ts"
},
"files": [
"dist",
"src"
],
"scripts": {
"typecheck": "tsgo --noEmit",
"build": "tsc",
"test": "bun test --timeout 30000"
},
"dependencies": {
"@aws-sdk/client-bedrock-runtime": "3.1005.0",
"@aws-sdk/credential-provider-ini": "3.972.31",
"@kilocode/kilo-gateway": "workspace:*",
"@kilocode/plugin": "workspace:*",
"@lancedb/lancedb": "0.26.2",
"@qdrant/js-client-rest": "1.17.0",
"async-mutex": "0.5.0",
"chokidar": "4.0.3",
"glob": "13.0.6",
"hono": "catalog:",
"hono-openapi": "catalog:",
"ignore": "7.0.5",
"minimatch": "10.2.5",
"openai": "6.27.0",
"p-limit": "7.3.0",
"tree-sitter-wasms": "0.1.13",
"uuid": "14.0.0",
"web-tree-sitter": "0.25.10",
"zod": "catalog:"
},
"devDependencies": {
"@tsconfig/node22": "catalog:",
"@types/node": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
}
}
packages/kilo-indexing/src/config.ts
+150
View File
@@ -0,0 +1,150 @@
import z from "zod"
import type { IndexingConfigInput } from "./indexing/config-manager"
import type { EmbedderProvider } from "./indexing/interfaces/manager"
const providers = [
"openai",
"ollama",
"openai-compatible",
"gemini",
"mistral",
"vercel-ai-gateway",
"bedrock",
"openrouter",
"voyage",
] as const satisfies readonly EmbedderProvider[]
export const IndexingConfig = z
.object({
enabled: z.boolean().optional().describe("Enable codebase indexing"),
provider: z.enum(providers).optional().describe("Embedding provider to use for codebase indexing"),
model: z.string().optional().describe("Embedding model ID (uses provider default if omitted)"),
dimension: z
.number()
.int()
.positive()
.optional()
.describe("Override embedding vector dimension (auto-detected from model if omitted)"),
vectorStore: z.enum(["lancedb", "qdrant"]).optional().describe("Vector store backend (default: qdrant)"),
openai: z
.object({ apiKey: z.string().optional() })
.strict()
.optional()
.describe("OpenAI embedding provider options"),
ollama: z
.object({ baseUrl: z.string().optional() })
.strict()
.optional()
.describe("Ollama embedding provider options"),
"openai-compatible": z
.object({
baseUrl: z.string().optional(),
apiKey: z.string().optional(),
})
.strict()
.optional()
.describe("OpenAI-compatible embedding provider options"),
gemini: z
.object({ apiKey: z.string().optional() })
.strict()
.optional()
.describe("Gemini embedding provider options"),
mistral: z
.object({ apiKey: z.string().optional() })
.strict()
.optional()
.describe("Mistral embedding provider options"),
"vercel-ai-gateway": z
.object({ apiKey: z.string().optional() })
.strict()
.optional()
.describe("Vercel AI Gateway embedding provider options"),
bedrock: z
.object({
region: z.string().optional(),
profile: z.string().optional(),
})
.strict()
.optional()
.describe("AWS Bedrock embedding provider options"),
openrouter: z
.object({
apiKey: z.string().optional(),
specificProvider: z.string().optional(),
})
.strict()
.optional()
.describe("OpenRouter embedding provider options"),
voyage: z
.object({ apiKey: z.string().optional() })
.strict()
.optional()
.describe("Voyage embedding provider options"),
qdrant: z
.object({
url: z.string().optional(),
apiKey: z.string().optional(),
})
.strict()
.optional()
.describe("Qdrant vector store connection options"),
lancedb: z
.object({ directory: z.string().optional() })
.strict()
.optional()
.describe("LanceDB vector store options"),
searchMinScore: z
.number()
.min(0)
.max(1)
.optional()
.describe("Minimum similarity score for search results (default: 0.4)"),
searchMaxResults: z.number().int().positive().optional().describe("Maximum number of search results (default: 50)"),
embeddingBatchSize: z
.number()
.int()
.positive()
.optional()
.describe("Number of code segments per embedding batch (default: 60)"),
scannerMaxBatchRetries: z
.number()
.int()
.positive()
.optional()
.describe("Maximum retry attempts for failed embedding batches (default: 3)"),
})
.strict()
.meta({ ref: "IndexingConfig" })
export type IndexingConfig = z.infer<typeof IndexingConfig>
export function toIndexingConfigInput(cfg: IndexingConfig | undefined): IndexingConfigInput {
const provider = cfg?.provider ?? "openai"
return {
enabled: cfg?.enabled ?? false,
embedderProvider: provider,
vectorStoreProvider: cfg?.vectorStore,
modelId: cfg?.model,
modelDimension: cfg?.dimension,
lancedbVectorStoreDirectory: cfg?.lancedb?.directory,
qdrantUrl: cfg?.qdrant?.url,
qdrantApiKey: cfg?.qdrant?.apiKey,
searchMinScore: cfg?.searchMinScore,
searchMaxResults: cfg?.searchMaxResults,
embeddingBatchSize: cfg?.embeddingBatchSize,
scannerMaxBatchRetries: cfg?.scannerMaxBatchRetries,
openAiKey: cfg?.openai?.apiKey,
ollamaBaseUrl: cfg?.ollama?.baseUrl,
openAiCompatibleBaseUrl: cfg?.["openai-compatible"]?.baseUrl,
openAiCompatibleApiKey: cfg?.["openai-compatible"]?.apiKey,
geminiApiKey: cfg?.gemini?.apiKey,
mistralApiKey: cfg?.mistral?.apiKey,
vercelAiGatewayApiKey: cfg?.["vercel-ai-gateway"]?.apiKey,
bedrockRegion: cfg?.bedrock?.region,
bedrockProfile: cfg?.bedrock?.profile,
openRouterApiKey: cfg?.openrouter?.apiKey,
openRouterSpecificProvider: cfg?.openrouter?.specificProvider,
voyageApiKey: cfg?.voyage?.apiKey,
}
}
packages/kilo-indexing/src/detect.ts
+98
View File
@@ -0,0 +1,98 @@
export const INDEXING_PLUGIN_NAMES = ["kilo-indexing", "@kilocode/kilo-indexing"] as const
// RATIONALE: PluginSpec is string | [string, Record] — accept both forms.
type Candidate = string | readonly [string, ...unknown[]]
const names = new Set<string>(INDEXING_PLUGIN_NAMES)
const pathRx = /^[A-Za-z]:[\\/]/
export function normalizePluginName(value: string): string {
if (!value) return ""
if (value.startsWith("file://")) {
return normalizePath(fromFileUrl(value))
}
if (isPathSpecifier(value)) {
return normalizePath(value)
}
return normalizePackage(value)
}
function specifier(value: Candidate): string {
return typeof value === "string" ? value : value[0]
}
export function isIndexingPlugin(value: Candidate): boolean {
return names.has(normalizePluginName(specifier(value)))
}
export function hasIndexingPlugin(values?: readonly Candidate[]): boolean {
return values?.some(isIndexingPlugin) ?? false
}
function stripVersion(value: string): string {
if (!value.startsWith("@")) {
const at = value.lastIndexOf("@")
return at > 0 ? value.slice(0, at) : value
}
const slash = value.indexOf("/")
if (slash === -1) return value
const at = value.indexOf("@", slash)
return at === -1 ? value : value.slice(0, at)
}
function normalizePackage(value: string): string {
return stripVersion(value)
}
function isPathSpecifier(value: string): boolean {
if (value.startsWith("@")) return false
if (value.startsWith(".") || value.startsWith("/") || value.startsWith("\\")) return true
if (pathRx.test(value)) return true
if (!value.includes("/") && !value.includes("\\")) return false
const normalized = value.replaceAll("\\", "/")
if (normalized.includes("/node_modules/")) return true
if (normalized.includes("/.opencode/") || normalized.includes("/.kilo/") || normalized.includes("/.kilocode/")) {
return true
}
return /\.[cm]?[jt]s$/.test(normalized)
}
function normalizePath(value: string): string {
const parts = value.split(/[\\/]+/).filter(Boolean)
const idx = parts.lastIndexOf("node_modules")
if (idx >= 0) {
const head = parts[idx + 1]
if (head?.startsWith("@")) {
const tail = parts[idx + 2]
if (tail) return `${head}/${tail}`
}
if (head) return head
}
const scoped = parts.findIndex((part, i) => part === "@kilocode" && parts[i + 1] === "kilo-indexing")
if (scoped >= 0) return "@kilocode/kilo-indexing"
const workspace = parts.findIndex((part, i) => part === "packages" && parts[i + 1] === "kilo-indexing")
if (workspace >= 0) return "@kilocode/kilo-indexing"
return stem(value)
}
function fromFileUrl(value: string): string {
const url = new URL(value)
const path = decodeURIComponent(url.pathname)
if (/^\/[A-Za-z]:\//.test(path)) return path.slice(1)
if (url.host) return `//${url.host}${path}`
return path
}
function stem(value: string): string {
const part = value.split(/[\\/]+/).filter(Boolean).at(-1) ?? value
const dot = part.lastIndexOf(".")
if (dot <= 0) return part
return part.slice(0, dot)
}
packages/kilo-indexing/src/file/ignore.ts
+76
View File
@@ -0,0 +1,76 @@
import { minimatch } from "minimatch"
export namespace FileIgnore {
const folders = new Set([
"node_modules",
"bower_components",
".pnpm-store",
"vendor",
".npm",
"dist",
"build",
"out",
".next",
"target",
"bin",
"obj",
".git",
".svn",
".hg",
".vscode",
".idea",
".turbo",
".output",
"desktop",
".sst",
".cache",
".webkit-cache",
"__pycache__",
".pytest_cache",
"mypy_cache",
".history",
".gradle",
])
const files = [
"**/*.swp",
"**/*.swo",
"**/*.pyc",
"**/.DS_Store",
"**/Thumbs.db",
"**/logs/**",
"**/tmp/**",
"**/temp/**",
"**/*.log",
"**/coverage/**",
"**/.nyc_output/**",
]
export const PATTERNS = [...files, ...folders]
export function match(
filePath: string,
opts?: {
extra?: string[]
whitelist?: string[]
},
) {
const normalized = filePath.replaceAll("\\", "/")
for (const pattern of opts?.whitelist || []) {
if (minimatch(normalized, pattern, { dot: true })) return false
}
const parts = normalized.split("/")
for (const part of parts) {
if (folders.has(part)) return true
}
const extra = opts?.extra || []
for (const pattern of [...files, ...extra]) {
if (minimatch(normalized, pattern, { dot: true })) return true
}
return false
}
}
packages/kilo-indexing/src/headers.ts
+6
View File
@@ -0,0 +1,6 @@
import { getDefaultHeaders } from "@kilocode/kilo-gateway"
/**
* Default headers for KiloCode requests
*/
export const DEFAULT_HEADERS = getDefaultHeaders()
packages/kilo-indexing/src/index.ts
+13
View File
@@ -0,0 +1,13 @@
export { KiloIndexingPlugin, default } from "./plugin.js"
export { IndexingConfig, toIndexingConfigInput } from "./config.js"
export { hasIndexingPlugin, isIndexingPlugin, normalizePluginName, INDEXING_PLUGIN_NAMES } from "./detect.js"
export {
INDEXING_STATUS_STATES,
IndexingStatus,
IndexingStatusState,
disabledIndexingStatus,
normalizeIndexingStatus,
} from "./status.js"
export type { IndexingConfig as IndexingConfigInfo } from "./config.js"
export type { IndexingStatus as IndexingStatusInfo, IndexingStatusState as IndexingStatusStateInfo } from "./status.js"
packages/kilo-indexing/src/indexing/cache-manager.ts
+80
View File
@@ -0,0 +1,80 @@
import { createHash } from "crypto"
import fs from "fs/promises"
import path from "path"
import type { ICacheManager } from "./interfaces/cache"
import { Log } from "../util/log"
const log = Log.create({ service: "indexing-cache" })
/**
* Manages the file-hash cache for code indexing.
*
* RATIONALE: Replaced vscode.ExtensionContext storage and vscode.workspace.fs
* with plain filesystem access so the cache manager works outside VS Code.
*/
export class CacheManager implements ICacheManager {
private readonly cachePath: string
private fileHashes: Record<string, string> = {}
private saveTimer: ReturnType<typeof setTimeout> | undefined
constructor(
private readonly cacheDirectory: string,
private readonly workspacePath: string,
) {
const hash = createHash("sha256").update(workspacePath).digest("hex")
this.cachePath = path.join(cacheDirectory, `roo-index-cache-${hash}.json`)
}
async initialize(): Promise<void> {
try {
const raw = await fs.readFile(this.cachePath, "utf-8")
this.fileHashes = JSON.parse(raw)
} catch {
this.fileHashes = {}
}
}
private scheduleSave(): void {
if (this.saveTimer) clearTimeout(this.saveTimer)
this.saveTimer = setTimeout(() => this.performSave(), 1500)
}
private async performSave(): Promise<void> {
try {
await fs.mkdir(path.dirname(this.cachePath), { recursive: true })
const tmp = `${this.cachePath}.tmp`
await fs.writeFile(tmp, JSON.stringify(this.fileHashes), "utf-8")
await fs.rename(tmp, this.cachePath)
} catch (err) {
log.error("failed to save cache", { err })
}
}
async clearCacheFile(): Promise<void> {
try {
this.fileHashes = {}
await fs.mkdir(path.dirname(this.cachePath), { recursive: true })
await fs.writeFile(this.cachePath, "{}", "utf-8")
} catch (err) {
log.error("failed to clear cache file", { err })
}
}
getHash(filePath: string): string | undefined {
return this.fileHashes[filePath]
}
updateHash(filePath: string, hash: string): void {
this.fileHashes[filePath] = hash
this.scheduleSave()
}
deleteHash(filePath: string): void {
delete this.fileHashes[filePath]
this.scheduleSave()
}
getAllHashes(): Record<string, string> {
return { ...this.fileHashes }
}
}
packages/kilo-indexing/src/indexing/config-bridge.ts
+2
View File
@@ -0,0 +1,2 @@
export { toIndexingConfigInput } from "../config"
export type { IndexingConfig } from "../config"
packages/kilo-indexing/src/indexing/config-manager.ts
+312
View File
@@ -0,0 +1,312 @@
import type { EmbedderProvider } from "./interfaces/manager"
import type { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/config"
import { DEFAULT_SEARCH_MIN_SCORE, DEFAULT_MAX_SEARCH_RESULTS } from "./constants"
import { getDefaultModelId, getModelDimension, getModelScoreThreshold } from "./model-registry"
import { isEmbeddingProfileEqual, resolveEmbeddingProfile } from "./embedding-profile"
/**
* Raw input fed to CodeIndexConfigManager from the host environment.
* The host (CLI, extension, tests) builds this object and passes it in;
* the config manager never reads storage or secrets directly.
*/
export interface IndexingConfigInput {
enabled: boolean
embedderProvider: EmbedderProvider
vectorStoreProvider?: "lancedb" | "qdrant"
lancedbVectorStoreDirectory?: string
modelId?: string
modelDimension?: number
qdrantUrl?: string
qdrantApiKey?: string
searchMinScore?: number
searchMaxResults?: number
embeddingBatchSize?: number
scannerMaxBatchRetries?: number
openAiKey?: string
ollamaBaseUrl?: string
openAiCompatibleBaseUrl?: string
openAiCompatibleApiKey?: string
geminiApiKey?: string
mistralApiKey?: string
vercelAiGatewayApiKey?: string
bedrockRegion?: string
bedrockProfile?: string
openRouterApiKey?: string
openRouterSpecificProvider?: string
voyageApiKey?: string
}
/**
* Manages configuration state and validation for the code indexing feature.
*
* RATIONALE: Replaced the legacy ContextProxy/getGlobalState/getSecret approach
* with a plain IndexingConfigInput object supplied by the host. The manager
* owns no storage; it only validates and projects the input into the shapes the
* rest of the indexing engine expects.
*/
export class CodeIndexConfigManager {
private enabled = false
private embedderProvider: EmbedderProvider = "openai"
private vectorStoreProvider: "lancedb" | "qdrant" = "qdrant"
private lancedbVectorStoreDirectory?: string
private modelId?: string
private modelDimension?: number
private openAiOptions?: { apiKey: string }
private ollamaOptions?: { baseUrl: string; modelId?: string }
private openAiCompatibleOptions?: { baseUrl: string; apiKey: string }
private geminiOptions?: { apiKey: string }
private mistralOptions?: { apiKey: string }
private vercelAiGatewayOptions?: { apiKey: string }
private bedrockOptions?: { region: string; profile?: string }
private openRouterOptions?: { apiKey: string; specificProvider?: string }
private voyageOptions?: { apiKey: string }
private qdrantUrl?: string = "http://localhost:6333"
private qdrantApiKey?: string
private searchMinScore?: number
private searchMaxResults?: number
private embeddingBatchSize?: number
private scannerMaxBatchRetries?: number
constructor(input: IndexingConfigInput) {
this.applyInput(input)
}
/**
* Applies new configuration input. Returns whether a restart is needed.
*/
public loadConfiguration(input: IndexingConfigInput): { requiresRestart: boolean } {
const snapshot = this.captureSnapshot()
this.applyInput(input)
const requiresRestart = this.doesConfigChangeRequireRestart(snapshot)
return { requiresRestart }
}
private applyInput(input: IndexingConfigInput): void {
this.enabled = input.enabled
this.embedderProvider = input.embedderProvider
this.vectorStoreProvider = input.vectorStoreProvider ?? "qdrant"
this.lancedbVectorStoreDirectory = input.lancedbVectorStoreDirectory
this.qdrantUrl = input.qdrantUrl ?? "http://localhost:6333"
this.qdrantApiKey = input.qdrantApiKey
this.searchMinScore = input.searchMinScore
this.searchMaxResults = input.searchMaxResults
this.embeddingBatchSize = input.embeddingBatchSize
this.scannerMaxBatchRetries = input.scannerMaxBatchRetries
this.modelId = input.modelId
// Validate and set model dimension
if (input.modelDimension !== undefined && input.modelDimension !== null) {
const dimension = Number(input.modelDimension)
this.modelDimension = !isNaN(dimension) && dimension > 0 ? dimension : undefined
} else {
this.modelDimension = undefined
}
this.openAiOptions = input.openAiKey ? { apiKey: input.openAiKey } : undefined
const url = input.ollamaBaseUrl ?? (input.embedderProvider === "ollama" ? "http://localhost:11434" : undefined)
this.ollamaOptions = url ? { baseUrl: url, modelId: input.modelId } : undefined
this.openAiCompatibleOptions =
input.openAiCompatibleBaseUrl && input.openAiCompatibleApiKey
? { baseUrl: input.openAiCompatibleBaseUrl, apiKey: input.openAiCompatibleApiKey }
: undefined
this.geminiOptions = input.geminiApiKey ? { apiKey: input.geminiApiKey } : undefined
this.mistralOptions = input.mistralApiKey ? { apiKey: input.mistralApiKey } : undefined
this.vercelAiGatewayOptions = input.vercelAiGatewayApiKey ? { apiKey: input.vercelAiGatewayApiKey } : undefined
this.bedrockOptions = input.bedrockRegion
? { region: input.bedrockRegion, profile: input.bedrockProfile }
: undefined
this.openRouterOptions = input.openRouterApiKey
? { apiKey: input.openRouterApiKey, specificProvider: input.openRouterSpecificProvider }
: undefined
this.voyageOptions = input.voyageApiKey ? { apiKey: input.voyageApiKey } : undefined
}
private captureSnapshot(): PreviousConfigSnapshot {
return {
enabled: this.enabled,
configured: this.isConfigured(),
embedderProvider: this.embedderProvider,
vectorStoreProvider: this.vectorStoreProvider,
lancedbVectorStoreDirectory: this.lancedbVectorStoreDirectory,
modelId: this.modelId,
modelDimension: this.modelDimension,
openAiKey: this.openAiOptions?.apiKey ?? "",
ollamaBaseUrl: this.ollamaOptions?.baseUrl ?? "",
openAiCompatibleBaseUrl: this.openAiCompatibleOptions?.baseUrl ?? "",
openAiCompatibleApiKey: this.openAiCompatibleOptions?.apiKey ?? "",
geminiApiKey: this.geminiOptions?.apiKey ?? "",
mistralApiKey: this.mistralOptions?.apiKey ?? "",
vercelAiGatewayApiKey: this.vercelAiGatewayOptions?.apiKey ?? "",
bedrockRegion: this.bedrockOptions?.region ?? "",
bedrockProfile: this.bedrockOptions?.profile ?? "",
openRouterApiKey: this.openRouterOptions?.apiKey ?? "",
openRouterSpecificProvider: this.openRouterOptions?.specificProvider ?? "",
voyageApiKey: this.voyageOptions?.apiKey ?? "",
qdrantUrl: this.qdrantUrl ?? "",
qdrantApiKey: this.qdrantApiKey ?? "",
}
}
public isConfigured(): boolean {
const provider = this.embedderProvider
const qdrant = this.qdrantUrl
const isLancedb = this.vectorStoreProvider === "lancedb"
// LanceDB doesn't need a qdrant URL; qdrant does
const hasStore = isLancedb || !!qdrant
if (provider === "openai") return !!(this.openAiOptions?.apiKey && hasStore)
if (provider === "ollama") return !!(this.ollamaOptions?.baseUrl && hasStore)
if (provider === "openai-compatible")
return !!(this.openAiCompatibleOptions?.baseUrl && this.openAiCompatibleOptions?.apiKey && hasStore)
if (provider === "gemini") return !!(this.geminiOptions?.apiKey && hasStore)
if (provider === "mistral") return !!(this.mistralOptions?.apiKey && hasStore)
if (provider === "vercel-ai-gateway") return !!(this.vercelAiGatewayOptions?.apiKey && hasStore)
if (provider === "bedrock") return !!(this.bedrockOptions?.region && hasStore)
if (provider === "openrouter") return !!(this.openRouterOptions?.apiKey && hasStore)
if (provider === "voyage") return !!(this.voyageOptions?.apiKey && hasStore)
return false
}
doesConfigChangeRequireRestart(prev: PreviousConfigSnapshot): boolean {
const nowConfigured = this.isConfigured()
const prevEnabled = prev.enabled ?? false
const prevConfigured = prev.configured ?? false
const prevProvider = prev.embedderProvider ?? "openai"
// Enable/disable transitions
if ((!prevEnabled || !prevConfigured) && this.enabled && nowConfigured) return true
if (prevEnabled && !this.enabled) return true
if ((!prevEnabled || !prevConfigured) && (!this.enabled || !nowConfigured)) return false
if (!this.enabled) return false
// Provider change
if (prevProvider !== this.embedderProvider) return true
// Vector store provider change
if ((prev.vectorStoreProvider ?? "qdrant") !== this.vectorStoreProvider) return true
// LanceDB path change
if (
this.vectorStoreProvider === "lancedb" &&
(prev.lancedbVectorStoreDirectory ?? "") !== (this.lancedbVectorStoreDirectory ?? "")
)
return true
// Auth changes
if ((prev.openAiKey ?? "") !== (this.openAiOptions?.apiKey ?? "")) return true
if ((prev.ollamaBaseUrl ?? "") !== (this.ollamaOptions?.baseUrl ?? "")) return true
if (
(prev.openAiCompatibleBaseUrl ?? "") !== (this.openAiCompatibleOptions?.baseUrl ?? "") ||
(prev.openAiCompatibleApiKey ?? "") !== (this.openAiCompatibleOptions?.apiKey ?? "")
)
return true
if ((prev.geminiApiKey ?? "") !== (this.geminiOptions?.apiKey ?? "")) return true
if ((prev.mistralApiKey ?? "") !== (this.mistralOptions?.apiKey ?? "")) return true
if ((prev.vercelAiGatewayApiKey ?? "") !== (this.vercelAiGatewayOptions?.apiKey ?? "")) return true
if (
(prev.bedrockRegion ?? "") !== (this.bedrockOptions?.region ?? "") ||
(prev.bedrockProfile ?? "") !== (this.bedrockOptions?.profile ?? "")
)
return true
if ((prev.openRouterApiKey ?? "") !== (this.openRouterOptions?.apiKey ?? "")) return true
if ((prev.openRouterSpecificProvider ?? "") !== (this.openRouterOptions?.specificProvider ?? "")) return true
if ((prev.voyageApiKey ?? "") !== (this.voyageOptions?.apiKey ?? "")) return true
// Qdrant connection changes
if ((prev.qdrantUrl ?? "") !== (this.qdrantUrl ?? "") || (prev.qdrantApiKey ?? "") !== (this.qdrantApiKey ?? ""))
return true
if (this.hasEmbeddingProfileChanged(prevProvider, prev.modelId, prev.modelDimension)) return true
return false
}
private hasEmbeddingProfileChanged(
prevProvider: EmbedderProvider,
prevModelId?: string,
prevModelDimension?: number,
): boolean {
const prev = resolveEmbeddingProfile(prevProvider, prevModelId, prevModelDimension)
const cur = resolveEmbeddingProfile(this.embedderProvider, this.modelId, this.modelDimension)
if (prev && cur) return !isEmbeddingProfileEqual(prev, cur)
const prevId = prevModelId ?? getDefaultModelId(prevProvider)
const curId = this.modelId ?? getDefaultModelId(this.embedderProvider)
if (prevProvider === this.embedderProvider && prevId === curId) return false
return true
}
public getConfig(): CodeIndexConfig {
return {
isConfigured: this.isConfigured(),
embedderProvider: this.embedderProvider,
vectorStoreProvider: this.vectorStoreProvider ?? "qdrant",
lancedbVectorStoreDirectoryPlaceholder: this.lancedbVectorStoreDirectory,
modelId: this.modelId,
modelDimension: this.modelDimension,
openAiOptions: this.openAiOptions,
ollamaOptions: this.ollamaOptions,
openAiCompatibleOptions: this.openAiCompatibleOptions,
geminiOptions: this.geminiOptions,
mistralOptions: this.mistralOptions,
vercelAiGatewayOptions: this.vercelAiGatewayOptions,
bedrockOptions: this.bedrockOptions,
openRouterOptions: this.openRouterOptions,
voyageOptions: this.voyageOptions,
qdrantUrl: this.qdrantUrl,
qdrantApiKey: this.qdrantApiKey,
searchMinScore: this.currentSearchMinScore,
searchMaxResults: this.currentSearchMaxResults,
embeddingBatchSize: this.currentEmbeddingBatchSize,
scannerMaxBatchRetries: this.currentScannerMaxBatchRetries,
}
}
public get isFeatureEnabled(): boolean {
return this.enabled
}
public get isFeatureConfigured(): boolean {
return this.isConfigured()
}
public get currentEmbedderProvider(): EmbedderProvider {
return this.embedderProvider
}
public get qdrantConfig(): { url?: string; apiKey?: string } {
return { url: this.qdrantUrl, apiKey: this.qdrantApiKey }
}
public get currentModelId(): string | undefined {
return this.modelId
}
public get currentModelDimension(): number | undefined {
const id = this.modelId ?? getDefaultModelId(this.embedderProvider)
const dim = getModelDimension(this.embedderProvider, id)
if (!dim && this.modelDimension && this.modelDimension > 0) return this.modelDimension
return dim
}
public get currentSearchMinScore(): number {
if (this.searchMinScore !== undefined) return this.searchMinScore
const id = this.modelId ?? getDefaultModelId(this.embedderProvider)
return getModelScoreThreshold(this.embedderProvider, id) ?? DEFAULT_SEARCH_MIN_SCORE
}
public get currentSearchMaxResults(): number {
return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS
}
public get currentEmbeddingBatchSize(): number | undefined {
return this.embeddingBatchSize
}
public get currentScannerMaxBatchRetries(): number | undefined {
return this.scannerMaxBatchRetries
}
}
packages/kilo-indexing/src/indexing/constants/index.ts
+69
View File
@@ -0,0 +1,69 @@
/**
* Codebase Index Constants
*/
export const CODEBASE_INDEX_DEFAULTS = {
MIN_SEARCH_RESULTS: 10,
MAX_SEARCH_RESULTS: 200,
DEFAULT_SEARCH_RESULTS: 50,
SEARCH_RESULTS_STEP: 10,
MIN_SEARCH_SCORE: 0,
MAX_SEARCH_SCORE: 1,
DEFAULT_SEARCH_MIN_SCORE: 0.4,
SEARCH_SCORE_STEP: 0.05,
MIN_EMBEDDING_BATCH_SIZE: 10,
MAX_EMBEDDING_BATCH_SIZE: 200,
DEFAULT_EMBEDDING_BATCH_SIZE: 60,
EMBEDDING_BATCH_SIZE_STEP: 10,
MIN_SCANNER_MAX_BATCH_RETRIES: 1,
MAX_SCANNER_MAX_BATCH_RETRIES: 10,
DEFAULT_SCANNER_MAX_BATCH_RETRIES: 3,
SCANNER_MAX_BATCH_RETRIES_STEP: 1,
} as const
/**Parser */
export const MAX_BLOCK_CHARS = 1000
export const MIN_BLOCK_CHARS = 50
export const MIN_CHUNK_REMAINDER_CHARS = 200 // Minimum characters for the *next* chunk after a split
export const MAX_CHARS_TOLERANCE_FACTOR = 1.15 // 15% tolerance for max chars
/**Search */
export const DEFAULT_SEARCH_MIN_SCORE = CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_MIN_SCORE
export const DEFAULT_MAX_SEARCH_RESULTS = CODEBASE_INDEX_DEFAULTS.DEFAULT_SEARCH_RESULTS
/**File Watcher */
export const QDRANT_CODE_BLOCK_NAMESPACE = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
export const MAX_FILE_SIZE_BYTES = 1 * 1024 * 1024 // 1MB
/**Directory Scanner */
export const MAX_LIST_FILES_LIMIT_CODE_INDEX = 50_000
export const BATCH_SEGMENT_THRESHOLD = 60 // Number of code segments to batch for embeddings/upserts
export const MAX_BATCH_RETRIES = 3
export const INITIAL_RETRY_DELAY_MS = 500
export const PARSING_CONCURRENCY = 10
export const MAX_PENDING_BATCHES = 20 // Maximum number of batches to accumulate before waiting
/**Manager Recovery */
export const MAX_MANAGER_RECOVERY_ATTEMPTS = 3
export const INITIAL_MANAGER_RECOVERY_DELAY_MS = 500
/**Embedder Validation */
export const REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS = 15_000
export const REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES = 0
export const OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS = 120_000
/**OpenAI Embedder */
export const MAX_BATCH_TOKENS = 100000
export const MAX_ITEM_TOKENS = 8191
export const BATCH_PROCESSING_CONCURRENCY = 10
/**Gemini Embedder */
export const GEMINI_MAX_ITEM_TOKENS = 2048
/**Managed Indexing */
export const MANAGED_MAX_CHUNK_CHARS = 1000
export const MANAGED_MIN_CHUNK_CHARS = 50
export const MANAGED_OVERLAP_LINES = 5
export const MANAGED_BATCH_SIZE = 60
export const MANAGED_FILE_WATCH_DEBOUNCE_MS = 500
export const MANAGED_MAX_CONCURRENT_FILES = 10
export const MANAGED_MAX_CONCURRENT_BATCHES = 50
packages/kilo-indexing/src/indexing/embedders/bedrock.ts
+306
View File
@@ -0,0 +1,306 @@
import { BedrockRuntimeClient, InvokeModelCommand, type InvokeModelCommandInput } from "@aws-sdk/client-bedrock-runtime"
import { fromIni } from "@aws-sdk/credential-provider-ini"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces"
import {
MAX_BATCH_TOKENS,
MAX_ITEM_TOKENS,
MAX_BATCH_RETRIES as MAX_RETRIES,
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
} from "../constants"
import { getDefaultModelId } from "../model-registry"
import { withValidationErrorHandling, formatEmbeddingError, type HttpError } from "../shared/validation-helpers"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-bedrock" })
/**
* Amazon Bedrock implementation of the embedder interface with batching and rate limiting
*/
export class BedrockEmbedder implements IEmbedder {
private bedrockClient: BedrockRuntimeClient
private readonly defaultModelId: string
/**
* Creates a new Amazon Bedrock embedder
* @param region AWS region for Bedrock service (required)
* @param profile AWS profile name for credentials (optional - uses default credential chain if not provided)
* @param modelId Optional model ID override
*/
constructor(
private readonly region: string,
private readonly profile?: string,
modelId?: string,
) {
if (!region) {
throw new Error("Region is required for AWS Bedrock embedder")
}
// If profile is specified, use that profile.
// Otherwise omit credentials so the AWS SDK uses the default provider chain.
const cfg = this.profile
? {
region: this.region,
credentials: fromIni({ profile: this.profile }),
}
: {
region: this.region,
}
this.bedrockClient = new BedrockRuntimeClient(cfg)
this.defaultModelId = modelId || getDefaultModelId("bedrock")
}
/**
* Creates embeddings for the given texts with batching and rate limiting
* @param texts Array of text strings to embed
* @param model Optional model identifier
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.defaultModelId
const allEmbeddings: number[][] = []
const usage = { promptTokens: 0, totalTokens: 0 }
const remainingTexts = [...texts]
while (remainingTexts.length > 0) {
const currentBatch: string[] = []
let currentBatchTokens = 0
const processedIndices: number[] = []
for (let i = 0; i < remainingTexts.length; i++) {
const text = remainingTexts[i]
const itemTokens = Math.ceil(text.length / 4)
if (itemTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${i} exceeds token limit (${itemTokens} > ${MAX_ITEM_TOKENS})`)
processedIndices.push(i)
continue
}
if (currentBatchTokens + itemTokens <= MAX_BATCH_TOKENS) {
currentBatch.push(text)
currentBatchTokens += itemTokens
processedIndices.push(i)
} else {
break
}
}
// Remove processed items from remainingTexts (in reverse order to maintain correct indices)
for (let i = processedIndices.length - 1; i >= 0; i--) {
remainingTexts.splice(processedIndices[i], 1)
}
if (currentBatch.length > 0) {
const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse)
allEmbeddings.push(...batchResult.embeddings)
usage.promptTokens += batchResult.usage.promptTokens
usage.totalTokens += batchResult.usage.totalTokens
}
}
return { embeddings: allEmbeddings, usage }
}
/**
* Helper method to handle batch embedding with retries and exponential backoff
* @param batchTexts Array of texts to embed in this batch
* @param model Model identifier to use
* @returns Promise resolving to embeddings and usage statistics
*/
private async _embedBatchWithRetries(
batchTexts: string[],
model: string,
): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> {
for (let attempts = 0; attempts < MAX_RETRIES; attempts++) {
try {
const embeddings: number[][] = []
let totalPromptTokens = 0
let totalTokens = 0
// Process each text in the batch
// Note: Amazon Titan models typically don't support batch embedding in a single request
// So we process them individually
for (const text of batchTexts) {
const embedding = await this._invokeEmbeddingModel(text, model)
embeddings.push(embedding.embedding)
totalPromptTokens += embedding.inputTextTokenCount || 0
totalTokens += embedding.inputTextTokenCount || 0
}
return {
embeddings,
usage: {
promptTokens: totalPromptTokens,
totalTokens,
},
}
} catch (error: any) {
const hasMoreAttempts = attempts < MAX_RETRIES - 1
// Check if it's a rate limit error
if (error.name === "ThrottlingException" && hasMoreAttempts) {
const delayMs = INITIAL_DELAY_MS * Math.pow(2, attempts)
log.warn(`Rate limit hit, retrying in ${delayMs}ms (attempt ${attempts + 1}/${MAX_RETRIES})`)
await new Promise((resolve) => setTimeout(resolve, delayMs))
continue
}
log.error("Bedrock embedder batch error", {
err: error instanceof Error ? error.message : String(error),
location: "BedrockEmbedder:_embedBatchWithRetries",
attempt: attempts + 1,
})
// Format and throw the error
throw formatEmbeddingError(error, MAX_RETRIES)
}
}
throw new Error(`Embedding failed after ${MAX_RETRIES} attempts`)
}
/**
* Invokes the embedding model for a single text
* @param text The text to embed
* @param model The model identifier to use
* @returns Promise resolving to embedding and token count
*/
private async _invokeEmbeddingModel(
text: string,
model: string,
): Promise<{ embedding: number[]; inputTextTokenCount?: number }> {
let requestBody: any
let modelId = model
// Prepare the request body based on the model
if (model.startsWith("amazon.nova-2-multimodal")) {
// Nova multimodal embeddings use a task-based format with embeddingParams
// Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/embeddings-nova.html
requestBody = {
taskType: "SINGLE_EMBEDDING",
singleEmbeddingParams: {
embeddingPurpose: "GENERIC_INDEX",
embeddingDimension: 1024, // Nova supports 1024 or 3072
text: {
truncationMode: "END",
value: text,
},
},
}
} else if (model.startsWith("amazon.titan-embed")) {
requestBody = {
inputText: text,
}
} else if (model.startsWith("cohere.embed")) {
requestBody = {
texts: [text],
input_type: "search_document", // or "search_query" depending on use case
}
} else {
// Default to Titan format
requestBody = {
inputText: text,
}
}
const params: InvokeModelCommandInput = {
modelId,
body: JSON.stringify(requestBody),
contentType: "application/json",
accept: "application/json",
}
const command = new InvokeModelCommand(params)
const response = await this.bedrockClient.send(command)
// Parse the response
const responseBody = JSON.parse(new TextDecoder().decode(response.body))
// Extract embedding based on model type
if (model.startsWith("amazon.nova-2-multimodal")) {
// Nova multimodal returns { embeddings: [{ embedding: [...] }] }
// Reference: AWS Bedrock documentation
return {
embedding: responseBody.embeddings?.[0]?.embedding || responseBody.embedding,
inputTextTokenCount: responseBody.inputTextTokenCount,
}
} else if (model.startsWith("amazon.titan-embed")) {
return {
embedding: responseBody.embedding,
inputTextTokenCount: responseBody.inputTextTokenCount,
}
} else if (model.startsWith("cohere.embed")) {
return {
embedding: responseBody.embeddings[0],
// Cohere doesn't provide token count in response
}
} else {
// Default to Titan format
return {
embedding: responseBody.embedding,
inputTextTokenCount: responseBody.inputTextTokenCount,
}
}
}
/**
* Validates the Bedrock embedder configuration by attempting a minimal embedding request
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(async () => {
try {
// Test with a minimal embedding request
const result = await this._invokeEmbeddingModel("test", this.defaultModelId)
// Check if we got a valid response
if (!result.embedding || result.embedding.length === 0) {
return {
valid: false,
error: "Bedrock returned an invalid response format",
}
}
return { valid: true }
} catch (error: any) {
// Check for specific AWS errors
if (error.name === "UnrecognizedClientException") {
return {
valid: false,
error: "Invalid AWS credentials for Bedrock",
}
}
if (error.name === "AccessDeniedException") {
return {
valid: false,
error: "Access denied to Bedrock embedding model",
}
}
if (error.name === "ResourceNotFoundException") {
return {
valid: false,
error: `Bedrock model '${this.defaultModelId}' not found`,
}
}
log.error("Bedrock embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "BedrockEmbedder:validateConfiguration",
})
throw error
}
}, "bedrock")
}
get embedderInfo(): EmbedderInfo {
return {
name: "bedrock",
}
}
}
packages/kilo-indexing/src/indexing/embedders/gemini.ts
+90
View File
@@ -0,0 +1,90 @@
import { OpenAICompatibleEmbedder } from "./openai-compatible"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import { GEMINI_MAX_ITEM_TOKENS } from "../constants"
import { Log } from "../../util/log"
import { getDefaultModelId } from "../model-registry"
const log = Log.create({ service: "embedder-gemini" })
/**
* Gemini embedder implementation that wraps the OpenAI Compatible embedder
* with configuration for Google's Gemini embedding API.
*
* Supported models:
* - text-embedding-004 (dimension: 768)
* - gemini-embedding-001 (dimension: 3072)
*/
export class GeminiEmbedder implements IEmbedder {
private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder
private static readonly GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
private readonly modelId: string
/**
* Creates a new Gemini embedder
* @param apiKey The Gemini API key for authentication
* @param modelId The model ID to use (defaults to gemini-embedding-001)
*/
constructor(apiKey: string, modelId?: string) {
if (!apiKey) {
throw new Error("API key is required for Gemini embedder")
}
// Use provided model or default
this.modelId = modelId || getDefaultModelId("gemini")
// Create an OpenAI Compatible embedder with Gemini's configuration
this.openAICompatibleEmbedder = new OpenAICompatibleEmbedder(
GeminiEmbedder.GEMINI_BASE_URL,
apiKey,
this.modelId,
GEMINI_MAX_ITEM_TOKENS,
)
}
/**
* Creates embeddings for the given texts using Gemini's embedding API
* @param texts Array of text strings to embed
* @param model Optional model identifier (uses constructor model if not provided)
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
try {
// Use the provided model or fall back to the instance's model
const modelToUse = model || this.modelId
return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse)
} catch (error) {
log.error("Gemini embedder error in createEmbeddings", {
err: error instanceof Error ? error.message : String(error),
location: "GeminiEmbedder:createEmbeddings",
})
throw error
}
}
/**
* Validates the Gemini embedder configuration by delegating to the underlying OpenAI-compatible embedder
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
try {
// Delegate validation to the OpenAI-compatible embedder
// The error messages will be specific to Gemini since we're using Gemini's base URL
return await this.openAICompatibleEmbedder.validateConfiguration()
} catch (error) {
log.error("Gemini embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "GeminiEmbedder:validateConfiguration",
})
throw error
}
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "gemini",
}
}
}
packages/kilo-indexing/src/indexing/embedders/mistral.ts
+90
View File
@@ -0,0 +1,90 @@
import { OpenAICompatibleEmbedder } from "./openai-compatible"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import { MAX_ITEM_TOKENS } from "../constants"
import { Log } from "../../util/log"
import { getDefaultModelId } from "../model-registry"
const log = Log.create({ service: "embedder-mistral" })
/**
* Mistral embedder implementation that wraps the OpenAI Compatible embedder
* with configuration for Mistral's embedding API.
*
* Supported models:
* - codestral-embed-2505 (dimension: 1536)
* - mistral-embed (dimension: 1024)
*/
export class MistralEmbedder implements IEmbedder {
private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder
private static readonly MISTRAL_BASE_URL = "https://api.mistral.ai/v1"
private readonly modelId: string
/**
* Creates a new Mistral embedder
* @param apiKey The Mistral API key for authentication
* @param modelId The model ID to use (defaults to codestral-embed-2505)
*/
constructor(apiKey: string, modelId?: string) {
if (!apiKey) {
throw new Error("API key is required for Mistral embedder")
}
// Use provided model or default
this.modelId = modelId || getDefaultModelId("mistral")
// Create an OpenAI Compatible embedder with Mistral's configuration
this.openAICompatibleEmbedder = new OpenAICompatibleEmbedder(
MistralEmbedder.MISTRAL_BASE_URL,
apiKey,
this.modelId,
MAX_ITEM_TOKENS, // This is the max token limit (8191), not the embedding dimension
)
}
/**
* Creates embeddings for the given texts using Mistral's embedding API
* @param texts Array of text strings to embed
* @param model Optional model identifier (uses constructor model if not provided)
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
try {
// Use the provided model or fall back to the instance's model
const modelToUse = model || this.modelId
return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse)
} catch (error) {
log.error("Mistral embedder error in createEmbeddings", {
err: error instanceof Error ? error.message : String(error),
location: "MistralEmbedder:createEmbeddings",
})
throw error
}
}
/**
* Validates the Mistral embedder configuration by delegating to the underlying OpenAI-compatible embedder
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
try {
// Delegate validation to the OpenAI-compatible embedder
// The error messages will be specific to Mistral since we're using Mistral's base URL
return await this.openAICompatibleEmbedder.validateConfiguration()
} catch (error) {
log.error("Mistral embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "MistralEmbedder:validateConfiguration",
})
throw error
}
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "mistral",
}
}
}
packages/kilo-indexing/src/indexing/embedders/ollama.ts
+270
View File
@@ -0,0 +1,270 @@
import type { EmbedderInfo, EmbeddingResponse, IEmbedder } from "../interfaces"
import { getModelQueryPrefix } from "../model-registry"
import { MAX_ITEM_TOKENS, OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS } from "../constants"
import { withValidationErrorHandling, sanitizeErrorMessage } from "../shared/validation-helpers"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-ollama" })
type OllamaEmbeddingResult = {
embeddings?: number[][]
}
type OllamaModel = {
name?: string
}
type OllamaModelsResult = {
models?: OllamaModel[]
}
/**
* Implements the IEmbedder interface using a local Ollama instance.
*/
export class CodeIndexOllamaEmbedder implements IEmbedder {
private readonly baseUrl: string
private readonly defaultModelId: string
private readonly dimensions?: number
constructor(baseUrl: string, modelId?: string, dimension?: number) {
let normalizedUrl = baseUrl || "http://localhost:11434"
// Normalize the baseUrl by removing all trailing slashes
normalizedUrl = normalizedUrl.replace(/\/+$/, "")
this.baseUrl = normalizedUrl
this.defaultModelId = modelId || "nomic-embed-text:latest"
this.dimensions = dimension
}
/**
* Creates embeddings for the given texts using the specified Ollama model.
* @param texts - An array of strings to embed.
* @param model - Optional model ID to override the default.
* @returns A promise that resolves to an EmbeddingResponse containing the embeddings and usage data.
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.defaultModelId
const dimensions = this.dimensions
const url = `${this.baseUrl}/api/embed` // Endpoint as specified
// Apply model-specific query prefix if required
const queryPrefix = getModelQueryPrefix("ollama", modelToUse)
const processedTexts = queryPrefix
? texts.map((text, index) => {
// Prevent double-prefixing
if (text.startsWith(queryPrefix)) {
return text
}
const prefixedText = `${queryPrefix}${text}`
const estimatedTokens = Math.ceil(prefixedText.length / 4)
if (estimatedTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${index} with prefix exceeds token limit (${estimatedTokens} > ${MAX_ITEM_TOKENS})`)
// Return original text if adding prefix would exceed limit
return text
}
return prefixedText
})
: texts
try {
// Add timeout to prevent indefinite hanging
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS)
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
model: modelToUse,
input: processedTexts,
dimensions,
}),
signal: controller.signal,
})
clearTimeout(timeoutId)
if (!response.ok) {
let errorBody = "Could not read error body"
try {
errorBody = await response.text()
} catch (e) {
// Ignore error reading body
}
throw new Error(`Ollama request failed: ${response.status} ${response.statusText} - ${errorBody}`)
}
const data = (await response.json()) as OllamaEmbeddingResult
// Extract embeddings using 'embeddings' key as requested
const embeddings = data.embeddings
if (!embeddings || !Array.isArray(embeddings)) {
throw new Error("Invalid Ollama response structure: missing embeddings array")
}
return {
embeddings: embeddings,
}
} catch (error: unknown) {
log.error("Ollama embedding failed", {
err: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
location: "OllamaEmbedder:createEmbeddings",
})
// Handle specific error types with better messages
if (error instanceof Error && error.name === "AbortError") {
throw new Error("Connection to embedding service failed (timeout)")
} else if (
error instanceof Error &&
(error.message.includes("fetch failed") || ("code" in error && error.code === "ECONNREFUSED"))
) {
throw new Error(`Ollama service is not running at ${this.baseUrl}`)
} else if (error instanceof Error && "code" in error && error.code === "ENOTFOUND") {
throw new Error(`Ollama host not found at ${this.baseUrl}`)
}
// Re-throw a more specific error for the caller
throw new Error(`Ollama embedding failed: ${error instanceof Error ? error.message : String(error)}`)
}
}
/**
* Validates the Ollama embedder configuration by checking service availability and model existence
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(
async () => {
// First check if Ollama service is running by trying to list models
const modelsUrl = `${this.baseUrl}/api/tags`
// Add timeout to prevent indefinite hanging
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS)
const modelsResponse = await fetch(modelsUrl, {
method: "GET",
headers: {
"Content-Type": "application/json",
},
signal: controller.signal,
})
clearTimeout(timeoutId)
if (!modelsResponse.ok) {
if (modelsResponse.status === 404) {
return {
valid: false,
error: `Ollama service is not running at ${this.baseUrl}`,
}
}
return {
valid: false,
error: `Ollama service unavailable at ${this.baseUrl} (status ${modelsResponse.status})`,
}
}
// Check if the specific model exists
const modelsData = (await modelsResponse.json()) as OllamaModelsResult
const models = modelsData.models ?? []
// Check both with and without :latest suffix
const modelExists = models.some((m) => {
const modelName = m.name ?? ""
return (
modelName === this.defaultModelId ||
modelName === `${this.defaultModelId}:latest` ||
modelName === this.defaultModelId.replace(":latest", "")
)
})
if (!modelExists) {
const availableModels = models.map((m) => m.name ?? "").join(", ")
return {
valid: false,
error: `Model '${this.defaultModelId}' not found. Available models: ${availableModels}`,
}
}
// Try a test embedding to ensure the model works for embeddings
const testUrl = `${this.baseUrl}/api/embed`
// Add timeout for test request too
const testController = new AbortController()
const testTimeoutId = setTimeout(() => testController.abort(), OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS)
const testResponse = await fetch(testUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
model: this.defaultModelId,
input: ["test"],
}),
signal: testController.signal,
})
clearTimeout(testTimeoutId)
if (!testResponse.ok) {
return {
valid: false,
error: `Model '${this.defaultModelId}' is not capable of generating embeddings`,
}
}
return { valid: true }
},
"ollama",
{
beforeStandardHandling: (error: any) => {
// Handle Ollama-specific connection errors
if (
error?.message?.includes("fetch failed") ||
error?.code === "ECONNREFUSED" ||
error?.message?.includes("ECONNREFUSED")
) {
log.error("Ollama connection failed", {
err: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
location: "OllamaEmbedder:validateConfiguration:connectionFailed",
})
return {
valid: false,
error: `Ollama service is not running at ${this.baseUrl}`,
}
} else if (error?.code === "ENOTFOUND" || error?.message?.includes("ENOTFOUND")) {
log.error("Ollama host not found", {
err: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
location: "OllamaEmbedder:validateConfiguration:hostNotFound",
})
return {
valid: false,
error: `Ollama host not found at ${this.baseUrl}`,
}
} else if (error?.name === "AbortError") {
log.error("Ollama connection timeout", {
err: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
location: "OllamaEmbedder:validateConfiguration:timeout",
})
// Handle timeout
return {
valid: false,
error: "Connection to embedding service failed (timeout)",
}
}
// Let standard handling take over
return undefined
},
},
)
}
get embedderInfo(): EmbedderInfo {
return {
name: "ollama",
}
}
}
packages/kilo-indexing/src/indexing/embedders/openai-compatible.ts
+484
View File
@@ -0,0 +1,484 @@
import { OpenAI } from "openai"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import {
MAX_BATCH_TOKENS,
MAX_ITEM_TOKENS,
MAX_BATCH_RETRIES as MAX_RETRIES,
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "../constants"
import { getDefaultModelId, getModelQueryPrefix } from "../model-registry"
import { withValidationErrorHandling, type HttpError, formatEmbeddingError } from "../shared/validation-helpers"
import { Mutex } from "async-mutex"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-openai-compatible" })
interface EmbeddingItem {
embedding: string | number[]
[key: string]: any
}
interface OpenAIEmbeddingResponse {
data: EmbeddingItem[]
usage?: {
prompt_tokens?: number
total_tokens?: number
}
}
/**
* OpenAI Compatible implementation of the embedder interface with batching and rate limiting.
* This embedder allows using any OpenAI-compatible API endpoint by specifying a custom baseURL.
*/
export class OpenAICompatibleEmbedder implements IEmbedder {
private embeddingsClient: OpenAI
private readonly defaultModelId: string
private readonly baseUrl: string
private readonly apiKey: string
private readonly isFullUrl: boolean
private readonly maxItemTokens: number
// Global rate limiting state shared across all instances
private static globalRateLimitState = {
isRateLimited: false,
rateLimitResetTime: 0,
consecutiveRateLimitErrors: 0,
lastRateLimitError: 0,
// Mutex to ensure thread-safe access to rate limit state
mutex: new Mutex(),
}
/**
* Creates a new OpenAI Compatible embedder
* @param baseUrl The base URL for the OpenAI-compatible API endpoint
* @param apiKey The API key for authentication
* @param modelId Optional model identifier (defaults to "text-embedding-3-small")
* @param maxItemTokens Optional maximum tokens per item (defaults to MAX_ITEM_TOKENS)
*/
constructor(baseUrl: string, apiKey: string, modelId?: string, maxItemTokens?: number) {
if (!baseUrl) {
throw new Error("Base URL is required for OpenAI-compatible embedder")
}
if (!apiKey) {
throw new Error("API key is required for OpenAI-compatible embedder")
}
this.baseUrl = baseUrl
this.apiKey = apiKey
try {
this.embeddingsClient = new OpenAI({
baseURL: baseUrl,
apiKey: apiKey,
})
} catch (error) {
throw error instanceof Error ? error : new Error(String(error))
}
this.defaultModelId = modelId || getDefaultModelId("openai-compatible")
// Cache the URL type check for performance
this.isFullUrl = this.isFullEndpointUrl(baseUrl)
this.maxItemTokens = maxItemTokens || MAX_ITEM_TOKENS
}
/**
* Creates embeddings for the given texts with batching and rate limiting
* @param texts Array of text strings to embed
* @param model Optional model identifier
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.defaultModelId
// Apply model-specific query prefix if required
const queryPrefix = getModelQueryPrefix("openai-compatible", modelToUse)
const processedTexts = queryPrefix
? texts.map((text, index) => {
// Prevent double-prefixing
if (text.startsWith(queryPrefix)) {
return text
}
const prefixedText = `${queryPrefix}${text}`
const estimatedTokens = Math.ceil(prefixedText.length / 4)
if (estimatedTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${index} with prefix exceeds token limit (${estimatedTokens} > ${MAX_ITEM_TOKENS})`)
// Return original text if adding prefix would exceed limit
return text
}
return prefixedText
})
: texts
const allEmbeddings: number[][] = []
const usage = { promptTokens: 0, totalTokens: 0 }
const remainingTexts = [...processedTexts]
while (remainingTexts.length > 0) {
const currentBatch: string[] = []
let currentBatchTokens = 0
const processedIndices: number[] = []
for (let i = 0; i < remainingTexts.length; i++) {
const text = remainingTexts[i]
const itemTokens = Math.ceil(text.length / 4)
if (itemTokens > this.maxItemTokens) {
log.warn(`Text at index ${i} exceeds token limit (${itemTokens} > ${this.maxItemTokens})`)
processedIndices.push(i)
continue
}
if (currentBatchTokens + itemTokens <= MAX_BATCH_TOKENS) {
currentBatch.push(text)
currentBatchTokens += itemTokens
processedIndices.push(i)
} else {
break
}
}
// Remove processed items from remainingTexts (in reverse order to maintain correct indices)
for (let i = processedIndices.length - 1; i >= 0; i--) {
remainingTexts.splice(processedIndices[i]!, 1)
}
if (currentBatch.length > 0) {
const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse)
allEmbeddings.push(...batchResult.embeddings)
usage.promptTokens += batchResult.usage.promptTokens
usage.totalTokens += batchResult.usage.totalTokens
}
}
return { embeddings: allEmbeddings, usage }
}
/**
* Determines if the provided URL is a full endpoint URL or a base URL that needs the endpoint appended by the SDK.
* Uses smart pattern matching for known providers while accepting we can't cover all possible patterns.
* @param url The URL to check
* @returns true if it's a full endpoint URL, false if it's a base URL
*/
private isFullEndpointUrl(url: string): boolean {
// Known patterns for major providers
const patterns = [
// Azure OpenAI: /deployments/{deployment-name}/embeddings
/\/deployments\/[^\/]+\/embeddings(\?|$)/,
// Azure Databricks: /serving-endpoints/{endpoint-name}/invocations
/\/serving-endpoints\/[^\/]+\/invocations(\?|$)/,
// Direct endpoints: ends with /embeddings (before query params)
/\/embeddings(\?|$)/,
// Some providers use /embed instead of /embeddings
/\/embed(\?|$)/,
]
return patterns.some((pattern) => pattern.test(url))
}
/**
* Makes a direct HTTP request to the embeddings endpoint
* Used when the user provides a full endpoint URL (e.g., Azure OpenAI with query parameters)
* @param url The full endpoint URL
* @param batchTexts Array of texts to embed
* @param model Model identifier to use
* @returns Promise resolving to OpenAI-compatible response
*/
private async makeDirectEmbeddingRequest(
url: string,
batchTexts: string[],
model: string,
signal?: AbortSignal,
): Promise<OpenAIEmbeddingResponse> {
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
// Azure OpenAI uses 'api-key' header, while OpenAI uses 'Authorization'
// We'll try 'api-key' first for Azure compatibility
"api-key": this.apiKey,
Authorization: `Bearer ${this.apiKey}`,
},
body: JSON.stringify({
input: batchTexts,
model: model,
encoding_format: "base64",
}),
signal,
})
if (!response || !response.ok) {
const status = response?.status || 0
let errorText = "No response"
try {
if (response && typeof response.text === "function") {
errorText = await response.text()
} else if (response) {
errorText = `Error ${status}`
}
} catch {
// Ignore text parsing errors
errorText = `Error ${status}`
}
const error = new Error(`HTTP ${status}: ${errorText}`) as HttpError
error.status = status || response?.status || 0
throw error
}
try {
return (await response.json()) as OpenAIEmbeddingResponse
} catch (e) {
const error = new Error(`Failed to parse response JSON`) as HttpError
error.status = response.status
throw error
}
}
/**
* Helper method to handle batch embedding with retries and exponential backoff
* @param batchTexts Array of texts to embed in this batch
* @param model Model identifier to use
* @returns Promise resolving to embeddings and usage statistics
*/
private async _embedBatchWithRetries(
batchTexts: string[],
model: string,
): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> {
// Use cached value for performance
const isFullUrl = this.isFullUrl
for (let attempts = 0; attempts < MAX_RETRIES; attempts++) {
// Check global rate limit before attempting request
await this.waitForGlobalRateLimit()
try {
let response: OpenAIEmbeddingResponse
if (isFullUrl) {
// Use direct HTTP request for full endpoint URLs
response = await this.makeDirectEmbeddingRequest(this.baseUrl, batchTexts, model)
} else {
// Use OpenAI SDK for base URLs
response = (await this.embeddingsClient.embeddings.create({
input: batchTexts,
model: model,
// OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256
// when processing numeric arrays, which breaks compatibility with models using larger dimensions.
// By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves.
encoding_format: "base64",
})) as OpenAIEmbeddingResponse
}
// Convert base64 embeddings to float32 arrays
const processedEmbeddings = response.data.map((item: EmbeddingItem) => {
if (typeof item.embedding === "string") {
const buffer = Buffer.from(item.embedding, "base64")
// Create Float32Array view over the buffer
const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
return {
...item,
embedding: Array.from(float32Array),
}
}
return item
})
// Replace the original data with processed embeddings
response.data = processedEmbeddings
const embeddings = response.data.map((item) => item.embedding as number[])
return {
embeddings: embeddings,
usage: {
promptTokens: response.usage?.prompt_tokens || 0,
totalTokens: response.usage?.total_tokens || 0,
},
}
} catch (error) {
log.error("OpenAI Compatible embedder batch error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenAICompatibleEmbedder:_embedBatchWithRetries",
attempt: attempts + 1,
})
const hasMoreAttempts = attempts < MAX_RETRIES - 1
// Check if it's a rate limit error
const httpError = error as HttpError
if (httpError?.status === 429) {
// Update global rate limit state
await this.updateGlobalRateLimitState(httpError)
if (hasMoreAttempts) {
// Calculate delay based on global rate limit state
const baseDelay = INITIAL_DELAY_MS * Math.pow(2, attempts)
const globalDelay = await this.getGlobalRateLimitDelay()
const delayMs = Math.max(baseDelay, globalDelay)
log.warn(`Rate limit hit, retrying in ${delayMs}ms (attempt ${attempts + 1}/${MAX_RETRIES})`)
await new Promise((resolve) => setTimeout(resolve, delayMs))
continue
}
}
// Format and throw the error
throw formatEmbeddingError(error, MAX_RETRIES)
}
}
throw new Error(`Embedding failed after ${MAX_RETRIES} attempts`)
}
/**
* Validates the OpenAI-compatible embedder configuration by testing endpoint connectivity and API key
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(async () => {
try {
// Test with a minimal embedding request
const testTexts = ["test"]
const modelToUse = this.defaultModelId
let response: OpenAIEmbeddingResponse
if (this.isFullUrl) {
// Test direct HTTP request for full endpoint URLs
const ctl = new AbortController()
const timer = setTimeout(() => ctl.abort(), REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS)
try {
response = await this.makeDirectEmbeddingRequest(this.baseUrl, testTexts, modelToUse, ctl.signal)
} finally {
clearTimeout(timer)
}
} else {
// Test using OpenAI SDK for base URLs
response = (await this.embeddingsClient.embeddings.create(
{
input: testTexts,
model: modelToUse,
encoding_format: "base64",
},
{
timeout: REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
maxRetries: REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
},
)) as OpenAIEmbeddingResponse
}
// Check if we got a valid response
if (!response?.data || response.data.length === 0) {
return {
valid: false,
error: "Invalid response from embedding endpoint",
}
}
return { valid: true }
} catch (error) {
log.error("OpenAI Compatible embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenAICompatibleEmbedder:validateConfiguration",
})
throw error
}
}, "openai-compatible")
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "openai-compatible",
}
}
/**
* Waits if there's an active global rate limit
*/
private async waitForGlobalRateLimit(): Promise<void> {
const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire()
try {
const state = OpenAICompatibleEmbedder.globalRateLimitState
if (state.isRateLimited && state.rateLimitResetTime > Date.now()) {
const waitTime = state.rateLimitResetTime - Date.now()
// Silent wait - no logging to prevent flooding
release() // Release mutex before waiting
await new Promise((resolve) => setTimeout(resolve, waitTime))
return
}
// Reset rate limit if time has passed
if (state.isRateLimited && state.rateLimitResetTime <= Date.now()) {
state.isRateLimited = false
state.consecutiveRateLimitErrors = 0
}
} finally {
// Only release if we haven't already
try {
release()
} catch {
// Already released
}
}
}
/**
* Updates global rate limit state when a 429 error occurs
*/
private async updateGlobalRateLimitState(error: HttpError): Promise<void> {
const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire()
try {
const state = OpenAICompatibleEmbedder.globalRateLimitState
const now = Date.now()
// Increment consecutive rate limit errors
if (now - state.lastRateLimitError < 60000) {
// Within 1 minute
state.consecutiveRateLimitErrors++
} else {
state.consecutiveRateLimitErrors = 1
}
state.lastRateLimitError = now
// Calculate exponential backoff based on consecutive errors
const baseDelay = 5000 // 5 seconds base
const maxDelay = 300000 // 5 minutes max
const exponentialDelay = Math.min(baseDelay * Math.pow(2, state.consecutiveRateLimitErrors - 1), maxDelay)
// Set global rate limit
state.isRateLimited = true
state.rateLimitResetTime = now + exponentialDelay
// Silent rate limit activation - no logging to prevent flooding
} finally {
release()
}
}
/**
* Gets the current global rate limit delay
*/
private async getGlobalRateLimitDelay(): Promise<number> {
const release = await OpenAICompatibleEmbedder.globalRateLimitState.mutex.acquire()
try {
const state = OpenAICompatibleEmbedder.globalRateLimitState
if (state.isRateLimited && state.rateLimitResetTime > Date.now()) {
return state.rateLimitResetTime - Date.now()
}
return 0
} finally {
release()
}
}
}
packages/kilo-indexing/src/indexing/embedders/openai.ts
+204
View File
@@ -0,0 +1,204 @@
import { OpenAI } from "openai"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces"
import {
MAX_BATCH_TOKENS,
MAX_ITEM_TOKENS,
MAX_BATCH_RETRIES as MAX_RETRIES,
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "../constants"
import { getModelQueryPrefix } from "../model-registry"
import { withValidationErrorHandling, formatEmbeddingError, type HttpError } from "../shared/validation-helpers"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-openai" })
/**
* OpenAI implementation of the embedder interface with batching and rate limiting
*/
export class OpenAiEmbedder implements IEmbedder {
private embeddingsClient: OpenAI
private readonly defaultModelId: string
/**
* Creates a new OpenAI embedder
* @param apiKey The OpenAI API key for authentication
* @param modelId Optional model identifier (defaults to "text-embedding-3-small")
*/
constructor(apiKey: string, modelId?: string) {
try {
this.embeddingsClient = new OpenAI({ apiKey })
} catch (error) {
throw error instanceof Error ? error : new Error(String(error))
}
this.defaultModelId = modelId || "text-embedding-3-small"
}
/**
* Creates embeddings for the given texts with batching and rate limiting
* @param texts Array of text strings to embed
* @param model Optional model identifier
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.defaultModelId
// Apply model-specific query prefix if required
const queryPrefix = getModelQueryPrefix("openai", modelToUse)
const processedTexts = queryPrefix
? texts.map((text, index) => {
// Prevent double-prefixing
if (text.startsWith(queryPrefix)) {
return text
}
const prefixedText = `${queryPrefix}${text}`
const estimatedTokens = Math.ceil(prefixedText.length / 4)
if (estimatedTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${index} with prefix exceeds token limit (${estimatedTokens} > ${MAX_ITEM_TOKENS})`)
// Return original text if adding prefix would exceed limit
return text
}
return prefixedText
})
: texts
const allEmbeddings: number[][] = []
const usage = { promptTokens: 0, totalTokens: 0 }
const remainingTexts = [...processedTexts]
while (remainingTexts.length > 0) {
const currentBatch: string[] = []
let currentBatchTokens = 0
const processedIndices: number[] = []
for (let i = 0; i < remainingTexts.length; i++) {
const text = remainingTexts[i]
const itemTokens = Math.ceil(text.length / 4)
if (itemTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${i} exceeds token limit (${itemTokens} > ${MAX_ITEM_TOKENS})`)
processedIndices.push(i)
continue
}
if (currentBatchTokens + itemTokens <= MAX_BATCH_TOKENS) {
currentBatch.push(text)
currentBatchTokens += itemTokens
processedIndices.push(i)
} else {
break
}
}
// Remove processed items from remainingTexts (in reverse order to maintain correct indices)
for (let i = processedIndices.length - 1; i >= 0; i--) {
remainingTexts.splice(processedIndices[i], 1)
}
if (currentBatch.length > 0) {
const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse)
allEmbeddings.push(...batchResult.embeddings)
usage.promptTokens += batchResult.usage.promptTokens
usage.totalTokens += batchResult.usage.totalTokens
}
}
return { embeddings: allEmbeddings, usage }
}
/**
* Helper method to handle batch embedding with retries and exponential backoff
* @param batchTexts Array of texts to embed in this batch
* @param model Model identifier to use
* @returns Promise resolving to embeddings and usage statistics
*/
private async _embedBatchWithRetries(
batchTexts: string[],
model: string,
): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> {
for (let attempts = 0; attempts < MAX_RETRIES; attempts++) {
try {
const response = await this.embeddingsClient.embeddings.create({
input: batchTexts,
model: model,
})
return {
embeddings: response.data.map((item: any) => item.embedding),
usage: {
promptTokens: response.usage?.prompt_tokens || 0,
totalTokens: response.usage?.total_tokens || 0,
},
}
} catch (error: any) {
const hasMoreAttempts = attempts < MAX_RETRIES - 1
// Check if it's a rate limit error
const httpError = error as HttpError
if (httpError?.status === 429 && hasMoreAttempts) {
const delayMs = INITIAL_DELAY_MS * Math.pow(2, attempts)
log.warn(`Rate limit hit, retrying in ${delayMs}ms (attempt ${attempts + 1}/${MAX_RETRIES})`)
await new Promise((resolve) => setTimeout(resolve, delayMs))
continue
}
log.error("OpenAI embedder batch error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenAiEmbedder:_embedBatchWithRetries",
attempt: attempts + 1,
})
// Format and throw the error
throw formatEmbeddingError(error, MAX_RETRIES)
}
}
throw new Error(`Embedding failed after ${MAX_RETRIES} attempts`)
}
/**
* Validates the OpenAI embedder configuration by attempting a minimal embedding request
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(async () => {
try {
// Test with a minimal embedding request
const response = await this.embeddingsClient.embeddings.create(
{
input: ["test"],
model: this.defaultModelId,
},
{
timeout: REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
maxRetries: REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
},
)
// Check if we got a valid response
if (!response.data || response.data.length === 0) {
return {
valid: false,
error: "OpenAI returned an invalid response format",
}
}
return { valid: true }
} catch (error) {
log.error("OpenAI embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenAiEmbedder:validateConfiguration",
})
throw error
}
}, "openai")
}
get embedderInfo(): EmbedderInfo {
return {
name: "openai",
}
}
}
packages/kilo-indexing/src/indexing/embedders/openrouter.ts
+427
View File
@@ -0,0 +1,427 @@
import { OpenAI } from "openai"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import {
MAX_BATCH_TOKENS,
MAX_ITEM_TOKENS,
MAX_BATCH_RETRIES as MAX_RETRIES,
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "../constants"
import { getDefaultModelId, getModelQueryPrefix } from "../model-registry"
import { withValidationErrorHandling, type HttpError, formatEmbeddingError } from "../shared/validation-helpers"
import { Mutex } from "async-mutex"
import { DEFAULT_HEADERS } from "../../headers"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-openrouter" })
// Default provider name when no specific provider is selected
export const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]"
interface EmbeddingItem {
embedding: string | number[]
[key: string]: any
}
interface OpenRouterEmbeddingResponse {
data: EmbeddingItem[]
usage?: {
prompt_tokens?: number
total_tokens?: number
}
}
/**
* OpenRouter implementation of the embedder interface with batching and rate limiting.
* OpenRouter provides an OpenAI-compatible API that gives access to hundreds of models
* through a single endpoint, automatically handling fallbacks and cost optimization.
*/
export class OpenRouterEmbedder implements IEmbedder {
private embeddingsClient: OpenAI
private readonly defaultModelId: string
private readonly apiKey: string
private readonly maxItemTokens: number
private readonly baseUrl: string = "https://openrouter.ai/api/v1"
private readonly specificProvider?: string
private readonly dimensions?: number
// Global rate limiting state shared across all instances
private static globalRateLimitState = {
isRateLimited: false,
rateLimitResetTime: 0,
consecutiveRateLimitErrors: 0,
lastRateLimitError: 0,
// Mutex to ensure thread-safe access to rate limit state
mutex: new Mutex(),
}
/**
* Creates a new OpenRouter embedder
* @param apiKey The API key for authentication
* @param modelId Optional model identifier (defaults to "openai/text-embedding-3-large")
* @param maxItemTokens Optional maximum tokens per item (defaults to MAX_ITEM_TOKENS)
* @param specificProvider Optional specific provider to route requests to
* @param dimensions Optional embedding dimensions override
*/
constructor(
apiKey: string,
modelId?: string,
maxItemTokens?: number,
specificProvider?: string,
dimensions?: number,
) {
if (!apiKey) {
throw new Error("API key is required for OpenRouter embedder")
}
this.apiKey = apiKey
// Only set specificProvider if it's not the default value
this.specificProvider =
specificProvider && specificProvider !== OPENROUTER_DEFAULT_PROVIDER_NAME ? specificProvider : undefined
try {
this.embeddingsClient = new OpenAI({
baseURL: this.baseUrl,
apiKey: apiKey,
defaultHeaders: DEFAULT_HEADERS,
})
} catch (error) {
throw error instanceof Error ? error : new Error(String(error))
}
this.defaultModelId = modelId || getDefaultModelId("openrouter")
this.maxItemTokens = maxItemTokens || MAX_ITEM_TOKENS
this.dimensions = dimensions
}
/**
* Creates embeddings for the given texts with batching and rate limiting
* @param texts Array of text strings to embed
* @param model Optional model identifier
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.defaultModelId
// Apply model-specific query prefix if required
const queryPrefix = getModelQueryPrefix("openrouter", modelToUse)
const processedTexts = queryPrefix
? texts.map((text, index) => {
// Prevent double-prefixing
if (text.startsWith(queryPrefix)) {
return text
}
const prefixedText = `${queryPrefix}${text}`
const estimatedTokens = Math.ceil(prefixedText.length / 4)
if (estimatedTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${index} with prefix exceeds token limit (${estimatedTokens} > ${MAX_ITEM_TOKENS})`)
// Return original text if adding prefix would exceed limit
return text
}
return prefixedText
})
: texts
const allEmbeddings: number[][] = []
const usage = { promptTokens: 0, totalTokens: 0 }
const remainingTexts = [...processedTexts]
while (remainingTexts.length > 0) {
const currentBatch: string[] = []
let currentBatchTokens = 0
const processedIndices: number[] = []
for (let i = 0; i < remainingTexts.length; i++) {
const text = remainingTexts[i]
if (text === undefined) {
continue
}
const itemTokens = Math.ceil(text.length / 4)
if (itemTokens > this.maxItemTokens) {
log.warn(`Text at index ${i} exceeds token limit (${itemTokens} > ${this.maxItemTokens})`)
processedIndices.push(i)
continue
}
if (currentBatchTokens + itemTokens <= MAX_BATCH_TOKENS) {
currentBatch.push(text)
currentBatchTokens += itemTokens
processedIndices.push(i)
} else {
break
}
}
// Remove processed items from remainingTexts (in reverse order to maintain correct indices)
for (let i = processedIndices.length - 1; i >= 0; i--) {
const idx = processedIndices[i]
if (idx === undefined) {
continue
}
remainingTexts.splice(idx, 1)
}
if (currentBatch.length > 0) {
const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse)
allEmbeddings.push(...batchResult.embeddings)
usage.promptTokens += batchResult.usage.promptTokens
usage.totalTokens += batchResult.usage.totalTokens
}
}
return { embeddings: allEmbeddings, usage }
}
/**
* Helper method to handle batch embedding with retries and exponential backoff
* @param batchTexts Array of texts to embed in this batch
* @param model Model identifier to use
* @returns Promise resolving to embeddings and usage statistics
*/
private async _embedBatchWithRetries(
batchTexts: string[],
model: string,
): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> {
for (let attempts = 0; attempts < MAX_RETRIES; attempts++) {
// Check global rate limit before attempting request
await this.waitForGlobalRateLimit()
try {
// Build the request parameters
const requestParams: any = {
input: batchTexts,
model: model,
// OpenAI package (as of v4.78.1) has a parsing issue that truncates embedding dimensions to 256
// when processing numeric arrays, which breaks compatibility with models using larger dimensions.
// By requesting base64 encoding, we bypass the package's parser and handle decoding ourselves.
encoding_format: "base64",
}
if (this.dimensions !== undefined) {
requestParams.dimensions = this.dimensions
}
// Add provider routing if a specific provider is set
if (this.specificProvider) {
requestParams.provider = {
order: [this.specificProvider],
only: [this.specificProvider],
allow_fallbacks: false,
}
}
const response = (await this.embeddingsClient.embeddings.create(requestParams)) as OpenRouterEmbeddingResponse
// Convert base64 embeddings to float32 arrays
const processedEmbeddings = response.data.map((item: EmbeddingItem) => {
if (typeof item.embedding === "string") {
const buffer = Buffer.from(item.embedding, "base64")
// Create Float32Array view over the buffer
const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
return {
...item,
embedding: Array.from(float32Array),
}
}
return item
})
// Replace the original data with processed embeddings
response.data = processedEmbeddings
const embeddings = response.data.map((item) => item.embedding as number[])
return {
embeddings: embeddings,
usage: {
promptTokens: response.usage?.prompt_tokens || 0,
totalTokens: response.usage?.total_tokens || 0,
},
}
} catch (error) {
log.error("OpenRouter embedder batch error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenRouterEmbedder:_embedBatchWithRetries",
attempt: attempts + 1,
})
const hasMoreAttempts = attempts < MAX_RETRIES - 1
// Check if it's a rate limit error
const httpError = error as HttpError
if (httpError?.status === 429) {
// Update global rate limit state
await this.updateGlobalRateLimitState(httpError)
if (hasMoreAttempts) {
// Calculate delay based on global rate limit state
const baseDelay = INITIAL_DELAY_MS * Math.pow(2, attempts)
const globalDelay = await this.getGlobalRateLimitDelay()
const delayMs = Math.max(baseDelay, globalDelay)
log.warn(`Rate limit hit, retrying in ${delayMs}ms (attempt ${attempts + 1}/${MAX_RETRIES})`)
await new Promise((resolve) => setTimeout(resolve, delayMs))
continue
}
}
// Format and throw the error
throw formatEmbeddingError(error, MAX_RETRIES)
}
}
throw new Error(`Embedding failed after ${MAX_RETRIES} attempts`)
}
/**
* Validates the OpenRouter embedder configuration by testing API connectivity
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(async () => {
try {
// Test with a minimal embedding request
const testTexts = ["test"]
const modelToUse = this.defaultModelId
// Build the request parameters
const requestParams: any = {
input: testTexts,
model: modelToUse,
encoding_format: "base64",
}
if (this.dimensions !== undefined) {
requestParams.dimensions = this.dimensions
}
// Add provider routing if a specific provider is set
if (this.specificProvider) {
requestParams.provider = {
order: [this.specificProvider],
only: [this.specificProvider],
allow_fallbacks: false,
}
}
const response = (await this.embeddingsClient.embeddings.create(requestParams, {
timeout: REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
maxRetries: REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
})) as OpenRouterEmbeddingResponse
// Check if we got a valid response
if (!response?.data || response.data.length === 0) {
return {
valid: false,
error: "Invalid response from OpenRouter embedding endpoint",
}
}
return { valid: true }
} catch (error) {
log.error("OpenRouter embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "OpenRouterEmbedder:validateConfiguration",
})
throw error
}
}, "openrouter")
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "openrouter",
}
}
/**
* Waits if there's an active global rate limit
*/
private async waitForGlobalRateLimit(): Promise<void> {
const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire()
let mutexReleased = false
try {
const state = OpenRouterEmbedder.globalRateLimitState
if (state.isRateLimited && state.rateLimitResetTime > Date.now()) {
const waitTime = state.rateLimitResetTime - Date.now()
// Silent wait - no logging to prevent flooding
release()
mutexReleased = true
await new Promise((resolve) => setTimeout(resolve, waitTime))
return
}
// Reset rate limit if time has passed
if (state.isRateLimited && state.rateLimitResetTime <= Date.now()) {
state.isRateLimited = false
state.consecutiveRateLimitErrors = 0
}
} finally {
// Only release if we haven't already
if (!mutexReleased) {
release()
}
}
}
/**
* Updates global rate limit state when a 429 error occurs
*/
private async updateGlobalRateLimitState(error: HttpError): Promise<void> {
const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire()
try {
const state = OpenRouterEmbedder.globalRateLimitState
const now = Date.now()
// Increment consecutive rate limit errors
if (now - state.lastRateLimitError < 60000) {
// Within 1 minute
state.consecutiveRateLimitErrors++
} else {
state.consecutiveRateLimitErrors = 1
}
state.lastRateLimitError = now
// Calculate exponential backoff based on consecutive errors
const baseDelay = 5000 // 5 seconds base
const maxDelay = 300000 // 5 minutes max
const exponentialDelay = Math.min(baseDelay * Math.pow(2, state.consecutiveRateLimitErrors - 1), maxDelay)
// Set global rate limit
state.isRateLimited = true
state.rateLimitResetTime = now + exponentialDelay
// Silent rate limit activation - no logging to prevent flooding
} finally {
release()
}
}
/**
* Gets the current global rate limit delay
*/
private async getGlobalRateLimitDelay(): Promise<number> {
const release = await OpenRouterEmbedder.globalRateLimitState.mutex.acquire()
try {
const state = OpenRouterEmbedder.globalRateLimitState
if (state.isRateLimited && state.rateLimitResetTime > Date.now()) {
return state.rateLimitResetTime - Date.now()
}
return 0
} finally {
release()
}
}
}
packages/kilo-indexing/src/indexing/embedders/vercel-ai-gateway.ts
+98
View File
@@ -0,0 +1,98 @@
import { OpenAICompatibleEmbedder } from "./openai-compatible"
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import { MAX_ITEM_TOKENS } from "../constants"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-vercel-ai-gateway" })
/**
* Vercel AI Gateway embedder implementation that wraps the OpenAI Compatible embedder
* with configuration for Vercel AI Gateway's embedding API.
*
* Supported models:
* - openai/text-embedding-3-small (dimension: 1536)
* - openai/text-embedding-3-large (dimension: 3072)
* - openai/text-embedding-ada-002 (dimension: 1536)
* - cohere/embed-v4.0 (dimension: 1024)
* - google/gemini-embedding-001 (dimension: 768)
* - google/text-embedding-005 (dimension: 768)
* - google/text-multilingual-embedding-002 (dimension: 768)
* - amazon/titan-embed-text-v2 (dimension: 1024)
* - mistral/codestral-embed (dimension: 1536)
* - mistral/mistral-embed (dimension: 1024)
*/
export class VercelAiGatewayEmbedder implements IEmbedder {
private readonly openAICompatibleEmbedder: OpenAICompatibleEmbedder
private static readonly VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"
private static readonly DEFAULT_MODEL = "openai/text-embedding-3-large"
private readonly modelId: string
/**
* Creates a new Vercel AI Gateway embedder
* @param apiKey The Vercel AI Gateway API key for authentication
* @param modelId The model ID to use (defaults to mistral/codestral-embed)
*/
constructor(apiKey: string, modelId?: string) {
if (!apiKey) {
throw new Error("API key is required for Vercel AI Gateway embedder")
}
// Use provided model or default
this.modelId = modelId || VercelAiGatewayEmbedder.DEFAULT_MODEL
// Create an OpenAI Compatible embedder with Vercel AI Gateway's configuration
this.openAICompatibleEmbedder = new OpenAICompatibleEmbedder(
VercelAiGatewayEmbedder.VERCEL_AI_GATEWAY_BASE_URL,
apiKey,
this.modelId,
MAX_ITEM_TOKENS,
)
}
/**
* Creates embeddings for the given texts using Vercel AI Gateway's embedding API
* @param texts Array of text strings to embed
* @param model Optional model identifier (uses constructor model if not provided)
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
try {
// Use the provided model or fall back to the instance's model
const modelToUse = model || this.modelId
return await this.openAICompatibleEmbedder.createEmbeddings(texts, modelToUse)
} catch (error) {
log.error("Vercel AI Gateway embedder error in createEmbeddings", {
err: error instanceof Error ? error.message : String(error),
location: "VercelAiGatewayEmbedder:createEmbeddings",
})
throw error
}
}
/**
* Validates the Vercel AI Gateway embedder configuration by delegating to the underlying OpenAI-compatible embedder
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
try {
// Delegate validation to the OpenAI-compatible embedder
// The error messages will be specific to Vercel AI Gateway since we're using Vercel's base URL
return await this.openAICompatibleEmbedder.validateConfiguration()
} catch (error) {
log.error("Vercel AI Gateway embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "VercelAiGatewayEmbedder:validateConfiguration",
})
throw error
}
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "vercel-ai-gateway",
}
}
}
packages/kilo-indexing/src/indexing/embedders/voyage.ts
+265
View File
@@ -0,0 +1,265 @@
import type { IEmbedder, EmbeddingResponse, EmbedderInfo } from "../interfaces/embedder"
import {
MAX_BATCH_TOKENS,
MAX_ITEM_TOKENS,
MAX_BATCH_RETRIES as MAX_RETRIES,
INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "../constants"
import { getModelQueryPrefix } from "../model-registry"
import { withValidationErrorHandling, formatEmbeddingError, type HttpError } from "../shared/validation-helpers"
import { Log } from "../../util/log"
const log = Log.create({ service: "embedder-voyage" })
/**
* Response structure from Voyage AI embedding API
*/
interface VoyageEmbeddingItem {
embedding: number[]
index: number
}
interface VoyageEmbeddingResponse {
data: VoyageEmbeddingItem[]
model: string
usage?: {
total_tokens?: number
}
}
/**
* Voyage AI embedder implementation using the native Voyage API.
*
* Voyage AI provides high-quality embedding models including code-specific models.
* API endpoint: https://api.voyageai.com/v1/embeddings
*
* Supported models:
* - voyage-code-3 (dimension: 1024, code-optimized)
* - voyage-4-large (dimension: 1024)
* - voyage-4 (dimension: 1024)
* - voyage-4-lite (dimension: 1024)
* - voyage-finance-2 (dimension: 1024)
* - voyage-law-2 (dimension: 1024)
*/
export class VoyageEmbedder implements IEmbedder {
private static readonly VOYAGE_BASE_URL = "https://api.voyageai.com/v1/embeddings"
private static readonly DEFAULT_MODEL = "voyage-code-3"
private readonly apiKey: string
private readonly modelId: string
/**
* Creates a new Voyage AI embedder
* @param apiKey The Voyage AI API key for authentication
* @param modelId The model ID to use (defaults to voyage-code-3)
*/
constructor(apiKey: string, modelId?: string) {
if (!apiKey) {
throw new Error("API key is required for Voyage embedder")
}
this.apiKey = apiKey
this.modelId = modelId || VoyageEmbedder.DEFAULT_MODEL
}
/**
* Creates embeddings for the given texts using Voyage AI's embedding API
* @param texts Array of text strings to embed
* @param model Optional model identifier (uses constructor model if not provided)
* @returns Promise resolving to embedding response
*/
async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> {
const modelToUse = model || this.modelId
// Apply model-specific query prefix if required
const queryPrefix = getModelQueryPrefix("voyage", modelToUse)
const processedTexts = queryPrefix
? texts.map((text, index) => {
// Prevent double-prefixing
if (text.startsWith(queryPrefix)) {
return text
}
const prefixedText = `${queryPrefix}${text}`
const estimatedTokens = Math.ceil(prefixedText.length / 4)
if (estimatedTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${index} with prefix exceeds token limit (${estimatedTokens} > ${MAX_ITEM_TOKENS})`)
// Return original text if adding prefix would exceed limit
return text
}
return prefixedText
})
: texts
const allEmbeddings: number[][] = []
const usage = { promptTokens: 0, totalTokens: 0 }
const remainingTexts = [...processedTexts]
while (remainingTexts.length > 0) {
const currentBatch: string[] = []
let currentBatchTokens = 0
const processedIndices: number[] = []
for (let i = 0; i < remainingTexts.length; i++) {
const text = remainingTexts[i]!
const itemTokens = Math.ceil(text.length / 4)
if (itemTokens > MAX_ITEM_TOKENS) {
log.warn(`Text at index ${i} exceeds token limit (${itemTokens} > ${MAX_ITEM_TOKENS})`)
processedIndices.push(i)
continue
}
if (currentBatchTokens + itemTokens <= MAX_BATCH_TOKENS) {
currentBatch.push(text)
currentBatchTokens += itemTokens
processedIndices.push(i)
} else {
break
}
}
// Remove processed items from remainingTexts (in reverse order to maintain correct indices)
for (let i = processedIndices.length - 1; i >= 0; i--) {
remainingTexts.splice(processedIndices[i]!, 1)
}
if (currentBatch.length > 0) {
const batchResult = await this._embedBatchWithRetries(currentBatch, modelToUse)
allEmbeddings.push(...batchResult.embeddings)
usage.promptTokens += batchResult.usage.promptTokens
usage.totalTokens += batchResult.usage.totalTokens
}
}
return { embeddings: allEmbeddings, usage }
}
/**
* Helper method to handle batch embedding with retries and exponential backoff
* @param batchTexts Array of texts to embed in this batch
* @param model Model identifier to use
* @returns Promise resolving to embeddings and usage statistics
*/
private async _embedBatchWithRetries(
batchTexts: string[],
model: string,
): Promise<{ embeddings: number[][]; usage: { promptTokens: number; totalTokens: number } }> {
for (let attempts = 0; attempts < MAX_RETRIES; attempts++) {
try {
const response = await fetch(VoyageEmbedder.VOYAGE_BASE_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
},
body: JSON.stringify({
input: batchTexts,
model: model,
input_type: "document", // For indexing, we use "document" type
}),
})
if (!response.ok) {
const errorBody = await response.text().catch(() => "Unknown error")
const error = new Error(`HTTP ${response.status}: ${errorBody}`) as HttpError
error.status = response.status
throw error
}
const result = (await response.json()) as VoyageEmbeddingResponse
// Sort by index to ensure correct order
const sortedData = [...result.data].sort((a, b) => a.index - b.index)
return {
embeddings: sortedData.map((item) => item.embedding),
usage: {
promptTokens: result.usage?.total_tokens || 0,
totalTokens: result.usage?.total_tokens || 0,
},
}
} catch (error: any) {
const hasMoreAttempts = attempts < MAX_RETRIES - 1
// Check if it's a rate limit error
const httpError = error as HttpError
if (httpError?.status === 429 && hasMoreAttempts) {
const delayMs = INITIAL_DELAY_MS * Math.pow(2, attempts)
log.warn(`Rate limit hit, retrying in ${delayMs}ms (attempt ${attempts + 1}/${MAX_RETRIES})`)
await new Promise((resolve) => setTimeout(resolve, delayMs))
continue
}
log.error("Voyage AI embedder batch error", {
err: error instanceof Error ? error.message : String(error),
location: "VoyageEmbedder:_embedBatchWithRetries",
attempt: attempts + 1,
})
// Format and throw the error
throw formatEmbeddingError(error, MAX_RETRIES)
}
}
throw new Error(`Embedding failed after ${MAX_RETRIES} attempts`)
}
/**
* Validates the Voyage AI embedder configuration by attempting a minimal embedding request
* @returns Promise resolving to validation result with success status and optional error message
*/
async validateConfiguration(): Promise<{ valid: boolean; error?: string }> {
return withValidationErrorHandling(async () => {
try {
const ctl = new AbortController()
const timer = setTimeout(() => ctl.abort(), REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS)
const response = await fetch(VoyageEmbedder.VOYAGE_BASE_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${this.apiKey}`,
},
body: JSON.stringify({
input: ["test"],
model: this.modelId,
}),
signal: ctl.signal,
}).finally(() => clearTimeout(timer))
if (!response.ok) {
const errorBody = await response.text().catch(() => "Unknown error")
const error = new Error(`HTTP ${response.status}: ${errorBody}`) as HttpError
error.status = response.status
throw error
}
const result = (await response.json()) as VoyageEmbeddingResponse
// Check if we got a valid response
if (!result.data || result.data.length === 0) {
return {
valid: false,
error: "Voyage AI returned an invalid response format",
}
}
return { valid: true }
} catch (error) {
log.error("Voyage AI embedder validation error", {
err: error instanceof Error ? error.message : String(error),
location: "VoyageEmbedder:validateConfiguration",
})
throw error
}
}, "voyage")
}
/**
* Returns information about this embedder
*/
get embedderInfo(): EmbedderInfo {
return {
name: "voyage",
}
}
}
packages/kilo-indexing/src/indexing/embedding-profile.ts
+35
View File
@@ -0,0 +1,35 @@
import type { EmbedderProvider } from "./interfaces/manager"
import { getDefaultModelId, getModelDimension } from "./model-registry"
export interface EmbeddingProfile {
provider: EmbedderProvider
modelId: string
dimension: number
}
function parseDimension(value?: number): number | undefined {
if (value === undefined || value === null) return undefined
const dim = Number(value)
if (!Number.isFinite(dim) || dim <= 0) return undefined
return dim
}
export function resolveEmbeddingProfile(
provider: EmbedderProvider,
modelId?: string,
modelDimension?: number,
): EmbeddingProfile | undefined {
const id = modelId ?? getDefaultModelId(provider)
const dim = getModelDimension(provider, id) ?? parseDimension(modelDimension)
if (!dim) return undefined
return {
provider,
modelId: id,
dimension: dim,
}
}
export function isEmbeddingProfileEqual(a?: EmbeddingProfile, b?: EmbeddingProfile): boolean {
if (!a || !b) return false
return a.provider === b.provider && a.modelId === b.modelId && a.dimension === b.dimension
}
packages/kilo-indexing/src/indexing/index.ts
+35
View File
@@ -0,0 +1,35 @@
export { CodeIndexManager } from "./manager"
export { CodeIndexConfigManager, type IndexingConfigInput } from "./config-manager"
export { CodeIndexStateManager, type IndexingState } from "./state-manager"
export { CodeIndexSearchService } from "./search-service"
export { CodeIndexOrchestrator } from "./orchestrator"
export { CodeIndexServiceFactory } from "./service-factory"
export { CacheManager } from "./cache-manager"
export { Emitter, type Disposable } from "./runtime"
export type { ICodeIndexManager, IndexProgressUpdate, EmbedderProvider } from "./interfaces/manager"
export type {
IndexingTelemetryEvent,
IndexingTelemetryMode,
IndexingTelemetryReporter,
IndexingTelemetrySource,
IndexingTelemetryTrigger,
} from "./interfaces/telemetry"
export type { CodeIndexConfig, PreviousConfigSnapshot } from "./interfaces/config"
export type { IEmbedder, EmbeddingResponse, EmbedderInfo, AvailableEmbedders } from "./interfaces/embedder"
export type { IVectorStore, VectorStoreSearchResult, PointStruct, Payload } from "./interfaces/vector-store"
export type {
ICodeParser,
IDirectoryScanner,
IFileWatcher,
CodeBlock,
FileProcessingResult,
BatchProcessingSummary,
} from "./interfaces/file-processor"
export type { ICacheManager } from "./interfaces/cache"
packages/kilo-indexing/src/indexing/interfaces/cache.ts
+6
View File
@@ -0,0 +1,6 @@
export interface ICacheManager {
getHash(filePath: string): string | undefined
updateHash(filePath: string, hash: string): void
deleteHash(filePath: string): void
getAllHashes(): Record<string, string>
}
packages/kilo-indexing/src/indexing/interfaces/config.ts
+56
View File
@@ -0,0 +1,56 @@
import type { EmbedderProvider } from "./manager"
/**
* Configuration state for the code indexing feature.
*
* RATIONALE: Replaced the legacy ApiHandlerOptions / ContextProxy types with
* indexing-local option shapes so the package does not depend on the
* extension's configuration plumbing.
*/
export interface CodeIndexConfig {
isConfigured: boolean
embedderProvider: EmbedderProvider
vectorStoreProvider?: "lancedb" | "qdrant"
lancedbVectorStoreDirectoryPlaceholder?: string
modelId?: string
modelDimension?: number
openAiOptions?: { apiKey: string }
ollamaOptions?: { baseUrl: string; modelId?: string }
openAiCompatibleOptions?: { baseUrl: string; apiKey: string }
geminiOptions?: { apiKey: string }
mistralOptions?: { apiKey: string }
vercelAiGatewayOptions?: { apiKey: string }
bedrockOptions?: { region: string; profile?: string }
openRouterOptions?: { apiKey: string; specificProvider?: string }
voyageOptions?: { apiKey: string }
qdrantUrl?: string
qdrantApiKey?: string
searchMinScore?: number
searchMaxResults?: number
embeddingBatchSize?: number
scannerMaxBatchRetries?: number
}
export type PreviousConfigSnapshot = {
enabled: boolean
configured: boolean
embedderProvider: EmbedderProvider
vectorStoreProvider?: "lancedb" | "qdrant"
lancedbVectorStoreDirectory?: string
modelId?: string
modelDimension?: number
openAiKey?: string
ollamaBaseUrl?: string
openAiCompatibleBaseUrl?: string
openAiCompatibleApiKey?: string
geminiApiKey?: string
mistralApiKey?: string
vercelAiGatewayApiKey?: string
bedrockRegion?: string
bedrockProfile?: string
openRouterApiKey?: string
openRouterSpecificProvider?: string
voyageApiKey?: string
qdrantUrl?: string
qdrantApiKey?: string
}
packages/kilo-indexing/src/indexing/interfaces/embedder.ts
+44
View File
@@ -0,0 +1,44 @@
/**
* Interface for code index embedders.
* This interface is implemented by both OpenAI and Ollama embedders.
*/
export interface IEmbedder {
/**
* Creates embeddings for the given texts.
* @param texts Array of text strings to create embeddings for
* @param model Optional model ID to use for embeddings
* @returns Promise resolving to an EmbeddingResponse
*/
createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse>
/**
* Validates the embedder configuration by testing connectivity and credentials.
* @returns Promise resolving to validation result with success status and optional error message
*/
validateConfiguration(): Promise<{ valid: boolean; error?: string }>
get embedderInfo(): EmbedderInfo
}
export interface EmbeddingResponse {
embeddings: number[][]
usage?: {
promptTokens: number
totalTokens: number
}
}
export type AvailableEmbedders =
| "openai"
| "ollama"
| "openai-compatible"
| "gemini"
| "mistral"
| "vercel-ai-gateway"
| "bedrock"
| "openrouter"
| "voyage"
export interface EmbedderInfo {
name: AvailableEmbedders
}
packages/kilo-indexing/src/indexing/interfaces/file-processor.ts
+74
View File
@@ -0,0 +1,74 @@
import type { PointStruct } from "./vector-store"
import type { Disposable, Emitter } from "../runtime"
import type { IndexingTelemetryMode } from "./telemetry"
export interface ICodeParser {
parseFile(
filePath: string,
options?: {
minBlockLines?: number
maxBlockLines?: number
content?: string
fileHash?: string
},
): Promise<CodeBlock[]>
}
export interface IDirectoryScanner {
scanDirectory(
directory: string,
onError?: (error: Error) => void,
onFilesIndexed?: (indexedCount: number) => void,
onFileParsed?: () => void,
mode?: IndexingTelemetryMode,
): Promise<{
stats: {
processed: number
skipped: number
}
totalBlockCount: number
}>
updateBatchSegmentThreshold(newThreshold: number): void
}
export interface IFileWatcher extends Disposable {
initialize(): Promise<void>
updateBatchSegmentThreshold(newThreshold: number): void
setCollecting(collecting: boolean): void
readonly onDidStartBatchProcessing: Emitter<string[]>
readonly onBatchProgressUpdate: Emitter<{
processedInBatch: number
totalInBatch: number
currentFile?: string
}>
readonly onDidFinishBatchProcessing: Emitter<BatchProcessingSummary>
processFile(filePath: string): Promise<FileProcessingResult>
}
export interface BatchProcessingSummary {
processedFiles: FileProcessingResult[]
batchError?: Error
}
export interface FileProcessingResult {
path: string
status: "success" | "skipped" | "error" | "processed_for_batching" | "local_error"
error?: Error
reason?: string
newHash?: string
pointsToUpsert?: PointStruct[]
}
export interface CodeBlock {
file_path: string
identifier: string | null
type: string
start_line: number
end_line: number
content: string
fileHash: string
segmentHash: string
}
packages/kilo-indexing/src/indexing/interfaces/index.ts
+7
View File
@@ -0,0 +1,7 @@
export * from "./embedder"
export * from "./vector-store"
export * from "./file-processor"
export * from "./manager"
export * from "./config"
export * from "./cache"
export * from "./telemetry"
packages/kilo-indexing/src/indexing/interfaces/manager.ts
+55
View File
@@ -0,0 +1,55 @@
import type { VectorStoreSearchResult } from "./vector-store"
import type { Emitter } from "../runtime"
import type { IndexingTelemetryEvent } from "./telemetry"
export interface ICodeIndexManager {
onProgressUpdate: Emitter<{
systemStatus: IndexingState
message?: string
processedItems: number
totalItems: number
currentItemUnit: string
gitBranch?: string
manifest?: { totalFiles: number; totalChunks: number; lastUpdated: string }
}>
onTelemetry: Emitter<IndexingTelemetryEvent>
readonly state: IndexingState
readonly isFeatureEnabled: boolean
readonly isFeatureConfigured: boolean
loadConfiguration(): Promise<void>
startIndexing(): Promise<void>
stopWatcher(): void
clearIndexData(): Promise<void>
searchIndex(query: string, directoryPrefix?: string): Promise<VectorStoreSearchResult[]>
getCurrentStatus(): {
systemStatus: IndexingState
message?: string
processedItems: number
totalItems: number
currentItemUnit: string
}
dispose(): void
}
export type IndexingState = "Standby" | "Indexing" | "Indexed" | "Error"
export type EmbedderProvider =
| "openai"
| "ollama"
| "openai-compatible"
| "gemini"
| "mistral"
| "vercel-ai-gateway"
| "bedrock"
| "openrouter"
| "voyage"
export interface IndexProgressUpdate {
systemStatus: IndexingState
message?: string
processedBlockCount?: number
totalBlockCount?: number
}
packages/kilo-indexing/src/indexing/interfaces/telemetry.ts
+58
View File
@@ -0,0 +1,58 @@
import type { EmbedderProvider } from "./manager"
export type IndexingTelemetryTrigger = "background" | "manual"
export type IndexingTelemetryMode = "full" | "incremental"
export type IndexingTelemetrySource = "scan" | "watcher"
export type IndexingVectorStore = "lancedb" | "qdrant"
export type IndexingTelemetryMeta = {
provider: EmbedderProvider
vectorStore: IndexingVectorStore
modelId?: string
}
export type IndexingTelemetryEvent =
| (IndexingTelemetryMeta & {
type: "started"
source: "scan"
trigger: IndexingTelemetryTrigger
mode?: IndexingTelemetryMode
})
| (IndexingTelemetryMeta & {
type: "completed"
source: "scan"
trigger: IndexingTelemetryTrigger
mode: IndexingTelemetryMode
filesIndexed: number
filesDiscovered: number
totalBlocks: number
batchErrors: number
})
| (IndexingTelemetryMeta & {
type: "file_count"
source: "scan"
mode: IndexingTelemetryMode
discovered: number
candidate: number
})
| (IndexingTelemetryMeta & {
type: "batch_retry"
source: IndexingTelemetrySource
mode: IndexingTelemetryMode
attempt: number
maxRetries: number
batchSize: number
error: string
})
| (IndexingTelemetryMeta & {
type: "error"
source: IndexingTelemetrySource
location: string
error: string
mode?: IndexingTelemetryMode
trigger?: IndexingTelemetryTrigger
retryCount?: number
maxRetries?: number
})
export type IndexingTelemetryReporter = (event: IndexingTelemetryEvent) => void
packages/kilo-indexing/src/indexing/interfaces/vector-store.ts
+97
View File
@@ -0,0 +1,97 @@
/**
* Interface for vector database clients
*/
export type PointStruct = {
id: string
vector: number[]
payload: Record<string, any>
}
export interface IVectorStore {
/**
* Initializes the vector store
* @returns Promise resolving to boolean indicating if a new collection was created
*/
initialize(): Promise<boolean>
/**
* Upserts points into the vector store
* @param points Array of points to upsert
*/
upsertPoints(points: PointStruct[]): Promise<void>
/**
* Searches for similar vectors
* @param queryVector Vector to search for
* @param directoryPrefix Optional directory prefix to filter results
* @param minScore Optional minimum score threshold
* @param maxResults Optional maximum number of results to return
* @returns Promise resolving to search results
*/
search(
queryVector: number[],
directoryPrefix?: string,
minScore?: number,
maxResults?: number,
): Promise<VectorStoreSearchResult[]>
/**
* Deletes points by file path
* @param filePath Path of the file to delete points for
*/
deletePointsByFilePath(filePath: string): Promise<void>
/**
* Deletes points by multiple file paths
* @param filePaths Array of file paths to delete points for
*/
deletePointsByMultipleFilePaths(filePaths: string[]): Promise<void>
/**
* Clears all points from the collection
*/
clearCollection(): Promise<void>
/**
* Deletes the entire collection.
*/
deleteCollection(): Promise<void>
/**
* Checks if the collection exists
* @returns Promise resolving to boolean indicating if the collection exists
*/
collectionExists(): Promise<boolean>
/**
* Checks if the collection exists and has indexed points
* @returns Promise resolving to boolean indicating if the collection exists and has points
*/
hasIndexedData(): Promise<boolean>
/**
* Marks the indexing process as complete by storing metadata
* Should be called after a successful full workspace scan or incremental scan
*/
markIndexingComplete(): Promise<void>
/**
* Marks the indexing process as incomplete by storing metadata
* Should be called at the start of indexing to indicate work in progress
*/
markIndexingIncomplete(): Promise<void>
}
export interface VectorStoreSearchResult {
id: string | number
score: number
payload?: Payload | null
}
export interface Payload {
filePath: string
codeChunk: string
startLine: number
endLine: number
[key: string]: any
}
packages/kilo-indexing/src/indexing/manager.ts
+517
View File
@@ -0,0 +1,517 @@
import type { VectorStoreSearchResult } from "./interfaces"
import type { IndexingState } from "./interfaces/manager"
import type { IndexingTelemetryEvent, IndexingTelemetryMeta, IndexingTelemetryTrigger } from "./interfaces/telemetry"
import { CodeIndexConfigManager, type IndexingConfigInput } from "./config-manager"
import { INITIAL_MANAGER_RECOVERY_DELAY_MS, MAX_MANAGER_RECOVERY_ATTEMPTS } from "./constants"
import { CodeIndexStateManager } from "./state-manager"
import { CodeIndexServiceFactory } from "./service-factory"
import { CodeIndexSearchService } from "./search-service"
import { CodeIndexOrchestrator } from "./orchestrator"
import { CacheManager } from "./cache-manager"
import { Emitter } from "./runtime"
import { Log } from "../util/log"
import { loadIgnore } from "./shared/load-ignore"
import { sanitizeErrorMessage } from "./shared/validation-helpers"
const log = Log.create({ service: "indexing-manager" })
/**
* RATIONALE: Removed the static singleton Map and vscode.ExtensionContext.
* The manager is now constructed directly with a workspace path and cache
* directory. The host (CLI, extension) is responsible for managing instances
* per workspace.
*/
export class CodeIndexManager {
private _configManager: CodeIndexConfigManager | undefined
private readonly _stateManager: CodeIndexStateManager
private readonly _telemetry = new Emitter<IndexingTelemetryEvent>()
private _serviceFactory: CodeIndexServiceFactory | undefined
private _orchestrator: CodeIndexOrchestrator | undefined
private _searchService: CodeIndexSearchService | undefined
private _cacheManager: CacheManager | undefined
private _isRecoveringFromError = false
private _retryTimer: ReturnType<typeof setTimeout> | undefined
private _retryResolve: (() => void) | undefined
private _retryTask: Promise<void> | undefined
private _retryAttempt = 0
private _retryMaxAttempts = MAX_MANAGER_RECOVERY_ATTEMPTS
private _retryInitialDelayMs = INITIAL_MANAGER_RECOVERY_DELAY_MS
constructor(
public readonly workspacePath: string,
private readonly cacheDirectory: string,
) {
this._stateManager = new CodeIndexStateManager()
}
public get onProgressUpdate() {
return this._stateManager.onProgressUpdate
}
public get onTelemetry() {
return this._telemetry
}
private getTelemetryMeta(): IndexingTelemetryMeta | undefined {
if (!this._configManager) {
return undefined
}
const cfg = this._configManager.getConfig()
return {
provider: cfg.embedderProvider,
vectorStore: cfg.vectorStoreProvider ?? "qdrant",
modelId: cfg.modelId,
}
}
private emitStart(trigger: IndexingTelemetryTrigger): void {
const meta = this.getTelemetryMeta()
if (!meta) {
return
}
this._telemetry.fire({
...meta,
type: "started",
source: "scan",
trigger,
})
}
private emitError(location: string, err: unknown, trigger?: IndexingTelemetryTrigger): void {
const meta = this.getTelemetryMeta()
if (!meta) {
return
}
const msg = err instanceof Error ? err.message : String(err)
this._telemetry.fire({
...meta,
type: "error",
source: "scan",
location,
trigger,
error: sanitizeErrorMessage(msg),
})
}
private clearRetryTimer(): void {
if (!this._retryTimer) {
this._retryResolve = undefined
return
}
clearTimeout(this._retryTimer)
this._retryTimer = undefined
this._retryResolve?.()
this._retryResolve = undefined
}
private resetRetryState(): void {
this._retryAttempt = 0
this.clearRetryTimer()
}
private async waitForRetry(delay: number): Promise<void> {
await new Promise<void>((resolve) => {
this._retryResolve = resolve
this._retryTimer = setTimeout(() => {
this._retryTimer = undefined
this._retryResolve = undefined
resolve()
}, delay)
})
}
private handleTelemetry(event: IndexingTelemetryEvent): void {
this._telemetry.fire(event)
if (event.type === "completed") {
this.resetRetryState()
return
}
if (event.type !== "error") return
if (event.location !== "orchestrator:startIndexing") return
if (!this.isFeatureEnabled || !this.isFeatureConfigured) return
if (this._retryTask || this._isRecoveringFromError) return
if (this._retryAttempt >= this._retryMaxAttempts) {
log.warn("indexing recovery retries exhausted", {
workspacePath: this.workspacePath,
attempts: this._retryAttempt,
maxAttempts: this._retryMaxAttempts,
})
return
}
void this.recoverFromError(event.trigger ?? "background")
}
private async runRecovery(trigger: IndexingTelemetryTrigger, attempt: number): Promise<void> {
this._isRecoveringFromError = true
this._retryAttempt = attempt
log.info("starting indexing error recovery attempt", {
workspacePath: this.workspacePath,
attempt,
maxAttempts: this._retryMaxAttempts,
trigger,
})
if (!this._configManager || !this._cacheManager) {
log.warn("indexing recovery skipped: manager not initialized", {
workspacePath: this.workspacePath,
})
this._isRecoveringFromError = false
return
}
this._stateManager.setSystemState("Standby", "")
try {
await this._recreateServices()
this.emitStart(trigger)
await this._orchestrator!.startIndexing(trigger)
} catch (err) {
log.error("indexing recovery attempt failed", {
err,
attempt,
})
this.emitError("manager:recoverFromError", err, trigger)
this._stateManager.setSystemState(
"Error",
`Failed during recovery: ${err instanceof Error ? err.message : String(err)}`,
)
}
const failed = this._orchestrator?.state === "Error" || this.getCurrentStatus().systemStatus === "Error"
if (!failed) {
this.resetRetryState()
this._isRecoveringFromError = false
log.info("completed indexing error recovery", {
workspacePath: this.workspacePath,
attempt,
})
return
}
if (attempt >= this._retryMaxAttempts) {
this._isRecoveringFromError = false
log.warn("indexing recovery reached max attempts", {
workspacePath: this.workspacePath,
attempts: attempt,
maxAttempts: this._retryMaxAttempts,
})
return
}
const delay = this._retryInitialDelayMs * Math.pow(2, attempt - 1)
await this.waitForRetry(delay)
this._isRecoveringFromError = false
return this.runRecovery(trigger, attempt + 1)
}
private assertInitialized() {
if (!this._configManager || !this._orchestrator || !this._searchService || !this._cacheManager) {
throw new Error("CodeIndexManager not initialized. Call initialize() first.")
}
}
public get state(): IndexingState {
if (!this.isFeatureEnabled) return "Standby"
return this._orchestrator?.state ?? this._stateManager.state
}
public get isFeatureEnabled(): boolean {
return this._configManager?.isFeatureEnabled ?? false
}
public get isFeatureConfigured(): boolean {
return this._configManager?.isFeatureConfigured ?? false
}
public get isInitialized(): boolean {
try {
this.assertInitialized()
return true
} catch (e) {
log.warn(`CodeIndexManager not initialized: ${e}`)
return false
}
}
public async initialize(input: IndexingConfigInput): Promise<{ requiresRestart: boolean }> {
if (!this._configManager) {
this._configManager = new CodeIndexConfigManager(input)
log.info("created indexing config manager", { workspacePath: this.workspacePath })
}
const { requiresRestart } = this._configManager.loadConfiguration(input)
log.info("loaded indexing configuration", {
workspacePath: this.workspacePath,
featureEnabled: this.isFeatureEnabled,
featureConfigured: this.isFeatureConfigured,
requiresRestart,
provider: this._configManager.currentEmbedderProvider,
vectorStore: this._configManager.getConfig().vectorStoreProvider,
})
if (!this.isFeatureEnabled) {
log.info("indexing disabled by configuration", { workspacePath: this.workspacePath })
this._orchestrator?.stopWatcher()
return { requiresRestart }
}
if (!this.workspacePath) {
log.info("indexing unavailable: no workspace path")
this._stateManager.setSystemState("Standby", "No workspace folder open")
return { requiresRestart }
}
if (!this.isFeatureConfigured) {
log.info("indexing enabled but not configured", {
workspacePath: this.workspacePath,
provider: this._configManager.currentEmbedderProvider,
})
this._orchestrator?.cancelIndexing()
this._stateManager.setSystemState(
"Standby",
"Code indexing is not configured. Save your settings to start indexing.",
)
return { requiresRestart }
}
if (!this._cacheManager) {
log.info("initializing indexing cache", { cacheDirectory: this.cacheDirectory })
this._cacheManager = new CacheManager(this.cacheDirectory, this.workspacePath)
await this._cacheManager.initialize()
log.info("indexing cache initialized", { cacheDirectory: this.cacheDirectory })
}
const needsServiceRecreation = !this._serviceFactory || requiresRestart
log.info("evaluated indexing service lifecycle", {
needsServiceRecreation,
requiresRestart,
hasServiceFactory: !!this._serviceFactory,
})
if (needsServiceRecreation) {
try {
log.info("recreating indexing services", { workspacePath: this.workspacePath })
await this._recreateServices()
log.info("indexing services recreated", { workspacePath: this.workspacePath })
} catch (err) {
log.error("failed to recreate services", { err })
this.emitError("manager:initialize", err, "background")
this._stateManager.setSystemState(
"Error",
`Failed to initialize: ${err instanceof Error ? err.message : String(err)}`,
)
throw err
}
}
const shouldStartOrRestart =
requiresRestart || (needsServiceRecreation && (!this._orchestrator || this._orchestrator.state !== "Indexing"))
if (shouldStartOrRestart) {
log.info("starting background indexing", {
workspacePath: this.workspacePath,
requiresRestart,
orchestratorState: this._orchestrator?.state,
})
this.emitStart("background")
// Fire and forget — indexing is a long-running background process
this._orchestrator?.startIndexing("background")
}
return { requiresRestart }
}
public async startIndexing(): Promise<void> {
if (!this.isFeatureEnabled) return
log.info("manual indexing start requested", { workspacePath: this.workspacePath })
const currentStatus = this.getCurrentStatus()
if (currentStatus.systemStatus === "Error") {
log.info("recovering from indexing error state before restart", {
workspacePath: this.workspacePath,
message: currentStatus.message,
})
this.resetRetryState()
await this.recoverFromError("manual")
return
}
this.assertInitialized()
this.emitStart("manual")
log.info("delegating manual indexing start to orchestrator", { workspacePath: this.workspacePath })
await this._orchestrator!.startIndexing("manual")
}
public stopWatcher(): void {
if (!this.isFeatureEnabled) return
this._orchestrator?.stopWatcher()
}
public cancelIndexing(): void {
if (!this.isFeatureEnabled) return
this._orchestrator?.cancelIndexing()
}
public updateBatchSegmentThreshold(newThreshold: number): void {
this._orchestrator?.updateBatchSegmentThreshold(newThreshold)
}
public async recoverFromError(trigger: IndexingTelemetryTrigger = "background"): Promise<void> {
if (this._retryTask) {
await this._retryTask
return
}
const attempt = this._retryAttempt + 1
if (attempt > this._retryMaxAttempts) {
log.warn("indexing recovery skipped: retry budget exhausted", {
workspacePath: this.workspacePath,
attempts: this._retryAttempt,
maxAttempts: this._retryMaxAttempts,
})
return
}
const task = this.runRecovery(trigger, attempt).finally(() => {
this._retryTask = undefined
this._isRecoveringFromError = false
this.clearRetryTimer()
})
this._retryTask = task
await task
}
public dispose(): void {
this.clearRetryTimer()
this._retryTask = undefined
// RATIONALE: cancelIndexing() sets _cancelRequested and calls stopWatcher() +
// scanner.cancel(), which cooperatively aborts any in-flight scan. Using only
// stopWatcher() left the orchestrator's _runScan() unaware it should exit.
this._orchestrator?.cancelIndexing()
this._stateManager.dispose()
this._telemetry.dispose()
}
public async clearIndexData(): Promise<void> {
if (!this.isFeatureEnabled) return
this.assertInitialized()
await this._orchestrator!.clearIndexData()
await this._cacheManager!.clearCacheFile()
}
public clearErrorState(): void {
this._stateManager.setSystemState("Standby", "")
}
public getCurrentStatus() {
const status = this._stateManager.getCurrentStatus()
return { ...status, workspacePath: this.workspacePath }
}
public async searchIndex(query: string, directoryPrefix?: string): Promise<VectorStoreSearchResult[]> {
if (!this.isFeatureEnabled) return []
this.assertInitialized()
return this._searchService!.searchIndex(query, directoryPrefix)
}
private async _recreateServices(): Promise<void> {
log.info("starting indexing service recreation", { workspacePath: this.workspacePath })
this._orchestrator?.stopWatcher()
this._orchestrator = undefined
this._searchService = undefined
this._serviceFactory = new CodeIndexServiceFactory(
this._configManager!,
this.workspacePath,
this._cacheManager!,
this.cacheDirectory,
(event) => this.handleTelemetry(event),
)
const ignoreInstance = await loadIgnore(this.workspacePath)
const config = this._configManager!.getConfig()
const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices(
this._cacheManager!,
ignoreInstance,
)
log.info("created indexing services", {
workspacePath: this.workspacePath,
provider: embedder.embedderInfo.name,
vectorStore: config.vectorStoreProvider,
model: config.modelId ?? "default",
})
const shouldValidate = embedder && embedder.embedderInfo.name === config.embedderProvider
if (shouldValidate) {
log.info("validating embedder configuration", {
workspacePath: this.workspacePath,
provider: embedder.embedderInfo.name,
})
const validationResult = await this._serviceFactory.validateEmbedder(embedder)
if (!validationResult.valid) {
const errorMessage = validationResult.error || "Embedder configuration validation failed"
this._stateManager.setSystemState("Error", errorMessage)
throw new Error(errorMessage)
}
log.info("embedder configuration validated", {
workspacePath: this.workspacePath,
provider: embedder.embedderInfo.name,
})
}
this._orchestrator = new CodeIndexOrchestrator(
this._configManager!,
this._stateManager,
this.workspacePath,
this._cacheManager!,
vectorStore,
scanner,
fileWatcher,
(event) => this.handleTelemetry(event),
)
this._searchService = new CodeIndexSearchService(this._configManager!, this._stateManager, embedder, vectorStore)
this._stateManager.setSystemState("Standby", "")
log.info("indexing services are ready", { workspacePath: this.workspacePath })
}
public async handleSettingsChange(input: IndexingConfigInput): Promise<void> {
if (!this._configManager) return
const { requiresRestart } = this._configManager.loadConfiguration(input)
log.info("processed indexing settings change", {
workspacePath: this.workspacePath,
featureEnabled: this.isFeatureEnabled,
featureConfigured: this.isFeatureConfigured,
requiresRestart,
})
if (!this.isFeatureEnabled) {
this._orchestrator?.stopWatcher()
this._stateManager.setSystemState("Standby", "Code indexing is disabled")
return
}
if (requiresRestart && this.isFeatureEnabled && this.isFeatureConfigured) {
try {
if (!this._cacheManager) {
this._cacheManager = new CacheManager(this.cacheDirectory, this.workspacePath)
await this._cacheManager.initialize()
}
await this._recreateServices()
} catch (err) {
log.error("failed to recreate services on settings change", { err })
throw err
}
}
}
}
packages/kilo-indexing/src/indexing/model-registry.ts
+88
View File
@@ -0,0 +1,88 @@
/**
* Indexing-local embedding model metadata registry.
*
* RATIONALE: The legacy codebase imported model metadata from a shared module
* (`shared/embeddingModels`) that does not exist in this package. Rather than
* recreating the full legacy module, we keep a focused registry of the models
* the indexing engine needs to know about — primarily for dimension resolution,
* default model selection, and score thresholds.
*/
import type { EmbedderProvider } from "./interfaces/manager"
interface ModelProfile {
dimension: number
scoreThreshold?: number
queryPrefix?: string
}
// ASSUMPTION: These dimensions and defaults match the provider documentation
// as of 2025-Q1. Update when providers ship new embedding models.
const profiles: Record<string, Record<string, ModelProfile>> = {
openai: {
"text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4 },
"text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4 },
"text-embedding-ada-002": { dimension: 1536, scoreThreshold: 0.4 },
},
ollama: {
"nomic-embed-text": { dimension: 768, scoreThreshold: 0.3, queryPrefix: "search_query: " },
"mxbai-embed-large": { dimension: 1024, scoreThreshold: 0.3 },
"all-minilm": { dimension: 384, scoreThreshold: 0.3 },
},
gemini: {
"gemini-embedding-001": { dimension: 3072, scoreThreshold: 0.35 },
"text-embedding-004": { dimension: 768, scoreThreshold: 0.35 },
"embedding-001": { dimension: 768, scoreThreshold: 0.35 },
},
mistral: {
"codestral-embed-2505": { dimension: 1536, scoreThreshold: 0.35 },
"codestral-embed": { dimension: 1536, scoreThreshold: 0.35 },
"mistral-embed": { dimension: 1024, scoreThreshold: 0.35 },
},
voyage: {
"voyage-code-3": { dimension: 1024, scoreThreshold: 0.35 },
"voyage-3": { dimension: 1024, scoreThreshold: 0.35 },
"voyage-3-lite": { dimension: 512, scoreThreshold: 0.35 },
},
bedrock: {
"amazon.titan-embed-text-v2:0": { dimension: 1024, scoreThreshold: 0.35 },
"amazon.titan-embed-text-v1": { dimension: 1536, scoreThreshold: 0.35 },
"cohere.embed-english-v3": { dimension: 1024, scoreThreshold: 0.35 },
},
openrouter: {
"openai/text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4 },
"openai/text-embedding-3-large": { dimension: 3072, scoreThreshold: 0.4 },
},
"openai-compatible": {},
"vercel-ai-gateway": {
"text-embedding-3-small": { dimension: 1536, scoreThreshold: 0.4 },
},
}
const defaults: Record<string, string> = {
openai: "text-embedding-3-small",
ollama: "nomic-embed-text",
gemini: "gemini-embedding-001",
mistral: "codestral-embed-2505",
voyage: "voyage-code-3",
bedrock: "amazon.titan-embed-text-v2:0",
openrouter: "openai/text-embedding-3-small",
"openai-compatible": "",
"vercel-ai-gateway": "text-embedding-3-small",
}
export function getDefaultModelId(provider: EmbedderProvider): string {
return defaults[provider] ?? ""
}
export function getModelDimension(provider: EmbedderProvider, modelId: string): number | undefined {
return profiles[provider]?.[modelId]?.dimension
}
export function getModelScoreThreshold(provider: EmbedderProvider, modelId: string): number | undefined {
return profiles[provider]?.[modelId]?.scoreThreshold
}
export function getModelQueryPrefix(provider: EmbedderProvider, modelId: string): string | undefined {
return profiles[provider]?.[modelId]?.queryPrefix
}
packages/kilo-indexing/src/indexing/orchestrator.ts
+396
View File
@@ -0,0 +1,396 @@
import path from "path"
import type { CodeIndexConfigManager } from "./config-manager"
import { CodeIndexStateManager, type IndexingState } from "./state-manager"
import type { IFileWatcher, BatchProcessingSummary } from "./interfaces"
import type {
IndexingTelemetryEvent,
IndexingTelemetryMeta,
IndexingTelemetryMode,
IndexingTelemetryReporter,
IndexingTelemetrySource,
IndexingTelemetryTrigger,
} from "./interfaces/telemetry"
import type { IVectorStore } from "./interfaces/vector-store"
import { DirectoryScanner } from "./processors"
import type { CacheManager } from "./cache-manager"
import type { Disposable } from "./runtime"
import { Log } from "../util/log"
import { sanitizeErrorMessage } from "./shared/validation-helpers"
const log = Log.create({ service: "indexing-orchestrator" })
export class CodeIndexOrchestrator {
private _fileWatcherSubscriptions: Disposable[] = []
private _isProcessing = false
private _cancelRequested = false
constructor(
private readonly configManager: CodeIndexConfigManager,
private readonly stateManager: CodeIndexStateManager,
private readonly workspacePath: string,
private readonly cacheManager: CacheManager,
private readonly vectorStore: IVectorStore,
private readonly scanner: DirectoryScanner,
private readonly fileWatcher: IFileWatcher,
private readonly onTelemetry?: IndexingTelemetryReporter,
) {}
private getTelemetryMeta(): IndexingTelemetryMeta {
const cfg = this.configManager.getConfig()
return {
provider: cfg.embedderProvider,
vectorStore: cfg.vectorStoreProvider ?? "qdrant",
modelId: cfg.modelId,
}
}
private emitTelemetry(event: IndexingTelemetryEvent): void {
this.onTelemetry?.(event)
}
private emitError(
location: string,
err: unknown,
source: IndexingTelemetrySource,
trigger?: IndexingTelemetryTrigger,
mode?: IndexingTelemetryMode,
): void {
const msg = err instanceof Error ? err.message : String(err)
this.emitTelemetry({
...this.getTelemetryMeta(),
type: "error",
source,
location,
trigger,
mode,
error: sanitizeErrorMessage(msg),
})
}
public updateBatchSegmentThreshold(newThreshold: number): void {
this.scanner.updateBatchSegmentThreshold(newThreshold)
this.fileWatcher.updateBatchSegmentThreshold(newThreshold)
}
private async _startWatcher(): Promise<void> {
if (!this.configManager.isFeatureConfigured) {
throw new Error("Cannot start watcher: Service not configured.")
}
log.info("starting file watcher", { workspacePath: this.workspacePath })
this.stateManager.setSystemState("Indexing", "Initializing file watcher...")
try {
await this.fileWatcher.initialize()
log.info("file watcher initialized", { workspacePath: this.workspacePath })
this._fileWatcherSubscriptions = [
this.fileWatcher.onDidStartBatchProcessing.on((paths) => {
log.info("file watcher batch started", {
workspacePath: this.workspacePath,
filesInBatch: paths.length,
})
if (this.stateManager.state !== "Indexing") {
this.stateManager.setSystemState("Indexing", "Processing file changes...")
}
}),
this.fileWatcher.onBatchProgressUpdate.on(({ processedInBatch, totalInBatch, currentFile }) => {
this.stateManager.reportFileQueueProgress(
processedInBatch,
totalInBatch,
currentFile ? path.basename(currentFile) : undefined,
)
if (processedInBatch === totalInBatch) {
log.info("file watcher batch completed", {
workspacePath: this.workspacePath,
totalInBatch,
})
if (totalInBatch > 0) {
this.stateManager.setSystemState("Indexed", "File changes processed. Index up-to-date.")
} else if (this.stateManager.state === "Indexing") {
this.stateManager.setSystemState("Indexed", "Index up-to-date. File queue empty.")
}
}
}),
this.fileWatcher.onDidFinishBatchProcessing.on((summary: BatchProcessingSummary) => {
if (summary.batchError) {
log.error("batch processing failed", { err: summary.batchError })
}
}),
]
this.fileWatcher.setCollecting(false)
log.info("file watcher is initialized in drain-only mode", { workspacePath: this.workspacePath })
} catch (err) {
log.error("failed to start file watcher", { err })
throw err
}
}
public async startIndexing(trigger: IndexingTelemetryTrigger = "background"): Promise<void> {
log.info("indexing start requested", {
workspacePath: this.workspacePath,
state: this.stateManager.state,
featureConfigured: this.configManager.isFeatureConfigured,
trigger,
})
if (!this.workspacePath) {
this.stateManager.setSystemState("Error", "Indexing requires a workspace folder.")
log.warn("start rejected: no workspace path")
return
}
if (!this.configManager.isFeatureConfigured) {
this.stateManager.setSystemState("Standby", "Missing configuration. Save your settings to start indexing.")
log.warn("start rejected: missing configuration")
return
}
if (
this._isProcessing ||
(this.stateManager.state !== "Standby" &&
this.stateManager.state !== "Error" &&
this.stateManager.state !== "Indexed")
) {
log.warn("start rejected", { state: this.stateManager.state })
return
}
this._cancelRequested = false
this._isProcessing = true
this.stateManager.setSystemState("Indexing", "Initializing services...")
let started = false
let source: IndexingTelemetrySource = "watcher"
let mode: IndexingTelemetryMode | undefined
try {
await this._startWatcher()
if (this._cancelRequested) {
this.stateManager.setSystemState("Standby", "Indexing cancelled.")
return
}
source = "scan"
const collectionCreated = await this.vectorStore.initialize()
log.info("vector store initialized", { workspacePath: this.workspacePath, collectionCreated })
started = true
if (this._cancelRequested) {
this.stateManager.setSystemState("Standby", "Indexing cancelled.")
return
}
if (collectionCreated) {
await this.cacheManager.clearCacheFile()
log.info("cleared indexing cache after new collection creation", { workspacePath: this.workspacePath })
}
const hasExistingData = await this.vectorStore.hasIndexedData()
log.info("checked vector store indexed data", {
workspacePath: this.workspacePath,
hasExistingData,
collectionCreated,
})
if (this._cancelRequested) {
this.stateManager.setSystemState("Standby", "Indexing cancelled.")
return
}
mode = hasExistingData && !collectionCreated ? "incremental" : "full"
if (mode === "incremental") {
log.info("collection has existing data, running incremental scan")
this.stateManager.setSystemState("Indexing", "Checking for new or modified files...")
await this.vectorStore.markIndexingIncomplete()
await this._runScan(mode, trigger)
} else {
log.info("running full scan", {
workspacePath: this.workspacePath,
hasExistingData,
collectionCreated,
})
this.stateManager.setSystemState("Indexing", "Services ready. Starting workspace scan...")
await this.vectorStore.markIndexingIncomplete()
await this._runScan(mode, trigger)
}
} catch (err) {
log.error("error during indexing", { err })
this.emitError("orchestrator:startIndexing", err, source, trigger, mode)
if (started) {
log.info("indexing failed after starting; preserving cache for retry")
} else {
log.info("failed to connect to vector store; preserving cache for future incremental scan")
}
const msg = err instanceof Error ? err.message : "Unknown error"
this.stateManager.setSystemState("Error", `Failed during initial scan: ${msg}`)
this.stopWatcher()
} finally {
this._isProcessing = false
log.info("indexing start flow finished", {
workspacePath: this.workspacePath,
state: this.stateManager.state,
})
}
}
private async _runScan(mode: IndexingTelemetryMode, trigger: IndexingTelemetryTrigger): Promise<void> {
if (this._cancelRequested) {
log.info("scan skipped: cancellation was requested", { workspacePath: this.workspacePath, mode })
return
}
log.info("starting workspace scan", { workspacePath: this.workspacePath, mode })
let cumulativeFilesIndexed = 0
let cumulativeFilesFound = 0
const batchErrors: Error[] = []
const handleFileParsed = () => {
cumulativeFilesFound += 1
this.stateManager.reportFileProgress(cumulativeFilesIndexed, cumulativeFilesFound)
}
const handleFilesIndexed = (indexedCount: number) => {
cumulativeFilesIndexed += indexedCount
this.stateManager.reportFileProgress(cumulativeFilesIndexed, cumulativeFilesFound)
}
const result = await this.scanner.scanDirectory(
this.workspacePath,
(batchError: Error) => {
log.error(`error during ${mode} scan batch`, { err: batchError })
batchErrors.push(batchError)
},
handleFilesIndexed,
handleFileParsed,
mode,
)
log.info("workspace scan completed", {
workspacePath: this.workspacePath,
mode,
filesDiscovered: cumulativeFilesFound,
filesIndexed: cumulativeFilesIndexed,
scanProcessed: result.stats.processed,
scanSkipped: result.stats.skipped,
totalBlocks: result.totalBlockCount,
batchErrorCount: batchErrors.length,
})
if (this._cancelRequested || this.scanner.isCancelled) {
this._isProcessing = false
if (this.stateManager.state !== "Error") {
this.stateManager.setSystemState("Standby", "Indexing cancelled.")
}
log.info("workspace scan cancelled", { workspacePath: this.workspacePath, mode })
return
}
if (mode === "full") {
// Validate full scan results
if (cumulativeFilesIndexed === 0 && cumulativeFilesFound > 0) {
const first = batchErrors.at(0)
const msg = first ? first.message : "No blocks were indexed"
throw new Error(`Indexing failed: ${msg}`)
}
if (batchErrors.length > 0) {
const failureRate = (cumulativeFilesFound - cumulativeFilesIndexed) / cumulativeFilesFound
if (failureRate > 0.1) {
const first = batchErrors.at(0)
const msg = first ? first.message : "Unknown batch error"
throw new Error(
`Indexing partially failed: Only ${cumulativeFilesIndexed} of ${cumulativeFilesFound} files were indexed. ${msg}`,
)
}
}
}
this.fileWatcher.setCollecting(true)
await this.vectorStore.markIndexingComplete()
this.stateManager.setSystemState("Indexed", "File watcher started. Index up-to-date.")
log.info("workspace scan finalized", {
workspacePath: this.workspacePath,
mode,
filesIndexed: cumulativeFilesIndexed,
filesDiscovered: cumulativeFilesFound,
})
this.emitTelemetry({
...this.getTelemetryMeta(),
type: "completed",
source: "scan",
trigger,
mode,
filesIndexed: cumulativeFilesIndexed,
filesDiscovered: cumulativeFilesFound,
totalBlocks: result.totalBlockCount,
batchErrors: batchErrors.length,
})
}
public stopWatcher(): void {
log.info("stopping file watcher", { workspacePath: this.workspacePath })
this.fileWatcher.dispose()
this.scanner.cancel()
for (const sub of this._fileWatcherSubscriptions) sub.dispose()
this._fileWatcherSubscriptions = []
if (this.stateManager.state !== "Error") {
this.stateManager.setSystemState("Standby", "File watcher stopped.")
}
this._isProcessing = false
log.info("file watcher stopped", { workspacePath: this.workspacePath, state: this.stateManager.state })
}
public cancelIndexing(): void {
log.info("cancelling indexing", { workspacePath: this.workspacePath })
this._cancelRequested = true
this.scanner.cancel()
this.stopWatcher()
this.stateManager.setSystemState("Standby", "Indexing cancelled.")
this._isProcessing = false
log.info("indexing cancelled", { workspacePath: this.workspacePath })
}
public async clearIndexData(): Promise<void> {
this._isProcessing = true
log.info("clearing index data", { workspacePath: this.workspacePath })
try {
this.stopWatcher()
try {
if (this.configManager.isFeatureConfigured) {
await this.vectorStore.deleteCollection()
} else {
log.warn("service not configured, skipping vector collection clear")
}
} catch (err: any) {
log.error("failed to clear vector collection", { err })
this.stateManager.setSystemState("Error", `Failed to clear vector collection: ${err.message}`)
}
await this.cacheManager.clearCacheFile()
if (this.stateManager.state !== "Error") {
this.stateManager.setSystemState("Standby", "Index data cleared successfully.")
}
} finally {
this._isProcessing = false
log.info("finished clearing index data", {
workspacePath: this.workspacePath,
state: this.stateManager.state,
})
}
}
public get state(): IndexingState {
return this.stateManager.state
}
}
packages/kilo-indexing/src/indexing/processors/file-watcher.ts
+694
View File
@@ -0,0 +1,694 @@
import { watch as chokidarWatch, type FSWatcher as ChokidarFSWatcher } from "chokidar"
import { stat, readFile } from "fs/promises"
import { createHash } from "crypto"
import path from "path"
import { v5 as uuidv5 } from "uuid"
import type { Ignore } from "ignore"
import { Emitter, type Disposable } from "../runtime"
import {
QDRANT_CODE_BLOCK_NAMESPACE,
MAX_FILE_SIZE_BYTES,
BATCH_SEGMENT_THRESHOLD,
MAX_BATCH_RETRIES,
INITIAL_RETRY_DELAY_MS,
} from "../constants"
import { scannerExtensions } from "../shared/supported-extensions"
import {
type IFileWatcher,
type FileProcessingResult,
type IEmbedder,
type IVectorStore,
type PointStruct,
type BatchProcessingSummary,
} from "../interfaces"
import type { IndexingTelemetryMeta, IndexingTelemetryReporter } from "../interfaces/telemetry"
import { codeParser } from "./parser"
import { CacheManager } from "../cache-manager"
import {
generateNormalizedAbsolutePath,
generateRelativeFilePath,
generateRelativeIgnorePath,
} from "../shared/get-relative-path"
import { FileIgnore } from "../../file/ignore"
import { Log } from "../../util/log"
import { sanitizeErrorMessage } from "../shared/validation-helpers"
const log = Log.create({ service: "file-watcher" })
/**
* Implementation of the file watcher interface.
*
* RATIONALE: Uses chokidar instead of vscode.workspace.createFileSystemWatcher
* so the watcher works outside VS Code (CLI, tests, headless).
*/
export class FileWatcher implements IFileWatcher {
private ignoreInstance?: Ignore
private watcher?: ChokidarFSWatcher
private accumulatedEvents: Map<string, { path: string; type: "create" | "change" | "delete" }> = new Map()
private batchProcessDebounceTimer?: NodeJS.Timeout
private readonly BATCH_DEBOUNCE_DELAY_MS = 500
private readonly FILE_PROCESSING_CONCURRENCY_LIMIT = 10
private batchSegmentThreshold: number
private maxBatchRetries: number
private collecting = true
private draining = false
private ready?: Promise<void>
public readonly onDidStartBatchProcessing = new Emitter<string[]>()
public readonly onBatchProgressUpdate = new Emitter<{
processedInBatch: number
totalInBatch: number
currentFile?: string
}>()
public readonly onDidFinishBatchProcessing = new Emitter<BatchProcessingSummary>()
constructor(
private workspacePath: string,
private readonly cacheManager: CacheManager,
private embedder?: IEmbedder,
private vectorStore?: IVectorStore,
ignoreInstance?: Ignore,
batchSegmentThreshold?: number,
maxBatchRetries?: number,
private readonly onTelemetry?: IndexingTelemetryReporter,
private readonly telemetryMeta?: IndexingTelemetryMeta,
) {
if (ignoreInstance) {
this.ignoreInstance = ignoreInstance
}
this.batchSegmentThreshold = batchSegmentThreshold ?? BATCH_SEGMENT_THRESHOLD
this.maxBatchRetries = maxBatchRetries ?? MAX_BATCH_RETRIES
}
private emitRetry(attempt: number, batchSize: number, err: unknown): void {
if (!this.onTelemetry || !this.telemetryMeta) {
return
}
const msg = err instanceof Error ? err.message : String(err)
this.onTelemetry({
...this.telemetryMeta,
type: "batch_retry",
source: "watcher",
mode: "incremental",
attempt,
maxRetries: this.maxBatchRetries,
batchSize,
error: sanitizeErrorMessage(msg),
})
}
private emitError(location: string, err: unknown, retryCount?: number): void {
if (!this.onTelemetry || !this.telemetryMeta) {
return
}
const msg = err instanceof Error ? err.message : String(err)
this.onTelemetry({
...this.telemetryMeta,
type: "error",
source: "watcher",
mode: "incremental",
location,
error: sanitizeErrorMessage(msg),
retryCount,
maxRetries: this.maxBatchRetries,
})
}
/**
* Initializes the file watcher using chokidar.
*
* RATIONALE: chokidar watches the filesystem directly using native OS events,
* removing the dependency on VS Code's file system watcher API.
*/
async initialize(): Promise<void> {
if (this.ready) {
await this.ready
return
}
log.info("initializing file watcher", { workspacePath: this.workspacePath })
this.watcher = chokidarWatch(this.workspacePath, {
ignored: (filePath: string) => {
const relativeFilePath = generateRelativeIgnorePath(filePath, this.workspacePath)
if (!relativeFilePath) return false
if (FileIgnore.match(relativeFilePath)) return true
return this.ignoreInstance?.ignores(relativeFilePath) ?? false
},
persistent: true,
ignoreInitial: true,
})
this.watcher.on("add", (filePath) => this.handleFileEvent(filePath, "create"))
this.watcher.on("change", (filePath) => this.handleFileEvent(filePath, "change"))
this.watcher.on("unlink", (filePath) => this.handleFileEvent(filePath, "delete"))
this.ready = new Promise((resolve, reject) => {
this.watcher?.once("ready", resolve)
this.watcher?.once("error", reject)
})
await this.ready
log.info("file watcher ready", { workspacePath: this.workspacePath })
}
setCollecting(collecting: boolean): void {
this.collecting = collecting
log.info("updated watcher collection mode", {
workspacePath: this.workspacePath,
collecting,
pendingEvents: this.accumulatedEvents.size,
})
if (collecting) this.scheduleBatchProcessing()
}
/**
* Updates the batch segment threshold.
*/
updateBatchSegmentThreshold(newThreshold: number): void {
this.batchSegmentThreshold = newThreshold
}
/**
* Disposes the file watcher and cleans up resources.
*/
dispose(): void {
this.watcher?.close()
if (this.batchProcessDebounceTimer) {
clearTimeout(this.batchProcessDebounceTimer)
}
this.onDidStartBatchProcessing.dispose()
this.onBatchProgressUpdate.dispose()
this.onDidFinishBatchProcessing.dispose()
this.accumulatedEvents.clear()
this.ready = undefined
}
/**
* Handles a file event from chokidar by accumulating it and scheduling batch processing.
*/
private handleFileEvent(filePath: string, type: "create" | "change" | "delete"): void {
if (!this.shouldIndex(filePath)) return
this.accumulatedEvents.set(filePath, { path: filePath, type })
if (!this.collecting) return
this.scheduleBatchProcessing()
}
/**
* Schedules batch processing with debounce.
*/
private scheduleBatchProcessing(): void {
if (!this.collecting) return
if (this.batchProcessDebounceTimer) {
clearTimeout(this.batchProcessDebounceTimer)
}
this.batchProcessDebounceTimer = setTimeout(() => this.triggerBatchProcessing(), this.BATCH_DEBOUNCE_DELAY_MS)
}
/**
* Triggers processing of accumulated events.
*/
private async triggerBatchProcessing(): Promise<void> {
if (this.draining || this.accumulatedEvents.size === 0 || !this.collecting) {
return
}
this.draining = true
log.info("starting watcher event drain", {
workspacePath: this.workspacePath,
pendingEvents: this.accumulatedEvents.size,
})
while (this.collecting && this.accumulatedEvents.size > 0) {
const eventsToProcess = new Map(this.accumulatedEvents)
this.accumulatedEvents.clear()
const filePathsInBatch = Array.from(eventsToProcess.keys())
this.onDidStartBatchProcessing.fire(filePathsInBatch)
await this.processBatch(eventsToProcess)
}
this.draining = false
log.info("completed watcher event drain", { workspacePath: this.workspacePath })
}
private shouldIndex(filePath: string) {
const relativeFilePath = generateRelativeIgnorePath(filePath, this.workspacePath)
if (!relativeFilePath) return false
const ext = path.extname(filePath).toLowerCase()
if (FileIgnore.match(relativeFilePath)) return false
if (this.ignoreInstance?.ignores(relativeFilePath)) return false
return scannerExtensions.includes(ext) || !path.extname(filePath)
}
/**
* Handles deletion phase of batch processing.
*
* Deletes vector store points for explicitly deleted files and for files
* that changed (old points are cleared before re-upserting new ones).
*/
private async _handleBatchDeletions(
batchResults: FileProcessingResult[],
processedCountInBatch: number,
totalFilesInBatch: number,
pathsToExplicitlyDelete: string[],
filesToUpsertDetails: Array<{ path: string; originalType: "create" | "change" }>,
): Promise<{ overallBatchError?: Error; clearedPaths: Set<string>; processedCount: number }> {
let overallBatchError: Error | undefined
const allPathsToClearFromDB = new Set<string>(pathsToExplicitlyDelete)
for (const fileDetail of filesToUpsertDetails) {
if (fileDetail.originalType === "change") {
allPathsToClearFromDB.add(fileDetail.path)
}
}
if (allPathsToClearFromDB.size > 0 && this.vectorStore) {
try {
await this.vectorStore.deletePointsByMultipleFilePaths(Array.from(allPathsToClearFromDB))
for (const path of pathsToExplicitlyDelete) {
this.cacheManager.deleteHash(path)
batchResults.push({ path, status: "success" })
processedCountInBatch++
this.onBatchProgressUpdate.fire({
processedInBatch: processedCountInBatch,
totalInBatch: totalFilesInBatch,
currentFile: path,
})
}
} catch (error: any) {
const errorStatus = error?.status || error?.response?.status || error?.statusCode
const errorMessage = error instanceof Error ? error.message : String(error)
log.error("batch deletion failed", {
error: sanitizeErrorMessage(errorMessage),
location: "deletePointsByMultipleFilePaths",
errorType: "deletion_error",
errorStatus,
})
this.emitError("file-watcher:deletePointsByMultipleFilePaths", error)
overallBatchError = error as Error
for (const path of pathsToExplicitlyDelete) {
batchResults.push({ path, status: "error", error: error as Error })
processedCountInBatch++
this.onBatchProgressUpdate.fire({
processedInBatch: processedCountInBatch,
totalInBatch: totalFilesInBatch,
currentFile: path,
})
}
}
}
return { overallBatchError, clearedPaths: allPathsToClearFromDB, processedCount: processedCountInBatch }
}
/**
* Processes individual files, parses them, creates embeddings, and collects
* the resulting points for a later batch upsert.
*/
private async _processFilesAndPrepareUpserts(
filesToUpsertDetails: Array<{ path: string; originalType: "create" | "change" }>,
batchResults: FileProcessingResult[],
processedCountInBatch: number,
totalFilesInBatch: number,
pathsToExplicitlyDelete: string[],
): Promise<{
pointsForBatchUpsert: PointStruct[]
successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }>
processedCount: number
}> {
const pointsForBatchUpsert: PointStruct[] = []
const successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }> = []
const filesToProcessConcurrently = [...filesToUpsertDetails]
for (let i = 0; i < filesToProcessConcurrently.length; i += this.FILE_PROCESSING_CONCURRENCY_LIMIT) {
const chunkToProcess = filesToProcessConcurrently.slice(i, i + this.FILE_PROCESSING_CONCURRENCY_LIMIT)
const chunkProcessingPromises = chunkToProcess.map(async (fileDetail) => {
this.onBatchProgressUpdate.fire({
processedInBatch: processedCountInBatch,
totalInBatch: totalFilesInBatch,
currentFile: fileDetail.path,
})
try {
const result = await this.processFile(fileDetail.path)
return { path: fileDetail.path, result: result, error: undefined }
} catch (e) {
const error = e as Error
log.error(`unhandled exception processing file ${fileDetail.path}`, { error })
return { path: fileDetail.path, result: undefined, error: error }
}
})
const settledChunkResults = await Promise.allSettled(chunkProcessingPromises)
for (const settledResult of settledChunkResults) {
let resultPath: string | undefined
if (settledResult.status === "fulfilled") {
const { path, result, error: directError } = settledResult.value
resultPath = path
if (directError) {
batchResults.push({ path, status: "error", error: directError })
} else if (result) {
if (result.status === "skipped" || result.status === "local_error") {
batchResults.push(result)
} else if (result.status === "processed_for_batching" && result.pointsToUpsert) {
pointsForBatchUpsert.push(...result.pointsToUpsert)
if (result.path && result.newHash) {
successfullyProcessedForUpsert.push({ path: result.path, newHash: result.newHash })
} else if (result.path && !result.newHash) {
successfullyProcessedForUpsert.push({ path: result.path })
}
} else {
batchResults.push({
path,
status: "error",
error: new Error(`Unexpected result status from processFile: ${result.status} for file ${path}`),
})
}
} else {
batchResults.push({
path,
status: "error",
error: new Error(`Fulfilled promise with no result or error for file ${path}`),
})
}
} else {
const error = settledResult.reason as Error
const rejectedPath = (settledResult.reason as any)?.path || "unknown"
log.error("a file processing promise was rejected", { error })
batchResults.push({
path: rejectedPath,
status: "error",
error: error,
})
}
if (!pathsToExplicitlyDelete.includes(resultPath || "")) {
processedCountInBatch++
}
this.onBatchProgressUpdate.fire({
processedInBatch: processedCountInBatch,
totalInBatch: totalFilesInBatch,
currentFile: resultPath,
})
}
}
return {
pointsForBatchUpsert,
successfullyProcessedForUpsert,
processedCount: processedCountInBatch,
}
}
/**
* Executes batch upsert operations against the vector store with retry logic.
*/
private async _executeBatchUpsertOperations(
pointsForBatchUpsert: PointStruct[],
successfullyProcessedForUpsert: Array<{ path: string; newHash?: string }>,
batchResults: FileProcessingResult[],
overallBatchError?: Error,
): Promise<Error | undefined> {
if (pointsForBatchUpsert.length > 0 && this.vectorStore && !overallBatchError) {
try {
for (let i = 0; i < pointsForBatchUpsert.length; i += this.batchSegmentThreshold) {
const batch = pointsForBatchUpsert.slice(i, i + this.batchSegmentThreshold)
let retryCount = 0
let upsertError: Error | undefined
while (retryCount < this.maxBatchRetries) {
try {
await this.vectorStore.upsertPoints(batch)
break
} catch (error) {
upsertError = error as Error
retryCount++
if (retryCount === this.maxBatchRetries) {
log.error("upsert retry exhausted", {
error: sanitizeErrorMessage(upsertError.message),
location: "upsertPoints",
errorType: "upsert_retry_exhausted",
retryCount: this.maxBatchRetries,
})
this.emitError("file-watcher:upsert_retry_exhausted", upsertError, this.maxBatchRetries)
throw new Error(`Failed to upsert batch after ${this.maxBatchRetries} retries: ${upsertError.message}`)
}
this.emitRetry(retryCount, batch.length, upsertError)
await new Promise((resolve) => setTimeout(resolve, INITIAL_RETRY_DELAY_MS * Math.pow(2, retryCount - 1)))
}
}
}
for (const { path, newHash } of successfullyProcessedForUpsert) {
if (newHash) {
this.cacheManager.updateHash(path, newHash)
}
batchResults.push({ path, status: "success" })
}
} catch (error) {
const err = error as Error
overallBatchError = overallBatchError || err
this.emitError("file-watcher:batch_upsert_error", err)
log.error("batch upsert error", {
error: sanitizeErrorMessage(err.message),
location: "executeBatchUpsertOperations",
errorType: "batch_upsert_error",
affectedFiles: successfullyProcessedForUpsert.length,
})
for (const { path } of successfullyProcessedForUpsert) {
batchResults.push({ path, status: "error", error: err })
}
}
} else if (overallBatchError && pointsForBatchUpsert.length > 0) {
for (const { path } of successfullyProcessedForUpsert) {
batchResults.push({ path, status: "error", error: overallBatchError })
}
}
return overallBatchError
}
/**
* Processes a batch of accumulated events through three phases:
* 1. Handle deletions (remove old points from vector store)
* 2. Process files and prepare upserts (parse, embed)
* 3. Execute batch upsert operations
*/
private async processBatch(
eventsToProcess: Map<string, { path: string; type: "create" | "change" | "delete" }>,
): Promise<void> {
const batchResults: FileProcessingResult[] = []
let processedCountInBatch = 0
const totalFilesInBatch = eventsToProcess.size
let overallBatchError: Error | undefined
// Initial progress update
this.onBatchProgressUpdate.fire({
processedInBatch: 0,
totalInBatch: totalFilesInBatch,
currentFile: undefined,
})
// Categorize events
const pathsToExplicitlyDelete: string[] = []
const filesToUpsertDetails: Array<{ path: string; originalType: "create" | "change" }> = []
for (const event of eventsToProcess.values()) {
if (event.type === "delete") {
pathsToExplicitlyDelete.push(event.path)
} else {
filesToUpsertDetails.push({
path: event.path,
originalType: event.type,
})
}
}
log.info("processing file watcher batch", {
workspacePath: this.workspacePath,
batchSize: totalFilesInBatch,
deletes: pathsToExplicitlyDelete.length,
upserts: filesToUpsertDetails.length,
})
// Phase 1: Handle deletions
const { overallBatchError: deletionError, processedCount: deletionCount } = await this._handleBatchDeletions(
batchResults,
processedCountInBatch,
totalFilesInBatch,
pathsToExplicitlyDelete,
filesToUpsertDetails,
)
overallBatchError = deletionError
processedCountInBatch = deletionCount
// Phase 2: Process files and prepare upserts
const {
pointsForBatchUpsert,
successfullyProcessedForUpsert,
processedCount: upsertCount,
} = await this._processFilesAndPrepareUpserts(
filesToUpsertDetails,
batchResults,
processedCountInBatch,
totalFilesInBatch,
pathsToExplicitlyDelete,
)
processedCountInBatch = upsertCount
// Phase 3: Execute batch upsert
overallBatchError = await this._executeBatchUpsertOperations(
pointsForBatchUpsert,
successfullyProcessedForUpsert,
batchResults,
overallBatchError,
)
// Finalize
this.onDidFinishBatchProcessing.fire({
processedFiles: batchResults,
batchError: overallBatchError,
})
const successCount = batchResults.filter((item) => item.status === "success").length
const skippedCount = batchResults.filter((item) => item.status === "skipped").length
const errorCount = batchResults.filter((item) => item.status === "error" || item.status === "local_error").length
log.info("completed file watcher batch", {
workspacePath: this.workspacePath,
batchSize: totalFilesInBatch,
successCount,
skippedCount,
errorCount,
hasBatchError: !!overallBatchError,
})
this.onBatchProgressUpdate.fire({
processedInBatch: totalFilesInBatch,
totalInBatch: totalFilesInBatch,
})
if (this.accumulatedEvents.size === 0) {
this.onBatchProgressUpdate.fire({
processedInBatch: 0,
totalInBatch: 0,
currentFile: undefined,
})
}
}
/**
* Processes a single file: checks ignore rules, reads content, computes hash,
* parses code blocks, creates embeddings, and returns points for batch upsert.
*/
async processFile(filePath: string): Promise<FileProcessingResult> {
try {
// Check if file is in an ignored directory
const relativeFilePath = generateRelativeIgnorePath(filePath, this.workspacePath)
if (!relativeFilePath) {
return {
path: filePath,
status: "skipped" as const,
reason: "File path is outside workspace",
}
}
if (FileIgnore.match(relativeFilePath)) {
return {
path: filePath,
status: "skipped" as const,
reason: "File is in an ignored directory",
}
}
// Check if file should be ignored by root .gitignore / .kilocodeignore rules.
if (this.ignoreInstance && this.ignoreInstance.ignores(relativeFilePath)) {
return {
path: filePath,
status: "skipped" as const,
reason: "File is ignored by .gitignore or .kilocodeignore",
}
}
// Check file size
const fileStat = await stat(filePath)
if (fileStat.size > MAX_FILE_SIZE_BYTES) {
return {
path: filePath,
status: "skipped" as const,
reason: "File is too large",
}
}
// Read file content
const content = await readFile(filePath, "utf-8")
// Calculate hash
const newHash = createHash("sha256").update(content).digest("hex")
// Check if file has changed
if (this.cacheManager.getHash(filePath) === newHash) {
return {
path: filePath,
status: "skipped" as const,
reason: "File has not changed",
}
}
// Parse file
const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash })
// Prepare points for batch processing
let pointsToUpsert: PointStruct[] = []
if (this.embedder && blocks.length > 0) {
const texts = blocks.map((block) => block.content)
const { embeddings } = await this.embedder.createEmbeddings(texts)
if (embeddings.length !== blocks.length) {
return {
path: filePath,
status: "local_error" as const,
error: new Error(
`Embedding count mismatch for ${filePath}: expected ${blocks.length}, got ${embeddings.length}`,
),
}
}
pointsToUpsert = blocks.map((block, index) => {
const vector = embeddings[index]!
const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath)
const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE)
return {
id: pointId,
vector,
payload: {
filePath: generateRelativeFilePath(normalizedAbsolutePath, this.workspacePath),
codeChunk: block.content,
startLine: block.start_line,
endLine: block.end_line,
segmentHash: block.segmentHash,
},
}
})
}
return {
path: filePath,
status: "processed_for_batching" as const,
newHash,
pointsToUpsert,
}
} catch (error) {
return {
path: filePath,
status: "local_error" as const,
error: error as Error,
}
}
}
}
packages/kilo-indexing/src/indexing/processors/index.ts
+3
View File
@@ -0,0 +1,3 @@
export * from "./parser"
export * from "./scanner"
export * from "./file-watcher"
packages/kilo-indexing/src/indexing/processors/parser.ts
+557
View File
@@ -0,0 +1,557 @@
import { readFile } from "fs/promises"
import { createHash } from "crypto"
import * as path from "path"
import { Node } from "web-tree-sitter"
import { type LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser"
import { parseMarkdown } from "../../tree-sitter/markdownParser"
import type { ICodeParser, CodeBlock } from "../interfaces"
import { scannerExtensions, shouldUseFallbackChunking } from "../shared/supported-extensions"
import { MAX_BLOCK_CHARS, MIN_BLOCK_CHARS, MIN_CHUNK_REMAINDER_CHARS, MAX_CHARS_TOLERANCE_FACTOR } from "../constants"
import { Log } from "../../util/log"
import { sanitizeErrorMessage } from "../shared/validation-helpers"
const log = Log.create({ service: "indexing-parser" })
/**
* Implementation of the code parser interface
*/
export class CodeParser implements ICodeParser {
private loadedParsers: LanguageParser = {}
private pendingLoads: Map<string, Promise<LanguageParser>> = new Map()
private failedParsers: Set<string> = new Set()
private parserFallbackNotified: Set<string> = new Set()
// Markdown files are now supported using the custom markdown parser
// which extracts headers and sections for semantic indexing
private _fallbackForUnavailableParser(
ext: string,
filePath: string,
content: string,
fileHash: string,
seenSegmentHashes: Set<string>,
error?: unknown,
): CodeBlock[] {
this.failedParsers.add(ext)
if (!this.parserFallbackNotified.has(ext)) {
this.parserFallbackNotified.add(ext)
log.warn("tree-sitter parser unavailable, using fallback chunking", {
ext,
filePath,
reason: sanitizeErrorMessage(error instanceof Error ? error.message : String(error ?? "parser unavailable")),
})
}
return this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes)
}
/**
* Parses a code file into code blocks
* @param filePath Path to the file to parse
* @param options Optional parsing options
* @returns Promise resolving to array of code blocks
*/
async parseFile(
filePath: string,
options?: {
content?: string
fileHash?: string
},
): Promise<CodeBlock[]> {
// Get file extension
const ext = path.extname(filePath).toLowerCase()
// Skip if not a supported language
if (!this.isSupportedLanguage(ext)) {
return []
}
// Get file content
let content: string
let fileHash: string
if (options?.content) {
content = options.content
fileHash = options.fileHash || this.createFileHash(content)
} else {
try {
content = await readFile(filePath, "utf8")
fileHash = this.createFileHash(content)
} catch (error) {
log.error(`error reading file ${filePath}`, { err: error })
return []
}
}
// Parse the file
return this.parseContent(filePath, content, fileHash)
}
/**
* Checks if a language is supported
* @param extension File extension
* @returns Boolean indicating if the language is supported
*/
private isSupportedLanguage(extension: string): boolean {
return scannerExtensions.includes(extension)
}
/**
* Creates a hash for a file
* @param content File content
* @returns Hash string
*/
private createFileHash(content: string): string {
return createHash("sha256").update(content).digest("hex")
}
/**
* Parses file content into code blocks
* @param filePath Path to the file
* @param content File content
* @param fileHash File hash
* @returns Array of code blocks
*/
private async parseContent(filePath: string, content: string, fileHash: string): Promise<CodeBlock[]> {
const ext = path.extname(filePath).slice(1).toLowerCase()
const seenSegmentHashes = new Set<string>()
// Handle markdown files specially
if (ext === "md" || ext === "markdown") {
return this.parseMarkdownContent(filePath, content, fileHash, seenSegmentHashes)
}
// Check if this extension should use fallback chunking
if (shouldUseFallbackChunking(`.${ext}`)) {
return this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes)
}
if (this.failedParsers.has(ext)) {
return this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes)
}
// Check if we already have the parser loaded
if (!this.loadedParsers[ext]) {
const pendingLoad = this.pendingLoads.get(ext)
if (pendingLoad) {
try {
await pendingLoad
} catch (error) {
return this._fallbackForUnavailableParser(ext, filePath, content, fileHash, seenSegmentHashes, error)
}
} else {
const loadPromise = loadRequiredLanguageParsers([filePath])
this.pendingLoads.set(ext, loadPromise)
try {
const newParsers = await loadPromise
if (newParsers) {
this.loadedParsers = { ...this.loadedParsers, ...newParsers }
}
} catch (error) {
return this._fallbackForUnavailableParser(ext, filePath, content, fileHash, seenSegmentHashes, error)
} finally {
this.pendingLoads.delete(ext)
}
}
}
const language = this.loadedParsers[ext]
if (!language) {
return this._fallbackForUnavailableParser(ext, filePath, content, fileHash, seenSegmentHashes)
}
const tree = language.parser.parse(content)
// We don't need to get the query string from languageQueries since it's already loaded
// in the language object
const captures = tree ? language.query.captures(tree.rootNode) : []
// Check if captures are empty
if (captures.length === 0) {
if (content.length >= MIN_BLOCK_CHARS) {
// Perform fallback chunking if content is large enough
const blocks = this._performFallbackChunking(filePath, content, fileHash, seenSegmentHashes)
return blocks
} else {
// Return empty if content is too small for fallback
return []
}
}
const results: CodeBlock[] = []
// Process captures if not empty
const queue: Node[] = Array.from(captures).map((capture: any) => capture.node)
while (queue.length > 0) {
const currentNode = queue.shift()!
// const lineSpan = currentNode.endPosition.row - currentNode.startPosition.row + 1 // Removed as per lint error
// Check if the node meets the minimum character requirement
if (currentNode.text.length >= MIN_BLOCK_CHARS) {
// If it also exceeds the maximum character limit, try to break it down
if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) {
if (currentNode.children.filter((child) => child !== null).length > 0) {
// If it has children, process them instead
queue.push(...currentNode.children.filter((child) => child !== null))
} else {
// If it's a leaf node, chunk it
const chunkedBlocks = this._chunkLeafNodeByLines(currentNode, filePath, fileHash, seenSegmentHashes)
results.push(...chunkedBlocks)
}
} else {
// Node meets min chars and is within max chars, create a block
const identifier =
currentNode.childForFieldName("name")?.text ||
currentNode.children.find((c) => c?.type === "identifier")?.text ||
null
const type = currentNode.type
const start_line = currentNode.startPosition.row + 1
const end_line = currentNode.endPosition.row + 1
const content = currentNode.text
const contentPreview = content.slice(0, 100)
const segmentHash = createHash("sha256")
.update(`${filePath}-${start_line}-${end_line}-${content.length}-${contentPreview}`)
.digest("hex")
if (!seenSegmentHashes.has(segmentHash)) {
seenSegmentHashes.add(segmentHash)
results.push({
file_path: filePath,
identifier,
type,
start_line,
end_line,
content,
segmentHash,
fileHash,
})
}
}
}
// Nodes smaller than minBlockChars are ignored
}
return results
}
/**
* Common helper function to chunk text by lines, avoiding tiny remainders.
*/
private _chunkTextByLines(
lines: string[],
filePath: string,
fileHash: string,
chunkType: string,
seenSegmentHashes: Set<string>,
baseStartLine: number = 1, // 1-based start line of the *first* line in the `lines` array
): CodeBlock[] {
const chunks: CodeBlock[] = []
let currentChunkLines: string[] = []
let currentChunkLength = 0
let chunkStartLineIndex = 0 // 0-based index within the `lines` array
const effectiveMaxChars = MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR
const finalizeChunk = (endLineIndex: number) => {
if (currentChunkLength >= MIN_BLOCK_CHARS && currentChunkLines.length > 0) {
const chunkContent = currentChunkLines.join("\n")
const startLine = baseStartLine + chunkStartLineIndex
const endLine = baseStartLine + endLineIndex
const contentPreview = chunkContent.slice(0, 100)
const segmentHash = createHash("sha256")
.update(`${filePath}-${startLine}-${endLine}-${chunkContent.length}-${contentPreview}`)
.digest("hex")
if (!seenSegmentHashes.has(segmentHash)) {
seenSegmentHashes.add(segmentHash)
chunks.push({
file_path: filePath,
identifier: null,
type: chunkType,
start_line: startLine,
end_line: endLine,
content: chunkContent,
segmentHash,
fileHash,
})
}
}
currentChunkLines = []
currentChunkLength = 0
chunkStartLineIndex = endLineIndex + 1
}
const createSegmentBlock = (segment: string, originalLineNumber: number, startCharIndex: number) => {
const segmentPreview = segment.slice(0, 100)
const segmentHash = createHash("sha256")
.update(
`${filePath}-${originalLineNumber}-${originalLineNumber}-${startCharIndex}-${segment.length}-${segmentPreview}`,
)
.digest("hex")
if (!seenSegmentHashes.has(segmentHash)) {
seenSegmentHashes.add(segmentHash)
chunks.push({
file_path: filePath,
identifier: null,
type: `${chunkType}_segment`,
start_line: originalLineNumber,
end_line: originalLineNumber,
content: segment,
segmentHash,
fileHash,
})
}
}
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
const lineLength = line.length + (i < lines.length - 1 ? 1 : 0) // +1 for newline, except last line
const originalLineNumber = baseStartLine + i
// Handle oversized lines (longer than effectiveMaxChars)
if (lineLength > effectiveMaxChars) {
// Finalize any existing normal chunk before processing the oversized line
if (currentChunkLines.length > 0) {
finalizeChunk(i - 1)
}
// Split the oversized line into segments
let remainingLineContent = line
let currentSegmentStartChar = 0
while (remainingLineContent.length > 0) {
const segment = remainingLineContent.substring(0, MAX_BLOCK_CHARS)
remainingLineContent = remainingLineContent.substring(MAX_BLOCK_CHARS)
createSegmentBlock(segment, originalLineNumber, currentSegmentStartChar)
currentSegmentStartChar += MAX_BLOCK_CHARS
}
// Update chunkStartLineIndex to continue processing from the next line
chunkStartLineIndex = i + 1
continue
}
// Handle normally sized lines
if (currentChunkLength > 0 && currentChunkLength + lineLength > effectiveMaxChars) {
// Re-balancing Logic
let splitIndex = i - 1
let remainderLength = 0
for (let j = i; j < lines.length; j++) {
remainderLength += lines[j].length + (j < lines.length - 1 ? 1 : 0)
}
if (
currentChunkLength >= MIN_BLOCK_CHARS &&
remainderLength < MIN_CHUNK_REMAINDER_CHARS &&
currentChunkLines.length > 1
) {
for (let k = i - 2; k >= chunkStartLineIndex; k--) {
const potentialChunkLines = lines.slice(chunkStartLineIndex, k + 1)
const potentialChunkLength = potentialChunkLines.join("\n").length + 1
const potentialNextChunkLines = lines.slice(k + 1)
const potentialNextChunkLength = potentialNextChunkLines.join("\n").length + 1
if (potentialChunkLength >= MIN_BLOCK_CHARS && potentialNextChunkLength >= MIN_CHUNK_REMAINDER_CHARS) {
splitIndex = k
break
}
}
}
finalizeChunk(splitIndex)
if (i >= chunkStartLineIndex) {
currentChunkLines.push(line)
currentChunkLength += lineLength
} else {
i = chunkStartLineIndex - 1
continue
}
} else {
currentChunkLines.push(line)
currentChunkLength += lineLength
}
}
// Process the last remaining chunk
if (currentChunkLines.length > 0) {
finalizeChunk(lines.length - 1)
}
return chunks
}
private _performFallbackChunking(
filePath: string,
content: string,
fileHash: string,
seenSegmentHashes: Set<string>,
): CodeBlock[] {
const lines = content.split("\n")
return this._chunkTextByLines(lines, filePath, fileHash, "fallback_chunk", seenSegmentHashes)
}
private _chunkLeafNodeByLines(
node: Node,
filePath: string,
fileHash: string,
seenSegmentHashes: Set<string>,
): CodeBlock[] {
const lines = node.text.split("\n")
const baseStartLine = node.startPosition.row + 1
return this._chunkTextByLines(
lines,
filePath,
fileHash,
node.type, // Use the node's type
seenSegmentHashes,
baseStartLine,
)
}
/**
* Helper method to process markdown content sections with consistent chunking logic
*/
private processMarkdownSection(
lines: string[],
filePath: string,
fileHash: string,
type: string,
seenSegmentHashes: Set<string>,
startLine: number,
identifier: string | null = null,
): CodeBlock[] {
const content = lines.join("\n")
if (content.trim().length < MIN_BLOCK_CHARS) {
return []
}
// Check if content needs chunking (either total size or individual line size)
const needsChunking =
content.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR ||
lines.some((line) => line.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR)
if (needsChunking) {
// Apply chunking for large content or oversized lines
const chunks = this._chunkTextByLines(lines, filePath, fileHash, type, seenSegmentHashes, startLine)
// Preserve identifier in all chunks if provided
if (identifier) {
chunks.forEach((chunk) => {
chunk.identifier = identifier
})
}
return chunks
}
// Create a single block for normal-sized content with no oversized lines
const endLine = startLine + lines.length - 1
const contentPreview = content.slice(0, 100)
const segmentHash = createHash("sha256")
.update(`${filePath}-${startLine}-${endLine}-${content.length}-${contentPreview}`)
.digest("hex")
if (!seenSegmentHashes.has(segmentHash)) {
seenSegmentHashes.add(segmentHash)
return [
{
file_path: filePath,
identifier,
type,
start_line: startLine,
end_line: endLine,
content,
segmentHash,
fileHash,
},
]
}
return []
}
private parseMarkdownContent(
filePath: string,
content: string,
fileHash: string,
seenSegmentHashes: Set<string>,
): CodeBlock[] {
const lines = content.split("\n")
const markdownCaptures = parseMarkdown(content) || []
if (markdownCaptures.length === 0) {
// No headers found, process entire content
return this.processMarkdownSection(lines, filePath, fileHash, "markdown_content", seenSegmentHashes, 1)
}
const results: CodeBlock[] = []
let lastProcessedLine = 0
// Process content before the first header
if (markdownCaptures.length > 0) {
const firstHeaderLine = markdownCaptures[0].node.startPosition.row
if (firstHeaderLine > 0) {
const preHeaderLines = lines.slice(0, firstHeaderLine)
const preHeaderBlocks = this.processMarkdownSection(
preHeaderLines,
filePath,
fileHash,
"markdown_content",
seenSegmentHashes,
1,
)
results.push(...preHeaderBlocks)
}
}
// Process markdown captures (headers and sections)
for (let i = 0; i < markdownCaptures.length; i += 2) {
const nameCapture = markdownCaptures[i]
// Ensure we don't go out of bounds when accessing the next capture
if (i + 1 >= markdownCaptures.length) break
const definitionCapture = markdownCaptures[i + 1]
if (!definitionCapture) continue
const startLine = definitionCapture.node.startPosition.row + 1
const endLine = definitionCapture.node.endPosition.row + 1
const sectionLines = lines.slice(startLine - 1, endLine)
// Extract header level for type classification
const headerMatch = nameCapture.name.match(/\.h(\d)$/)
const headerLevel = headerMatch ? parseInt(headerMatch[1]) : 1
const headerText = nameCapture.node.text
const sectionBlocks = this.processMarkdownSection(
sectionLines,
filePath,
fileHash,
`markdown_header_h${headerLevel}`,
seenSegmentHashes,
startLine,
headerText,
)
results.push(...sectionBlocks)
lastProcessedLine = endLine
}
// Process any remaining content after the last header section
if (lastProcessedLine < lines.length) {
const remainingLines = lines.slice(lastProcessedLine)
const remainingBlocks = this.processMarkdownSection(
remainingLines,
filePath,
fileHash,
"markdown_content",
seenSegmentHashes,
lastProcessedLine + 1,
)
results.push(...remainingBlocks)
}
return results
}
}
// Export a singleton instance for convenience
export const codeParser = new CodeParser()
packages/kilo-indexing/src/indexing/processors/scanner.ts
+662
View File
@@ -0,0 +1,662 @@
import type { Ignore } from "ignore"
import { stat, readFile } from "fs/promises"
import path from "path"
import { glob } from "glob"
import {
generateNormalizedAbsolutePath,
generateRelativeFilePath,
generateRelativeIgnorePath,
} from "../shared/get-relative-path"
import { scannerExtensions } from "../shared/supported-extensions"
import type { CodeBlock, ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../interfaces"
import { createHash } from "crypto"
import { v5 as uuidv5 } from "uuid"
import pLimit from "p-limit"
import { Mutex } from "async-mutex"
import { CacheManager } from "../cache-manager"
import {
QDRANT_CODE_BLOCK_NAMESPACE,
MAX_FILE_SIZE_BYTES,
BATCH_SEGMENT_THRESHOLD,
MAX_BATCH_RETRIES,
INITIAL_RETRY_DELAY_MS,
PARSING_CONCURRENCY,
BATCH_PROCESSING_CONCURRENCY,
MAX_PENDING_BATCHES,
} from "../constants"
import { FileIgnore } from "../../file/ignore"
import { Log } from "../../util/log"
import { sanitizeErrorMessage } from "../shared/validation-helpers"
import type { IndexingTelemetryMeta, IndexingTelemetryMode, IndexingTelemetryReporter } from "../interfaces/telemetry"
const log = Log.create({ service: "indexing-scanner" })
export class DirectoryScanner implements IDirectoryScanner {
private _cancelled = false
private batchSegmentThreshold: number
private maxBatchRetries: number
constructor(
private readonly embedder: IEmbedder,
private readonly vectorStore: IVectorStore,
private readonly codeParser: ICodeParser,
private readonly cacheManager: CacheManager,
private readonly ignoreInstance: Ignore,
batchSegmentThreshold?: number,
maxBatchRetries?: number,
private readonly onTelemetry?: IndexingTelemetryReporter,
private readonly telemetryMeta?: IndexingTelemetryMeta,
) {
this.batchSegmentThreshold = batchSegmentThreshold ?? BATCH_SEGMENT_THRESHOLD
this.maxBatchRetries = maxBatchRetries ?? MAX_BATCH_RETRIES
}
private emitFileCount(mode: IndexingTelemetryMode, discovered: number, candidate: number): void {
if (!this.onTelemetry || !this.telemetryMeta) {
return
}
this.onTelemetry({
...this.telemetryMeta,
type: "file_count",
source: "scan",
mode,
discovered,
candidate,
})
}
private emitRetry(mode: IndexingTelemetryMode, attempt: number, batchSize: number, err: unknown): void {
if (!this.onTelemetry || !this.telemetryMeta) {
return
}
const msg = err instanceof Error ? err.message : String(err)
this.onTelemetry({
...this.telemetryMeta,
type: "batch_retry",
source: "scan",
mode,
attempt,
maxRetries: this.maxBatchRetries,
batchSize,
error: sanitizeErrorMessage(msg),
})
}
private emitError(mode: IndexingTelemetryMode, location: string, err: unknown, retryCount?: number): void {
if (!this.onTelemetry || !this.telemetryMeta) {
return
}
const msg = err instanceof Error ? err.message : String(err)
this.onTelemetry({
...this.telemetryMeta,
type: "error",
source: "scan",
mode,
location,
error: sanitizeErrorMessage(msg),
retryCount,
maxRetries: this.maxBatchRetries,
})
}
/**
* Request cooperative cancellation of any in-flight scanning work.
* The scanDirectory and batch operations periodically check this flag
* and will exit as soon as practical.
*/
public cancel(): void {
this._cancelled = true
}
public get isCancelled(): boolean {
return this._cancelled
}
/**
* Updates the batch segment threshold
* @param newThreshold New batch segment threshold value
*/
public updateBatchSegmentThreshold(newThreshold: number): void {
this.batchSegmentThreshold = newThreshold
}
/**
* Recursively scans a directory for code blocks in supported files.
* @param directoryPath The directory to scan
* @param onError Optional error handler callback
* @param onBlocksIndexed Optional callback when blocks are indexed
* @param onFileParsed Optional callback when a file is parsed
* @returns Promise with processing stats and total block count
*/
public async scanDirectory(
directory: string,
onError?: (error: Error) => void,
onFilesIndexed?: (indexedCount: number) => void,
onFileParsed?: () => void,
mode: IndexingTelemetryMode = "full",
): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> {
// reset cooperative cancel flag on new full scan
this._cancelled = false
const directoryPath = directory
// Use the directory path directly as the workspace root
const scanWorkspace = directoryPath
log.info("starting directory scan", { workspacePath: scanWorkspace })
// Get all files recursively, filtering out ignored directories via glob
const allPaths = await glob("**/*", {
cwd: directoryPath,
absolute: true,
nodir: true,
dot: false,
ignore: FileIgnore.PATTERNS,
maxDepth: Infinity,
})
// Filter by supported extensions, ignore patterns, and excluded directories
const supportedPaths = allPaths.filter((filePath) => {
const ext = path.extname(filePath).toLowerCase()
const relativeFilePath = generateRelativeIgnorePath(filePath, scanWorkspace)
if (!relativeFilePath) {
return false
}
// Check if file is in an ignored directory using FileIgnore
if (FileIgnore.match(relativeFilePath)) {
return false
}
return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
})
log.info("discovered candidate files for indexing", {
workspacePath: scanWorkspace,
discoveredFiles: allPaths.length,
supportedFiles: supportedPaths.length,
})
this.emitFileCount(mode, allPaths.length, supportedPaths.length)
// Initialize tracking variables
const processedFiles = new Set<string>()
let processedCount = 0
let skippedCount = 0
// Initialize parallel processing tools
const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing
const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing
const mutex = new Mutex()
// Shared batch accumulators (protected by mutex)
let currentBatchBlocks: CodeBlock[] = []
let currentBatchTexts: string[] = []
let currentBatchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[] = []
const batched = new Map<string, string>()
let failed = false
const activeBatchPromises = new Set<Promise<void>>()
let pendingBatchCount = 0
// Initialize block counter
let totalBlockCount = 0
const queueBatch = async (
batchBlocks: CodeBlock[],
batchTexts: string[],
batchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[],
): Promise<void> => {
while (!this._cancelled) {
const release = await mutex.acquire()
let wait: Promise<void> | null = null
try {
if (pendingBatchCount < MAX_PENDING_BATCHES) {
pendingBatchCount++
const batchPromise = batchLimiter(() =>
this.processBatch(
batchBlocks,
batchTexts,
batchFileInfos,
scanWorkspace,
mode,
onError,
onFilesIndexed,
() => {
failed = true
},
),
)
activeBatchPromises.add(batchPromise)
// Clean up completed promises to prevent memory accumulation
batchPromise.finally(() => {
activeBatchPromises.delete(batchPromise)
pendingBatchCount--
})
return
}
wait = activeBatchPromises.size > 0 ? Promise.race(activeBatchPromises) : Promise.resolve()
} finally {
release()
}
await wait
}
}
// Process all files in parallel with concurrency control
const parsePromises = supportedPaths.map((filePath) =>
parseLimiter(async () => {
// Early exit if cancellation requested
if (this._cancelled) {
return
}
try {
// Check file size
const stats = await stat(filePath)
if (this._cancelled) {
return
}
if (stats.size > MAX_FILE_SIZE_BYTES) {
skippedCount++ // Skip large files
return
}
// Read file content using fs/promises
const content = await readFile(filePath, "utf-8")
if (this._cancelled) {
return
}
// Calculate current hash
const currentFileHash = createHash("sha256").update(content).digest("hex")
processedFiles.add(filePath)
// Check against cache
const cachedFileHash = this.cacheManager.getHash(filePath)
const isNewFile = !cachedFileHash
if (cachedFileHash === currentFileHash) {
// File is unchanged
skippedCount++
return
}
// File is new or changed - parse it using the injected parser function
const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash })
if (this._cancelled) {
return
}
const fileBlockCount = blocks.length
onFileParsed?.()
processedCount++
// Process embeddings if configured
if (this.embedder && this.vectorStore && blocks.length > 0) {
// Add to batch accumulators
let addedBlocksFromFile = false
let queued = false
const info = {
filePath,
fileHash: currentFileHash,
isNew: isNewFile,
}
for (const block of blocks) {
if (this._cancelled) break
const trimmedContent = block.content.trim()
if (trimmedContent) {
const nextBatch = await (async () => {
const release = await mutex.acquire()
try {
if (this._cancelled) {
// Abort adding more items if cancelled
return null
}
currentBatchBlocks.push(block)
currentBatchTexts.push(trimmedContent)
addedBlocksFromFile = true
// Check if batch threshold is met
if (currentBatchBlocks.length < this.batchSegmentThreshold) {
return null
}
// Copy current batch data and clear accumulators
const batchBlocks = [...currentBatchBlocks]
const batchTexts = [...currentBatchTexts]
// RATIONALE: Include the current file metadata before the flush snapshot
// so threshold-triggered batches still run delete updates for this file.
const batchFileInfos = queued ? [...currentBatchFileInfos] : [...currentBatchFileInfos, info]
queued = true
currentBatchBlocks = []
currentBatchTexts = []
currentBatchFileInfos = []
return {
batchBlocks,
batchTexts,
batchFileInfos,
}
} finally {
release()
}
})()
if (!nextBatch) {
continue
}
await queueBatch(nextBatch.batchBlocks, nextBatch.batchTexts, nextBatch.batchFileInfos)
}
}
// Add file info once per file (outside the block loop)
if (addedBlocksFromFile) {
const release = await mutex.acquire()
try {
totalBlockCount += fileBlockCount
batched.set(filePath, currentFileHash)
if (!queued) {
currentBatchFileInfos.push(info)
queued = true
}
} finally {
release()
}
}
} else {
// Only update hash if not being processed in a batch
this.cacheManager.updateHash(filePath, currentFileHash)
}
} catch (error) {
log.error(`Error processing file ${filePath} in workspace ${scanWorkspace}`, {
error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined,
location: "scanDirectory:processFile",
})
if (onError) {
onError(
error instanceof Error
? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`)
: new Error(`Unknown error processing file ${filePath} (Workspace: ${scanWorkspace})`),
)
}
}
}),
)
// Wait for all parsing to complete
await Promise.all(parsePromises)
log.info("finished parsing scan candidates", {
workspacePath: scanWorkspace,
processedCount,
skippedCount,
pendingBatches: pendingBatchCount,
cancelled: this._cancelled,
})
// Process any remaining items in batch
const finalBatch = await (async () => {
const release = await mutex.acquire()
try {
if (this._cancelled || currentBatchBlocks.length === 0) {
return null
}
// Copy current batch data and clear accumulators
const batchBlocks = [...currentBatchBlocks]
const batchTexts = [...currentBatchTexts]
const batchFileInfos = [...currentBatchFileInfos]
currentBatchBlocks = []
currentBatchTexts = []
currentBatchFileInfos = []
return {
batchBlocks,
batchTexts,
batchFileInfos,
}
} finally {
release()
}
})()
if (finalBatch) {
await queueBatch(finalBatch.batchBlocks, finalBatch.batchTexts, finalBatch.batchFileInfos)
}
// Short-circuit if cancelled before handling deletions
if (this._cancelled) {
log.info("directory scan cancelled", {
workspacePath: scanWorkspace,
processedCount,
skippedCount,
totalBlockCount,
})
return {
stats: {
processed: processedCount,
skipped: skippedCount,
},
totalBlockCount,
}
} else {
await Promise.all(activeBatchPromises)
}
if (!failed) {
for (const [filePath, fileHash] of batched.entries()) {
this.cacheManager.updateHash(filePath, fileHash)
}
}
if (failed && batched.size > 0) {
log.warn("skipping cache hash updates due failed batch", {
workspacePath: scanWorkspace,
affectedFiles: batched.size,
})
}
// Handle deleted files
const oldHashes = this.cacheManager.getAllHashes()
for (const cachedFilePath of Object.keys(oldHashes)) {
if (!processedFiles.has(cachedFilePath)) {
// File was deleted or is no longer supported/indexed
if (this.vectorStore) {
try {
await this.vectorStore.deletePointsByFilePath(cachedFilePath)
this.cacheManager.deleteHash(cachedFilePath)
} catch (error: any) {
const errorStatus = error?.status || error?.response?.status || error?.statusCode
const errorMessage = error instanceof Error ? error.message : String(error)
log.error(`Failed to delete points for ${cachedFilePath} in workspace ${scanWorkspace}`, {
error: sanitizeErrorMessage(errorMessage),
stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined,
location: "scanDirectory:deleteRemovedFiles",
errorStatus,
})
if (onError) {
// Report error to error handler
onError(
error instanceof Error
? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${cachedFilePath})`)
: new Error(`Unknown error deleting points for ${cachedFilePath} (Workspace: ${scanWorkspace})`),
)
}
}
}
}
}
log.info("directory scan complete", {
workspacePath: scanWorkspace,
processedCount,
skippedCount,
totalBlockCount,
})
return {
stats: {
processed: processedCount,
skipped: skippedCount,
},
totalBlockCount,
}
}
private async processBatch(
batchBlocks: CodeBlock[],
batchTexts: string[],
batchFileInfos: { filePath: string; fileHash: string; isNew: boolean }[],
scanWorkspace: string,
mode: IndexingTelemetryMode,
onError?: (error: Error) => void,
onFilesIndexed?: (indexedCount: number) => void,
onBatchFailed?: () => void,
): Promise<void> {
// Respect cooperative cancellation
if (this._cancelled || batchBlocks.length === 0) return
if (batchBlocks.length === 0) {
log.debug("Skipping empty batch processing")
return
}
log.debug(`Starting to process batch of ${batchBlocks.length} blocks in workspace ${scanWorkspace}`)
let attempts = 0
let success = false
let lastError: Error | null = null
while (attempts < this.maxBatchRetries && !success) {
attempts++
if (this._cancelled) return
log.debug(`Processing batch attempt ${attempts}/${this.maxBatchRetries} for ${batchBlocks.length} blocks`)
try {
// --- Deletion Step ---
log.debug("Starting deletion step for modified files")
const uniqueFilePaths = [
...new Set(
batchFileInfos
.filter((info) => !info.isNew) // Only modified files (not new)
.map((info) => info.filePath),
),
]
log.debug(`Identified ${uniqueFilePaths.length} modified files to delete points for`)
if (uniqueFilePaths.length > 0) {
try {
await this.vectorStore.deletePointsByMultipleFilePaths(uniqueFilePaths)
log.debug(`Successfully deleted points for ${uniqueFilePaths.length} files`)
} catch (deleteError: any) {
const errorStatus = deleteError?.status || deleteError?.response?.status || deleteError?.statusCode
const errorMessage = deleteError instanceof Error ? deleteError.message : String(deleteError)
log.error(
`Failed to delete points for ${uniqueFilePaths.length} files before upsert in workspace ${scanWorkspace}`,
{
error: sanitizeErrorMessage(errorMessage),
stack: deleteError instanceof Error ? sanitizeErrorMessage(deleteError.stack || "") : undefined,
location: "processBatch:deletePointsByMultipleFilePaths",
fileCount: uniqueFilePaths.length,
errorStatus,
},
)
// Re-throw with workspace context
throw new Error(
`Failed to delete points for ${uniqueFilePaths.length} files. Workspace: ${scanWorkspace}. ${errorMessage}`,
{ cause: deleteError },
)
}
}
// --- End Deletion Step ---
// Create embeddings for batch
if (this._cancelled) return
log.debug(`Creating embeddings for ${batchTexts.length} texts`)
const { embeddings } = await this.embedder.createEmbeddings(batchTexts)
log.debug(`Successfully created ${embeddings.length} embeddings`)
// Prepare points for Qdrant
log.debug("Preparing points for Qdrant upsert")
const points = batchBlocks.map((block, index) => {
const vector = embeddings[index]
if (!vector) {
throw new Error(`Missing embedding for block at index ${index}`)
}
const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, scanWorkspace)
// Use segmentHash for unique ID generation to handle multiple segments from same line
const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE)
return {
id: pointId,
vector,
payload: {
filePath: generateRelativeFilePath(normalizedAbsolutePath, scanWorkspace),
codeChunk: block.content,
startLine: block.start_line,
endLine: block.end_line,
segmentHash: block.segmentHash,
},
}
})
log.debug(`Prepared ${points.length} points for Qdrant`)
// Upsert points to Qdrant
if (this._cancelled) return
log.debug("Starting Qdrant upsert")
await this.vectorStore.upsertPoints(points)
log.debug("Completed Qdrant upsert")
onFilesIndexed?.(batchFileInfos.length)
success = true
log.debug(`Successfully processed batch of ${batchBlocks.length} blocks after ${attempts} attempt(s)`)
} catch (error) {
lastError = error as Error
log.error(`Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}`, {
error: sanitizeErrorMessage(error instanceof Error ? error.message : String(error)),
stack: error instanceof Error ? sanitizeErrorMessage(error.stack || "") : undefined,
location: "processBatch:retry",
attemptNumber: attempts,
batchSize: batchBlocks.length,
})
if (attempts < this.maxBatchRetries) {
this.emitRetry(mode, attempts, batchBlocks.length, error)
const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1)
log.debug(`Retrying batch in ${delay}ms`)
await new Promise((resolve) => setTimeout(resolve, delay))
}
}
}
if (!success && lastError) {
log.error(`Failed to process batch after ${this.maxBatchRetries} attempts`)
this.emitError(mode, "scanner:processBatch", lastError, this.maxBatchRetries)
onBatchFailed?.()
if (onError) {
// Preserve the original error message from embedders which now have detailed messages
const errorMessage = lastError.message || "Unknown error"
onError(new Error(`Failed to process batch after ${this.maxBatchRetries} retries: ${errorMessage}`))
}
}
}
}
packages/kilo-indexing/src/indexing/runtime.ts
+33
View File
@@ -0,0 +1,33 @@
/**
* Runtime adapter interfaces for the indexing package.
*
* RATIONALE: Decouple the indexing engine from any host environment (VS Code, CLI, etc.)
* by expressing all external capabilities as injectable contracts.
*/
/**
* Minimal typed event emitter that replaces vscode.EventEmitter.
* Consumers subscribe via `on()` and receive a dispose function.
*/
export class Emitter<T> {
private listeners = new Set<(value: T) => void>()
on(listener: (value: T) => void): Disposable {
this.listeners.add(listener)
return { dispose: () => this.listeners.delete(listener) }
}
fire(value: T): void {
for (const listener of this.listeners) {
listener(value)
}
}
dispose(): void {
this.listeners.clear()
}
}
export interface Disposable {
dispose(): void
}
packages/kilo-indexing/src/indexing/search-service.ts
+46
View File
@@ -0,0 +1,46 @@
import path from "path"
import type { VectorStoreSearchResult } from "./interfaces"
import type { IEmbedder } from "./interfaces/embedder"
import type { IVectorStore } from "./interfaces/vector-store"
import type { CodeIndexConfigManager } from "./config-manager"
import type { CodeIndexStateManager } from "./state-manager"
import { Log } from "../util/log"
const log = Log.create({ service: "indexing-search" })
export class CodeIndexSearchService {
constructor(
private readonly configManager: CodeIndexConfigManager,
private readonly stateManager: CodeIndexStateManager,
private readonly embedder: IEmbedder,
private readonly vectorStore: IVectorStore,
) {}
public async searchIndex(query: string, directoryPrefix?: string): Promise<VectorStoreSearchResult[]> {
if (!this.configManager.isFeatureEnabled || !this.configManager.isFeatureConfigured) {
throw new Error("Code index feature is disabled or not configured.")
}
const minScore = this.configManager.currentSearchMinScore
const maxResults = this.configManager.currentSearchMaxResults
const currentState = this.stateManager.getCurrentStatus().systemStatus
if (currentState !== "Indexed" && currentState !== "Indexing") {
throw new Error(`Code index is not ready for search. Current state: ${currentState}`)
}
try {
const embeddingResponse = await this.embedder.createEmbeddings([query])
const vector = embeddingResponse?.embeddings[0]
if (!vector) {
throw new Error("Failed to generate embedding for query.")
}
const normalizedPrefix = directoryPrefix ? path.normalize(directoryPrefix) : undefined
return await this.vectorStore.search(vector, normalizedPrefix, minScore, maxResults)
} catch (err) {
log.error("search failed", { err })
throw err
}
}
}
packages/kilo-indexing/src/indexing/service-factory.ts
+273
View File
@@ -0,0 +1,273 @@
import type { Ignore } from "ignore"
import path from "path"
import { getDefaultModelId } from "./model-registry"
import { resolveEmbeddingProfile } from "./embedding-profile"
import { OpenAiEmbedder } from "./embedders/openai"
import { CodeIndexOllamaEmbedder } from "./embedders/ollama"
import { OpenAICompatibleEmbedder } from "./embedders/openai-compatible"
import { GeminiEmbedder } from "./embedders/gemini"
import { MistralEmbedder } from "./embedders/mistral"
import { VercelAiGatewayEmbedder } from "./embedders/vercel-ai-gateway"
import { BedrockEmbedder } from "./embedders/bedrock"
import { OpenRouterEmbedder } from "./embedders/openrouter"
import { VoyageEmbedder } from "./embedders/voyage"
import { QdrantVectorStore } from "./vector-store/qdrant-client"
import { LanceDBVectorStore } from "./vector-store/lancedb-vector-store"
import { codeParser, DirectoryScanner, FileWatcher } from "./processors"
import type { ICodeParser, IEmbedder, IFileWatcher, IVectorStore } from "./interfaces"
import type { CodeIndexConfigManager } from "./config-manager"
import type { CacheManager } from "./cache-manager"
import type { IndexingTelemetryMeta, IndexingTelemetryReporter } from "./interfaces/telemetry"
import {
BATCH_SEGMENT_THRESHOLD,
OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "./constants"
import { Log } from "../util/log"
const log = Log.create({ service: "indexing-factory" })
function timeout(provider: string): number {
if (provider === "ollama") return OLLAMA_EMBEDDER_REQUEST_TIMEOUT_MS
return REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS
}
/**
* Factory class responsible for creating and configuring code indexing service dependencies.
*
* RATIONALE: Removed vscode.ExtensionContext, Package, RooIgnoreController, and
* LanceDBManager inputs. All batch sizing, retry counts, vector-store selection,
* and model selection now come from the injected CodeIndexConfigManager.
*/
export class CodeIndexServiceFactory {
constructor(
private readonly configManager: CodeIndexConfigManager,
private readonly workspacePath: string,
private readonly cacheManager: CacheManager,
private readonly cacheDirectory: string,
private readonly onTelemetry?: IndexingTelemetryReporter,
) {}
private getTelemetryMeta(): IndexingTelemetryMeta {
const cfg = this.configManager.getConfig()
return {
provider: cfg.embedderProvider,
vectorStore: cfg.vectorStoreProvider ?? "qdrant",
modelId: cfg.modelId,
}
}
public createEmbedder(): IEmbedder {
const config = this.configManager.getConfig()
const provider = config.embedderProvider
if (provider === "openai") {
if (!config.openAiOptions?.apiKey) throw new Error("OpenAI API key is required for embedding.")
return new OpenAiEmbedder(config.openAiOptions.apiKey, config.modelId)
}
if (provider === "ollama") {
if (!config.ollamaOptions?.baseUrl) throw new Error("Ollama base URL is required for embedding.")
return new CodeIndexOllamaEmbedder(config.ollamaOptions.baseUrl, config.modelId, config.modelDimension)
}
if (provider === "openai-compatible") {
if (!config.openAiCompatibleOptions?.baseUrl || !config.openAiCompatibleOptions?.apiKey)
throw new Error("OpenAI-compatible base URL and API key are required.")
return new OpenAICompatibleEmbedder(
config.openAiCompatibleOptions.baseUrl,
config.openAiCompatibleOptions.apiKey,
config.modelId,
)
}
if (provider === "gemini") {
if (!config.geminiOptions?.apiKey) throw new Error("Gemini API key is required for embedding.")
return new GeminiEmbedder(config.geminiOptions.apiKey, config.modelId)
}
if (provider === "mistral") {
if (!config.mistralOptions?.apiKey) throw new Error("Mistral API key is required for embedding.")
return new MistralEmbedder(config.mistralOptions.apiKey, config.modelId)
}
if (provider === "vercel-ai-gateway") {
if (!config.vercelAiGatewayOptions?.apiKey)
throw new Error("Vercel AI Gateway API key is required for embedding.")
return new VercelAiGatewayEmbedder(config.vercelAiGatewayOptions.apiKey, config.modelId)
}
if (provider === "bedrock") {
if (!config.bedrockOptions?.region) throw new Error("Bedrock region is required for embedding.")
return new BedrockEmbedder(config.bedrockOptions.region, config.bedrockOptions.profile, config.modelId)
}
if (provider === "openrouter") {
if (!config.openRouterOptions?.apiKey) throw new Error("OpenRouter API key is required for embedding.")
return new OpenRouterEmbedder(
config.openRouterOptions.apiKey,
config.modelId,
undefined,
config.openRouterOptions.specificProvider,
config.modelDimension,
)
}
if (provider === "voyage") {
if (!config.voyageOptions?.apiKey) throw new Error("Voyage API key is required for embedding.")
return new VoyageEmbedder(config.voyageOptions.apiKey, config.modelId)
}
throw new Error(`Unsupported embedder provider: ${provider}`)
}
public async validateEmbedder(embedder: IEmbedder): Promise<{ valid: boolean; error?: string }> {
const ms = timeout(embedder.embedderInfo.name)
let timer: ReturnType<typeof setTimeout> | undefined
const wait = embedder.validateConfiguration()
const fail = new Promise<{ valid: boolean; error?: string }>((resolve) => {
timer = setTimeout(
() =>
resolve({
valid: false,
error:
embedder.embedderInfo.name === "ollama"
? "Connection to embedding service failed (timeout)"
: "Connection failed. Please check the endpoint URL and network connectivity.",
}),
ms,
)
})
try {
log.info("validating embedder", { provider: embedder.embedderInfo.name })
const result = await Promise.race([wait, fail])
if (result.valid) {
log.info("embedder validation succeeded", { provider: embedder.embedderInfo.name })
}
if (!result.valid) {
log.warn("embedder validation failed", {
provider: embedder.embedderInfo.name,
error: result.error,
})
}
return result
} catch (err) {
log.error("embedder validation failed", { err })
return {
valid: false,
error: err instanceof Error ? err.message : "Configuration validation error",
}
} finally {
if (timer) clearTimeout(timer)
}
}
public createVectorStore(): IVectorStore {
const config = this.configManager.getConfig()
const profile = resolveEmbeddingProfile(config.embedderProvider, config.modelId, config.modelDimension)
if (!profile || profile.dimension <= 0) {
throw new Error(
`Cannot determine vector dimension for model "${config.modelId ?? getDefaultModelId(config.embedderProvider)}" with provider "${config.embedderProvider}". ` +
(config.embedderProvider === "openai-compatible"
? "Please set the model dimension explicitly."
: "Check your model configuration."),
)
}
if (config.vectorStoreProvider === "lancedb") {
const dbDir = config.lancedbVectorStoreDirectoryPlaceholder ?? path.join(this.cacheDirectory, "lancedb")
log.info("creating vector store", {
provider: config.embedderProvider,
vectorStore: "lancedb",
model: profile.modelId,
vectorSize: profile.dimension,
dbDir,
})
return new LanceDBVectorStore(this.workspacePath, profile.dimension, dbDir, profile)
}
if (!config.qdrantUrl) throw new Error("Qdrant URL is required.")
log.info("creating vector store", {
provider: config.embedderProvider,
vectorStore: "qdrant",
model: profile.modelId,
vectorSize: profile.dimension,
})
return new QdrantVectorStore(this.workspacePath, config.qdrantUrl, profile.dimension, config.qdrantApiKey, profile)
}
public createDirectoryScanner(
embedder: IEmbedder,
vectorStore: IVectorStore,
parser: ICodeParser,
ignoreInstance: Ignore,
): DirectoryScanner {
const config = this.configManager.getConfig()
const meta = this.getTelemetryMeta()
return new DirectoryScanner(
embedder,
vectorStore,
parser,
this.cacheManager,
ignoreInstance,
config.embeddingBatchSize,
config.scannerMaxBatchRetries,
this.onTelemetry,
meta,
)
}
public createFileWatcher(
embedder: IEmbedder,
vectorStore: IVectorStore,
cacheManager: CacheManager,
ignoreInstance: Ignore,
): IFileWatcher {
const config = this.configManager.getConfig()
const meta = this.getTelemetryMeta()
return new FileWatcher(
this.workspacePath,
cacheManager,
embedder,
vectorStore,
ignoreInstance,
config.embeddingBatchSize,
config.scannerMaxBatchRetries,
this.onTelemetry,
meta,
)
}
public createServices(
cacheManager: CacheManager,
ignoreInstance: Ignore,
): {
embedder: IEmbedder
vectorStore: IVectorStore
parser: ICodeParser
scanner: DirectoryScanner
fileWatcher: IFileWatcher
} {
if (!this.configManager.isFeatureConfigured) {
throw new Error("Code indexing is not configured. Save your settings to start indexing.")
}
const config = this.configManager.getConfig()
log.info("creating indexing services", {
workspacePath: this.workspacePath,
provider: config.embedderProvider,
vectorStore: config.vectorStoreProvider,
model: config.modelId ?? getDefaultModelId(config.embedderProvider),
configured: config.isConfigured,
})
const embedder = this.createEmbedder()
const vectorStore = this.createVectorStore()
const parser = codeParser
const scanner = this.createDirectoryScanner(embedder, vectorStore, parser, ignoreInstance)
const fileWatcher = this.createFileWatcher(embedder, vectorStore, cacheManager, ignoreInstance)
log.info("indexing services created", {
workspacePath: this.workspacePath,
provider: embedder.embedderInfo.name,
})
return { embedder, vectorStore, parser, scanner, fileWatcher }
}
}
packages/kilo-indexing/src/indexing/shared/get-relative-path.ts
+46
View File
@@ -0,0 +1,46 @@
import path from "path"
/**
* Generates a normalized absolute path from a given file path and workspace root.
* Handles path resolution and normalization to ensure consistent absolute paths.
*
* @param filePath - The file path to normalize (can be relative or absolute)
* @param workspaceRoot - The root directory of the workspace (required)
* @returns The normalized absolute path
*/
export function generateNormalizedAbsolutePath(filePath: string, workspaceRoot: string): string {
// Resolve the path to make it absolute if it's relative
const resolvedPath = path.resolve(workspaceRoot, filePath)
// Normalize to handle any . or .. segments and duplicate slashes
return path.normalize(resolvedPath)
}
/**
* Generates a relative file path from a normalized absolute path and workspace root.
* Ensures consistent relative path generation across different platforms.
*
* @param normalizedAbsolutePath - The normalized absolute path to convert
* @param workspaceRoot - The root directory of the workspace (required)
* @returns The relative path from workspaceRoot to the file
*/
export function generateRelativeFilePath(normalizedAbsolutePath: string, workspaceRoot: string): string {
// Generate the relative path
const relativePath = path.relative(workspaceRoot, normalizedAbsolutePath)
// Normalize to ensure consistent path separators
return path.normalize(relativePath)
}
/**
* Generates a relative path that is safe to pass to ignore().ignores().
*
* ignore() only accepts paths that are strictly relative to workspaceRoot.
* It rejects ".", empty, absolute, and parent-prefixed paths.
*/
export function generateRelativeIgnorePath(normalizedAbsolutePath: string, workspaceRoot: string): string | undefined {
const relativePath = generateRelativeFilePath(normalizedAbsolutePath, workspaceRoot)
if (!relativePath || relativePath === ".") return
if (path.isAbsolute(relativePath)) return
if (relativePath === "..") return
if (relativePath.startsWith(`..${path.sep}`)) return
return relativePath
}
packages/kilo-indexing/src/indexing/shared/load-ignore.ts
+37
View File
@@ -0,0 +1,37 @@
import fs from "fs/promises"
import ignore, { type Ignore } from "ignore"
import path from "path"
const files = [".gitignore", ".kilocodeignore"] as const
function notFound(err: unknown): boolean {
if (!err || typeof err !== "object") {
return false
}
return "code" in err && err.code === "ENOENT"
}
async function read(root: string, name: string): Promise<string | undefined> {
return fs.readFile(path.join(root, name), "utf8").catch((err) => {
if (notFound(err)) {
return undefined
}
throw err
})
}
export async function loadIgnore(root: string): Promise<Ignore> {
const ig = ignore()
for (const name of files) {
const txt = await read(root, name)
if (!txt?.trim()) {
continue
}
ig.add(txt)
ig.add(name)
}
return ig
}
packages/kilo-indexing/src/indexing/shared/supported-extensions.ts
+56
View File
@@ -0,0 +1,56 @@
import { extensions as allExtensions } from "../../tree-sitter"
// Include all extensions including markdown for the scanner
export const scannerExtensions = allExtensions
/**
* Extensions that should always use fallback chunking instead of tree-sitter parsing.
*
* These are either formats with no query/parser wiring yet, text-like build/doc files,
* or languages where AST chunking is intentionally disabled for indexing stability.
*
* NOTE: Any extension listed here must also appear in `src/tree-sitter/index.ts`.
* Keep this list explicit so broad-support formats do not rely on parser-load failures
* to become indexable.
*/
export const fallbackExtensions = [
// Shell and build scripts
".bash",
".bazel",
".bzl",
".build",
".gradle",
".ninja",
".sh",
".zsh",
// Languages with no query or parser wiring yet
".dart",
".elm",
".m",
".mm",
".ql",
".r",
".res",
".resi",
".sql",
".vb",
".yaml",
".yml",
// Documentation and query formats
".rst",
// Known unstable or intentionally disabled AST chunking
".scala",
".swift",
]
/**
* Check if a file extension should use fallback chunking
* @param extension File extension (including the dot)
* @returns true if the extension should use fallback chunking
*/
export function shouldUseFallbackChunking(extension: string): boolean {
return fallbackExtensions.includes(extension.toLowerCase())
}
packages/kilo-indexing/src/indexing/shared/validation-helpers.ts
+215
View File
@@ -0,0 +1,215 @@
import { Log } from "../../util/log"
const log = Log.create({ service: "indexing-validation" })
/**
* Sanitizes error messages by removing sensitive information like file paths and URLs
* @param errorMessage The error message to sanitize
* @returns The sanitized error message
*/
export function sanitizeErrorMessage(errorMessage: string): string {
if (!errorMessage || typeof errorMessage !== "string") {
return String(errorMessage)
}
let sanitized = errorMessage
// Replace URLs first (http, https, ftp, file protocols)
// This needs to be done before file paths to avoid partial replacements
sanitized = sanitized.replace(
/(?:https?|ftp|file):\/\/(?:localhost|[\w\-\.]+)(?::\d+)?(?:\/[\w\-\.\/\?\&\=\#]*)?/gi,
"[REDACTED_URL]",
)
// Replace email addresses
sanitized = sanitized.replace(/[\w\-\.]+@[\w\-\.]+\.\w+/g, "[REDACTED_EMAIL]")
// Replace IP addresses (IPv4)
sanitized = sanitized.replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, "[REDACTED_IP]")
// Replace file paths in quotes (handles paths with spaces)
sanitized = sanitized.replace(/"[^"]*(?:\/|\\)[^"]*"/g, '"[REDACTED_PATH]"')
// Replace file paths (Unix and Windows style)
// Matches paths like /Users/username/path, C:\Users\path, ./relative/path, ../relative/path
sanitized = sanitized.replace(
/(?:\/[\w\-\.]+)+(?:\/[\w\-\.\s]*)*|(?:[A-Za-z]:\\[\w\-\.\\]+)|(?:\.{1,2}\/[\w\-\.\/]+)/g,
"[REDACTED_PATH]",
)
// Replace port numbers that appear after colons (e.g., :11434, :8080)
// Do this after URLs to avoid double replacement
sanitized = sanitized.replace(/(?<!REDACTED_URL\]):(\d{2,5})\b/g, ":[REDACTED_PORT]")
return sanitized
}
/**
* HTTP error interface for embedder errors
*/
export interface HttpError extends Error {
status?: number
response?: {
status?: number
}
}
/**
* Common error types that can occur during embedder validation
*/
export interface ValidationError {
status?: number
message?: string
name?: string
code?: string
}
/**
* Maps HTTP status codes to appropriate error messages
*/
export function getErrorMessageForStatus(status: number | undefined, embedderType: string): string | undefined {
switch (status) {
case 401:
case 403:
return "Authentication failed. Please check your API key."
case 404:
return embedderType === "openai"
? "The specified embedding model is not available."
: "Invalid endpoint URL. Please verify the endpoint."
case 429:
return "Service is temporarily unavailable due to rate limiting. Please try again later."
default:
if (status && status >= 400 && status < 600) {
return "Configuration error. Please verify your embedder settings."
}
return undefined
}
}
/**
* Extracts status code from various error formats
*/
export function extractStatusCode(error: any): number | undefined {
// Direct status property
if (error?.status) return error.status
// Response status property
if (error?.response?.status) return error.response.status
// Extract from error message (e.g., "HTTP 404: Not Found")
if (error?.message) {
const match = error.message.match(/HTTP (\d+):/)
if (match) {
return parseInt(match[1], 10)
}
}
return undefined
}
/**
* Extracts error message from various error formats
*/
export function extractErrorMessage(error: any): string {
if (error?.message) {
return error.message
}
if (typeof error === "string") {
return error
}
if (error && typeof error === "object" && "toString" in error) {
try {
return String(error)
} catch {
return "Unknown error"
}
}
return "Unknown error"
}
/**
* Standard validation error handler for embedder configuration validation
* Returns a consistent error response based on the error type
*/
export function handleValidationError(
error: any,
embedderType: string,
customHandlers?: {
beforeStandardHandling?: (error: any) => { valid: boolean; error: string } | undefined
},
): { valid: boolean; error: string } {
// Allow custom handling first
if (customHandlers?.beforeStandardHandling) {
const customResult = customHandlers.beforeStandardHandling(error)
if (customResult) return customResult
}
const statusCode = extractStatusCode(error)
const errorMessage = extractErrorMessage(error)
// Check for status-based errors first
const statusError = getErrorMessageForStatus(statusCode, embedderType)
if (statusError) {
return { valid: false, error: statusError }
}
// Check for connection errors
if (errorMessage) {
if (
errorMessage.includes("ENOTFOUND") ||
errorMessage.includes("ECONNREFUSED") ||
errorMessage.includes("ETIMEDOUT") ||
errorMessage === "AbortError" ||
errorMessage.includes("HTTP 0:") ||
errorMessage === "No response"
) {
return { valid: false, error: "Connection failed. Please check the endpoint URL and network connectivity." }
}
if (errorMessage.includes("Failed to parse response JSON")) {
return { valid: false, error: "Received an invalid response from the embedding service." }
}
}
// For generic errors, preserve the original error message if it's not a standard one
if (errorMessage && errorMessage !== "Unknown error") {
return { valid: false, error: errorMessage }
}
// Fallback to generic error
return { valid: false, error: "Configuration error. Please verify your embedder settings." }
}
/**
* Wraps an async validation function with standard error handling
*/
export async function withValidationErrorHandling<T extends { valid: boolean; error?: string }>(
validationFn: () => Promise<T>,
embedderType: string,
customHandlers?: Parameters<typeof handleValidationError>[2],
): Promise<{ valid: boolean; error?: string }> {
try {
return await validationFn()
} catch (error) {
return handleValidationError(error, embedderType, customHandlers)
}
}
/**
* Formats an embedding error message based on the error type and context
*/
export function formatEmbeddingError(error: any, maxRetries: number): Error {
const errorMessage = extractErrorMessage(error)
const statusCode = extractStatusCode(error)
if (statusCode === 401) {
return new Error("Authentication failed. Please check your API key.")
}
if (statusCode) {
return new Error(`Embedding request failed after ${maxRetries} attempts with status ${statusCode}: ${errorMessage}`)
}
return new Error(`Embedding request failed after ${maxRetries} attempts: ${errorMessage}`)
}
packages/kilo-indexing/src/indexing/state-manager.ts
+104
View File
@@ -0,0 +1,104 @@
import { Emitter } from "./runtime"
export type IndexingState = "Standby" | "Indexing" | "Indexed" | "Error"
export class CodeIndexStateManager {
private _systemStatus: IndexingState = "Standby"
private _statusMessage = ""
private _processedFiles = 0
private _totalFiles = 0
private _percent = 0
private _gitBranch?: string
private _manifest?: {
totalFiles: number
totalChunks: number
lastUpdated: string
}
private _progressEmitter = new Emitter<ReturnType<typeof this.getCurrentStatus>>()
public readonly onProgressUpdate = this._progressEmitter
public get state(): IndexingState {
return this._systemStatus
}
public getCurrentStatus() {
return {
systemStatus: this._systemStatus,
message: this._statusMessage,
processedItems: this._processedFiles,
totalItems: this._totalFiles,
currentItemUnit: "files",
percent: this._percent,
gitBranch: this._gitBranch,
manifest: this._manifest,
}
}
public setSystemState(
newState: IndexingState,
message?: string,
manifest?: {
totalFiles: number
totalChunks: number
lastUpdated: string
},
gitBranch?: string,
): void {
const stateChanged = newState !== this._systemStatus || (message !== undefined && message !== this._statusMessage)
if (!stateChanged) return
this._systemStatus = newState
if (message !== undefined) this._statusMessage = message
if (manifest !== undefined) this._manifest = manifest
if (gitBranch !== undefined) this._gitBranch = gitBranch
if (newState !== "Indexing") {
this._percent = newState === "Indexed" ? 100 : 0
if (newState === "Standby" && message === undefined) this._statusMessage = "Ready."
if (newState === "Indexed" && message === undefined) this._statusMessage = "Index up-to-date."
if (newState === "Error" && message === undefined) this._statusMessage = "An error occurred."
}
if (newState !== "Indexed") {
this._manifest = undefined
}
this._progressEmitter.fire(this.getCurrentStatus())
}
public reportFileProgress(processedFiles: number, totalFiles: number, currentFileBasename?: string): void {
const percent = totalFiles > 0 ? Math.min(100, Math.round((processedFiles / totalFiles) * 100)) : 0
const progressChanged =
processedFiles !== this._processedFiles || totalFiles !== this._totalFiles || percent !== this._percent
if (!progressChanged && this._systemStatus === "Indexing") return
this._processedFiles = processedFiles
this._totalFiles = totalFiles
this._percent = percent
const message =
totalFiles > 0
? `Indexed ${processedFiles} / ${totalFiles} files (${percent}%).${currentFileBasename ? ` Current: ${currentFileBasename}` : ""}`
: "Indexing files..."
const oldStatus = this._systemStatus
const oldMessage = this._statusMessage
this._systemStatus = "Indexing"
this._statusMessage = message
if (oldStatus !== this._systemStatus || oldMessage !== this._statusMessage || progressChanged) {
this._progressEmitter.fire(this.getCurrentStatus())
}
}
public reportFileQueueProgress(processedFiles: number, totalFiles: number, currentFileBasename?: string): void {
this.reportFileProgress(processedFiles, totalFiles, currentFileBasename)
}
public dispose(): void {
this._progressEmitter.dispose()
}
}
packages/kilo-indexing/src/indexing/vector-store/lancedb-loader.ts
+9
View File
@@ -0,0 +1,9 @@
const env = "KILO_LANCEDB_PATH"
export function resolveLanceDBSpecifier() {
return process.env[env] || "@lancedb/lancedb"
}
export async function loadLanceDB() {
return import(resolveLanceDBSpecifier())
}
packages/kilo-indexing/src/indexing/vector-store/lancedb-vector-store.ts
+636
View File
@@ -0,0 +1,636 @@
import { createHash } from "crypto"
import * as path from "path"
import type { Connection, Table, VectorQuery } from "@lancedb/lancedb"
import type { IVectorStore } from "../interfaces/vector-store"
import type { Payload, VectorStoreSearchResult } from "../interfaces"
import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE } from "../constants"
import fs from "fs"
import { Log } from "../../util/log"
import type { EmbeddingProfile } from "../embedding-profile"
import { loadLanceDB } from "./lancedb-loader"
const log = Log.create({ service: "lancedb-store" })
const KEY = {
size: "vector_size",
complete: "indexing_complete",
provider: "embedding_provider",
model: "embedding_model_id",
dimension: "embedding_dimension",
}
/**
* Local implementation of the vector store using LanceDB
*/
export class LanceDBVectorStore implements IVectorStore {
private readonly vectorSize: number
private readonly dbPath: string
private readonly workspacePath: string
private readonly profile: EmbeddingProfile
private db: Connection | null = null
private table: Table | null = null
private readonly vectorTableName = "vector"
private readonly metadataTableName = "metadata"
private lancedbModule: any = null
constructor(workspacePath: string, vectorSize: number, dbDirectory: string, profile?: EmbeddingProfile) {
this.vectorSize = vectorSize
this.workspacePath = workspacePath
this.profile =
profile ??
({
provider: "openai",
modelId: "",
dimension: vectorSize,
} as EmbeddingProfile)
const basename = path.basename(workspacePath)
// Generate database directory name from workspace path
const hash = createHash("sha256").update(workspacePath).digest("hex")
const dbName = `${basename}-${hash.substring(0, 16)}`
// Set up database path
this.dbPath = path.join(dbDirectory, dbName)
}
/**
* Dynamically loads the LanceDB module.
* @returns The LanceDB module.
*/
private async loadLanceDBModule(): Promise<any> {
if (this.lancedbModule) {
return this.lancedbModule
}
try {
this.lancedbModule = await loadLanceDB()
return this.lancedbModule
} catch (error: unknown) {
log.error("Failed to load LanceDB module", { error })
throw new Error(`Failed to load LanceDB module: ${(error as Error).message}`)
}
}
/**
* Gets or connects to the LanceDB database.
* @returns The LanceDB connection.
*/
private async getDb(): Promise<Connection> {
if (this.db) {
return this.db
}
const lancedb = await this.loadLanceDBModule()
// Create parent directory if needed
if (!fs.existsSync(this.dbPath)) {
fs.mkdirSync(this.dbPath, { recursive: true })
}
this.db = await lancedb.connect(this.dbPath)
return this.db as Connection
}
/**
* Gets or opens the vector table.
* @returns The LanceDB table.
*/
private async getTable(): Promise<Table> {
if (this.table) {
return this.table
}
const db = await this.getDb()
try {
// Try to open existing table
const table = await db.openTable(this.vectorTableName)
this.table = table
return table
} catch (error) {
// Table doesn't exist, will be created in initialize()
throw new Error(`Table ${this.vectorTableName} does not exist`)
}
}
/**
* Creates sample data for the vector table schema.
* @returns An array containing sample data.
*/
private _createSampleData() {
return [
{
id: "sample",
vector: new Array(this.vectorSize).fill(0),
filePath: "sample",
codeChunk: "sample",
startLine: 0,
endLine: 0,
},
]
}
/**
* Creates metadata for the vector size.
* @returns An array containing metadata.
*/
private _createMetadataData() {
return [
{
key: KEY.size,
value: this.vectorSize,
},
{
key: KEY.provider,
value: this.profile.provider,
},
{
key: KEY.model,
value: this.profile.modelId,
},
{
key: KEY.dimension,
value: this.profile.dimension,
},
{
key: KEY.complete,
value: false,
},
]
}
/**
* Creates the vector table and deletes the sample data.
* @param db The LanceDB connection.
*/
private async _createVectorTable(db: Connection): Promise<void> {
this.table = await db.createTable(this.vectorTableName, this._createSampleData())
if (this.table) {
await this.table.delete("id = 'sample'")
}
}
/**
* Creates the metadata table.
* @param db The LanceDB connection.
*/
private async _createMetadataTable(db: Connection): Promise<void> {
await db.createTable(this.metadataTableName, this._createMetadataData())
}
/**
* Drops a table if it exists.
* @param db The LanceDB connection.
* @param tableName The name of the table to drop.
*/
private async _dropTableIfExists(db: Connection, tableName: string): Promise<void> {
const tableNames = await db.tableNames()
if (tableNames.includes(tableName)) {
await db.dropTable(tableName)
}
}
/**
* Retrieves the stored vector size from the metadata table.
* @param db The LanceDB connection.
* @returns The stored vector size, or null if not found.
*/
private async _getStoredVectorSize(db: Connection): Promise<number | null> {
try {
const value = await this._getMetadataValue(db, KEY.size)
if (value === undefined) return null
const dim = this._parseNumber(value)
return dim ?? null
} catch (error) {
log.warn("Failed to read metadata table", { error })
return null
}
}
private isValidMetadataKey(key: string): boolean {
return Object.values(KEY).includes(key as (typeof KEY)[keyof typeof KEY])
}
private _parseNumber(value: unknown): number | undefined {
const dim = Number(value)
if (!Number.isFinite(dim) || dim <= 0) return undefined
return dim
}
private async _getMetadataValue(db: Connection, key: string): Promise<unknown | undefined> {
if (!this.isValidMetadataKey(key)) {
throw new Error(`Invalid metadata key: ${key}`)
}
const metadataTable = await db.openTable(this.metadataTableName)
const rows = await metadataTable.query().where(`key = '${key}'`).toArray()
return rows.length > 0 ? rows[0].value : undefined
}
private async _getStoredEmbeddingProfile(db: Connection): Promise<EmbeddingProfile | undefined> {
try {
const provider = await this._getMetadataValue(db, KEY.provider)
const modelId = await this._getMetadataValue(db, KEY.model)
const dimension = await this._getMetadataValue(db, KEY.dimension)
if (typeof provider !== "string" || typeof modelId !== "string") return undefined
const dim = this._parseNumber(dimension)
if (!dim) return undefined
return {
provider: provider as EmbeddingProfile["provider"],
modelId,
dimension: dim,
}
} catch (error) {
log.warn("Failed to read embedding profile metadata", { error })
return undefined
}
}
private _isEmbeddingProfileMatch(profile: EmbeddingProfile): boolean {
return (
profile.provider === this.profile.provider &&
profile.modelId === this.profile.modelId &&
profile.dimension === this.profile.dimension
)
}
async initialize(): Promise<boolean> {
try {
await this.closeConnect()
const db = await this.getDb()
const tableNames = await db.tableNames()
const vectorTableExists = tableNames.includes(this.vectorTableName)
const metadataTableExists = tableNames.includes(this.metadataTableName)
let needsRecreation = false
if (!vectorTableExists) {
await this._createVectorTable(db)
await this._createMetadataTable(db)
log.info("LanceDB store initialized", {
workspacePath: this.workspacePath,
dbPath: this.dbPath,
created: true,
vectorSize: this.vectorSize,
})
return true
}
this.table = await db.openTable(this.vectorTableName)
const storedVectorSize = metadataTableExists ? await this._getStoredVectorSize(db) : null
const pointCount = await this.table.countRows()
if (storedVectorSize === null || storedVectorSize !== this.vectorSize) {
needsRecreation = true
}
if (!needsRecreation && pointCount > 0) {
const storedProfile = metadataTableExists ? await this._getStoredEmbeddingProfile(db) : undefined
if (!storedProfile || !this._isEmbeddingProfileMatch(storedProfile)) {
needsRecreation = true
}
}
if (needsRecreation) {
await this._dropTableIfExists(db, this.vectorTableName)
await this._dropTableIfExists(db, this.metadataTableName)
await this._createVectorTable(db)
await this._createMetadataTable(db)
this.optimizeTable()
log.info("LanceDB store reinitialized for embedding profile change", {
workspacePath: this.workspacePath,
dbPath: this.dbPath,
created: true,
vectorSize: this.vectorSize,
})
return true
}
this.optimizeTable()
log.info("LanceDB store initialized", {
workspacePath: this.workspacePath,
dbPath: this.dbPath,
created: false,
vectorSize: this.vectorSize,
})
return false
} catch (error) {
log.error("Failed to initialize LanceDB store", { error })
throw new Error(`Failed to initialize LanceDB store: ${(error as Error).message}`, { cause: error })
}
}
async upsertPoints(
points: Array<{
id: string
vector: number[]
payload: Record<string, any>
}>,
): Promise<void> {
if (points.length === 0) {
return
}
const table = await this.getTable()
const valids = points.filter((point) => this.isPayloadValid(point.payload))
if (valids.length === 0) {
return
}
try {
// Convert points to LanceDB format
const lanceData = valids.map((point) => ({
id: point.id,
vector: point.vector,
filePath: point.payload.filePath,
codeChunk: point.payload.codeChunk,
startLine: point.payload.startLine,
endLine: point.payload.endLine,
}))
// Delete existing points with same IDs first
const existingIds = lanceData.map((d) => d.id)
if (existingIds.length > 0) {
const bad = existingIds.find((id) => !this.isValidId(id))
if (bad) {
throw new Error(`Invalid point id format: ${bad}`)
}
const escapedIds = existingIds.map((id) => `'${this.escapeSqlString(id)}'`).join(", ")
const idFilter = `id IN (${escapedIds})`
await table.delete(idFilter)
}
// Insert new data
await table.add(lanceData)
} catch (error) {
log.error("Failed to upsert points", { error })
throw error
}
}
// Temporary till lancedb implements parameter support
// https://github.com/lance-format/lance/issues/2160
private escapeSqlString(value: string): string {
return value.replace(/'/g, "''")
}
private isValidId(id: string): boolean {
// ASSUMPTION: Point IDs are uuidv5 values produced by scanner and file watcher.
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)
}
private escapeSqlLikePattern(pattern: string): string {
let escaped = this.escapeSqlString(pattern)
escaped = escaped.replace(/\\/g, "\\\\")
escaped = escaped.replace(/%/g, "\\%").replace(/_/g, "\\_")
return escaped
}
private isPayloadValid(payload: Record<string, unknown> | null | undefined): payload is Payload {
if (!payload) {
return false
}
const validKeys = ["filePath", "codeChunk", "startLine", "endLine"]
const hasValidKeys = validKeys.every((key) => key in payload)
return hasValidKeys
}
async search(
queryVector: number[],
directoryPrefix?: string,
minScore?: number,
maxResults?: number,
): Promise<VectorStoreSearchResult[]> {
try {
const table = await this.getTable()
const actualMinScore = minScore ?? DEFAULT_SEARCH_MIN_SCORE
const actualMaxResults = maxResults ?? DEFAULT_MAX_SEARCH_RESULTS
// Build filter condition
let filter = ""
if (directoryPrefix) {
const escapedPrefix = this.escapeSqlLikePattern(directoryPrefix)
filter = `\`filePath\` LIKE '${escapedPrefix}%'`
}
// Perform vector search with distance range filtering
let searchQuery = (await table.search(queryVector)) as VectorQuery
if (filter !== "") {
searchQuery = searchQuery.where(filter)
}
searchQuery = searchQuery
.distanceType("cosine")
.distanceRange(0, 1 - actualMinScore)
.limit(actualMaxResults)
const list = await searchQuery.toArray()
const results = list.map((result: any) => ({
id: result.id,
score: 1 - result._distance, // Convert distance to similarity score
payload: {
filePath: result.filePath,
codeChunk: result.codeChunk,
startLine: result.startLine,
endLine: result.endLine,
} as Payload,
}))
return results
} catch (error) {
log.error("Failed to search points", { error })
throw error
}
}
async deletePointsByFilePath(filePath: string): Promise<void> {
return this.deletePointsByMultipleFilePaths([filePath])
}
async deletePointsByMultipleFilePaths(filePaths: string[]): Promise<void> {
if (filePaths.length === 0) {
return
}
try {
const table = await this.getTable()
const workspaceRoot = this.workspacePath
const normalizedPaths = filePaths.map((fp) =>
path.normalize(path.isAbsolute(fp) ? path.relative(workspaceRoot, fp) : fp),
)
// Create filter condition for multiple file paths
const escapedPaths = normalizedPaths.map((fp) => `'${this.escapeSqlString(fp)}'`).join(", ")
const filterCondition = `\`filePath\` IN (${escapedPaths})`
await table.delete(filterCondition)
} catch (error) {
log.error("Failed to delete points by file paths", { error })
throw error
}
}
async deleteCollection(): Promise<void> {
await this.closeConnect()
try {
if (fs.existsSync(this.dbPath)) {
fs.rmSync(this.dbPath, { recursive: true, force: true })
}
} catch (error) {
// If file deletion fails, try to clear the collection and metadata table
try {
const db = await this.getDb()
await this._dropTableIfExists(db, this.vectorTableName)
await this._dropTableIfExists(db, this.metadataTableName)
} catch (clearError) {
log.error("Failed to clear collection and metadata", { error: clearError })
}
throw error
}
}
async clearCollection(): Promise<void> {
try {
const table = await this.getTable()
// Delete all records from the table
await table.delete("true") // Delete all records
// Also clear metadata table
try {
const db = await this.getDb()
const tableNames = await db.tableNames()
if (tableNames.includes(this.metadataTableName)) {
const metadataTable = await db.openTable(this.metadataTableName)
await metadataTable.delete("true")
}
} catch (metadataError) {
log.warn("Failed to clear metadata table", { error: metadataError })
}
// Run optimization to clean up disk space after clearing
await this.optimizeTable()
} catch (error) {
log.error("Failed to clear collection", { error })
throw error
}
}
async collectionExists(): Promise<boolean> {
try {
const db = await this.getDb()
const tableNames = await db.tableNames()
return tableNames.includes(this.vectorTableName)
} catch (error) {
return false
}
}
private async closeConnect(): Promise<void> {
if (this.table) {
this.table = null
}
if (this.db) {
await this.db.close()
this.db = null
}
}
/**
* Optimizes the table to reduce disk space usage and improve performance.
* This method performs compaction, pruning of old versions, and index optimization.
* Should be called periodically to prevent unbounded disk space growth.
*/
async optimizeTable(): Promise<void> {
try {
const table = await this.getTable()
await table.optimize({
cleanupOlderThan: new Date(),
deleteUnverified: false,
})
} catch (error) {
log.error("Failed to optimize table", { error })
}
}
/**
* Checks if the collection exists and has indexed points
* @returns Promise resolving to boolean indicating if the collection exists and has points
*/
async hasIndexedData(): Promise<boolean> {
try {
const db = await this.getDb()
const table = await this.getTable()
const pointCount = await table.countRows()
if (pointCount === 0) {
log.info("LanceDB has no indexed data", {
workspacePath: this.workspacePath,
reason: "points_zero",
})
return false
}
const metadataTable = await db.openTable(this.metadataTableName)
const metadataResults = await metadataTable.query().where(`key = '${KEY.complete}'`).toArray()
const indexed = metadataResults.length > 0 ? metadataResults[0].value : false
log.info("LanceDB indexing metadata evaluated", {
workspacePath: this.workspacePath,
pointCount,
indexed,
})
return indexed
} catch (error) {
log.warn("Failed to check if collection has data", { error })
return false
}
}
private async _upsertMetadata(metadataTable: Table, key: string, value: unknown): Promise<void> {
if (!this.isValidMetadataKey(key)) {
throw new Error(`Invalid metadata key: ${key}`)
}
await metadataTable.delete(`key = '${key}'`)
await metadataTable.add([{ key, value }])
}
private async _persistEmbeddingProfile(metadataTable: Table): Promise<void> {
await this._upsertMetadata(metadataTable, KEY.provider, this.profile.provider)
await this._upsertMetadata(metadataTable, KEY.model, this.profile.modelId)
await this._upsertMetadata(metadataTable, KEY.dimension, this.profile.dimension)
await this._upsertMetadata(metadataTable, KEY.size, this.vectorSize)
}
/**
* Marks the indexing process as complete by storing metadata
* Should be called after a successful full workspace scan or incremental scan
*/
async markIndexingComplete(): Promise<void> {
try {
const db = await this.getDb()
const metadataTable = await db.openTable(this.metadataTableName)
await this._persistEmbeddingProfile(metadataTable)
await this._upsertMetadata(metadataTable, KEY.complete, true)
log.info("Marked indexing as complete")
} catch (error) {
log.error("Failed to mark indexing as complete", { error })
throw error
}
}
/**
* Marks the indexing process as incomplete by storing metadata
* Should be called at the start of indexing to indicate work in progress
*/
async markIndexingIncomplete(): Promise<void> {
try {
const db = await this.getDb()
const metadataTable = await db.openTable(this.metadataTableName)
await this._persistEmbeddingProfile(metadataTable)
await this._upsertMetadata(metadataTable, KEY.complete, false)
log.info("Marked indexing as incomplete (in progress)")
} catch (error) {
log.error("Failed to mark indexing as incomplete", { error })
throw error
}
}
}
packages/kilo-indexing/src/indexing/vector-store/qdrant-client.ts
+759
View File
@@ -0,0 +1,759 @@
import { QdrantClient, type Schemas } from "@qdrant/js-client-rest"
import { createHash } from "crypto"
import * as path from "path"
import type { IVectorStore } from "../interfaces/vector-store"
import type { Payload, VectorStoreSearchResult } from "../interfaces"
import { DEFAULT_MAX_SEARCH_RESULTS, DEFAULT_SEARCH_MIN_SCORE } from "../constants"
import { Log } from "../../util/log"
import type { EmbeddingProfile } from "../embedding-profile"
const log = Log.create({ service: "qdrant-store" })
const KEY = {
complete: "indexing_complete",
provider: "embedding_provider",
model: "embedding_model_id",
dimension: "embedding_dimension",
}
const METADATA_ID = "f946a536-9af4-4f1f-9f95-7d6efb4647d5"
/**
* Qdrant implementation of the vector store interface
*/
export class QdrantVectorStore implements IVectorStore {
private readonly vectorSize!: number
private readonly DISTANCE_METRIC = "Cosine"
private client: QdrantClient
private readonly collectionName: string
private readonly qdrantUrl: string = "http://localhost:6333"
private readonly workspacePath: string
private readonly profile: EmbeddingProfile
/**
* Creates a new Qdrant vector store
* @param workspacePath Path to the workspace
* @param url Optional URL to the Qdrant server
*/
constructor(workspacePath: string, url: string, vectorSize: number, apiKey?: string, profile?: EmbeddingProfile) {
// Parse the URL to determine the appropriate QdrantClient configuration
const parsedUrl = this.parseQdrantUrl(url)
// Store the resolved URL for our property
this.qdrantUrl = parsedUrl
this.workspacePath = workspacePath
try {
const urlObj = new URL(parsedUrl)
// Always use host-based configuration with explicit ports to avoid QdrantClient defaults
let port: number
let useHttps: boolean
if (urlObj.port) {
// Explicit port specified - use it and determine protocol
port = Number(urlObj.port)
useHttps = urlObj.protocol === "https:"
} else {
// No explicit port - use protocol defaults
if (urlObj.protocol === "https:") {
port = 443
useHttps = true
} else {
// http: or other protocols default to port 80
port = 80
useHttps = false
}
}
this.client = new QdrantClient({
host: urlObj.hostname,
https: useHttps,
port: port,
prefix: urlObj.pathname === "/" ? undefined : urlObj.pathname.replace(/\/+$/, ""),
apiKey,
headers: {
"User-Agent": "Kilo-Code",
},
})
} catch (urlError) {
// If URL parsing fails, fall back to URL-based config
// Note: This fallback won't correctly handle prefixes, but it's a last resort for malformed URLs.
this.client = new QdrantClient({
url: parsedUrl,
apiKey,
headers: {
"User-Agent": "Kilo-Code",
},
})
}
// Generate collection name from workspace path
const hash = createHash("sha256").update(workspacePath).digest("hex")
this.vectorSize = vectorSize
this.profile =
profile ??
({
provider: "openai",
modelId: "",
dimension: vectorSize,
} as EmbeddingProfile)
this.collectionName = `ws-${hash.substring(0, 16)}`
}
/**
* Parses and normalizes Qdrant server URLs to handle various input formats
* @param url Raw URL input from user
* @returns Properly formatted URL for QdrantClient
*/
private parseQdrantUrl(url: string | undefined): string {
// Handle undefined/null/empty cases
if (!url || url.trim() === "") {
return "http://localhost:6333"
}
const trimmedUrl = url.trim()
// Check if it starts with a protocol
if (!trimmedUrl.startsWith("http://") && !trimmedUrl.startsWith("https://") && !trimmedUrl.includes("://")) {
// No protocol - treat as hostname
return this.parseHostname(trimmedUrl)
}
try {
// Attempt to parse as complete URL - return as-is, let constructor handle ports
const parsedUrl = new URL(trimmedUrl)
return trimmedUrl
} catch {
// Failed to parse as URL - treat as hostname
return this.parseHostname(trimmedUrl)
}
}
/**
* Handles hostname-only inputs
* @param hostname Raw hostname input
* @returns Properly formatted URL with http:// prefix
*/
private parseHostname(hostname: string): string {
if (hostname.includes(":")) {
// Has port - add http:// prefix if missing
return hostname.startsWith("http") ? hostname : `http://${hostname}`
} else {
// No port - add http:// prefix without port (let constructor handle port assignment)
return `http://${hostname}`
}
}
private async getCollectionInfo(): Promise<Schemas["CollectionInfo"] | null> {
try {
const collectionInfo = await this.client.getCollection(this.collectionName)
return collectionInfo
} catch (error: unknown) {
if (error instanceof Error) {
log.warn(
`Warning during getCollectionInfo for "${this.collectionName}". Collection may not exist or another error occurred:`,
{ error: error.message },
)
}
return null
}
}
private metadataId(): string {
return METADATA_ID
}
private parseDimension(value: unknown): number | undefined {
const dim = Number(value)
if (!Number.isFinite(dim) || dim <= 0) return undefined
return dim
}
private async getMetadataPayload(): Promise<Record<string, unknown> | undefined> {
const metadataPoints = await this.client.retrieve(this.collectionName, {
ids: [this.metadataId()],
})
if (metadataPoints.length === 0) return undefined
const first = metadataPoints[0]
if (!first) return undefined
const payload = first.payload
if (!payload || typeof payload !== "object") return undefined
return payload as Record<string, unknown>
}
private getStoredProfile(payload?: Record<string, unknown>): EmbeddingProfile | undefined {
if (!payload) return undefined
const provider = payload[KEY.provider]
const modelId = payload[KEY.model]
const dimension = payload[KEY.dimension]
if (typeof provider !== "string" || typeof modelId !== "string") return undefined
const dim = this.parseDimension(dimension)
if (!dim) return undefined
return {
provider: provider as EmbeddingProfile["provider"],
modelId,
dimension: dim,
}
}
private isProfileMatch(profile: EmbeddingProfile): boolean {
return (
profile.provider === this.profile.provider &&
profile.modelId === this.profile.modelId &&
profile.dimension === this.profile.dimension
)
}
private async createCollection(): Promise<void> {
await this.client.createCollection(this.collectionName, {
vectors: {
size: this.vectorSize,
distance: this.DISTANCE_METRIC,
on_disk: true,
},
hnsw_config: {
m: 64,
ef_construct: 512,
on_disk: true,
},
})
}
private async recreateCollectionForProfile(stored?: EmbeddingProfile): Promise<boolean> {
const from = stored
? `${stored.provider}:${stored.modelId}:${stored.dimension}`
: "missing embedding metadata on populated collection"
const to = `${this.profile.provider}:${this.profile.modelId}:${this.profile.dimension}`
log.warn(`Collection ${this.collectionName} embedding profile changed (${from} -> ${to}). Recreating collection.`)
await this.client.deleteCollection(this.collectionName)
await new Promise((resolve) => setTimeout(resolve, 100))
const verificationInfo = await this.getCollectionInfo()
if (verificationInfo !== null) {
throw new Error("Embedding identity mismatch: collection still exists after deletion attempt")
}
await this.createCollection()
return true
}
/**
* Initializes the vector store
* @returns Promise resolving to boolean indicating if a new collection was created
*/
async initialize(): Promise<boolean> {
let created = false
try {
const collectionInfo = await this.getCollectionInfo()
if (collectionInfo === null) {
// Collection info not retrieved (assume not found or inaccessible), create it
await this.createCollection()
created = true
} else {
// Collection exists, check vector size
const vectorsConfig = collectionInfo.config?.params?.vectors
let existingVectorSize: number
if (typeof vectorsConfig === "number") {
existingVectorSize = vectorsConfig
} else if (
vectorsConfig &&
typeof vectorsConfig === "object" &&
"size" in vectorsConfig &&
typeof vectorsConfig.size === "number"
) {
existingVectorSize = vectorsConfig.size
} else {
existingVectorSize = 0 // Fallback for unknown configuration
}
if (existingVectorSize === this.vectorSize) {
const pointCount = collectionInfo.points_count ?? 0
if (pointCount === 0) {
created = false
} else {
const payload = await this.getMetadataPayload()
const profile = this.getStoredProfile(payload)
created =
!profile || !this.isProfileMatch(profile) ? await this.recreateCollectionForProfile(profile) : false
}
} else {
// Exists but wrong vector size, recreate with enhanced error handling
created = await this._recreateCollectionWithNewDimension(existingVectorSize)
}
}
// Create payload indexes
await this._createPayloadIndexes()
log.info("Qdrant collection ready", {
collection: this.collectionName,
created,
vectorSize: this.vectorSize,
url: this.qdrantUrl,
})
return created
} catch (error: any) {
const errorMessage = error?.message || error
log.error(`Failed to initialize Qdrant collection "${this.collectionName}"`, { error: errorMessage })
// If this is already a vector dimension mismatch error (identified by cause), re-throw it as-is
if (error instanceof Error && error.cause !== undefined) {
throw error
}
// Otherwise, provide a more user-friendly error message that includes the original error
throw new Error(`Failed to connect to Qdrant at ${this.qdrantUrl}: ${errorMessage}`)
}
}
/**
* Recreates the collection with a new vector dimension, handling failures gracefully.
* @param existingVectorSize The current vector size of the existing collection
* @returns Promise resolving to boolean indicating if a new collection was created
*/
private async _recreateCollectionWithNewDimension(existingVectorSize: number): Promise<boolean> {
log.warn(
`Collection ${this.collectionName} exists with vector size ${existingVectorSize}, but expected ${this.vectorSize}. Recreating collection.`,
)
let deletionSucceeded = false
let recreationAttempted = false
try {
// Step 1: Attempt to delete the existing collection
log.info(`Deleting existing collection ${this.collectionName}...`)
await this.client.deleteCollection(this.collectionName)
deletionSucceeded = true
log.info(`Successfully deleted collection ${this.collectionName}`)
// Step 2: Wait a brief moment to ensure deletion is processed
await new Promise((resolve) => setTimeout(resolve, 100))
// Step 3: Verify the collection is actually deleted
const verificationInfo = await this.getCollectionInfo()
if (verificationInfo !== null) {
throw new Error("Collection still exists after deletion attempt")
}
// Step 4: Create the new collection with correct dimensions
log.info(`Creating new collection ${this.collectionName} with vector size ${this.vectorSize}...`)
recreationAttempted = true
await this.createCollection()
log.info(`Successfully created new collection ${this.collectionName}`)
return true
} catch (recreationError) {
const errorMessage = recreationError instanceof Error ? recreationError.message : String(recreationError)
// Provide detailed error context based on what stage failed
let contextualErrorMessage: string
if (!deletionSucceeded) {
contextualErrorMessage = `Failed to delete existing collection with vector size ${existingVectorSize}. ${errorMessage}`
} else if (!recreationAttempted) {
contextualErrorMessage = `Deleted existing collection but failed verification step. ${errorMessage}`
} else {
contextualErrorMessage = `Deleted existing collection but failed to create new collection with vector size ${this.vectorSize}. ${errorMessage}`
}
log.error(
`CRITICAL: Failed to recreate collection ${this.collectionName} for dimension change (${existingVectorSize} -> ${this.vectorSize}). ${contextualErrorMessage}`,
)
// Create a comprehensive error message for the user
throw new Error(`Vector dimension mismatch: ${contextualErrorMessage}`, { cause: recreationError })
}
}
/**
* Creates payload indexes for the collection, handling errors gracefully.
*/
private async _createPayloadIndexes(): Promise<void> {
// Create index for the 'type' field to enable metadata filtering
try {
await this.client.createPayloadIndex(this.collectionName, {
field_name: "type",
field_schema: "keyword",
})
} catch (indexError: any) {
const errorMessage = (indexError?.message || "").toLowerCase()
if (!errorMessage.includes("already exists")) {
log.warn(`Could not create payload index for type on ${this.collectionName}`, {
details: indexError?.message || indexError,
})
}
}
// Create indexes for pathSegments fields
for (let i = 0; i <= 4; i++) {
try {
await this.client.createPayloadIndex(this.collectionName, {
field_name: `pathSegments.${i}`,
field_schema: "keyword",
})
} catch (indexError: any) {
const errorMessage = (indexError?.message || "").toLowerCase()
if (!errorMessage.includes("already exists")) {
log.warn(`Could not create payload index for pathSegments.${i} on ${this.collectionName}`, {
details: indexError?.message || indexError,
})
}
}
}
}
/**
* Upserts points into the vector store
* @param points Array of points to upsert
*/
async upsertPoints(
points: Array<{
id: string
vector: number[]
payload: Record<string, any>
}>,
): Promise<void> {
try {
const processedPoints = points.map((point) => {
if (point.payload?.filePath) {
const segments = point.payload.filePath.split(path.sep).filter(Boolean)
const pathSegments = segments.reduce((acc: Record<string, string>, segment: string, index: number) => {
acc[index.toString()] = segment
return acc
}, {})
return {
...point,
payload: {
...point.payload,
pathSegments,
},
}
}
return point
})
await this.client.upsert(this.collectionName, {
points: processedPoints,
wait: true,
})
} catch (error) {
log.error("Failed to upsert points", { error })
throw error
}
}
/**
* Checks if a payload is valid
* @param payload Payload to check
* @returns Boolean indicating if the payload is valid
*/
private isPayloadValid(payload: Record<string, unknown> | null | undefined): payload is Payload {
if (!payload) {
return false
}
const validKeys = ["filePath", "codeChunk", "startLine", "endLine"]
const hasValidKeys = validKeys.every((key) => key in payload)
return hasValidKeys
}
/**
* Searches for similar vectors
* @param queryVector Vector to search for
* @param directoryPrefix Optional directory prefix to filter results
* @param minScore Optional minimum score threshold
* @param maxResults Optional maximum number of results to return
* @returns Promise resolving to search results
*/
async search(
queryVector: number[],
directoryPrefix?: string,
minScore?: number,
maxResults?: number,
): Promise<VectorStoreSearchResult[]> {
try {
let filter:
| {
must: Array<{ key: string; match: { value: string } }>
must_not?: Array<{ key: string; match: { value: string } }>
}
| undefined = undefined
if (directoryPrefix) {
// Check if the path represents current directory
const normalizedPrefix = path.posix.normalize(directoryPrefix.replace(/\\/g, "/"))
// Note: path.posix.normalize("") returns ".", and normalize("./") returns "./"
if (normalizedPrefix === "." || normalizedPrefix === "./") {
// Don't create a filter - search entire workspace
filter = undefined
} else {
// Remove leading "./" from paths like "./src" to normalize them
const cleanedPrefix = path.posix.normalize(
normalizedPrefix.startsWith("./") ? normalizedPrefix.slice(2) : normalizedPrefix,
)
const segments = cleanedPrefix.split("/").filter(Boolean)
if (segments.length > 0) {
filter = {
must: segments.map((segment, index) => ({
key: `pathSegments.${index}`,
match: { value: segment },
})),
}
}
}
}
// Always exclude metadata points at query-time to avoid wasting top-k
const metadataExclusion = {
must_not: [{ key: "type", match: { value: "metadata" } }],
}
const mergedFilter = filter
? { ...filter, must_not: [...(filter.must_not || []), ...metadataExclusion.must_not] }
: metadataExclusion
const searchRequest = {
query: queryVector,
filter: mergedFilter,
score_threshold: minScore ?? DEFAULT_SEARCH_MIN_SCORE,
limit: maxResults ?? DEFAULT_MAX_SEARCH_RESULTS,
params: {
hnsw_ef: 128,
exact: false,
},
with_payload: {
include: ["filePath", "codeChunk", "startLine", "endLine", "pathSegments"],
},
}
const operationResult = await this.client.query(this.collectionName, searchRequest)
const filteredPoints = operationResult.points.filter((p: any) => this.isPayloadValid(p.payload))
return filteredPoints as VectorStoreSearchResult[]
} catch (error) {
log.error("Failed to search points", { error })
throw error
}
}
/**
* Deletes points by file path
* @param filePath Path of the file to delete points for
*/
async deletePointsByFilePath(filePath: string): Promise<void> {
return this.deletePointsByMultipleFilePaths([filePath])
}
async deletePointsByMultipleFilePaths(filePaths: string[]): Promise<void> {
if (filePaths.length === 0) {
return
}
try {
// First check if the collection exists
const collectionExists = await this.collectionExists()
if (!collectionExists) {
log.warn(`Skipping deletion - collection "${this.collectionName}" does not exist`)
return
}
const workspaceRoot = this.workspacePath
// Build filters using pathSegments to match the indexed fields
const filters = filePaths.map((filePath) => {
// IMPORTANT: Use the relative path to match what's stored in upsertPoints
// upsertPoints stores the relative filePath, not the absolute path
const relativePath = path.isAbsolute(filePath) ? path.relative(workspaceRoot, filePath) : filePath
// Normalize the relative path
const normalizedRelativePath = path.normalize(relativePath)
// Split the path into segments like we do in upsertPoints
const segments = normalizedRelativePath.split(path.sep).filter(Boolean)
// Create a filter that matches all segments of the path
// This ensures we only delete points that match the exact file path
const mustConditions = segments.map((segment, index) => ({
key: `pathSegments.${index}`,
match: { value: segment },
}))
return { must: mustConditions }
})
// Use 'should' to match any of the file paths (OR condition)
const filter = filters.length === 1 ? filters[0]! : { should: filters }
await this.client.delete(this.collectionName, {
filter,
wait: true,
})
} catch (error: any) {
// Extract more detailed error information
const errorMessage = error?.message || String(error)
const errorStatus = error?.status || error?.response?.status || error?.statusCode
const errorDetails = error?.response?.data || error?.data || ""
log.error(`Failed to delete points by file paths`, {
error: errorMessage,
status: errorStatus,
details: errorDetails,
collection: this.collectionName,
fileCount: filePaths.length,
// Include first few file paths for debugging (avoid logging too many)
samplePaths: filePaths.slice(0, 3),
})
throw error
}
}
/**
* Deletes the entire collection.
*/
async deleteCollection(): Promise<void> {
try {
// Check if collection exists before attempting deletion to avoid errors
if (await this.collectionExists()) {
await this.client.deleteCollection(this.collectionName)
}
} catch (error) {
log.error(`Failed to delete collection ${this.collectionName}`, { error })
throw error // Re-throw to allow calling code to handle it
}
}
/**
* Clears all points from the collection
*/
async clearCollection(): Promise<void> {
try {
await this.client.delete(this.collectionName, {
filter: {
must: [],
},
wait: true,
})
} catch (error) {
log.error("Failed to clear collection", { error })
throw error
}
}
/**
* Checks if the collection exists
* @returns Promise resolving to boolean indicating if the collection exists
*/
async collectionExists(): Promise<boolean> {
const collectionInfo = await this.getCollectionInfo()
return collectionInfo !== null
}
/**
* Checks if the collection exists and has indexed points
* @returns Promise resolving to boolean indicating if the collection exists and has points
*/
async hasIndexedData(): Promise<boolean> {
try {
const collectionInfo = await this.getCollectionInfo()
if (!collectionInfo) {
log.info("Qdrant collection has no indexed data", {
collection: this.collectionName,
reason: "collection_missing",
})
return false
}
// Check if the collection has any points indexed
const pointsCount = collectionInfo.points_count ?? 0
if (pointsCount === 0) {
log.info("Qdrant collection has no indexed data", {
collection: this.collectionName,
reason: "points_zero",
})
return false
}
// Check if the indexing completion marker exists
const payload = await this.getMetadataPayload()
// If marker exists, use it to determine completion status
if (payload) {
const indexed = payload[KEY.complete] === true
log.info("Qdrant indexing metadata evaluated", {
collection: this.collectionName,
pointsCount,
indexed,
})
return indexed
}
// Backward compatibility: No marker exists (old index or pre-marker version)
// Fall back to old logic - assume complete if collection has points
log.info("No indexing metadata marker found. Using backward compatibility mode (checking points_count > 0).")
return pointsCount > 0
} catch (error) {
log.warn("Failed to check if collection has data", { error })
return false
}
}
/**
* Marks the indexing process as complete by storing metadata
* Should be called after a successful full workspace scan or incremental scan
*/
async markIndexingComplete(): Promise<void> {
try {
await this.client.upsert(this.collectionName, {
points: [
{
id: this.metadataId(),
vector: new Array(this.vectorSize).fill(0),
payload: {
type: "metadata",
[KEY.complete]: true,
[KEY.provider]: this.profile.provider,
[KEY.model]: this.profile.modelId,
[KEY.dimension]: this.profile.dimension,
completed_at: Date.now(),
},
},
],
wait: true,
})
log.info("Marked indexing as complete")
} catch (error) {
log.error("Failed to mark indexing as complete", { error })
throw error
}
}
/**
* Marks the indexing process as incomplete by storing metadata
* Should be called at the start of indexing to indicate work in progress
*/
async markIndexingIncomplete(): Promise<void> {
try {
await this.client.upsert(this.collectionName, {
points: [
{
id: this.metadataId(),
vector: new Array(this.vectorSize).fill(0),
payload: {
type: "metadata",
[KEY.complete]: false,
[KEY.provider]: this.profile.provider,
[KEY.model]: this.profile.modelId,
[KEY.dimension]: this.profile.dimension,
started_at: Date.now(),
},
},
],
wait: true,
})
log.info("Marked indexing as incomplete (in progress)")
} catch (error) {
log.error("Failed to mark indexing as incomplete", { error })
throw error
}
}
}
packages/kilo-indexing/src/plugin.ts
+8
View File
@@ -0,0 +1,8 @@
import type { Plugin } from "@kilocode/plugin"
// RATIONALE: The host runtime owns lifecycle, routes, and native tool wiring.
// The plugin entry exists so workspaces can opt into indexing with a normal
// plugin specifier while keeping the engine and shims outside the plugin API.
export const KiloIndexingPlugin: Plugin = async () => ({})
export default KiloIndexingPlugin
packages/kilo-indexing/src/server/routes.ts
+27
View File
@@ -0,0 +1,27 @@
import { Hono } from "hono"
import { describeRoute, resolver } from "hono-openapi"
import { IndexingStatus, type IndexingStatus as Status } from "../status"
export function createIndexingRoutes(input: { current(): Promise<Status> }) {
return new Hono().get(
"/status",
describeRoute({
summary: "Get indexing status",
description: "Retrieve the current code indexing status for the active project.",
operationId: "indexing.status",
responses: {
200: {
description: "Indexing status",
content: {
"application/json": {
schema: resolver(IndexingStatus),
},
},
},
},
}),
async (c) => {
return c.json(await input.current())
},
)
}
packages/kilo-indexing/src/status.ts
+92
View File
@@ -0,0 +1,92 @@
import z from "zod"
import type { IndexingState } from "./indexing/interfaces/manager"
type StatusSource = {
readonly isFeatureEnabled: boolean
readonly isFeatureConfigured: boolean
getCurrentStatus(): {
systemStatus: IndexingState
message?: string
processedItems: number
totalItems: number
currentItemUnit: string
}
}
export const INDEXING_STATUS_STATES = ["Disabled", "In Progress", "Complete", "Error", "Standby"] as const
export const IndexingStatusState = z.enum(INDEXING_STATUS_STATES).meta({ ref: "IndexingStatusState" })
export type IndexingStatusState = z.infer<typeof IndexingStatusState>
export const IndexingStatus = z
.object({
state: IndexingStatusState,
message: z.string(),
processedFiles: z.number().int().nonnegative(),
totalFiles: z.number().int().nonnegative(),
percent: z.number().int().min(0).max(100),
})
.meta({ ref: "IndexingStatus" })
export type IndexingStatus = z.infer<typeof IndexingStatus>
export function disabledIndexingStatus(message = "Indexing disabled."): IndexingStatus {
return {
state: "Disabled",
message,
processedFiles: 0,
totalFiles: 0,
percent: 0,
}
}
export function normalizeIndexingStatus(manager: StatusSource): IndexingStatus {
const cfg = manager.getCurrentStatus()
const files = cfg.currentItemUnit === "files"
const processedFiles = files ? cfg.processedItems : 0
const totalFiles = files ? cfg.totalItems : 0
const percent = totalFiles > 0 ? Math.min(100, Math.max(0, Math.round((processedFiles / totalFiles) * 100))) : 0
if (!manager.isFeatureEnabled || !manager.isFeatureConfigured) {
return disabledIndexingStatus(cfg.message || "Indexing disabled.")
}
if (cfg.systemStatus === "Error") {
return {
state: "Error",
message: cfg.message || "Indexing failed.",
processedFiles,
totalFiles,
percent,
}
}
if (cfg.systemStatus === "Indexing") {
return {
state: "In Progress",
message: cfg.message || "Indexing in progress.",
processedFiles,
totalFiles,
percent,
}
}
if (cfg.systemStatus === "Standby") {
return {
state: "Standby",
message: cfg.message || "Indexing paused.",
processedFiles,
totalFiles,
percent,
}
}
return {
state: "Complete",
message: cfg.message || "Index up-to-date.",
processedFiles,
totalFiles,
percent: totalFiles > 0 ? percent : 100,
}
}
packages/kilo-indexing/src/tree-sitter/index.ts
+263
View File
@@ -0,0 +1,263 @@
import * as fs from "fs/promises"
import * as path from "path"
import type { LanguageParser } from "./languageParser"
import { loadRequiredLanguageParsers } from "./languageParser"
import { parseMarkdown } from "./markdownParser"
import type { QueryCapture } from "web-tree-sitter"
import { Log } from "../util/log"
const log = Log.create({ service: "tree-sitter" })
const METHOD_CAPTURE = ["definition.method", "definition.method.start"]
const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4
let currentMinComponentLines = DEFAULT_MIN_COMPONENT_LINES_VALUE
export function getMinComponentLines(): number {
return currentMinComponentLines
}
export function setMinComponentLines(value: number): void {
currentMinComponentLines = value
}
function shouldSkipMinLines(lineCount: number, capture: QueryCapture, _language: string) {
if (METHOD_CAPTURE.includes(capture.name)) {
// In OOP languages, method signatures are only one line and should not be ignored
return false
}
return lineCount < getMinComponentLines()
}
const extensions = [
// Shell and build systems
"bash",
"bazel",
"bzl",
"build",
"gradle",
"ninja",
"sh",
"zsh",
// Web and frontend
"css",
"ejs",
"erb",
"htm",
"html",
"js",
"jsx",
"ts",
"tsx",
"vue",
// Native and systems languages
"c",
"cpp",
"cs",
"go",
"h",
"hpp",
"m",
"mm",
"rs",
"swift",
"zig",
// JVM and BEAM languages
"ex",
"exs",
"java",
"kt",
"kts",
"scala",
// Scripting and application languages
"dart",
"el",
"elm",
"lua",
"php",
"py",
"r",
"rb",
"vb",
// Functional and specialized languages
"ml",
"mli",
"ql",
"rdl",
"res",
"resi",
"sol",
"tla",
// Data and documentation
"json",
"markdown",
"md",
"rst",
"sql",
"toml",
"yaml",
"yml",
].map((e) => `.${e}`)
export { extensions }
export async function parseSourceCodeDefinitionsForFile(filePath: string): Promise<string | undefined> {
try {
await fs.access(path.resolve(filePath))
} catch {
return "This file does not exist or you do not have permission to access it."
}
const ext = path.extname(filePath).toLowerCase()
if (!extensions.includes(ext)) {
return undefined
}
// Markdown files use a custom parser (no tree-sitter WASM needed)
if (ext === ".md" || ext === ".markdown") {
const fileContent = await fs.readFile(filePath, "utf8")
const lines = fileContent.split("\n")
const markdownCaptures = parseMarkdown(fileContent)
const markdownDefinitions = processCaptures(markdownCaptures, lines, "markdown")
if (markdownDefinitions) {
return `# ${path.basename(filePath)}\n${markdownDefinitions}`
}
return undefined
}
// For other file types, load parser and use tree-sitter
try {
const languageParsers = await loadRequiredLanguageParsers([filePath])
const definitions = await parseFile(filePath, languageParsers)
if (definitions) {
return `# ${path.basename(filePath)}\n${definitions}`
}
return undefined
} catch (error) {
const msg = error instanceof Error ? error.message : String(error)
const isUnsupported =
msg.startsWith("Unsupported language:") || msg.startsWith("Missing tree-sitter language WASM:")
if (!isUnsupported) {
throw error
}
log.debug("skipping AST definition extraction for fallback-only extension", {
filePath,
ext,
err: msg,
})
return undefined
}
}
function processCaptures(captures: QueryCapture[], lines: string[], language: string): string | null {
const needsHtmlFiltering = ["jsx", "tsx"].includes(language)
const isNotHtmlElement = (line: string): boolean => {
if (!needsHtmlFiltering) return true
const HTML_ELEMENTS = /^[^A-Z]*<\/?(?:div|span|button|input|h[1-6]|p|a|img|ul|li|form)\b/
const trimmedLine = line.trim()
return !HTML_ELEMENTS.test(trimmedLine)
}
if (captures.length === 0) {
return null
}
let formattedOutput = ""
captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row)
const processedLines = new Set<string>()
captures.forEach((capture) => {
const { node, name } = capture
if (!name.includes("definition") && !name.includes("name")) {
return
}
const definitionNode = name.includes("name") ? node.parent : node
if (!definitionNode) return
const startLine = definitionNode.startPosition.row
const endLine = definitionNode.endPosition.row
const lineCount = endLine - startLine + 1
if (shouldSkipMinLines(lineCount, capture, language)) {
return
}
const lineKey = `${startLine}-${endLine}`
if (processedLines.has(lineKey)) {
return
}
const startLineContent = lines[startLine]?.trim() ?? ""
if (name.includes("name.definition")) {
const componentName = node.text
if (!processedLines.has(lineKey) && componentName) {
formattedOutput += `${startLine + 1}--${endLine + 1} | ${lines[startLine]}\n`
processedLines.add(lineKey)
}
} else if (isNotHtmlElement(startLineContent)) {
formattedOutput += `${startLine + 1}--${endLine + 1} | ${lines[startLine]}\n`
processedLines.add(lineKey)
if (node.parent && node.parent.lastChild) {
const contextEnd = node.parent.lastChild.endPosition.row
const contextSpan = contextEnd - node.parent.startPosition.row + 1
if (contextSpan >= getMinComponentLines()) {
const rangeKey = `${node.parent.startPosition.row}-${contextEnd}`
if (!processedLines.has(rangeKey)) {
formattedOutput += `${node.parent.startPosition.row + 1}--${contextEnd + 1} | ${lines[node.parent.startPosition.row]}\n`
processedLines.add(rangeKey)
}
}
}
}
})
if (formattedOutput.length > 0) {
return formattedOutput
}
return null
}
async function parseFile(filePath: string, languageParsers: LanguageParser): Promise<string | null> {
const fileContent = await fs.readFile(filePath, "utf8")
const extLang = path.extname(filePath).toLowerCase().slice(1)
const { parser, query } = languageParsers[extLang] || {}
if (!parser || !query) {
return `Unsupported file type: ${filePath}`
}
try {
const tree = parser.parse(fileContent)
const captures = tree ? query.captures(tree.rootNode) : []
const lines = fileContent.split("\n")
return processCaptures(captures, lines, extLang)
} catch (error) {
log.error(`Error parsing file: ${filePath}`, {
err: error instanceof Error ? error.message : String(error),
})
return null
}
}
packages/kilo-indexing/src/tree-sitter/languageParser.ts
+299
View File
@@ -0,0 +1,299 @@
import * as path from "path"
import { existsSync } from "fs"
import type { Parser as ParserT, Language as LanguageT, Query as QueryT } from "web-tree-sitter"
import {
javascriptQuery,
typescriptQuery,
tsxQuery,
pythonQuery,
rustQuery,
goQuery,
cppQuery,
cQuery,
csharpQuery,
rubyQuery,
javaQuery,
phpQuery,
htmlQuery,
swiftQuery,
kotlinQuery,
cssQuery,
ocamlQuery,
solidityQuery,
tomlQuery,
vueQuery,
luaQuery,
systemrdlQuery,
tlaPlusQuery,
zigQuery,
embeddedTemplateQuery,
elispQuery,
elixirQuery,
} from "./queries"
import { Log } from "../util/log"
const log = Log.create({ service: "tree-sitter-parser" })
export interface LanguageParser {
[key: string]: {
parser: ParserT
query: QueryT
}
}
function resolveModulePath(specifier: string): string | undefined {
try {
return require.resolve(specifier)
} catch {
return
}
}
function uniquePaths(paths: Array<string | undefined>): string[] {
return [...new Set(paths.filter((item): item is string => !!item))]
}
function wasmDirectories(sourceDirectory?: string): string[] {
const baseDir = sourceDirectory || __dirname
const execDir = path.dirname(process.execPath)
const envDir = process.env.KILO_TREE_SITTER_WASM_DIR
const wasmPkg = resolveModulePath("tree-sitter-wasms/package.json")
const wasmOutDir = wasmPkg ? path.join(path.dirname(wasmPkg), "out") : undefined
return uniquePaths([
baseDir,
path.join(baseDir, "tree-sitter"),
execDir,
path.join(execDir, "tree-sitter"),
envDir,
wasmOutDir,
])
}
function resolveFromDirectories(file: string, dirs: string[]): string | undefined {
for (const dir of dirs) {
const candidate = path.join(dir, file)
if (!existsSync(candidate)) {
continue
}
return candidate
}
}
function resolveCoreRuntimeWasmPath(sourceDirectory?: string): string | undefined {
const dirs = wasmDirectories(sourceDirectory)
const localPath = resolveFromDirectories("tree-sitter.wasm", dirs)
if (localPath) {
return localPath
}
return resolveModulePath("web-tree-sitter/tree-sitter.wasm")
}
function resolveLanguageWasmPath(langName: string, sourceDirectory?: string) {
const dirs = wasmDirectories(sourceDirectory)
const fileName = `tree-sitter-${langName}.wasm`
const wasmPath = resolveFromDirectories(fileName, dirs)
const searchedPaths = dirs.map((dir) => path.join(dir, fileName))
return {
wasmPath,
searchedPaths,
}
}
async function loadLanguage(langName: string, sourceDirectory?: string) {
const { Language } = require("web-tree-sitter")
const resolved = resolveLanguageWasmPath(langName, sourceDirectory)
if (!resolved.wasmPath) {
log.error(`Failed to resolve language WASM: ${langName}`, {
searchedPaths: resolved.searchedPaths,
})
throw new Error(`Missing tree-sitter language WASM: ${langName}`)
}
try {
return await Language.load(resolved.wasmPath)
} catch (error) {
log.warn(`language WASM unavailable: ${langName}`, {
wasmPath: resolved.wasmPath,
err: error instanceof Error ? error.message : String(error),
})
throw error
}
}
let isParserInitialized = false
/*
RATIONALE: Uses web-tree-sitter WASM modules instead of native node bindings
to avoid architecture-specific build issues. Each language's grammar is loaded
from a .wasm file on demand based on the file extensions being parsed.
Sources:
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/README.md
- https://github.com/Gregoor/tree-sitter-wasms/blob/main/README.md
*/
export async function loadRequiredLanguageParsers(filesToParse: string[], sourceDirectory?: string) {
const { Parser, Query } = require("web-tree-sitter")
if (!isParserInitialized) {
try {
const runtimeWasmPath = resolveCoreRuntimeWasmPath(sourceDirectory)
await (runtimeWasmPath
? Parser.init({
locateFile() {
return runtimeWasmPath
},
})
: Parser.init())
isParserInitialized = true
} catch (error) {
log.error("Failed to initialize tree-sitter parser", {
err: error instanceof Error ? error.message : String(error),
})
throw error
}
}
const extensionsToLoad = new Set(filesToParse.map((file) => path.extname(file).toLowerCase().slice(1)))
const parsers: LanguageParser = {}
for (const ext of extensionsToLoad) {
let language: LanguageT
let query: QueryT
let parserKey = ext
switch (ext) {
case "js":
case "jsx":
case "json":
language = await loadLanguage("javascript", sourceDirectory)
query = new Query(language, javascriptQuery)
break
case "ts":
language = await loadLanguage("typescript", sourceDirectory)
query = new Query(language, typescriptQuery)
break
case "tsx":
language = await loadLanguage("tsx", sourceDirectory)
query = new Query(language, tsxQuery)
break
case "py":
language = await loadLanguage("python", sourceDirectory)
query = new Query(language, pythonQuery)
break
case "rs":
language = await loadLanguage("rust", sourceDirectory)
query = new Query(language, rustQuery)
break
case "go":
language = await loadLanguage("go", sourceDirectory)
query = new Query(language, goQuery)
break
case "cpp":
case "hpp":
language = await loadLanguage("cpp", sourceDirectory)
query = new Query(language, cppQuery)
break
case "c":
case "h":
language = await loadLanguage("c", sourceDirectory)
query = new Query(language, cQuery)
break
case "cs":
language = await loadLanguage("c_sharp", sourceDirectory)
query = new Query(language, csharpQuery)
break
case "rb":
language = await loadLanguage("ruby", sourceDirectory)
query = new Query(language, rubyQuery)
break
case "java":
language = await loadLanguage("java", sourceDirectory)
query = new Query(language, javaQuery)
break
case "php":
language = await loadLanguage("php", sourceDirectory)
query = new Query(language, phpQuery)
break
case "swift":
language = await loadLanguage("swift", sourceDirectory)
query = new Query(language, swiftQuery)
break
case "kt":
case "kts":
language = await loadLanguage("kotlin", sourceDirectory)
query = new Query(language, kotlinQuery)
break
case "css":
language = await loadLanguage("css", sourceDirectory)
query = new Query(language, cssQuery)
break
case "html":
language = await loadLanguage("html", sourceDirectory)
query = new Query(language, htmlQuery)
break
case "ml":
case "mli":
language = await loadLanguage("ocaml", sourceDirectory)
query = new Query(language, ocamlQuery)
break
case "scala":
language = await loadLanguage("scala", sourceDirectory)
query = new Query(language, luaQuery) // COMPAT: Uses Lua query until Scala is implemented
break
case "sol":
language = await loadLanguage("solidity", sourceDirectory)
query = new Query(language, solidityQuery)
break
case "toml":
language = await loadLanguage("toml", sourceDirectory)
query = new Query(language, tomlQuery)
break
case "vue":
language = await loadLanguage("vue", sourceDirectory)
query = new Query(language, vueQuery)
break
case "lua":
language = await loadLanguage("lua", sourceDirectory)
query = new Query(language, luaQuery)
break
case "rdl":
language = await loadLanguage("systemrdl", sourceDirectory)
query = new Query(language, systemrdlQuery)
break
case "tla":
language = await loadLanguage("tlaplus", sourceDirectory)
query = new Query(language, tlaPlusQuery)
break
case "zig":
language = await loadLanguage("zig", sourceDirectory)
query = new Query(language, zigQuery)
break
case "ejs":
case "erb":
parserKey = "embedded_template"
language = await loadLanguage("embedded_template", sourceDirectory)
query = new Query(language, embeddedTemplateQuery)
break
case "el":
language = await loadLanguage("elisp", sourceDirectory)
query = new Query(language, elispQuery)
break
case "ex":
case "exs":
language = await loadLanguage("elixir", sourceDirectory)
query = new Query(language, elixirQuery)
break
default:
throw new Error(`Unsupported language: ${ext}`)
}
const parser = new Parser()
parser.setLanguage(language)
parsers[parserKey] = { parser, query }
}
return parsers
}
packages/kilo-indexing/src/tree-sitter/markdownParser.ts
+180
View File
@@ -0,0 +1,180 @@
import type { QueryCapture } from "web-tree-sitter"
interface MockNode {
startPosition: {
row: number
}
endPosition: {
row: number
}
text: string
parent?: MockNode
}
interface MockCapture {
node: MockNode
name: string
patternIndex: number
}
/**
* Parse a markdown file and extract headers and section line ranges.
* Returns mock captures compatible with tree-sitter's QueryCapture format.
*/
export function parseMarkdown(content: string): QueryCapture[] {
if (!content || content.trim() === "") {
return []
}
const lines = content.split("\n")
const captures: MockCapture[] = []
const atxHeaderRegex = /^(#{1,6})\s+(.+)$/
const setextH1Regex = /^={3,}\s*$/
const setextH2Regex = /^-{3,}\s*$/
const validSetextTextRegex = /^\s*[^#<>!\[\]`\t]+[^\n]$/
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
// ATX headers (# Header)
const atxMatch = line.match(atxHeaderRegex)
if (atxMatch) {
const level = atxMatch[1].length
const text = atxMatch[2].trim()
const node: MockNode = {
startPosition: { row: i },
endPosition: { row: i },
text: text,
}
captures.push({
node,
name: `name.definition.header.h${level}`,
patternIndex: 0,
})
captures.push({
node,
name: `definition.header.h${level}`,
patternIndex: 0,
})
continue
}
// Setext headers (underlined)
if (i > 0) {
if (setextH1Regex.test(line) && validSetextTextRegex.test(lines[i - 1])) {
const text = lines[i - 1].trim()
const node: MockNode = {
startPosition: { row: i - 1 },
endPosition: { row: i },
text: text,
}
captures.push({
node,
name: "name.definition.header.h1",
patternIndex: 0,
})
captures.push({
node,
name: "definition.header.h1",
patternIndex: 0,
})
continue
}
if (setextH2Regex.test(line) && validSetextTextRegex.test(lines[i - 1])) {
const text = lines[i - 1].trim()
const node: MockNode = {
startPosition: { row: i - 1 },
endPosition: { row: i },
text: text,
}
captures.push({
node,
name: "name.definition.header.h2",
patternIndex: 0,
})
captures.push({
node,
name: "definition.header.h2",
patternIndex: 0,
})
continue
}
}
}
// Calculate section ranges
captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row)
const headerCaptures: MockCapture[][] = []
for (let i = 0; i < captures.length; i += 2) {
if (i + 1 < captures.length) {
headerCaptures.push([captures[i], captures[i + 1]])
} else {
headerCaptures.push([captures[i]])
}
}
// Update end positions for section ranges
for (let i = 0; i < headerCaptures.length; i++) {
const headerPair = headerCaptures[i]
if (i < headerCaptures.length - 1) {
const nextHeaderStartRow = headerCaptures[i + 1][0].node.startPosition.row
headerPair.forEach((capture) => {
capture.node.endPosition.row = nextHeaderStartRow - 1
})
} else {
headerPair.forEach((capture) => {
capture.node.endPosition.row = lines.length - 1
})
}
}
// Cast to QueryCapture[] — our MockCapture objects provide all properties
// that the consuming code uses (node.startPosition, node.endPosition, node.text, node.parent, name)
return headerCaptures.flat() as QueryCapture[]
}
export function formatMarkdownCaptures(captures: QueryCapture[], minSectionLines: number = 4): string | null {
if (captures.length === 0) {
return null
}
let formattedOutput = ""
for (let i = 1; i < captures.length; i += 2) {
const capture = captures[i]
const startLine = capture.node.startPosition.row
const endLine = capture.node.endPosition.row
const sectionLength = endLine - startLine + 1
if (sectionLength >= minSectionLines) {
let headerLevel = 1
const headerMatch = capture.name.match(/\.h(\d)$/)
if (headerMatch && headerMatch[1]) {
headerLevel = parseInt(headerMatch[1])
}
const headerPrefix = "#".repeat(headerLevel)
formattedOutput += `${startLine}--${endLine} | ${headerPrefix} ${capture.node.text}\n`
}
}
return formattedOutput.length > 0 ? formattedOutput : null
}
packages/kilo-indexing/src/tree-sitter/queries/c-sharp.ts
+65
View File
@@ -0,0 +1,65 @@
/*
C# Tree-Sitter Query Patterns
*/
export default `
; Using directives
(using_directive) @definition.using
; Namespace declarations (including file-scoped)
; Support both simple names (TestNamespace) and qualified names (My.Company.Module)
(namespace_declaration
name: (qualified_name) @name) @definition.namespace
(namespace_declaration
name: (identifier) @name) @definition.namespace
(file_scoped_namespace_declaration
name: (qualified_name) @name) @definition.namespace
(file_scoped_namespace_declaration
name: (identifier) @name) @definition.namespace
; Class declarations (including generic, static, abstract, partial, nested)
(class_declaration
name: (identifier) @name) @definition.class
; Interface declarations
(interface_declaration
name: (identifier) @name) @definition.interface
; Struct declarations
(struct_declaration
name: (identifier) @name) @definition.struct
; Enum declarations
(enum_declaration
name: (identifier) @name) @definition.enum
; Record declarations
(record_declaration
name: (identifier) @name) @definition.record
; Method declarations (including async, static, generic)
(method_declaration
name: (identifier) @name) @definition.method
; Property declarations
(property_declaration
name: (identifier) @name) @definition.property
; Event declarations
(event_declaration
name: (identifier) @name) @definition.event
; Delegate declarations
(delegate_declaration
name: (identifier) @name) @definition.delegate
; Attribute declarations
(attribute
name: (identifier) @name) @definition.attribute
; Generic type parameters
(type_parameter
name: (identifier) @name) @definition.type_parameter
; LINQ expressions
(query_expression) @definition.linq_expression
`
packages/kilo-indexing/src/tree-sitter/queries/c.ts
+85
View File
@@ -0,0 +1,85 @@
/*
C Language Constructs Supported by Tree-Sitter Parser:
1. Class-like Constructs:
- struct definitions (with fields)
- union definitions (with variants)
- enum definitions (with values)
- anonymous unions/structs
- aligned structs
2. Function-related Constructs:
- function definitions (with parameters)
- function declarations (prototypes)
- static functions
- function pointers
3. Type Definitions:
- typedef declarations (all types)
- function pointer typedefs
- struct/union typedefs
4. Variable Declarations:
- global variables
- static variables
- array declarations
- pointer declarations
5. Preprocessor Constructs:
- function-like macros
- object-like macros
- conditional compilation
*/
export default `
; Function definitions and declarations
(function_definition
declarator: (function_declarator
declarator: (identifier) @name.definition.function))
(declaration
type: (_)?
declarator: (function_declarator
declarator: (identifier) @name.definition.function
parameters: (parameter_list)?)?) @definition.function
(function_declarator
declarator: (identifier) @name.definition.function
parameters: (parameter_list)?) @definition.function
; Struct definitions
(struct_specifier
name: (type_identifier) @name.definition.struct) @definition.struct
; Union definitions
(union_specifier
name: (type_identifier) @name.definition.union) @definition.union
; Enum definitions
(enum_specifier
name: (type_identifier) @name.definition.enum) @definition.enum
; Typedef declarations
(type_definition
declarator: (type_identifier) @name.definition.type) @definition.type
; Global variables
(declaration
(storage_class_specifier)?
type: (_)
declarator: (identifier) @name.definition.variable) @definition.variable
(declaration
(storage_class_specifier)?
type: (_)
declarator: (init_declarator
declarator: (identifier) @name.definition.variable)) @definition.variable
; Object-like macros
(preproc_def
name: (identifier) @name.definition.macro) @definition.macro
; Function-like macros
(preproc_function_def
name: (identifier) @name.definition.macro) @definition.macro
`
packages/kilo-indexing/src/tree-sitter/queries/cpp.ts
+96
View File
@@ -0,0 +1,96 @@
/*
Supported C++ structures:
- struct/class/union declarations
- function/method declarations
- typedef declarations
- enum declarations
- namespace definitions
- template declarations
- macro definitions
- variable declarations
- constructors/destructors
- operator overloads
- friend declarations
- using declarations
*/
export default `
; Basic declarations
(struct_specifier
name: (type_identifier) @name.definition.class) @definition.class
(union_specifier
name: (type_identifier) @name.definition.class) @definition.class
; Function declarations (prototypes)
(declaration
type: (_)
declarator: (function_declarator
declarator: (identifier) @name.definition.function)) @definition.function
; Function definitions (with body)
(function_definition
type: (_)
declarator: (function_declarator
declarator: (identifier) @name.definition.function)) @definition.function
(function_definition
declarator: (function_declarator
declarator: (field_identifier) @name.definition.method)) @definition.method
(type_definition
type: (_)
declarator: (type_identifier) @name.definition.type) @definition.type
(class_specifier
name: (type_identifier) @name.definition.class) @definition.class
; Enum declarations
(enum_specifier
name: (type_identifier) @name.definition.enum) @definition.enum
; Namespace definitions
(namespace_definition
name: (namespace_identifier) @name.definition.namespace) @definition.namespace
(namespace_definition
body: (declaration_list
(namespace_definition
name: (namespace_identifier) @name.definition.namespace))) @definition.namespace
; Template declarations
(template_declaration
parameters: (template_parameter_list)
(class_specifier
name: (type_identifier) @name.definition.template.class)) @definition.template
; Macro definitions
(preproc_function_def
name: (identifier) @name.definition.macro) @definition.macro
; Variable declarations with initialization
(declaration
type: (_)
declarator: (init_declarator
declarator: (identifier) @name.definition.variable)) @definition.variable
; Constructor declarations
(function_definition
declarator: (function_declarator
declarator: (identifier) @name.definition.constructor)) @definition.constructor
; Destructor declarations
(function_definition
declarator: (function_declarator
declarator: (destructor_name) @name.definition.destructor)) @definition.destructor
; Operator overloads
(function_definition
declarator: (function_declarator
declarator: (operator_name) @name.definition.operator)) @definition.operator
; Friend declarations
(friend_declaration) @definition.friend
; Using declarations
(using_declaration) @definition.using
`
packages/kilo-indexing/src/tree-sitter/queries/css.ts
+71
View File
@@ -0,0 +1,71 @@
/*
CSS Tree-Sitter Query Patterns
*/
const cssQuery = String.raw`
; CSS rulesets and selectors
(rule_set
(selectors
(class_selector
(class_name) @name.definition.ruleset)) @_rule
(#match? @name.definition.ruleset "test-ruleset-definition"))
(rule_set
(selectors
(pseudo_class_selector
(class_selector
(class_name) @name.definition.selector))) @_selector
(#match? @name.definition.selector "test-selector-definition"))
; Media queries
(media_statement
(block
(rule_set
(selectors
(class_selector
(class_name) @name.definition.media_query)))) @_media
(#match? @name.definition.media_query "test-media-query-definition-container"))
; Keyframe animations
(keyframes_statement
(keyframes_name) @name.definition.keyframe) @_keyframe
(#match? @name.definition.keyframe "test-keyframe-definition-fade")
; Animation related classes
(rule_set
(selectors
(class_selector
(class_name) @name.definition.animation)) @_animation
(#match? @name.definition.animation "test-animation-definition"))
; Functions
(rule_set
(selectors
(class_selector
(class_name) @name.definition.function)) @_function
(#match? @name.definition.function "test-function-definition"))
; Variables (CSS custom properties)
(declaration
(property_name) @name.definition.variable) @_variable
(#match? @name.definition.variable "^--test-variable-definition")
; Import statements
(import_statement
(string_value) @name.definition.import) @_import
(#match? @name.definition.import "test-import-definition")
; Nested rulesets
(rule_set
(selectors
(class_selector
(class_name) @name.definition.nested_ruleset)) @_nested
(#match? @name.definition.nested_ruleset "test-nested-ruleset-definition"))
; Mixins (using CSS custom properties as a proxy)
(rule_set
(selectors
(class_selector
(class_name) @name.definition.mixin)) @_mixin
(#match? @name.definition.mixin "test-mixin-definition"))`
export default cssQuery
packages/kilo-indexing/src/tree-sitter/queries/elisp.ts
+40
View File
@@ -0,0 +1,40 @@
// Query patterns for Emacs Lisp
export const elispQuery = `
; Function definitions - capture only name and actual function node
((function_definition
name: (symbol) @name.definition.function) @_func
(#match? @name.definition.function "^[^;]"))
; Macro definitions - capture only name and actual macro node
((macro_definition
name: (symbol) @name.definition.macro) @_macro
(#match? @name.definition.macro "^[^;]"))
; Custom forms - match defcustom specifically and avoid comments
((list
. (symbol) @_def
. (symbol) @name.definition.custom) @_custom
(#eq? @_def "defcustom")
(#match? @name.definition.custom "^[^;]"))
; Face definitions - match defface specifically and avoid comments
((list
. (symbol) @_def
. (symbol) @name.definition.face) @_face
(#eq? @_def "defface")
(#match? @name.definition.face "^[^;]"))
; Group definitions - match defgroup specifically and avoid comments
((list
. (symbol) @_def
. (symbol) @name.definition.group) @_group
(#eq? @_def "defgroup")
(#match? @name.definition.group "^[^;]"))
; Advice definitions - match defadvice specifically and avoid comments
((list
. (symbol) @_def
. (symbol) @name.definition.advice) @_advice
(#eq? @_def "defadvice")
(#match? @name.definition.advice "^[^;]"))
`
packages/kilo-indexing/src/tree-sitter/queries/elixir.ts
+70
View File
@@ -0,0 +1,70 @@
export default String.raw`
; Module, Protocol, and Implementation definitions
(call
target: (identifier) @function
(arguments) @args
(do_block)?
(#match? @function "^(defmodule|defprotocol|defimpl)$")) @definition.module
; Function definitions
(call
target: (identifier) @function
(arguments) @args
(do_block)?
(#eq? @function "def")) @definition.function
; Macro definitions
(call
target: (identifier) @function
(arguments) @args
(do_block)?
(#eq? @function "defmacro")) @definition.macro
; Struct definitions
(call
target: (identifier) @function
(arguments (list))
(#eq? @function "defstruct")) @definition.struct
; Guard definitions
(call
target: (identifier) @function
(arguments) @args
(#eq? @function "defguard")) @definition.guard
; Behaviour callback definitions
(call
target: (identifier) @function
(arguments) @args
(#eq? @function "@callback")) @definition.behaviour
; Sigils
(sigil
(sigil_name)
(quoted_content)) @definition.sigil
; Module attributes
(unary_operator
operator: "@"
operand: (call)) @definition.attribute
; Test definitions with string name and map args
(call
target: (identifier) @function
(arguments
(string)
(map))
(#eq? @function "test")) @definition.test
; Pipeline operator usage
(binary_operator
operator: "|>"
left: (_) @left
right: (_) @right) @definition.pipeline
; For comprehensions with generator and filter clauses
(call
target: (identifier) @function
(arguments) @args
(do_block)?
(#eq? @function "for")) @definition.for_comprehension`
packages/kilo-indexing/src/tree-sitter/queries/embedded_template.ts
+19
View File
@@ -0,0 +1,19 @@
/*
Supported Embedded Template structures:
- Code blocks (class, module, method definitions)
- Output blocks (expressions)
- Comments
*/
export default `
; Code blocks - class, module, method definitions
(directive
(code) @name.definition.code) @definition.directive
; Output blocks - expressions
(output_directive
(code) @output.content) @output
; Comments - documentation and section markers
(comment_directive
(comment) @name.definition.comment) @definition.comment
`
packages/kilo-indexing/src/tree-sitter/queries/go.ts
+26
View File
@@ -0,0 +1,26 @@
/*
Go Tree-Sitter Query Patterns
Updated to capture full declarations instead of just identifiers
*/
export default `
; Function declarations - capture the entire declaration
(function_declaration) @name.definition.function
; Method declarations - capture the entire declaration
(method_declaration) @name.definition.method
; Type declarations (interfaces, structs, type aliases) - capture the entire declaration
(type_declaration) @name.definition.type
; Variable declarations - capture the entire declaration
(var_declaration) @name.definition.var
; Constant declarations - capture the entire declaration
(const_declaration) @name.definition.const
; Package clause
(package_clause) @name.definition.package
; Import declarations - capture the entire import block
(import_declaration) @name.definition.import
`
packages/kilo-indexing/src/tree-sitter/queries/html.ts
+51
View File
@@ -0,0 +1,51 @@
export default `
; Document structure
(document) @definition.document
; Elements with content
(element
(start_tag
(tag_name) @name.definition)
(#not-eq? @name.definition "script")
(#not-eq? @name.definition "style")) @definition.element
; Script elements
(script_element
(start_tag
(tag_name) @name.definition)) @definition.script
; Style elements
(style_element
(start_tag
(tag_name) @name.definition)) @definition.style
; Attributes
(attribute
(attribute_name) @name.definition) @definition.attribute
; Comments
(comment) @definition.comment
; Text content
(text) @definition.text
; Raw text content
(raw_text) @definition.raw_text
; Void elements (self-closing)
(element
(start_tag
(tag_name) @name.definition)
(#match? @name.definition "^(area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr)$")) @definition.void_element
; Self-closing tags
(self_closing_tag
(tag_name) @name.definition) @definition.self_closing_tag
; Doctype declarations
(doctype) @definition.doctype
; Multiple elements (parent with children)
(element
(element)+) @definition.nested_elements
`
packages/kilo-indexing/src/tree-sitter/queries/index.ts
+28
View File
@@ -0,0 +1,28 @@
export { solidityQuery } from "./solidity"
export { default as phpQuery } from "./php"
export { vueQuery } from "./vue"
export { default as typescriptQuery } from "./typescript"
export { default as tsxQuery } from "./tsx"
export { default as pythonQuery } from "./python"
export { default as javascriptQuery } from "./javascript"
export { default as javaQuery } from "./java"
export { default as rustQuery } from "./rust"
export { default as rubyQuery } from "./ruby"
export { default as cppQuery } from "./cpp"
export { default as cQuery } from "./c"
export { default as csharpQuery } from "./c-sharp"
export { default as goQuery } from "./go"
export { default as swiftQuery } from "./swift"
export { default as kotlinQuery } from "./kotlin"
export { default as cssQuery } from "./css"
export { default as elixirQuery } from "./elixir"
export { default as htmlQuery } from "./html"
export { default as luaQuery } from "./lua"
export { ocamlQuery } from "./ocaml"
export { tomlQuery } from "./toml"
export { default as systemrdlQuery } from "./systemrdl"
export { default as tlaPlusQuery } from "./tlaplus"
export { zigQuery } from "./zig"
export { default as embeddedTemplateQuery } from "./embedded_template"
export { elispQuery } from "./elisp"
export { scalaQuery } from "./scala"
packages/kilo-indexing/src/tree-sitter/queries/java.ts
+74
View File
@@ -0,0 +1,74 @@
/*
Query patterns for Java language structures
*/
export default `
; Module declarations
(module_declaration
name: (scoped_identifier) @name.definition.module) @definition.module
; Package declarations
((package_declaration
(scoped_identifier)) @name.definition.package) @definition.package
; Line comments
(line_comment) @definition.comment
; Class declarations
(class_declaration
name: (identifier) @name.definition.class) @definition.class
; Interface declarations
(interface_declaration
name: (identifier) @name.definition.interface) @definition.interface
; Enum declarations
(enum_declaration
name: (identifier) @name.definition.enum) @definition.enum
; Record declarations
(record_declaration
name: (identifier) @name.definition.record) @definition.record
; Annotation declarations
(annotation_type_declaration
name: (identifier) @name.definition.annotation) @definition.annotation
; Constructor declarations
(constructor_declaration
name: (identifier) @name.definition.constructor) @definition.constructor
; Method declarations
(method_declaration
type: (_) @definition.method.start ; kilocode_change
name: (identifier) @name.definition.method) @definition.method
; Inner class declarations
(class_declaration
(class_body
(class_declaration
name: (identifier) @name.definition.inner_class))) @definition.inner_class
; Static nested class declarations
(class_declaration
(class_body
(class_declaration
name: (identifier) @name.definition.static_nested_class))) @definition.static_nested_class
; Lambda expressions
(lambda_expression) @definition.lambda
; Field declarations
(field_declaration
(modifiers)?
type: (_)
declarator: (variable_declarator
name: (identifier) @name.definition.field)) @definition.field
; Import declarations
(import_declaration
(scoped_identifier) @name.definition.import) @definition.import
; Type parameters
(type_parameters
(type_parameter) @name.definition.type_parameter) @definition.type_parameter
`
packages/kilo-indexing/src/tree-sitter/queries/javascript.ts
+123
View File
@@ -0,0 +1,123 @@
/*
- class definitions
- method definitions (including decorated methods)
- named function declarations
- arrow functions and function expressions assigned to variables
- JSON object and array definitions (for JSON files)
- decorators and decorated elements
*/
export default `
(
(comment)* @doc
.
(method_definition
name: (property_identifier) @name) @definition.method
(#not-eq? @name "constructor")
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
(#select-adjacent! @doc @definition.method)
)
(
(comment)* @doc
.
[
(class
name: (_) @name)
(class_declaration
name: (_) @name)
] @definition.class
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
(#select-adjacent! @doc @definition.class)
)
(
(comment)* @doc
.
[
(function_declaration
name: (identifier) @name)
(generator_function_declaration
name: (identifier) @name)
] @definition.function
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
(#select-adjacent! @doc @definition.function)
)
(
(comment)* @doc
.
(lexical_declaration
(variable_declarator
name: (identifier) @name
value: [(arrow_function) (function_expression)]) @definition.function)
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
(#select-adjacent! @doc @definition.function)
)
(
(comment)* @doc
.
(variable_declaration
(variable_declarator
name: (identifier) @name
value: [(arrow_function) (function_expression)]) @definition.function)
(#strip! @doc "^[\\s\\*/]+|^[\\s\\*/]$")
(#select-adjacent! @doc @definition.function)
)
; JSON object definitions
(object) @object.definition
; JSON object key-value pairs
(pair
key: (string) @property.name.definition
value: [
(object) @object.value
(array) @array.value
(string) @string.value
(number) @number.value
(true) @boolean.value
(false) @boolean.value
(null) @null.value
]
) @property.definition
; JSON array definitions
(array) @array.definition
; Decorated method definitions
(
[
(method_definition
decorator: (decorator)
name: (property_identifier) @name) @definition.method
(method_definition
decorator: (decorator
(call_expression
function: (identifier) @decorator_name))
name: (property_identifier) @name) @definition.method
]
(#not-eq? @name "constructor")
)
; Decorated class definitions
(
[
(class
decorator: (decorator)
name: (_) @name) @definition.class
(class_declaration
decorator: (decorator)
name: (_) @name) @definition.class
]
)
; Capture method names in decorated classes
(
(class_declaration
decorator: (decorator)
body: (class_body
(method_definition
name: (property_identifier) @name) @definition.method))
(#not-eq? @name "constructor")
)
`
packages/kilo-indexing/src/tree-sitter/queries/kotlin.ts
+110
View File
@@ -0,0 +1,110 @@
/*
- class declarations (regular, data, abstract, sealed, enum, annotation)
- interface declarations
- function declarations (regular, suspend, extension)
- object declarations (including companion objects)
- property declarations and accessors
- type aliases and constructors
*/
export default `
; Type alias declarations
(type_alias
(type_identifier) @name.definition.type_alias
) @definition.type_alias
; Regular class declarations
(class_declaration
(type_identifier) @name.definition.class
) @definition.class
; Data class declarations
(class_declaration
(modifiers
(class_modifier) @_modifier (#eq? @_modifier "data"))
(type_identifier) @name.definition.data_class
) @definition.data_class
; Abstract class declarations
(class_declaration
(modifiers
(inheritance_modifier) @_modifier (#eq? @_modifier "abstract"))
(type_identifier) @name.definition.abstract_class
) @definition.abstract_class
; Sealed class declarations
(class_declaration
(modifiers
(class_modifier) @_modifier (#eq? @_modifier "sealed"))
(type_identifier) @name.definition.sealed_class
) @definition.sealed_class
; Enum class declarations
(class_declaration
(type_identifier)
(enum_class_body)
) @definition.enum_class
; Interface declarations
(class_declaration
(type_identifier) @name.definition.interface
) @definition.interface
; Regular function declarations
(function_declaration
(simple_identifier) @name.definition.function
) @definition.function
; Suspend function declarations
(function_declaration
(modifiers
(function_modifier) @_modifier (#eq? @_modifier "suspend"))
(simple_identifier) @name.definition.suspend_function
) @definition.suspend_function
; Object declarations
(object_declaration
(type_identifier) @name.definition.object
) @definition.object
; Companion object declarations
(companion_object) @definition.companion_object
; Annotation class declarations
(class_declaration
(modifiers
(class_modifier) @_modifier (#eq? @_modifier "annotation"))
(type_identifier) @name.definition.annotation_class
) @definition.annotation_class
; Extension function declarations
(function_declaration
(modifiers
(function_modifier) @_modifier (#eq? @_modifier "extension"))
(simple_identifier) @name.definition.extension_function
) @definition.extension_function
; Primary constructor declarations
(class_declaration
(primary_constructor) @definition.primary_constructor
)
; Secondary constructor declarations
(secondary_constructor) @definition.secondary_constructor
; Property declarations
(property_declaration
(variable_declaration
(simple_identifier) @name.definition.property)
) @definition.property
; Property declarations with accessors
(property_declaration
(variable_declaration
(simple_identifier) @name.definition.property)
(getter)? @definition.getter
(setter)? @definition.setter
) @definition.property_with_accessors
`
packages/kilo-indexing/src/tree-sitter/queries/lua.ts
+37
View File
@@ -0,0 +1,37 @@
/*
Supported Lua structures:
- function definitions (global, local, and method)
- table constructors
- variable declarations
- class-like structures
*/
export default String.raw`
; Function definitions
(function_definition_statement
name: (identifier) @name.definition.function) @definition.function
(function_definition_statement
name: (variable
table: (identifier)
field: (identifier) @name.definition.method)) @definition.method
(local_function_definition_statement
name: (identifier) @name.definition.function) @definition.function
; Table constructors (class-like structures)
(local_variable_declaration
(variable_list
(variable name: (identifier) @name.definition.table))
(expression_list
value: (table))) @definition.table
; Variable declarations
(variable_assignment
(variable_list
(variable name: (identifier) @name.definition.variable))) @definition.variable
; Local variable declarations
(local_variable_declaration
(variable_list
(variable name: (identifier) @name.definition.variable))) @definition.variable
`
packages/kilo-indexing/src/tree-sitter/queries/ocaml.ts
+31
View File
@@ -0,0 +1,31 @@
export const ocamlQuery = `
; Captures module definitions
(module_definition
(module_binding
name: (module_name) @name.definition)) @definition.module
; Captures type definitions
(type_definition
(type_binding
name: (type_constructor) @name.definition)) @definition.type
; Captures function definitions
(value_definition
(let_binding
pattern: (value_name) @name.definition
(parameter))) @definition.function
; Captures class definitions
(class_definition
(class_binding
name: (class_name) @name.definition)) @definition.class
; Captures method definitions
(method_definition
name: (method_name) @name.definition) @definition.method
; Captures value bindings
(value_definition
(let_binding
pattern: (value_name) @name.definition)) @definition.value
`
packages/kilo-indexing/src/tree-sitter/queries/php.ts
+172
View File
@@ -0,0 +1,172 @@
/*
PHP Tree-sitter Query - Standardized Version
This query file captures PHP language constructs for code navigation and analysis.
Each query pattern is organized by construct type and includes clear comments.
SUPPORTED LANGUAGE CONSTRUCTS:
------------------------------
1. CLASS DEFINITIONS
- Regular classes
- Abstract classes
- Final classes
- Readonly classes (PHP 8.2+)
2. INTERFACE & TRAIT DEFINITIONS
- Interfaces
- Traits
- Enums (PHP 8.1+)
3. FUNCTION & METHOD DEFINITIONS
- Global functions
- Class methods
- Static methods
- Abstract methods
- Final methods
- Arrow functions (PHP 7.4+)
4. PROPERTY DEFINITIONS
- Regular properties
- Static properties
- Readonly properties (PHP 8.1+)
- Constructor property promotion (PHP 8.0+)
5. OTHER LANGUAGE CONSTRUCTS
- Constants
- Namespaces
- Use statements (imports)
- Anonymous classes
- Attributes (PHP 8.0+)
- Match expressions (PHP 8.0+)
- Heredoc and nowdoc syntax
*/
export default `
;--------------------------
; 1. CLASS DEFINITIONS
;--------------------------
; Regular classes
(class_declaration
name: (name) @name.definition.class) @definition.class
; Abstract classes
(class_declaration
(abstract_modifier)
name: (name) @name.definition.abstract_class) @definition.abstract_class
; Final classes
(class_declaration
(final_modifier)
name: (name) @name.definition.final_class) @definition.final_class
; Readonly classes (PHP 8.2+)
(class_declaration
(readonly_modifier)
name: (name) @name.definition.readonly_class) @definition.readonly_class
;--------------------------
; 2. INTERFACE & TRAIT DEFINITIONS
;--------------------------
; Interfaces
(interface_declaration
name: (name) @name.definition.interface) @definition.interface
; Traits
(trait_declaration
name: (name) @name.definition.trait) @definition.trait
; Enums (PHP 8.1+)
(enum_declaration
name: (name) @name.definition.enum) @definition.enum
;--------------------------
; 3. FUNCTION & METHOD DEFINITIONS
;--------------------------
; Global functions
(function_definition
name: (name) @name.definition.function) @definition.function
; Regular methods
(method_declaration
name: (name) @name.definition.method) @definition.method
; Static methods
(method_declaration
(static_modifier)
name: (name) @name.definition.static_method) @definition.static_method
; Abstract methods
(method_declaration
(abstract_modifier)
name: (name) @name.definition.abstract_method) @definition.abstract_method
; Final methods
(method_declaration
(final_modifier)
name: (name) @name.definition.final_method) @definition.final_method
; Arrow functions (PHP 7.4+)
(arrow_function) @definition.arrow_function
;--------------------------
; 4. PROPERTY DEFINITIONS
;--------------------------
; Regular properties
(property_declaration
(property_element
(variable_name
(name) @name.definition.property))) @definition.property
; Static properties
(property_declaration
(static_modifier)
(property_element
(variable_name
(name) @name.definition.static_property))) @definition.static_property
; Readonly properties (PHP 8.1+)
(property_declaration
(readonly_modifier)
(property_element
(variable_name
(name) @name.definition.readonly_property))) @definition.readonly_property
; Constructor property promotion (PHP 8.0+)
(property_promotion_parameter
name: (variable_name
(name) @name.definition.promoted_property)) @definition.promoted_property
;--------------------------
; 5. OTHER LANGUAGE CONSTRUCTS
;--------------------------
; Constants
(const_declaration
(const_element
(name) @name.definition.constant)) @definition.constant
; Namespaces
(namespace_definition
name: (namespace_name) @name.definition.namespace) @definition.namespace
; Use statements (imports)
(namespace_use_declaration
(namespace_use_clause
(qualified_name) @name.definition.use)) @definition.use
; Anonymous classes
(object_creation_expression
(declaration_list)) @definition.anonymous_class
; Attributes (PHP 8.0+)
(attribute_group
(attribute
(name) @name.definition.attribute)) @definition.attribute
; Match expressions (PHP 8.0+)
(match_expression) @definition.match_expression
; Heredoc syntax
(heredoc) @definition.heredoc
; Nowdoc syntax
(nowdoc) @definition.nowdoc
`
packages/kilo-indexing/src/tree-sitter/queries/python.ts
+73
View File
@@ -0,0 +1,73 @@
/*
Python Tree-sitter Query Patterns
*/
export default `
; Class definitions (including decorated)
(class_definition
name: (identifier) @name.definition.class) @definition.class
(decorated_definition
definition: (class_definition
name: (identifier) @name.definition.class)) @definition.class
; Function and method definitions (including async and decorated)
(function_definition
name: (identifier) @name.definition.function) @definition.function
(decorated_definition
definition: (function_definition
name: (identifier) @name.definition.function)) @definition.function
; Lambda expressions
(expression_statement
(assignment
left: (identifier) @name.definition.lambda
right: (parenthesized_expression
(lambda)))) @definition.lambda
; Generator functions (functions containing yield)
(function_definition
name: (identifier) @name.definition.generator
body: (block
(expression_statement
(yield)))) @definition.generator
; Comprehensions
(expression_statement
(assignment
left: (identifier) @name.definition.comprehension
right: [
(list_comprehension)
(dictionary_comprehension)
(set_comprehension)
])) @definition.comprehension
; With statements
(with_statement) @definition.with_statement
; Try statements
(try_statement) @definition.try_statement
; Import statements
(import_from_statement) @definition.import
(import_statement) @definition.import
; Global/Nonlocal statements
(function_definition
body: (block
[(global_statement) (nonlocal_statement)])) @definition.scope
; Match case statements
(function_definition
body: (block
(match_statement))) @definition.match_case
; Type annotations
(typed_parameter
type: (type)) @definition.type_annotation
(expression_statement
(assignment
left: (identifier) @name.definition.type
type: (type))) @definition.type_annotation
`
packages/kilo-indexing/src/tree-sitter/queries/ruby.ts
+204
View File
@@ -0,0 +1,204 @@
/*
- method definitions (including singleton methods and aliases, with associated comments)
- class definitions (including singleton classes, with associated comments)
- module definitions
- constants
- global variables
- instance variables
- class variables
- symbols
- blocks, procs, and lambdas
- mixins (include, extend, prepend)
- metaprogramming constructs (define_method, method_missing)
- attribute accessors (attr_reader, attr_writer, attr_accessor)
- class macros (has_many, belongs_to, etc. in Rails-like code)
- exception handling (begin/rescue/ensure)
- keyword arguments
- splat operators
- hash rocket and JSON-style hashes
- string interpolation
- regular expressions
- Ruby 2.7+ pattern matching
- Ruby 3.0+ endless methods
- Ruby 3.1+ pin operator and shorthand hash syntax
*/
export default `
; Method definitions
(method
name: (identifier) @name.definition.method) @definition.method
; Singleton methods
(singleton_method
object: (_)
name: (identifier) @name.definition.method) @definition.method
; Method aliases
(alias
name: (_) @name.definition.method) @definition.method
; Class definitions
(class
name: [
(constant) @name.definition.class
(scope_resolution
name: (_) @name.definition.class)
]) @definition.class
; Singleton classes
(singleton_class
value: [
(constant) @name.definition.class
(scope_resolution
name: (_) @name.definition.class)
]) @definition.class
; Module definitions
(module
name: [
(constant) @name.definition.module
(scope_resolution
name: (_) @name.definition.module)
]) @definition.module
; Constants
(assignment
left: (constant) @name.definition.constant) @definition.constant
; Global variables
(global_variable) @definition.global_variable
; Instance variables
(instance_variable) @definition.instance_variable
; Class variables
(class_variable) @definition.class_variable
; Symbols
(simple_symbol) @definition.symbol
(hash_key_symbol) @definition.symbol
; Blocks
(block) @definition.block
(do_block) @definition.block
; Basic mixin statements - capture all include/extend/prepend calls
(call
method: (identifier) @_mixin_method
arguments: (argument_list
(constant) @name.definition.mixin)
(#match? @_mixin_method "^(include|extend|prepend)$")) @definition.mixin
; Mixin module definition
(module
name: (constant) @name.definition.mixin_module
(#match? @name.definition.mixin_module ".*Module$")) @definition.mixin_module
; Mixin-related methods
(method
name: (identifier) @name.definition.mixin_method
(#match? @name.definition.mixin_method "(included|extended|prepended)_method")) @definition.mixin_method
; Singleton class blocks
(singleton_class) @definition.singleton_class
; Class methods in singleton context
(singleton_method
object: (self)
name: (identifier) @name.definition.singleton_method) @definition.singleton_method
; Attribute accessors
(call
method: (identifier) @_attr_accessor
arguments: (argument_list
(_) @name.definition.attr_accessor)
(#eq? @_attr_accessor "attr_accessor")) @definition.attr_accessor
(call
method: (identifier) @_attr_reader
arguments: (argument_list
(_) @name.definition.attr_reader)
(#eq? @_attr_reader "attr_reader")) @definition.attr_reader
(call
method: (identifier) @_attr_writer
arguments: (argument_list
(_) @name.definition.attr_writer)
(#eq? @_attr_writer "attr_writer")) @definition.attr_writer
; Class macros (Rails-like)
(call
method: (identifier) @_macro_name
arguments: (argument_list
(_) @name.definition.class_macro)
(#match? @_macro_name "^(has_many|belongs_to|has_one|validates|scope|before_action|after_action)$")) @definition.class_macro
; Exception handling
(begin) @definition.begin
(rescue) @definition.rescue
(ensure) @definition.ensure
; Keyword arguments
(keyword_parameter
name: (identifier) @name.definition.keyword_parameter) @definition.keyword_parameter
; Splat operators
(splat_parameter) @definition.splat_parameter
(splat_argument) @definition.splat_argument
; Hash syntax variants
(pair
key: (_) @name.definition.hash_key) @definition.hash_pair
; String interpolation - capture the string with interpolation and surrounding context
(assignment
left: (identifier) @name.definition.string_var
right: (string
(interpolation))) @definition.string_interpolation
; Regular expressions - capture the regex pattern and assignment
(assignment
left: (identifier) @name.definition.regex_var
right: (regex)) @definition.regex_assignment
; Pattern matching - capture the entire case_match structure
(case_match) @definition.case_match
; Pattern matching - capture in_clause with hash pattern
(in_clause
pattern: (hash_pattern)) @definition.hash_pattern_clause
; Endless methods - capture the method definition with name and surrounding context
(comment) @_endless_method_comment
(#match? @_endless_method_comment "Ruby 3.0\\+ endless method")
(method
name: (identifier) @name.definition.endless_method
body: (binary
operator: "=")) @definition.endless_method
; Pin operator - capture the entire in_clause with variable_reference_pattern
(in_clause
pattern: (variable_reference_pattern)) @definition.pin_pattern_clause
; Shorthand hash syntax - capture the method containing shorthand hash
(comment) @_shorthand_hash_comment
(#match? @_shorthand_hash_comment "Ruby 3.1\\+ shorthand hash syntax")
(method
name: (identifier) @name.definition.shorthand_method) @definition.shorthand_method
; Shorthand hash syntax - capture the hash with shorthand syntax
(hash
(pair
(hash_key_symbol)
":")) @definition.shorthand_hash
; Capture larger contexts for features that need at least 4 lines
; Capture the entire program to include all comments and code
(program) @definition.program
; Capture all comments
(comment) @definition.comment
; Capture all method definitions
(method) @definition.method_all
`
packages/kilo-indexing/src/tree-sitter/queries/rust.ts
+80
View File
@@ -0,0 +1,80 @@
/*
Rust language structures for tree-sitter parsing
Captures all required constructs for tests
*/
export default `
; Function definitions (all types)
(function_item
name: (identifier) @name.definition.function) @definition.function
; Struct definitions (all types - standard, tuple, unit)
(struct_item
name: (type_identifier) @name.definition.struct) @definition.struct
; Enum definitions with variants
(enum_item
name: (type_identifier) @name.definition.enum) @definition.enum
; Trait definitions
(trait_item
name: (type_identifier) @name.definition.trait) @definition.trait
; Impl blocks (inherent implementation)
(impl_item
type: (type_identifier) @name.definition.impl) @definition.impl
; Trait implementations
(impl_item
trait: (type_identifier) @name.definition.impl_trait
type: (type_identifier) @name.definition.impl_for) @definition.impl_trait
; Module definitions
(mod_item
name: (identifier) @name.definition.module) @definition.module
; Macro definitions
(macro_definition
name: (identifier) @name.definition.macro) @definition.macro
; Attribute macros (for #[derive(...)] etc.)
(attribute_item
(attribute) @name.definition.attribute) @definition.attribute
; Type aliases
(type_item
name: (type_identifier) @name.definition.type_alias) @definition.type_alias
; Constants
(const_item
name: (identifier) @name.definition.constant) @definition.constant
; Static items
(static_item
name: (identifier) @name.definition.static) @definition.static
; Methods inside impl blocks
(impl_item
body: (declaration_list
(function_item
name: (identifier) @name.definition.method))) @definition.method_container
; Use declarations
(use_declaration) @definition.use_declaration
; Lifetime definitions
(lifetime
"'" @punctuation.lifetime
(identifier) @name.definition.lifetime) @definition.lifetime
; Where clauses
(where_clause
(where_predicate)*) @definition.where_clause
; Match expressions
(match_expression
value: (_) @match.value
body: (match_block)) @definition.match
; Unsafe blocks
(unsafe_block) @definition.unsafe_block
`
packages/kilo-indexing/src/tree-sitter/queries/scala.ts
+44
View File
@@ -0,0 +1,44 @@
export const scalaQuery = `
; Classes
(class_definition
name: (identifier) @name.definition) @definition.class
(class_definition
(modifiers)
name: (identifier) @name.definition) @definition.class
; Objects
(object_definition
name: (identifier) @name.definition) @definition.object
(object_definition
name: (identifier) @name.definition
extend: (extends_clause)?) @definition.object
; Traits
(trait_definition
name: (identifier) @name.definition) @definition.trait
; Methods
(function_definition
name: (identifier) @name.definition) @definition.method
; Values and Variables
(val_definition
pattern: (identifier) @name.definition) @definition.variable
(var_definition
pattern: (identifier) @name.definition) @definition.variable
(val_definition
(modifiers)
pattern: (identifier) @name.definition) @definition.variable
; Types
(type_definition
name: (type_identifier) @name.definition) @definition.type
; Package declarations
(package_clause
name: (package_identifier) @name.definition) @definition.namespace
`
packages/kilo-indexing/src/tree-sitter/queries/solidity.ts
+44
View File
@@ -0,0 +1,44 @@
export const solidityQuery = `
; Contract declarations
(contract_declaration
name: (identifier) @name.definition.contract) @definition.contract
(interface_declaration
name: (identifier) @name.definition.interface) @definition.interface
(library_declaration
name: (identifier) @name.definition.library) @definition.library
; Function declarations
(function_definition
name: (identifier) @name.definition.function) @definition.function
(modifier_definition
name: (identifier) @name.definition.modifier) @definition.modifier
(constructor_definition) @definition.constructor
(fallback_receive_definition
(visibility)
(state_mutability)) @definition.fallback
; Type declarations
(struct_declaration
name: (identifier) @name.definition.struct) @definition.struct
(enum_declaration
name: (identifier) @name.definition.enum) @definition.enum
(event_definition
name: (identifier) @name.definition.event) @definition.event
(error_declaration
name: (identifier) @name.definition.error) @definition.error
; Variable declarations
(state_variable_declaration
name: (identifier) @name.definition.variable) @definition.variable
; Using directives
(using_directive
(type_alias) @name.definition.using) @definition.using`
packages/kilo-indexing/src/tree-sitter/queries/swift.ts
+74
View File
@@ -0,0 +1,74 @@
/*
Swift Tree-Sitter Query Patterns
This file contains query patterns for Swift language constructs:
- class declarations - Captures standard, final, and open class definitions
- struct declarations - Captures standard and generic struct definitions
- protocol declarations - Captures protocol definitions with requirements
- extension declarations - Captures extensions for classes, structs, and protocols
- method declarations - Captures instance and type methods
- property declarations - Captures stored and computed properties
- initializer declarations - Captures designated and convenience initializers
- deinitializer declarations - Captures deinit methods
- subscript declarations - Captures subscript methods
- type alias declarations - Captures type alias definitions
Each query pattern is mapped to a specific test in parseSourceCodeDefinitions.swift.test.ts
*/
export default `
; Class declarations - captures standard, final, and open classes
(class_declaration
name: (type_identifier) @name) @definition.class
; Protocol declarations - captures protocols with requirements
(protocol_declaration
name: (type_identifier) @name) @definition.interface
; Method declarations in classes/structs/enums/extensions
(function_declaration
name: (simple_identifier) @name) @definition.method
; Static/class method declarations
(function_declaration
(modifiers
(property_modifier))
name: (simple_identifier) @name) @definition.static_method
; Initializers - captures designated initializers
(init_declaration
"init" @name) @definition.initializer
; Convenience initializers
(init_declaration
(modifiers (member_modifier))
"init" @name) @definition.convenience_initializer
; Deinitializers
(deinit_declaration
"deinit" @name) @definition.deinitializer
; Subscript declarations
(subscript_declaration
(parameter) @name) @definition.subscript
; Property declarations - captures stored properties
(property_declaration
(pattern) @name) @definition.property
; Computed property declarations with accessors
(property_declaration
(pattern)
(computed_property)) @definition.computed_property
; Type aliases
(typealias_declaration
name: (type_identifier) @name) @definition.type_alias
; Protocol property requirements
(protocol_property_declaration
name: (pattern) @name) @definition.protocol_property
; Protocol method requirements
(protocol_function_declaration
name: (simple_identifier) @name) @definition.protocol_method
`
packages/kilo-indexing/src/tree-sitter/queries/systemrdl.ts
+33
View File
@@ -0,0 +1,33 @@
/*
Supported SystemRDL structures:
- component declarations
- field declarations
- property assignments
- parameter declarations
- enum declarations
*/
export default `
; Component declarations
(component_named_def
type: (component_type)
id: (id) @name.definition.component) @definition.component
; Field declarations
(component_anon_def
type: (component_type (component_primary_type))
body: (component_body
(component_body_elem
(property_assignment)))) @definition.field
; Property declarations
(property_definition
(id) @name.definition.property) @definition.property
; Parameter declarations
(component_inst
id: (id) @name.definition.parameter) @definition.parameter
; Enum declarations
(enum_def
(id) @name.definition.enum) @definition.enum
`
packages/kilo-indexing/src/tree-sitter/queries/tlaplus.ts
+32
View File
@@ -0,0 +1,32 @@
/*
Supported TLA+ structures:
- modules with header, extends, constants, variables
- operator definitions with parameters and bodies
- function definitions with quantifier bounds
- let expressions with operator definitions
- case expressions with multiple arms
- variable and constant declarations
*/
export default `
; Module declarations
(module
name: (identifier) @name.definition.module) @definition.module
; Operator definitions with optional parameters
(operator_definition
name: (identifier) @name.definition.operator
parameter: (identifier)?) @definition.operator
; Function definitions with bounds
(function_definition
name: (identifier) @name.definition.function
(quantifier_bound)?) @definition.function
; Variable declarations
(variable_declaration
(identifier) @name.definition.variable) @definition.variable
; Constant declarations
(constant_declaration
(identifier) @name.definition.constant) @definition.constant
`
packages/kilo-indexing/src/tree-sitter/queries/toml.ts
+24
View File
@@ -0,0 +1,24 @@
// Query patterns for TOML syntax elements
export const tomlQuery = `
; Tables - capture the entire table node
(table) @definition
; Array tables - capture the entire array table node
(table_array_element) @definition
; Key-value pairs - capture the entire pair
(pair) @definition
; Arrays and inline tables
(array) @definition
(inline_table) @definition
; Basic values
(string) @definition
(integer) @definition
(float) @definition
(boolean) @definition
(offset_date_time) @definition
(local_date) @definition
(local_time) @definition
`
packages/kilo-indexing/src/tree-sitter/queries/tsx.ts
+87
View File
@@ -0,0 +1,87 @@
import typescriptQuery from "./typescript"
/**
* Tree-sitter Query for TSX Files
*
* This query captures React component definitions in TSX files:
* - Function Components
* - Class Components
* - Higher Order Components
* - Type Definitions
* - Props Interfaces
* - State Definitions
* - Generic Components
*/
export default `${typescriptQuery}
; Function Components - Both function declarations and arrow functions
(function_declaration
name: (identifier) @name) @definition.component
; Arrow Function Components
(variable_declaration
(variable_declarator
name: (identifier) @name
value: (arrow_function))) @definition.component
; Export Statement Components
(export_statement
(variable_declaration
(variable_declarator
name: (identifier) @name
value: (arrow_function)))) @definition.component
; Class Components
(class_declaration
name: (type_identifier) @name) @definition.class_component
; Interface Declarations
(interface_declaration
name: (type_identifier) @name) @definition.interface
; Type Alias Declarations
(type_alias_declaration
name: (type_identifier) @name) @definition.type
; HOC Components
(variable_declaration
(variable_declarator
name: (identifier) @name
value: (call_expression
function: (identifier)))) @definition.component
; JSX Component Usage - Capture all components in JSX
(jsx_element
open_tag: (jsx_opening_element
name: [(identifier) @component (member_expression) @component])) @definition.jsx_element
; Self-closing JSX elements
(jsx_self_closing_element
name: [(identifier) @component (member_expression) @component]) @definition.jsx_self_closing_element
; Capture all identifiers in JSX expressions that start with capital letters
(jsx_expression
(identifier) @jsx_component) @definition.jsx_component
; Capture all member expressions in JSX
(member_expression
object: (identifier) @object
property: (property_identifier) @property) @definition.member_component
; Capture components in conditional expressions
(ternary_expression
consequence: (parenthesized_expression
(jsx_element
open_tag: (jsx_opening_element
name: (identifier) @component)))) @definition.conditional_component
(ternary_expression
alternative: (jsx_self_closing_element
name: (identifier) @component)) @definition.conditional_component
; Generic Components
(function_declaration
name: (identifier) @name
type_parameters: (type_parameters)) @definition.generic_component
`
packages/kilo-indexing/src/tree-sitter/queries/typescript.ts
+123
View File
@@ -0,0 +1,123 @@
/*
- function signatures and declarations
- method signatures and definitions
- abstract method signatures
- class declarations (including abstract classes)
- module declarations
- arrow functions (lambda functions)
- switch/case statements with complex case blocks
- enum declarations with members
- namespace declarations
- utility types
- class members and properties
- constructor methods
- getter/setter methods
- async functions and arrow functions
*/
export default `
(function_signature
name: (identifier) @name.definition.function) @definition.function
(method_signature
name: (property_identifier) @name.definition.method) @definition.method
(abstract_method_signature
name: (property_identifier) @name.definition.method) @definition.method
(abstract_class_declaration
name: (type_identifier) @name.definition.class) @definition.class
(module
name: (identifier) @name.definition.module) @definition.module
(function_declaration
name: (identifier) @name.definition.function) @definition.function
(method_definition
name: (property_identifier) @name.definition.method) @definition.method
(class_declaration
name: (type_identifier) @name.definition.class) @definition.class
(call_expression
function: (identifier) @func_name
arguments: (arguments
(string) @name
[(arrow_function) (function_expression)]) @definition.test)
(#match? @func_name "^(describe|test|it)$")
(assignment_expression
left: (member_expression
object: (identifier) @obj
property: (property_identifier) @prop)
right: [(arrow_function) (function_expression)]) @definition.test
(#eq? @obj "exports")
(#eq? @prop "test")
(arrow_function) @definition.lambda
; Switch statements and case clauses
(switch_statement) @definition.switch
; Individual case clauses with their blocks
(switch_case) @definition.case
; Default clause
(switch_default) @definition.default
; Enum declarations
(enum_declaration
name: (identifier) @name.definition.enum) @definition.enum
; Decorator definitions with decorated class
(export_statement
decorator: (decorator
(call_expression
function: (identifier) @name.definition.decorator))
declaration: (class_declaration
name: (type_identifier) @name.definition.decorated_class)) @definition.decorated_class
; Explicitly capture class name in decorated class
(class_declaration
name: (type_identifier) @name.definition.class) @definition.class
; Namespace declarations
(internal_module
name: (identifier) @name.definition.namespace) @definition.namespace
; Interface declarations with generic type parameters and constraints
(interface_declaration
name: (type_identifier) @name.definition.interface
type_parameters: (type_parameters)?) @definition.interface
; Type alias declarations with generic type parameters and constraints
(type_alias_declaration
name: (type_identifier) @name.definition.type
type_parameters: (type_parameters)?) @definition.type
; Utility Types
(type_alias_declaration
name: (type_identifier) @name.definition.utility_type) @definition.utility_type
; Class Members and Properties
(public_field_definition
name: (property_identifier) @name.definition.property) @definition.property
; Constructor
(method_definition
name: (property_identifier) @name.definition.constructor
(#eq? @name.definition.constructor "constructor")) @definition.constructor
; Getter/Setter Methods
(method_definition
name: (property_identifier) @name.definition.accessor) @definition.accessor
; Async Functions
(function_declaration
name: (identifier) @name.definition.async_function) @definition.async_function
; Async Arrow Functions
(variable_declaration
(variable_declarator
name: (identifier) @name.definition.async_arrow
value: (arrow_function))) @definition.async_arrow
`
packages/kilo-indexing/src/tree-sitter/queries/vue.ts
+29
View File
@@ -0,0 +1,29 @@
export const vueQuery = `
; Top-level structure
(component) @component.definition
; Template section
(template_element) @template.definition
(template_element
(element
(start_tag
(tag_name) @element.name.definition))
(element
(start_tag
(attribute
(attribute_name) @attribute.name.definition)))
(element
(start_tag
(directive_attribute
(directive_name) @directive.name.definition))))
; Script section
(script_element) @script.definition
(script_element
(raw_text) @script.content.definition)
; Style section
(style_element) @style.definition
(style_element
(raw_text) @style.content.definition)
`
packages/kilo-indexing/src/tree-sitter/queries/zig.ts
+21
View File
@@ -0,0 +1,21 @@
export const zigQuery = `
; Functions
(function_declaration) @function.definition
; Structs and containers
(variable_declaration
(identifier) @name
(struct_declaration)
) @container.definition
; Enums
(variable_declaration
(identifier) @name
(enum_declaration)
) @container.definition
; Variables and constants
(variable_declaration
(identifier) @name
) @variable.definition
`
packages/kilo-indexing/src/util/log.ts
+74
View File
@@ -0,0 +1,74 @@
type Entry = {
debug(message?: unknown, extra?: Record<string, unknown>): void
info(message?: unknown, extra?: Record<string, unknown>): void
warn(message?: unknown, extra?: Record<string, unknown>): void
error(message?: unknown, extra?: Record<string, unknown>): void
tag(key: string, value: string): Entry
clone(): Entry
time(
message: string,
extra?: Record<string, unknown>,
): {
stop(): void
[Symbol.dispose](): void
}
}
const enabled = process.env.KILO_INDEXING_LOG === "1" || process.env.KILO_INDEXING_LOG === "true"
export namespace Log {
export type Logger = Entry
export function create(input: Record<string, unknown> = {}): Entry {
const tags = { ...input }
function write(level: string, message?: unknown, extra?: Record<string, unknown>) {
if (!enabled) return
const line = JSON.stringify({
level,
time: new Date().toISOString(),
message,
...tags,
...extra,
})
console.error(line)
}
const log: Entry = {
debug(message, extra) {
write("DEBUG", message, extra)
},
info(message, extra) {
write("INFO", message, extra)
},
warn(message, extra) {
write("WARN", message, extra)
},
error(message, extra) {
write("ERROR", message, extra)
},
tag(key, value) {
tags[key] = value
return log
},
clone() {
return create(tags)
},
time(message, extra) {
const start = Date.now()
const stop = () => {
write("INFO", message, { duration: Date.now() - start, ...extra })
}
return {
stop,
[Symbol.dispose]() {
stop()
},
}
},
}
return log
}
}
packages/kilo-indexing/test/kilocode/indexing/config-manager.test.ts
+66
View File
@@ -0,0 +1,66 @@
import { describe, expect, test } from "bun:test"
import { CodeIndexConfigManager, type IndexingConfigInput } from "../../../src/indexing/config-manager"
function createInput(input: Partial<IndexingConfigInput> = {}): IndexingConfigInput {
return {
enabled: true,
embedderProvider: "openai",
vectorStoreProvider: "lancedb",
openAiKey: "sk-test",
...input,
}
}
describe("CodeIndexConfigManager", () => {
test("uses default ollama base URL when omitted", () => {
const cfg = new CodeIndexConfigManager(
createInput({
embedderProvider: "ollama",
openAiKey: undefined,
ollamaBaseUrl: undefined,
}),
)
expect(cfg.isFeatureConfigured).toBe(true)
expect(cfg.getConfig().ollamaOptions?.baseUrl).toBe("http://localhost:11434")
})
test("defaults vector store to qdrant when omitted", () => {
const cfg = new CodeIndexConfigManager(createInput({ vectorStoreProvider: undefined }))
expect(cfg.getConfig().vectorStoreProvider).toBe("qdrant")
})
describe("loadConfiguration restart checks", () => {
test("requires restart when model changes with same dimension", () => {
const cfg = new CodeIndexConfigManager(createInput({ modelId: "text-embedding-3-small" }))
const result = cfg.loadConfiguration(createInput({ modelId: "text-embedding-ada-002" }))
expect(result.requiresRestart).toBe(true)
})
test("does not restart when default model is made explicit", () => {
const cfg = new CodeIndexConfigManager(createInput())
const result = cfg.loadConfiguration(createInput({ modelId: "text-embedding-3-small" }))
expect(result.requiresRestart).toBe(false)
})
test("requires restart when provider changes with same dimension", () => {
const cfg = new CodeIndexConfigManager(createInput({ modelId: "text-embedding-3-small" }))
const result = cfg.loadConfiguration(
createInput({
embedderProvider: "vercel-ai-gateway",
vercelAiGatewayApiKey: "kg-test",
openAiKey: undefined,
modelId: "text-embedding-3-small",
}),
)
expect(result.requiresRestart).toBe(true)
})
})
})
packages/kilo-indexing/test/kilocode/indexing/detect.test.ts
+60
View File
@@ -0,0 +1,60 @@
import { describe, expect, test } from "bun:test"
import { mkdtemp } from "node:fs/promises"
import { tmpdir } from "node:os"
import { hasIndexingPlugin, isIndexingPlugin, normalizePluginName } from "../../../src/detect"
describe("indexing plugin detection", () => {
test("bundles detect module for browser targets", async () => {
const dir = await mkdtemp(`${tmpdir()}/kilo-indexing-detect-`)
const result = await Bun.build({
entrypoints: [new URL("../../../src/detect.ts", import.meta.url).pathname],
minify: true,
outdir: dir,
target: "browser",
})
expect(result.success).toBe(true)
})
test("normalizes supported plugin forms", () => {
expect(normalizePluginName("kilo-indexing")).toBe("kilo-indexing")
expect(normalizePluginName("kilo-indexing@1.2.3")).toBe("kilo-indexing")
expect(normalizePluginName("@kilocode/kilo-indexing")).toBe("@kilocode/kilo-indexing")
expect(normalizePluginName("@kilocode/kilo-indexing@1.2.3")).toBe("@kilocode/kilo-indexing")
expect(normalizePluginName("../../packages/kilo-indexing")).toBe("@kilocode/kilo-indexing")
expect(normalizePluginName("file:///tmp/.opencode/plugin/kilo-indexing.js")).toBe("kilo-indexing")
expect(normalizePluginName("file:///tmp/node_modules/@kilocode/kilo-indexing/index.js")).toBe(
"@kilocode/kilo-indexing",
)
expect(normalizePluginName("file:///tmp/repo/packages/kilo-indexing/src/index.ts")).toBe("@kilocode/kilo-indexing")
})
test("detects supported indexing plugin specifiers", () => {
const values = [
"kilo-indexing",
"kilo-indexing@1.2.3",
"@kilocode/kilo-indexing",
"@kilocode/kilo-indexing@1.2.3",
"../../packages/kilo-indexing",
"file:///tmp/.opencode/plugin/kilo-indexing.js",
"file:///tmp/node_modules/@kilocode/kilo-indexing/index.js",
"file:///tmp/repo/packages/kilo-indexing/src/index.ts",
]
for (const value of values) {
expect(isIndexingPlugin(value)).toBe(true)
}
})
test("ignores unrelated plugin specifiers", () => {
expect(isIndexingPlugin("@kilocode/kilo-gateway")).toBe(false)
expect(isIndexingPlugin("file:///tmp/.opencode/plugin/index.js")).toBe(false)
expect(hasIndexingPlugin(["@kilocode/kilo-gateway", "foo@1.0.0"])).toBe(false)
})
test("detects indexing plugin in merged plugin lists", () => {
expect(
hasIndexingPlugin(["@kilocode/kilo-gateway", "file:///tmp/node_modules/@kilocode/kilo-indexing/index.js"]),
).toBe(true)
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/__helpers__/openai-mock.ts
+33
View File
@@ -0,0 +1,33 @@
// Shared mock state for the "openai" module used across multiple embedder test files.
// RATIONALE: mock.module() is process-wide in Bun - multiple test files calling
// mock.module("openai", ...) with separate mock functions causes cross-test interference.
// By sharing the mock function, whichever file's mock.module call wins, all tests
// still reference the same mockEmbeddingsCreate instance.
//
// Each test file must still call mock.module("openai", openAIMockFactory) directly
// because Bun only processes mock.module calls in the test file itself (not in imports).
import { mock } from "bun:test"
export const mockEmbeddingsCreate = mock()
let _constructorHook: ((config: any) => void) | undefined
export function setOpenAIConstructorHook(hook: ((config: any) => void) | undefined) {
_constructorHook = hook
}
export function openAIMockFactory() {
return {
OpenAI: class {
config: any
embeddings = { create: mockEmbeddingsCreate }
constructor(config: any) {
this.config = config
if (_constructorHook) {
_constructorHook(config)
}
}
},
}
}
packages/kilo-indexing/test/kilocode/indexing/embedders/bedrock.test.ts
+579
View File
@@ -0,0 +1,579 @@
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test"
// Set up AWS SDK mocks BEFORE importing modules that use them
const mockSend = mock()
mock.module("@aws-sdk/client-bedrock-runtime", () => ({
BedrockRuntimeClient: class {
send = mockSend
},
InvokeModelCommand: class {
constructor(public input: any) {}
},
}))
const mockFromEnv = mock(() => Promise.resolve({}))
const mockFromIni = mock(() => Promise.resolve({}))
mock.module("@aws-sdk/credential-provider-ini", () => ({
fromIni: mockFromIni,
}))
// Now import the module under test
import { BedrockEmbedder } from "../../../../src/indexing/embedders/bedrock"
import { MAX_ITEM_TOKENS } from "../../../../src/indexing/constants"
describe("BedrockEmbedder", () => {
let embedder: BedrockEmbedder
beforeEach(() => {
mockSend.mockReset()
embedder = new BedrockEmbedder("us-east-1", "test-profile", "amazon.titan-embed-text-v2:0")
})
describe("constructor", () => {
test("should initialize with provided region, profile and model", () => {
expect(embedder.embedderInfo.name).toBe("bedrock")
})
test("should require region", () => {
expect(() => new BedrockEmbedder("", "profile", "model")).toThrow("Region is required for AWS Bedrock embedder")
})
test("should use profile for credentials", () => {
mockFromEnv.mockReset()
mockFromIni.mockReset()
const inst = new BedrockEmbedder("us-west-2", "dev-profile")
expect(inst).toBeDefined()
expect(mockFromIni).toHaveBeenCalledWith({ profile: "dev-profile" })
expect(mockFromEnv).not.toHaveBeenCalled()
})
test("should use default credential chain when profile is not provided", () => {
mockFromEnv.mockReset()
mockFromIni.mockReset()
const inst = new BedrockEmbedder("us-west-2")
expect(inst).toBeDefined()
expect(mockFromIni).not.toHaveBeenCalled()
expect(mockFromEnv).not.toHaveBeenCalled()
})
})
describe("createEmbeddings", () => {
const testModelId = "amazon.titan-embed-text-v2:0"
test("should create embeddings for a single text with Titan model", async () => {
const testTexts = ["Hello world"]
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 2,
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(mockSend).toHaveBeenCalled()
const command = mockSend.mock.calls[0][0] as any
expect(command.input.modelId).toBe(testModelId)
const bodyStr =
typeof command.input.body === "string"
? command.input.body
: new TextDecoder().decode(command.input.body as Uint8Array)
expect(JSON.parse(bodyStr || "{}")).toEqual({
inputText: "Hello world",
})
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 2, totalTokens: 2 },
})
})
test("should create embeddings for multiple texts", async () => {
const testTexts = ["Hello world", "Another text"]
const mockResponses = [
{
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 2,
}),
),
},
{
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.4, 0.5, 0.6],
inputTextTokenCount: 3,
}),
),
},
]
mockSend.mockResolvedValueOnce(mockResponses[0]).mockResolvedValueOnce(mockResponses[1])
const result = await embedder.createEmbeddings(testTexts)
expect(mockSend).toHaveBeenCalledTimes(2)
expect(result).toEqual({
embeddings: [
[0.1, 0.2, 0.3],
[0.4, 0.5, 0.6],
],
usage: { promptTokens: 5, totalTokens: 5 },
})
})
test("should handle Cohere model format", async () => {
const cohereEmbedder = new BedrockEmbedder("us-east-1", "test-profile", "cohere.embed-english-v3")
const testTexts = ["Hello world"]
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embeddings: [[0.1, 0.2, 0.3]],
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await cohereEmbedder.createEmbeddings(testTexts)
const command = mockSend.mock.calls[0][0] as any
const bodyStr =
typeof command.input.body === "string"
? command.input.body
: new TextDecoder().decode(command.input.body as Uint8Array)
expect(JSON.parse(bodyStr || "{}")).toEqual({
texts: ["Hello world"],
input_type: "search_document",
})
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 0, totalTokens: 0 },
})
})
test("should create embeddings with Nova multimodal model", async () => {
const novaMultimodalEmbedder = new BedrockEmbedder(
"us-east-1",
"test-profile",
"amazon.nova-2-multimodal-embeddings-v1:0",
)
const testTexts = ["Hello world"]
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embeddings: [
{
embedding: [0.1, 0.2, 0.3],
},
],
inputTextTokenCount: 2,
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await novaMultimodalEmbedder.createEmbeddings(testTexts)
expect(mockSend).toHaveBeenCalled()
const command = mockSend.mock.calls[0][0] as any
expect(command.input.modelId).toBe("amazon.nova-2-multimodal-embeddings-v1:0")
const bodyStr =
typeof command.input.body === "string"
? command.input.body
: new TextDecoder().decode(command.input.body as Uint8Array)
expect(JSON.parse(bodyStr || "{}")).toEqual({
taskType: "SINGLE_EMBEDDING",
singleEmbeddingParams: {
embeddingPurpose: "GENERIC_INDEX",
embeddingDimension: 1024,
text: {
truncationMode: "END",
value: "Hello world",
},
},
})
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 2, totalTokens: 2 },
})
})
test("should handle Nova multimodal model with multiple texts", async () => {
const novaMultimodalEmbedder = new BedrockEmbedder(
"us-east-1",
"test-profile",
"amazon.nova-2-multimodal-embeddings-v1:0",
)
const testTexts = ["Hello world", "Another text"]
const mockResponses = [
{
body: new TextEncoder().encode(
JSON.stringify({
embeddings: [
{
embedding: [0.1, 0.2, 0.3],
},
],
inputTextTokenCount: 2,
}),
),
},
{
body: new TextEncoder().encode(
JSON.stringify({
embeddings: [
{
embedding: [0.4, 0.5, 0.6],
},
],
inputTextTokenCount: 3,
}),
),
},
]
mockSend.mockResolvedValueOnce(mockResponses[0]).mockResolvedValueOnce(mockResponses[1])
const result = await novaMultimodalEmbedder.createEmbeddings(testTexts)
expect(mockSend).toHaveBeenCalledTimes(2)
// Verify the request format for both texts
const firstCommand = mockSend.mock.calls[0][0] as any
const firstBodyStr =
typeof firstCommand.input.body === "string"
? firstCommand.input.body
: new TextDecoder().decode(firstCommand.input.body as Uint8Array)
expect(JSON.parse(firstBodyStr || "{}")).toEqual({
taskType: "SINGLE_EMBEDDING",
singleEmbeddingParams: {
embeddingPurpose: "GENERIC_INDEX",
embeddingDimension: 1024,
text: {
truncationMode: "END",
value: "Hello world",
},
},
})
const secondCommand = mockSend.mock.calls[1][0] as any
const secondBodyStr =
typeof secondCommand.input.body === "string"
? secondCommand.input.body
: new TextDecoder().decode(secondCommand.input.body as Uint8Array)
expect(JSON.parse(secondBodyStr || "{}")).toEqual({
taskType: "SINGLE_EMBEDDING",
singleEmbeddingParams: {
embeddingPurpose: "GENERIC_INDEX",
embeddingDimension: 1024,
text: {
truncationMode: "END",
value: "Another text",
},
},
})
expect(result).toEqual({
embeddings: [
[0.1, 0.2, 0.3],
[0.4, 0.5, 0.6],
],
usage: { promptTokens: 5, totalTokens: 5 },
})
})
test("should use custom model when provided", async () => {
const testTexts = ["Hello world"]
const customModel = "amazon.titan-embed-text-v1"
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 2,
}),
),
}
mockSend.mockResolvedValue(mockResponse)
await embedder.createEmbeddings(testTexts, customModel)
const command = mockSend.mock.calls[0][0] as any
expect(command.input.modelId).toBe(customModel)
})
test("should handle missing token count data gracefully", async () => {
const testTexts = ["Hello world"]
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 0, totalTokens: 0 },
})
})
describe("batching logic", () => {
test("should skip texts exceeding maximum token limit", async () => {
// Create a text that exceeds MAX_ITEM_TOKENS (4 characters ~ 1 token)
const oversizedText = "a".repeat(MAX_ITEM_TOKENS * 4 + 100)
const normalText = "normal text"
const testTexts = [normalText, oversizedText, "another normal"]
const mockResponses = [
{
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 3,
}),
),
},
{
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.4, 0.5, 0.6],
inputTextTokenCount: 3,
}),
),
},
]
mockSend.mockResolvedValueOnce(mockResponses[0]).mockResolvedValueOnce(mockResponses[1])
const result = await embedder.createEmbeddings(testTexts)
// Verify only normal texts were processed (oversized skipped)
expect(mockSend).toHaveBeenCalledTimes(2)
expect(result.embeddings).toHaveLength(2)
})
test("should handle all texts being skipped due to size", async () => {
const oversizedText = "a".repeat(MAX_ITEM_TOKENS * 4 + 100)
const testTexts = [oversizedText, oversizedText]
const result = await embedder.createEmbeddings(testTexts)
expect(mockSend).not.toHaveBeenCalled()
expect(result).toEqual({
embeddings: [],
usage: { promptTokens: 0, totalTokens: 0 },
})
})
})
describe("retry logic", () => {
// TODO: bun:test doesn't support fake timers
test.skip("should retry on throttling errors with exponential backoff", async () => {
const testTexts = ["Hello world"]
const throttlingError = new Error("Rate limit exceeded")
throttlingError.name = "ThrottlingException"
mockSend
.mockRejectedValueOnce(throttlingError)
.mockRejectedValueOnce(throttlingError)
.mockResolvedValueOnce({
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 2,
}),
),
})
const result = await embedder.createEmbeddings(testTexts)
expect(mockSend).toHaveBeenCalledTimes(3)
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 2, totalTokens: 2 },
})
})
test("should not retry on non-throttling errors", async () => {
const testTexts = ["Hello world"]
const authError = new Error("Unauthorized")
authError.name = "UnrecognizedClientException"
mockSend.mockRejectedValue(authError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Unauthorized",
)
expect(mockSend).toHaveBeenCalledTimes(1)
})
})
describe("error handling", () => {
test("should handle API errors gracefully", async () => {
const testTexts = ["Hello world"]
const apiError = new Error("API connection failed")
mockSend.mockRejectedValue(apiError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: API connection failed",
)
})
test("should handle empty text arrays", async () => {
const testTexts: string[] = []
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [],
usage: { promptTokens: 0, totalTokens: 0 },
})
expect(mockSend).not.toHaveBeenCalled()
})
test("should handle malformed API responses", async () => {
const testTexts = ["Hello world"]
const malformedResponse = {
body: new TextEncoder().encode("not json"),
}
mockSend.mockResolvedValue(malformedResponse)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow()
})
test("should handle AWS-specific errors", async () => {
const testTexts = ["Hello world"]
// Test UnrecognizedClientException
const authError = new Error("Invalid credentials")
authError.name = "UnrecognizedClientException"
mockSend.mockRejectedValueOnce(authError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Invalid credentials",
)
// Test AccessDeniedException
const accessError = new Error("Access denied")
accessError.name = "AccessDeniedException"
mockSend.mockRejectedValueOnce(accessError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Access denied",
)
// Test ResourceNotFoundException
const notFoundError = new Error("Model not found")
notFoundError.name = "ResourceNotFoundException"
mockSend.mockRejectedValueOnce(notFoundError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Model not found",
)
})
})
})
describe("validateConfiguration", () => {
test("should validate successfully with valid configuration", async () => {
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
embedding: [0.1, 0.2, 0.3],
inputTextTokenCount: 1,
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
expect(mockSend).toHaveBeenCalled()
})
test("should fail validation with authentication error", async () => {
const authError = new Error("Invalid credentials")
authError.name = "UnrecognizedClientException"
mockSend.mockRejectedValue(authError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Invalid AWS credentials for Bedrock")
})
test("should fail validation with access denied error", async () => {
const accessError = new Error("Access denied")
accessError.name = "AccessDeniedException"
mockSend.mockRejectedValue(accessError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Access denied to Bedrock embedding model")
})
test("should fail validation with model not found error", async () => {
const notFoundError = new Error("Model not found")
notFoundError.name = "ResourceNotFoundException"
mockSend.mockRejectedValue(notFoundError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("not found")
})
test("should fail validation with invalid response", async () => {
const mockResponse = {
body: new TextEncoder().encode(
JSON.stringify({
// Missing embedding field
inputTextTokenCount: 1,
}),
),
}
mockSend.mockResolvedValue(mockResponse)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Bedrock returned an invalid response format")
})
test("should fail validation with connection error", async () => {
const connectionError = new Error("ECONNREFUSED")
mockSend.mockRejectedValue(connectionError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Connection failed")
})
test("should fail validation with generic error", async () => {
const genericError = new Error("Unknown error")
mockSend.mockRejectedValue(genericError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Configuration error")
})
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/gemini.test.ts
+134
View File
@@ -0,0 +1,134 @@
import { describe, test, expect, beforeEach, mock } from "bun:test"
import { mockEmbeddingsCreate, openAIMockFactory } from "./__helpers__/openai-mock"
// RATIONALE: Test GeminiEmbedder through the real OpenAICompatibleEmbedder with mocked OpenAI SDK.
// Mocking the openai-compatible module with mock.module() is process-wide in Bun and would
// interfere with openai-compatible.test.ts which needs the real implementation.
mock.module("openai", openAIMockFactory)
import { GeminiEmbedder } from "../../../../src/indexing/embedders/gemini"
describe("GeminiEmbedder", () => {
let embedder: GeminiEmbedder
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
})
describe("constructor", () => {
test("should create an instance with default model", () => {
embedder = new GeminiEmbedder("test-gemini-api-key")
expect(embedder).toBeDefined()
})
test("should create an instance with specified model", () => {
embedder = new GeminiEmbedder("test-gemini-api-key", "text-embedding-004")
expect(embedder).toBeDefined()
})
test("should throw error when API key is not provided", () => {
expect(() => new GeminiEmbedder("")).toThrow("API key is required for Gemini embedder")
expect(() => new GeminiEmbedder(null as any)).toThrow("API key is required for Gemini embedder")
expect(() => new GeminiEmbedder(undefined as any)).toThrow("API key is required for Gemini embedder")
})
})
describe("embedderInfo", () => {
test("should return correct embedder info", () => {
embedder = new GeminiEmbedder("test-api-key")
expect(embedder.embedderInfo).toEqual({
name: "gemini",
})
})
})
describe("createEmbeddings", () => {
test("should use default model when no model parameter provided", async () => {
embedder = new GeminiEmbedder("test-api-key")
const texts = ["test text 1", "test text 2"]
const mockResponse = {
data: [{ embedding: [0.1, 0.2] }, { embedding: [0.3, 0.4] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(texts)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: texts,
model: "gemini-embedding-001",
encoding_format: "base64",
})
expect(result.embeddings).toEqual([
[0.1, 0.2],
[0.3, 0.4],
])
})
test("should use provided model parameter when specified", async () => {
embedder = new GeminiEmbedder("test-api-key", "text-embedding-004")
const texts = ["test text 1"]
const mockResponse = {
data: [{ embedding: [0.5, 0.6] }],
usage: { prompt_tokens: 5, total_tokens: 5 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(texts, "gemini-embedding-001")
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: texts,
model: "gemini-embedding-001",
encoding_format: "base64",
})
expect(result.embeddings).toEqual([[0.5, 0.6]])
})
test("should handle errors from embedding API", async () => {
embedder = new GeminiEmbedder("test-api-key")
const error = new Error("Embedding failed")
mockEmbeddingsCreate.mockRejectedValue(error)
await expect(embedder.createEmbeddings(["test text"])).rejects.toThrow()
})
})
describe("validateConfiguration", () => {
test("should validate successfully with valid configuration", async () => {
embedder = new GeminiEmbedder("test-api-key")
mockEmbeddingsCreate.mockResolvedValue({
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 2, total_tokens: 2 },
})
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
})
test("should fail validation with authentication error", async () => {
embedder = new GeminiEmbedder("test-api-key")
const authError = new Error("Invalid API key")
;(authError as any).status = 401
mockEmbeddingsCreate.mockRejectedValue(authError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Authentication failed. Please check your API key.")
})
test("should handle validation exceptions", async () => {
embedder = new GeminiEmbedder("test-api-key")
const error = new Error("ECONNREFUSED")
mockEmbeddingsCreate.mockRejectedValue(error)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Connection failed")
})
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/mistral.test.ts
+134
View File
@@ -0,0 +1,134 @@
import { describe, test, expect, beforeEach, mock } from "bun:test"
import { mockEmbeddingsCreate, openAIMockFactory } from "./__helpers__/openai-mock"
// RATIONALE: Test MistralEmbedder through the real OpenAICompatibleEmbedder with mocked OpenAI SDK.
// Mocking the openai-compatible module with mock.module() is process-wide in Bun and would
// interfere with openai-compatible.test.ts which needs the real implementation.
mock.module("openai", openAIMockFactory)
import { MistralEmbedder } from "../../../../src/indexing/embedders/mistral"
describe("MistralEmbedder", () => {
let embedder: MistralEmbedder
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
})
describe("constructor", () => {
test("should create an instance with default model", () => {
embedder = new MistralEmbedder("test-mistral-api-key")
expect(embedder).toBeDefined()
})
test("should create an instance with specified model", () => {
embedder = new MistralEmbedder("test-mistral-api-key", "custom-embed-model")
expect(embedder).toBeDefined()
})
test("should throw error when API key is not provided", () => {
expect(() => new MistralEmbedder("")).toThrow("API key is required for Mistral embedder")
expect(() => new MistralEmbedder(null as any)).toThrow("API key is required for Mistral embedder")
expect(() => new MistralEmbedder(undefined as any)).toThrow("API key is required for Mistral embedder")
})
})
describe("embedderInfo", () => {
test("should return correct embedder info", () => {
embedder = new MistralEmbedder("test-api-key")
expect(embedder.embedderInfo).toEqual({
name: "mistral",
})
})
})
describe("createEmbeddings", () => {
test("should use default model when no model parameter provided", async () => {
embedder = new MistralEmbedder("test-api-key")
const texts = ["test text 1", "test text 2"]
const mockResponse = {
data: [{ embedding: [0.1, 0.2] }, { embedding: [0.3, 0.4] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(texts)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: texts,
model: "codestral-embed-2505",
encoding_format: "base64",
})
expect(result.embeddings).toEqual([
[0.1, 0.2],
[0.3, 0.4],
])
})
test("should use provided model parameter when specified", async () => {
embedder = new MistralEmbedder("test-api-key", "custom-embed-model")
const texts = ["test text 1"]
const mockResponse = {
data: [{ embedding: [0.5, 0.6] }],
usage: { prompt_tokens: 5, total_tokens: 5 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(texts, "codestral-embed-2505")
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: texts,
model: "codestral-embed-2505",
encoding_format: "base64",
})
expect(result.embeddings).toEqual([[0.5, 0.6]])
})
test("should handle errors from embedding API", async () => {
embedder = new MistralEmbedder("test-api-key")
const error = new Error("Embedding failed")
mockEmbeddingsCreate.mockRejectedValue(error)
await expect(embedder.createEmbeddings(["test text"])).rejects.toThrow()
})
})
describe("validateConfiguration", () => {
test("should validate successfully with valid configuration", async () => {
embedder = new MistralEmbedder("test-api-key")
mockEmbeddingsCreate.mockResolvedValue({
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 2, total_tokens: 2 },
})
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
})
test("should fail validation with authentication error", async () => {
embedder = new MistralEmbedder("test-api-key")
const authError = new Error("Invalid API key")
;(authError as any).status = 401
mockEmbeddingsCreate.mockRejectedValue(authError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Authentication failed. Please check your API key.")
})
test("should handle validation exceptions", async () => {
embedder = new MistralEmbedder("test-api-key")
const error = new Error("ECONNREFUSED")
mockEmbeddingsCreate.mockRejectedValue(error)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Connection failed")
})
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/ollama.test.ts
+256
View File
@@ -0,0 +1,256 @@
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test"
import { CodeIndexOllamaEmbedder } from "../../../../src/indexing/embedders/ollama"
const mockFetch = mock() as unknown as typeof fetch
global.fetch = mockFetch
describe("CodeIndexOllamaEmbedder", () => {
let embedder: CodeIndexOllamaEmbedder
beforeEach(() => {
;(mockFetch as any).mockReset()
embedder = new CodeIndexOllamaEmbedder("http://localhost:11434", "nomic-embed-text")
})
afterEach(() => {
;(mockFetch as any).mockReset()
})
describe("constructor", () => {
test("should initialize with provided options", () => {
expect(embedder.embedderInfo.name).toBe("ollama")
})
test("should use default values when not provided", () => {
const embedderWithDefaults = new CodeIndexOllamaEmbedder("")
expect(embedderWithDefaults.embedderInfo.name).toBe("ollama")
})
test("should normalize URLs with trailing slashes", async () => {
const embedderWithTrailingSlash = new CodeIndexOllamaEmbedder("http://localhost:11434/", "nomic-embed-text")
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve({ models: [{ name: "nomic-embed-text" }] }),
} as Response),
)
await embedderWithTrailingSlash.validateConfiguration()
expect(mockFetch).toHaveBeenCalledWith(
"http://localhost:11434/api/tags",
expect.objectContaining({
method: "GET",
}),
)
})
test("should not modify URLs without trailing slashes", async () => {
const embedderWithoutTrailingSlash = new CodeIndexOllamaEmbedder("http://localhost:11434", "nomic-embed-text")
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve({ models: [{ name: "nomic-embed-text" }] }),
} as Response),
)
await embedderWithoutTrailingSlash.validateConfiguration()
expect(mockFetch).toHaveBeenCalledWith(
"http://localhost:11434/api/tags",
expect.objectContaining({
method: "GET",
}),
)
})
test("should handle multiple trailing slashes", async () => {
const embedderWithMultipleTrailingSlashes = new CodeIndexOllamaEmbedder(
"http://localhost:11434///",
"nomic-embed-text",
)
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () => Promise.resolve({ models: [{ name: "nomic-embed-text" }] }),
} as Response),
)
await embedderWithMultipleTrailingSlashes.validateConfiguration()
expect(mockFetch).toHaveBeenCalledWith(
"http://localhost:11434/api/tags",
expect.objectContaining({
method: "GET",
}),
)
})
})
describe("validateConfiguration", () => {
test("should validate successfully when service is available and model exists", async () => {
// Mock successful /api/tags call
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () =>
Promise.resolve({
models: [{ name: "nomic-embed-text:latest" }, { name: "llama2:latest" }],
}),
} as Response),
)
// Mock successful /api/embed test call
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () =>
Promise.resolve({
embeddings: [[0.1, 0.2, 0.3]],
}),
} as Response),
)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
expect(mockFetch).toHaveBeenCalledTimes(2)
// Check first call (GET /api/tags)
const firstCall = (mockFetch as any).mock.calls[0]
expect(firstCall[0]).toBe("http://localhost:11434/api/tags")
expect(firstCall[1]?.method).toBe("GET")
expect(firstCall[1]?.headers).toEqual({ "Content-Type": "application/json" })
expect(firstCall[1]?.signal).toBeDefined()
// Check second call (POST /api/embed)
const secondCall = (mockFetch as any).mock.calls[1]
expect(secondCall[0]).toBe("http://localhost:11434/api/embed")
expect(secondCall[1]?.method).toBe("POST")
expect(secondCall[1]?.headers).toEqual({ "Content-Type": "application/json" })
expect(secondCall[1]?.body).toBe(JSON.stringify({ model: "nomic-embed-text", input: ["test"] }))
expect(secondCall[1]?.signal).toBeDefined()
})
test("should fail validation when service is not available", async () => {
;(mockFetch as any).mockRejectedValueOnce(new Error("ECONNREFUSED"))
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Ollama service is not running")
})
test("should fail validation when tags endpoint returns 404", async () => {
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: false,
status: 404,
} as Response),
)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Ollama service is not running")
})
test("should fail validation when tags endpoint returns other error", async () => {
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: false,
status: 500,
} as Response),
)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Ollama service unavailable")
})
test("should fail validation when model does not exist", async () => {
// Mock successful /api/tags call with different models
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () =>
Promise.resolve({
models: [{ name: "llama2:latest" }, { name: "mistral:latest" }],
}),
} as Response),
)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("not found")
})
test("should fail validation when model exists but doesn't support embeddings", async () => {
// Mock successful /api/tags call
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: true,
status: 200,
json: () =>
Promise.resolve({
models: [{ name: "nomic-embed-text" }],
}),
} as Response),
)
// Mock failed /api/embed test call
;(mockFetch as any).mockImplementationOnce(() =>
Promise.resolve({
ok: false,
status: 400,
} as Response),
)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("not capable of generating embeddings")
})
test("should handle ECONNREFUSED errors", async () => {
;(mockFetch as any).mockRejectedValueOnce(new Error("ECONNREFUSED"))
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Ollama service is not running")
})
test("should handle ENOTFOUND errors", async () => {
;(mockFetch as any).mockRejectedValueOnce(new Error("ENOTFOUND"))
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toContain("Ollama host not found")
})
test("should handle generic network errors", async () => {
;(mockFetch as any).mockRejectedValueOnce(new Error("Network timeout"))
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Network timeout")
})
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/openai-compatible-rate-limit.test.ts
+153
View File
@@ -0,0 +1,153 @@
// TODO: These tests require fake timers (vitest.useFakeTimers) which bun:test doesn't support
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test"
import { mockEmbeddingsCreate, openAIMockFactory } from "./__helpers__/openai-mock"
mock.module("openai", openAIMockFactory)
import { OpenAICompatibleEmbedder } from "../../../../src/indexing/embedders/openai-compatible"
describe.skip("OpenAICompatibleEmbedder - Global Rate Limiting", () => {
const testBaseUrl = "https://api.openai.com/v1"
const testApiKey = "test-api-key"
const testModelId = "text-embedding-3-small"
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
mockEmbeddingsCreate.mockImplementation(() => mockEmbeddingsCreate)
// Reset global rate limit state
const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
;(embedder as any).constructor.globalRateLimitState = {
isRateLimited: false,
rateLimitResetTime: 0,
consecutiveRateLimitErrors: 0,
lastRateLimitError: 0,
mutex: (embedder as any).constructor.globalRateLimitState.mutex,
}
})
afterEach(() => {
mockEmbeddingsCreate.mockReset()
})
test("should apply global rate limiting across multiple batch requests", async () => {
const embedder1 = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const embedder2 = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
// First batch hits rate limit
const rateLimitError = new Error("Rate limit exceeded") as any
rateLimitError.status = 429
mockEmbeddingsCreate
.mockRejectedValueOnce(rateLimitError) // First attempt fails
.mockResolvedValue({
data: [{ embedding: "base64encodeddata" }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
// Start first batch request
const batch1Promise = embedder1.createEmbeddings(["test1"])
// Start second batch request while global rate limit is active
const batch2Promise = embedder2.createEmbeddings(["test2"])
// Check that global rate limit was set
const state = (embedder1 as any).constructor.globalRateLimitState
expect(state.isRateLimited).toBe(true)
expect(state.consecutiveRateLimitErrors).toBe(1)
// Both requests should complete
const [result1, result2] = await Promise.all([batch1Promise, batch2Promise])
expect(result1.embeddings).toHaveLength(1)
expect(result2.embeddings).toHaveLength(1)
})
test("should track consecutive rate limit errors", async () => {
const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const state = (embedder as any).constructor.globalRateLimitState
const rateLimitError = new Error("Rate limit exceeded") as any
rateLimitError.status = 429
// Test that consecutive errors increment when they happen quickly
// Mock multiple rate limit errors in a single request
mockEmbeddingsCreate
.mockRejectedValueOnce(rateLimitError) // First attempt
.mockRejectedValueOnce(rateLimitError) // Retry 1
.mockResolvedValueOnce({
data: [{ embedding: "base64encodeddata" }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
const promise1 = embedder.createEmbeddings(["test1"])
expect(state.consecutiveRateLimitErrors).toBe(1)
await promise1
// Verify the delay increases with consecutive errors
// Make another request immediately that also hits rate limit
mockEmbeddingsCreate.mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({
data: [{ embedding: "base64encodeddata" }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
// Store the current consecutive count before the next request
const previousCount = state.consecutiveRateLimitErrors
const promise2 = embedder.createEmbeddings(["test2"])
// Should have incremented from the previous count
expect(state.consecutiveRateLimitErrors).toBeGreaterThan(previousCount)
// Complete the second request
await promise2
})
test("should reset consecutive error count after time passes", async () => {
const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const state = (embedder as any).constructor.globalRateLimitState
// Manually set state to simulate previous errors
state.consecutiveRateLimitErrors = 3
state.lastRateLimitError = Date.now() - 70000 // 70 seconds ago
const rateLimitError = new Error("Rate limit exceeded") as any
rateLimitError.status = 429
mockEmbeddingsCreate.mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({
data: [{ embedding: "base64encodeddata" }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
// Trigger the updateGlobalRateLimitState method
await (embedder as any).updateGlobalRateLimitState(rateLimitError)
// Should reset to 1 since more than 60 seconds passed
expect(state.consecutiveRateLimitErrors).toBe(1)
})
test("should not exceed maximum delay of 5 minutes", async () => {
const embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const state = (embedder as any).constructor.globalRateLimitState
// Set state to simulate many consecutive errors
state.consecutiveRateLimitErrors = 10 // This would normally result in a very long delay
const rateLimitError = new Error("Rate limit exceeded") as any
rateLimitError.status = 429
// Trigger the updateGlobalRateLimitState method
await (embedder as any).updateGlobalRateLimitState(rateLimitError)
// Calculate the expected delay
const now = Date.now()
const delay = state.rateLimitResetTime - now
// Should be capped at 5 minutes (300000ms)
expect(delay).toBeLessThanOrEqual(300000)
expect(delay).toBeGreaterThan(0)
})
})
packages/kilo-indexing/test/kilocode/indexing/embedders/openai-compatible.test.ts
+912
View File
@@ -0,0 +1,912 @@
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test"
import {
MAX_ITEM_TOKENS,
REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
} from "../../../../src/indexing/constants"
import { mockEmbeddingsCreate, setOpenAIConstructorHook, openAIMockFactory } from "./__helpers__/openai-mock"
mock.module("openai", openAIMockFactory)
import { OpenAICompatibleEmbedder } from "../../../../src/indexing/embedders/openai-compatible"
// Mock global fetch
const mockFetch = mock() as any
global.fetch = mockFetch
describe("OpenAICompatibleEmbedder", () => {
let embedder: OpenAICompatibleEmbedder
const testBaseUrl = "https://api.example.com/v1"
const testApiKey = "test-api-key"
const testModelId = "text-embedding-3-small"
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
mockFetch.mockReset()
setOpenAIConstructorHook(undefined)
// Reset global rate limit state to prevent interference between tests
const tempEmbedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
;(tempEmbedder as any).constructor.globalRateLimitState = {
isRateLimited: false,
rateLimitResetTime: 0,
consecutiveRateLimitErrors: 0,
lastRateLimitError: 0,
mutex: (tempEmbedder as any).constructor.globalRateLimitState.mutex,
}
})
afterEach(() => {
mockEmbeddingsCreate.mockReset()
mockFetch.mockReset()
setOpenAIConstructorHook(undefined)
})
describe("constructor", () => {
test("should create embedder with valid configuration", () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
expect(embedder).toBeDefined()
})
test("should use default model when modelId is not provided", () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey)
expect(embedder).toBeDefined()
})
test("should throw error when baseUrl is missing", () => {
expect(() => new OpenAICompatibleEmbedder("", testApiKey, testModelId)).toThrow(
"Base URL is required for OpenAI-compatible embedder",
)
})
test("should throw error when apiKey is missing", () => {
expect(() => new OpenAICompatibleEmbedder(testBaseUrl, "", testModelId)).toThrow(
"API key is required for OpenAI-compatible embedder",
)
})
test("should throw error when both baseUrl and apiKey are missing", () => {
expect(() => new OpenAICompatibleEmbedder("", "", testModelId)).toThrow(
"Base URL is required for OpenAI-compatible embedder",
)
})
test("should handle API key with invalid characters (ByteString conversion error)", () => {
const invalidApiKey = "sk-test\u2022invalid" // Contains bullet character (U+2022)
const byteStringError = new Error(
"Cannot convert argument to a ByteString because the character at index 7 has a value of 8226 which is greater than 255.",
)
// Make the mock OpenAI constructor throw
setOpenAIConstructorHook(() => {
throw byteStringError
})
expect(() => new OpenAICompatibleEmbedder(testBaseUrl, invalidApiKey, testModelId)).toThrow(
"Cannot convert argument to a ByteString",
)
})
})
describe("embedderInfo", () => {
beforeEach(() => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
})
test("should return correct embedder info", () => {
const info = embedder.embedderInfo
expect(info).toEqual({
name: "openai-compatible",
})
})
})
describe("createEmbeddings", () => {
beforeEach(() => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
})
test("should create embeddings for single text", async () => {
const testTexts = ["Hello world"]
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: testTexts,
model: testModelId,
encoding_format: "base64",
})
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 10, totalTokens: 15 },
})
})
test("should create embeddings for multiple texts", async () => {
const testTexts = ["Hello world", "Goodbye world"]
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }, { embedding: [0.4, 0.5, 0.6] }],
usage: { prompt_tokens: 20, total_tokens: 30 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: testTexts,
model: testModelId,
encoding_format: "base64",
})
expect(result).toEqual({
embeddings: [
[0.1, 0.2, 0.3],
[0.4, 0.5, 0.6],
],
usage: { promptTokens: 20, totalTokens: 30 },
})
})
test("should use custom model when provided", async () => {
const testTexts = ["Hello world"]
const customModel = "custom-embedding-model"
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
await embedder.createEmbeddings(testTexts, customModel)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: testTexts,
model: customModel,
encoding_format: "base64",
})
})
test("should handle missing usage data gracefully", async () => {
const testTexts = ["Hello world"]
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: undefined,
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [[0.1, 0.2, 0.3]],
usage: { promptTokens: 0, totalTokens: 0 },
})
})
/**
* Test base64 conversion logic
*/
describe("base64 conversion", () => {
test("should convert base64 encoded embeddings to float arrays", async () => {
const testTexts = ["Hello world"]
// Create a Float32Array with test values that can be exactly represented in Float32
const testEmbedding = new Float32Array([0.25, 0.5, 0.75, 1.0])
// Convert to base64 string (simulating what OpenAI API returns)
const buffer = Buffer.from(testEmbedding.buffer)
const base64String = buffer.toString("base64")
const mockResponse = {
data: [{ embedding: base64String }], // Base64 string instead of array
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(mockEmbeddingsCreate).toHaveBeenCalledWith({
input: testTexts,
model: testModelId,
encoding_format: "base64",
})
// Verify the base64 string was converted back to the original float array
expect(result).toEqual({
embeddings: [[0.25, 0.5, 0.75, 1.0]],
usage: { promptTokens: 10, totalTokens: 15 },
})
})
test("should handle multiple base64 encoded embeddings", async () => {
const testTexts = ["Hello world", "Goodbye world"]
// Create test embeddings with values that can be exactly represented in Float32
const embedding1 = new Float32Array([0.25, 0.5, 0.75])
const embedding2 = new Float32Array([1.0, 1.25, 1.5])
// Convert to base64 strings
const base64String1 = Buffer.from(embedding1.buffer).toString("base64")
const base64String2 = Buffer.from(embedding2.buffer).toString("base64")
const mockResponse = {
data: [{ embedding: base64String1 }, { embedding: base64String2 }],
usage: { prompt_tokens: 20, total_tokens: 30 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [
[0.25, 0.5, 0.75],
[1.0, 1.25, 1.5],
],
usage: { promptTokens: 20, totalTokens: 30 },
})
})
test("should handle mixed base64 and array embeddings", async () => {
const testTexts = ["Hello world", "Goodbye world"]
// Create one base64 embedding and one regular array (edge case)
const embedding1 = new Float32Array([0.25, 0.5, 0.75])
const base64String1 = Buffer.from(embedding1.buffer).toString("base64")
const mockResponse = {
data: [
{ embedding: base64String1 }, // Base64 string
{ embedding: [1.0, 1.25, 1.5] }, // Regular array
],
usage: { prompt_tokens: 20, total_tokens: 30 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [
[0.25, 0.5, 0.75],
[1.0, 1.25, 1.5],
],
usage: { promptTokens: 20, totalTokens: 30 },
})
})
})
/**
* Test batching logic when texts exceed token limits
*/
describe("batching logic", () => {
test("should process texts in batches", async () => {
// Use normal sized texts that won't be skipped
const testTexts = ["text1", "text2", "text3"]
mockEmbeddingsCreate.mockResolvedValue({
data: [{ embedding: [0.1, 0.2, 0.3] }, { embedding: [0.4, 0.5, 0.6] }, { embedding: [0.7, 0.8, 0.9] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
await embedder.createEmbeddings(testTexts)
// Should be called once for normal texts
expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1)
})
test("should skip texts that exceed MAX_ITEM_TOKENS", async () => {
const normalText = "Hello world"
const oversizedText = "a".repeat(MAX_ITEM_TOKENS * 5) // Exceeds MAX_ITEM_TOKENS
const testTexts = [normalText, oversizedText, normalText]
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }, { embedding: [0.4, 0.5, 0.6] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
await embedder.createEmbeddings(testTexts)
// Should only process normal texts (1 call for 2 normal texts batched together)
expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1)
})
test("should return correct usage statistics", async () => {
const testTexts = ["text1", "text2"]
mockEmbeddingsCreate.mockResolvedValue({
data: [{ embedding: [0.1, 0.2, 0.3] }, { embedding: [0.4, 0.5, 0.6] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
const result = await embedder.createEmbeddings(testTexts)
expect(result.usage).toEqual({
promptTokens: 10,
totalTokens: 15,
})
})
})
/**
* Test retry logic with exponential backoff
*/
describe("retry logic", () => {
// TODO: bun:test doesn't support fake timers
test.skip("should retry on rate limit errors with exponential backoff", async () => {
const testTexts = ["Hello world"]
const rateLimitError = { status: 429, message: "Rate limit exceeded" }
// Create base64 encoded embedding for successful response
const testEmbedding = new Float32Array([0.25, 0.5, 0.75])
const base64String = Buffer.from(testEmbedding.buffer).toString("base64")
mockEmbeddingsCreate
.mockRejectedValueOnce(rateLimitError)
.mockRejectedValueOnce(rateLimitError)
.mockResolvedValueOnce({
data: [{ embedding: base64String }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
const result = await embedder.createEmbeddings(testTexts)
expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(3)
expect(result).toEqual({
embeddings: [[0.25, 0.5, 0.75]],
usage: { promptTokens: 10, totalTokens: 15 },
})
})
test("should not retry on non-rate-limit errors", async () => {
const testTexts = ["Hello world"]
const authError = new Error("Unauthorized")
;(authError as any).status = 401
mockEmbeddingsCreate.mockRejectedValue(authError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Authentication failed. Please check your API key.",
)
expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1)
})
test("should throw error immediately on non-retryable errors", async () => {
const testTexts = ["Hello world"]
const serverError = new Error("Internal server error")
;(serverError as any).status = 500
mockEmbeddingsCreate.mockRejectedValue(serverError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts with status 500: Internal server error",
)
expect(mockEmbeddingsCreate).toHaveBeenCalledTimes(1)
})
})
/**
* Test error handling scenarios
*/
describe("error handling", () => {
test("should handle API errors gracefully", async () => {
const testTexts = ["Hello world"]
const apiError = new Error("API connection failed")
mockEmbeddingsCreate.mockRejectedValue(apiError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: API connection failed",
)
})
test("should handle batch processing errors", async () => {
const testTexts = ["text1", "text2"]
const batchError = new Error("Batch processing failed")
mockEmbeddingsCreate.mockRejectedValue(batchError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Batch processing failed",
)
})
test("should handle empty text arrays", async () => {
const testTexts: string[] = []
const result = await embedder.createEmbeddings(testTexts)
expect(result).toEqual({
embeddings: [],
usage: { promptTokens: 0, totalTokens: 0 },
})
expect(mockEmbeddingsCreate).not.toHaveBeenCalled()
})
test("should handle malformed API responses", async () => {
const testTexts = ["Hello world"]
const malformedResponse = {
data: null,
usage: { prompt_tokens: 10, total_tokens: 15 },
}
mockEmbeddingsCreate.mockResolvedValue(malformedResponse)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow()
})
test("should provide specific authentication error message", async () => {
const testTexts = ["Hello world"]
const authError = new Error("Invalid API key")
;(authError as any).status = 401
mockEmbeddingsCreate.mockRejectedValue(authError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Authentication failed. Please check your API key.",
)
})
test("should provide detailed error message for HTTP errors", async () => {
const testTexts = ["Hello world"]
const httpError = new Error("Bad request")
;(httpError as any).status = 400
mockEmbeddingsCreate.mockRejectedValue(httpError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts with status 400: Bad request",
)
})
test("should handle errors without status codes", async () => {
const testTexts = ["Hello world"]
const networkError = new Error("Network timeout")
mockEmbeddingsCreate.mockRejectedValue(networkError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Network timeout",
)
})
test("should handle errors without message property", async () => {
const testTexts = ["Hello world"]
const weirdError = { toString: () => "Custom error object" }
mockEmbeddingsCreate.mockRejectedValue(weirdError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Custom error object",
)
})
test("should handle completely unknown error types", async () => {
const testTexts = ["Hello world"]
const unknownError = null
mockEmbeddingsCreate.mockRejectedValue(unknownError)
await expect(embedder.createEmbeddings(testTexts)).rejects.toThrow(
"Embedding request failed after 3 attempts: Unknown error",
)
})
})
/**
* Test to confirm OpenAI package bug with base64 encoding.
* This test verifies that when we request encoding_format: "base64",
* the OpenAI package returns unparsed base64 strings as expected.
*/
describe("OpenAI package base64 behavior verification", () => {
// TODO: bun:test doesn't support importActual
test.skip("should return unparsed base64 when encoding_format is base64", async () => {
// This test requires vi.importActual("openai") which is not available in bun:test
})
})
/**
* Test Azure OpenAI compatibility with helper functions for conciseness
*/
describe("Azure OpenAI compatibility", () => {
const azureUrl =
"https://myresource.openai.azure.com/openai/deployments/mymodel/embeddings?api-version=2024-02-01"
const baseUrl = "https://api.openai.com/v1"
// Helper to create mock fetch response
const createMockResponse = (data: any, status = 200, ok = true) => ({
ok,
status,
json: mock().mockResolvedValue(data),
text: mock().mockResolvedValue(status === 200 ? "" : "Error message"),
})
// Helper to create base64 embedding
const createBase64Embedding = (values: number[]) => {
const embedding = new Float32Array(values)
return Buffer.from(embedding.buffer).toString("base64")
}
// Helper to verify embedding values with floating-point tolerance
const expectEmbeddingValues = (actual: number[], expected: number[]) => {
expect(actual).toHaveLength(expected.length)
expected.forEach((val, i) => expect(actual[i]).toBeCloseTo(val, 5))
}
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
mockFetch.mockReset()
})
describe("URL detection", () => {
test.each([
["https://myresource.openai.azure.com/openai/deployments/mymodel/embeddings?api-version=2024-02-01", true],
["https://myresource.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings", true],
["https://api.openai.com/v1", false],
["https://api.example.com", false],
["http://localhost:8080", false],
])("should detect URL type correctly: %s -> %s", (url, expected) => {
const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId)
const isFullUrl = (embedder as any).isFullEndpointUrl(url)
expect(isFullUrl).toBe(expected)
})
// Edge cases where 'embeddings' or 'deployments' appear in non-endpoint contexts
test("should return false for URLs with 'embeddings' in non-endpoint contexts", () => {
const testUrls = [
"https://api.example.com/embeddings-service/v1",
"https://embeddings.example.com/api",
"https://api.example.com/v1/embeddings-api",
"https://my-embeddings-provider.com/v1",
]
testUrls.forEach((url) => {
const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId)
const isFullUrl = (embedder as any).isFullEndpointUrl(url)
expect(isFullUrl).toBe(false)
})
})
test("should return false for URLs with 'deployments' in non-endpoint contexts", () => {
const testUrls = [
"https://deployments.example.com/api",
"https://api.deployments.com/v1",
"https://my-deployments-service.com/api/v1",
"https://deployments-manager.example.com",
]
testUrls.forEach((url) => {
const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId)
const isFullUrl = (embedder as any).isFullEndpointUrl(url)
expect(isFullUrl).toBe(false)
})
})
test("should correctly identify actual endpoint URLs", () => {
const endpointUrls = [
"https://api.example.com/v1/embeddings",
"https://api.example.com/v1/embeddings?api-version=2024",
"https://myresource.openai.azure.com/openai/deployments/mymodel/embeddings",
"https://api.example.com/embed",
"https://api.example.com/embed?version=1",
]
endpointUrls.forEach((url) => {
const embedder = new OpenAICompatibleEmbedder(url, testApiKey, testModelId)
const isFullUrl = (embedder as any).isFullEndpointUrl(url)
expect(isFullUrl).toBe(true)
})
})
})
describe("direct HTTP requests", () => {
test("should use direct fetch for Azure URLs and SDK for base URLs", async () => {
const testTexts = ["Test text"]
const base64String = createBase64Embedding([0.1, 0.2, 0.3])
// Test Azure URL (direct fetch)
const azureEmbedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId)
const mockFetchResponse = createMockResponse({
data: [{ embedding: base64String }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
mockFetch.mockResolvedValue(mockFetchResponse as any)
const azureResult = await azureEmbedder.createEmbeddings(testTexts)
expect(mockFetch).toHaveBeenCalledWith(
azureUrl,
expect.objectContaining({
method: "POST",
headers: expect.objectContaining({
"api-key": testApiKey,
Authorization: `Bearer ${testApiKey}`,
}),
}),
)
expect(mockEmbeddingsCreate).not.toHaveBeenCalled()
expectEmbeddingValues(azureResult.embeddings[0], [0.1, 0.2, 0.3])
// Reset and test base URL (SDK)
mockEmbeddingsCreate.mockReset()
mockFetch.mockReset()
const baseEmbedder = new OpenAICompatibleEmbedder(baseUrl, testApiKey, testModelId)
mockEmbeddingsCreate.mockResolvedValue({
data: [{ embedding: [0.4, 0.5, 0.6] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
const baseResult = await baseEmbedder.createEmbeddings(testTexts)
expect(mockEmbeddingsCreate).toHaveBeenCalled()
expect(mockFetch).not.toHaveBeenCalled()
expect(baseResult.embeddings[0]).toEqual([0.4, 0.5, 0.6])
})
test.each([
[401, "Authentication failed. Please check your API key."],
[500, "Embedding request failed after 3 attempts"],
])("should handle HTTP errors: %d", async (status, expectedMessage) => {
const embedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId)
const mockResponse = createMockResponse({}, status, false)
mockFetch.mockResolvedValue(mockResponse as any)
await expect(embedder.createEmbeddings(["test"])).rejects.toThrow(expectedMessage)
})
// TODO: bun:test doesn't support fake timers
test.skip("should handle rate limiting with retries", async () => {
const embedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId)
const base64String = createBase64Embedding([0.1, 0.2, 0.3])
mockFetch
.mockResolvedValueOnce(createMockResponse({}, 429, false) as any)
.mockResolvedValueOnce(createMockResponse({}, 429, false) as any)
.mockResolvedValueOnce(
createMockResponse({
data: [{ embedding: base64String }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}) as any,
)
const result = await embedder.createEmbeddings(["test"])
expect(mockFetch).toHaveBeenCalledTimes(3)
expectEmbeddingValues(result.embeddings[0], [0.1, 0.2, 0.3])
})
test("should handle multiple embeddings and network errors", async () => {
const embedder = new OpenAICompatibleEmbedder(azureUrl, testApiKey, testModelId)
// Test multiple embeddings
const base64_1 = createBase64Embedding([0.25, 0.5])
const base64_2 = createBase64Embedding([0.75, 1.0])
const mockResponse = createMockResponse({
data: [{ embedding: base64_1 }, { embedding: base64_2 }],
usage: { prompt_tokens: 20, total_tokens: 30 },
})
mockFetch.mockResolvedValue(mockResponse as any)
const result = await embedder.createEmbeddings(["test1", "test2"])
expect(result.embeddings).toHaveLength(2)
expectEmbeddingValues(result.embeddings[0], [0.25, 0.5])
expectEmbeddingValues(result.embeddings[1], [0.75, 1.0])
// Test network error
const networkError = new Error("Network failed")
mockFetch.mockRejectedValue(networkError)
await expect(embedder.createEmbeddings(["test"])).rejects.toThrow("Embedding request failed after 3 attempts")
})
})
})
})
describe("URL detection", () => {
test("should detect Azure deployment URLs as full endpoints", async () => {
const embedder = new OpenAICompatibleEmbedder(
"https://myinstance.openai.azure.com/openai/deployments/my-deployment/embeddings?api-version=2023-05-15",
"test-key",
)
// The private method is tested indirectly through the createEmbeddings behavior
// If it's detected as a full URL, it will make a direct HTTP request
const localMockFetch = mock().mockResolvedValue({
ok: true,
json: async () => ({
data: [{ embedding: [0.1, 0.2] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}),
})
global.fetch = localMockFetch as any
await embedder.createEmbeddings(["test"])
// Should make direct HTTP request to the full URL
expect(localMockFetch).toHaveBeenCalledWith(
"https://myinstance.openai.azure.com/openai/deployments/my-deployment/embeddings?api-version=2023-05-15",
expect.any(Object),
)
// Restore the module-level mock
global.fetch = mockFetch
})
test("should detect /embed endpoints as full URLs", async () => {
const embedder = new OpenAICompatibleEmbedder("https://api.example.com/v1/embed", "test-key")
const localMockFetch = mock().mockResolvedValue({
ok: true,
json: async () => ({
data: [{ embedding: [0.1, 0.2] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
}),
})
global.fetch = localMockFetch as any
await embedder.createEmbeddings(["test"])
// Should make direct HTTP request to the full URL
expect(localMockFetch).toHaveBeenCalledWith("https://api.example.com/v1/embed", expect.any(Object))
// Restore the module-level mock
global.fetch = mockFetch
})
test("should treat base URLs without endpoint patterns as SDK URLs", async () => {
const embedder = new OpenAICompatibleEmbedder("https://api.openai.com/v1", "test-key")
// Mock the OpenAI SDK's embeddings.create method
const localMockCreate = mock().mockResolvedValue({
data: [{ embedding: [0.1, 0.2] }],
usage: { prompt_tokens: 10, total_tokens: 15 },
})
embedder["embeddingsClient"].embeddings = {
create: localMockCreate,
} as any
await embedder.createEmbeddings(["test"])
// Should use SDK which will append /embeddings
expect(localMockCreate).toHaveBeenCalled()
})
})
describe("validateConfiguration", () => {
let embedder: OpenAICompatibleEmbedder
beforeEach(() => {
mockEmbeddingsCreate.mockReset()
mockFetch.mockReset()
})
test("should validate successfully with valid configuration and base URL", async () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const mockResponse = {
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 2, total_tokens: 2 },
}
mockEmbeddingsCreate.mockResolvedValue(mockResponse)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
expect(mockEmbeddingsCreate).toHaveBeenCalledWith(
{
input: ["test"],
model: testModelId,
encoding_format: "base64",
},
{
timeout: REMOTE_EMBEDDER_VALIDATION_TIMEOUT_MS,
maxRetries: REMOTE_EMBEDDER_VALIDATION_MAX_RETRIES,
},
)
})
test("should validate successfully with full endpoint URL", async () => {
const fullUrl = "https://api.example.com/v1/embeddings"
embedder = new OpenAICompatibleEmbedder(fullUrl, testApiKey, testModelId)
mockFetch.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({
data: [{ embedding: [0.1, 0.2, 0.3] }],
usage: { prompt_tokens: 2, total_tokens: 2 },
}),
text: async () => "",
} as any)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(true)
expect(result.error).toBeUndefined()
expect(mockFetch).toHaveBeenCalledWith(
fullUrl,
expect.objectContaining({
method: "POST",
headers: expect.objectContaining({
Authorization: `Bearer ${testApiKey}`,
}),
}),
)
})
test("should fail validation with authentication error", async () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const authError = new Error("Invalid API key")
;(authError as any).status = 401
mockEmbeddingsCreate.mockRejectedValue(authError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Authentication failed. Please check your API key.")
})
test("should fail validation with connection error", async () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const connectionError = new Error("ECONNREFUSED")
mockEmbeddingsCreate.mockRejectedValue(connectionError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Connection failed. Please check the endpoint URL and network connectivity.")
})
test("should fail validation with invalid endpoint for full URL", async () => {
const fullUrl = "https://api.example.com/v1/embeddings"
embedder = new OpenAICompatibleEmbedder(fullUrl, testApiKey, testModelId)
mockFetch.mockResolvedValueOnce({
ok: false,
status: 404,
json: async () => ({ error: "Not found" }),
text: async () => "Not found",
} as any)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
// 404 for non-openai embedder returns "Invalid endpoint URL. Please verify the endpoint."
expect(result.error).toBe("Invalid endpoint URL. Please verify the endpoint.")
})
test("should fail validation with rate limit error", async () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const rateLimitError = new Error("Rate limit exceeded")
;(rateLimitError as any).status = 429
mockEmbeddingsCreate.mockRejectedValue(rateLimitError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Service is temporarily unavailable due to rate limiting. Please try again later.")
})
test("should fail validation with generic error", async () => {
embedder = new OpenAICompatibleEmbedder(testBaseUrl, testApiKey, testModelId)
const genericError = new Error("Unknown error")
;(genericError as any).status = 500
mockEmbeddingsCreate.mockRejectedValue(genericError)
const result = await embedder.validateConfiguration()
expect(result.valid).toBe(false)
expect(result.error).toBe("Configuration error. Please verify your embedder settings.")
})
})
})

Some files were not shown because too many files have changed in this diff Show More