refactor(codemode): simplify tool instructions

This commit is contained in:
Aiden Cline
2026-07-24 00:57:28 -05:00
parent d811cba574
commit ef30690016
6 changed files with 92 additions and 189 deletions
-35
View File
@@ -1,35 +0,0 @@
import { searchSignature } from "./packages/codemode/src"
import { CodeModeCatalog } from "./packages/core/src/codemode/catalog"
const prompt = (catalog: CodeModeCatalog.Summary, hasMoreTools: boolean) => {
return `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`.
Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`.
Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.<namespace>["tool-name"](input)\`.${hasMoreTools ? `
## Search
Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools:
- ${searchSignature}` : ""}
## Available tools`
}
export function render(catalog: CodeModeCatalog.Summary) {
const hasMoreTools = true
const tools = catalog.namespaces.flatMap((namespace) => {
const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools`
const label = (() => {
if (namespace.entries.length === namespace.count) return count
if (namespace.entries.length === 0) return `${count}, none shown`
return `${count}, ${namespace.entries.length} shown`
})()
return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)]
})
return `${prompt(catalog, hasMoreTools)}
${tools.join("\n")}`
}
+6 -2
View File
@@ -463,7 +463,7 @@ const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>): Tool<R> =>
const node = lookup(root, segments)
if (node === undefined) {
throw new ToolRuntimeError("UnknownTool", `Unknown tool '${segments.join(".")}'.`, [
"Use search({ query }) to find available described tools.",
"The tool may have been removed or renamed. Use search to find available tools.",
])
}
if (node.tool === undefined) {
@@ -536,7 +536,11 @@ export const make = <R>(
const input = yield* Effect.try({
try: () => decodeToolInput(tool, externalArgs[0]),
catch: (cause) =>
new ToolRuntimeError("InvalidToolInput", `Invalid input for tool '${name}': ${String(cause)}`),
new ToolRuntimeError(
"InvalidToolInput",
`Invalid input for tool '${name}': ${String(cause)}`,
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
),
})
const index = yield* recordAndObserve(name, input)
return yield* observeEnd(
+28
View File
@@ -85,6 +85,9 @@ describe("callable namespaces", () => {
const diagnostic = await failure(runtime, `return await tools.issues.missing({})`)
expect(diagnostic.kind).toBe("UnknownTool")
expect(diagnostic.message).toContain("Unknown tool 'issues.missing'")
expect(diagnostic.suggestions).toEqual([
"The tool may have been removed or renamed. Use search to find available tools.",
])
})
test("a namespace without its own tool stays non-callable", async () => {
@@ -95,6 +98,31 @@ describe("callable namespaces", () => {
})
})
describe("tool input diagnostics", () => {
const runtime = CodeMode.make({
tools: {
"notes.echo": Tool.make({
description: "Echo text",
input: Schema.Struct({ text: Schema.String }),
output: Schema.String,
execute: ({ text }) => Effect.succeed(text),
}),
},
})
test("a schema mismatch suggests searching for the current signature", async () => {
const diagnostic = await failure(runtime, `return await tools.notes.echo({ message: "hello" })`)
expect(diagnostic.kind).toBe("InvalidToolInput")
expect(diagnostic.suggestions).toEqual(["The signature may have changed. Use search to get the current signature."])
})
test("a wrong argument count keeps the existing error without a stale-signature hint", async () => {
const diagnostic = await failure(runtime, `return await tools.notes.echo()`)
expect(diagnostic.kind).toBe("InvalidToolInput")
expect(diagnostic.suggestions).toBeUndefined()
})
})
describe("blocked member names on tool paths", () => {
const runtime = CodeMode.make({
tools: {
+30 -93
View File
@@ -8,106 +8,44 @@ import { CodeMode } from "../codemode"
import { Instructions } from "../instructions/index"
import { CodeModeCatalog } from "./catalog"
const completeWorkflow = `## Workflow
// prettier-ignore
const prompt = (hasMoreTools: boolean) => `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`.
1. Pick a tool from the list under \`## Available tools\` - each line is the exact call signature; use it as-is rather than guessing segments.
2. Call it using the exact signature shown: \`const result = await tools.<namespace>.<tool>(input)\`; bracket notation and quotes are part of the path.
3. Return only the fields you need from structured results; narrow unknown results before reading fields, and avoid returning large raw payloads.`
Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`.
const partialWorkflow = `## Workflow
Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.<namespace>["tool-name"](input)\`.${hasMoreTools ? `
1. If needed, discover tools with the built-in search function: \`return search({ query: "<intent + key nouns>" })\`.
2. In the next execution, copy a returned path exactly, call it, and return only the needed fields.`
## Search
const language = `## Language
Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools:
Use common JavaScript data operations, functions, control flow, selected standard-library methods, and awaited tool calls. Built-ins include Date, RegExp, Map, Set, URL, URLSearchParams, and URI encoding helpers.
Modules/imports, classes, timers, fetch, eval, prototype access, and unlisted methods are unavailable. Use tools for external operations. Use await with try/catch.
Prefer explicit \`return\`; otherwise only the final top-level expression becomes the result.
Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to \`{}\`.`
- ${searchSignature}` : ""}
## Available tools`
export function render(catalog: CodeModeCatalog.Summary) {
const complete = catalog.shown === catalog.total
const sections: Array<string> = []
if (catalog.total === 0) return "No tools are currently available."
if (catalog.total === 0) {
sections.push("This is a restricted JavaScript language for calling tools, not a general-purpose runtime.")
} else {
const availability = (() => {
if (complete) return "listed below"
return "listed or searchable below"
})()
sections.push(
`This is a restricted JavaScript language for calling tools, not a general-purpose runtime. Inside the confined interpreter, \`tools\` contains the tools ${availability}; surrounding agent tools are not available.\nDo not infer or normalize tool names; use only exact signatures shown below or returned by search.`,
)
const tools = catalog.namespaces.flatMap((namespace) => {
const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools`
const label =
namespace.entries.length === namespace.count
? count
: namespace.entries.length === 0
? `${count}, none shown`
: `${count}, ${namespace.entries.length} shown`
return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)]
})
if (complete) sections.push(completeWorkflow)
if (!complete) sections.push(partialWorkflow)
return `${prompt(catalog.shown < catalog.total)}
const availabilityRule = (() => {
if (complete) return "- Only tools listed here are available; surrounding agent tools are not implicitly exposed."
return "- Only tools listed here or returned by the built-in `search` function are available; surrounding agent tools are not implicitly exposed."
})()
const rules = [
"## Rules",
"",
availabilityRule,
"- Filter, aggregate, and transform collections in code - never return them raw or call a tool per item across messages.",
"- A result typed `Promise<unknown>` may be structured data or text. Before reading fields, check that it is a non-null object and not an array; otherwise handle the returned text or primitive directly.",
'- Run independent calls in parallel: `await Promise.all(items.map((item) => tools.<namespace>.<tool>(item)))`, or use `tools.<namespace>["tool-name"](item)` when the listed signature uses bracket notation.',
"- Execution ends when the program returns; pending promises are interrupted, so await every call whose completion matters.",
"- `Object.keys(tools)` lists namespaces; `Object.keys(tools.<namespace>)` lists its tools; `for...in` works on both.",
]
if (!complete) {
rules.push(
'- Browse one namespace: `search({ query: "", namespace: "<name>" })`.',
"- If search returns `next`, repeat the same search with `offset: next.offset`.",
)
}
sections.push(rules.join("\n"))
}
sections.push(language)
if (catalog.total === 0) {
sections.push("No tools are currently available.")
return sections.join("\n\n")
}
const tools: Array<string> = []
if (complete) {
tools.push("## Available tools (COMPLETE list - every tool is shown below with its full call signature)", "")
} else {
tools.push(
`## Available tools (PARTIAL - ${catalog.shown} of ${catalog.total} shown; find the rest with search(...))`,
"",
)
}
for (const namespace of catalog.namespaces) {
const count = (() => {
if (namespace.count === 1) return "1 tool"
return `${namespace.count} tools`
})()
const label = (() => {
if (namespace.entries.length === namespace.count) return count
if (namespace.entries.length === 0) return `${count}, none shown`
return `${count}, ${namespace.entries.length} shown`
})()
tools.push(`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line))
}
if (!complete) tools.push("", "Search returns complete callable signatures:", `- ${searchSignature}`)
sections.push(tools.join("\n"))
return sections.join("\n\n")
${tools.join("\n")}`
}
export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatalog.Summary) {
const full = [
"The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.",
render(current),
].join("\n\n")
const full = `The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.
${render(current)}`
const previousComplete = previous.shown === previous.total
const currentComplete = current.shown === current.total
if (previousComplete !== currentComplete) return full
@@ -162,9 +100,10 @@ export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatal
const parts = ["The Code Mode tool catalog has changed."]
if (diff.added.length > 0) {
parts.push(
["New tools are available in addition to those previously listed:", ...diff.added.map((entry) => entry.line)].join(
"\n",
),
[
"New tools are available in addition to those previously listed:",
...diff.added.map((entry) => entry.line),
].join("\n"),
)
}
if (diff.changed.length > 0) {
@@ -200,9 +139,7 @@ const layer = Layer.effect(
return Service.of({
load: Effect.fn("CodeModeInstructions.load")(function* (selection) {
const entries = selection.info
? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? [])
: []
const entries = selection.info ? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? []) : []
const catalog = CodeModeCatalog.summarize(entries)
return Instructions.make<CodeModeCatalog.Summary>({
key: Instructions.Key.make("core/codemode"),
+26 -58
View File
@@ -22,10 +22,7 @@ const update = (
current: ReadonlyArray<CodeModeCatalog.Entry>,
budget?: number,
) =>
CodeModeInstructions.update(
CodeModeCatalog.summarize(previous, budget),
CodeModeCatalog.summarize(current, budget),
)
CodeModeInstructions.update(CodeModeCatalog.summarize(previous, budget), CodeModeCatalog.summarize(current, budget))
describe("CodeModeCatalog.summarize", () => {
test("retains namespace inventory without retaining tools outside the inline budget", () => {
@@ -63,62 +60,37 @@ describe("CodeModeCatalog.summarize", () => {
})
describe("CodeModeInstructions.render", () => {
test("inlines complete catalogs with markdown sections and placeholder-only call forms", () => {
test("inlines complete catalogs without search guidance", () => {
const instructions = render([lookup])
expect(instructions).toContain("## Available tools (COMPLETE list")
expect(instructions).toContain("## Available tools")
expect(instructions).toContain("- orders (1 tool)")
expect(instructions).toContain(` - ${lookup.signature} // Look up an order by ID`)
expect(instructions).not.toContain("search(")
expect(instructions).toContain("## Workflow")
expect(instructions).toContain("## Rules")
expect(instructions).toContain("## Language")
expect(instructions.indexOf("## Workflow")).toBeLessThan(instructions.indexOf("## Rules"))
expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Language"))
expect(instructions.indexOf("## Language")).toBeLessThan(instructions.indexOf("\n## Available tools (COMPLETE"))
expect(instructions).not.toContain("## Search")
expect(instructions).toContain("Do not infer or normalize tool names")
expect(instructions).toContain("bracket notation and quotes are part of the path")
expect(instructions).toContain("surrounding agent tools are not available")
expect(instructions).toContain("Only tools listed here are available")
expect(instructions).toContain("`const result = await tools.<namespace>.<tool>(input)`")
expect(instructions).toContain("check that it is a non-null object and not an array")
expect(instructions).not.toContain("tools.orders.lookup({")
expect(instructions).toContain("1. Pick a tool from the list under `## Available tools`")
expect(instructions).not.toContain("Browse one namespace")
expect(instructions).toContain('`tools.<namespace>["tool-name"](input)`')
})
test("describes the restricted runtime without overclaiming", () => {
test("describes the runtime and execution lifecycle concisely", () => {
const instructions = render([lookup])
expect(instructions).toContain("restricted JavaScript language for calling tools")
expect(instructions).toContain("not a general-purpose runtime")
for (const missing of ["Modules/imports", "classes", "fetch"]) {
expect(instructions).toContain(missing)
}
// Generators are supported by the interpreter and must not be listed as unavailable.
expect(instructions).not.toContain("generators")
expect(instructions).toContain("URL, URLSearchParams, and URI encoding helpers")
expect(instructions).toContain("Use tools for external operations")
expect(instructions).toContain("Run JavaScript to orchestrate tool calls and compose their results.")
expect(instructions).toContain("Imports, filesystem access, and timers are unavailable.")
expect(instructions).toContain("Do not use `fetch`; all API calls go through `tools`.")
expect(instructions).toContain(
"Prefer explicit `return`; otherwise only the final top-level expression becomes the result.",
)
expect(instructions).toContain(
"Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to `{}`.",
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
)
expect(instructions).toContain("any calls still pending when execution ends are interrupted")
expect(instructions).toContain("Run independent calls concurrently with `Promise.all`.")
})
test("switches to search-first guidance when the catalog exceeds the budget", () => {
test("adds search guidance when the catalog exceeds the budget", () => {
const partial = render([lookup], 0)
expect(partial).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))")
expect(partial).toContain("## Available tools")
expect(partial).toContain("- orders (1 tool, none shown)")
expect(partial).toContain(
'1. If needed, discover tools with the built-in search function: `return search({ query: "<intent + key nouns>" })`.',
)
expect(partial).toContain("In the next execution, copy a returned path exactly")
expect(partial).toContain("Only tools listed here or returned by the built-in `search` function")
expect(partial).toContain('- Browse one namespace: `search({ query: "", namespace: "<name>" })`.')
expect(partial).toContain("repeat the same search with `offset: next.offset`")
expect(partial).toContain("Search returns complete callable signatures:\n- search(input: {")
expect(partial).toContain("## Search")
expect(partial).toContain("Only some tool signatures are shown.")
expect(partial).toContain("- search(input: {")
expect(partial).toContain(" limit?: number,\n offset?: number,")
expect(partial).toContain("or returned by `search`")
expect(partial).not.toContain("tools.orders.lookup(input:")
})
@@ -133,7 +105,7 @@ describe("CodeModeInstructions.render", () => {
// Round 1 places alpha.cheap and beta.cheap; in round 2 alpha.expensive does not fit,
// which marks only alpha done - it must NOT prevent other namespaces from inlining.
const instructions = render([cheapAlpha, expensive, cheapBeta], 40)
expect(instructions).toContain("## Available tools (PARTIAL - 2 of 3 shown; find the rest with search(...))")
expect(instructions).toContain("## Search")
expect(instructions).toContain("- alpha (2 tools, 1 shown)")
expect(instructions).toContain(` - ${cheapAlpha.signature} // Cheap`)
expect(instructions).not.toContain("tools.alpha.expensive(")
@@ -148,18 +120,12 @@ describe("CodeModeInstructions.render", () => {
`tools.records.lookup(input: {\n /** ${"A detailed identifier description. ".repeat(20).trim()} */\n id: string,\n}): Promise<string>`,
)
const instructions = render([documented], 40)
expect(instructions).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))")
expect(instructions).toContain("- records (1 tool, none shown)")
expect(instructions).not.toContain("tools.records.lookup(input:")
})
test("renders the no-tools notice with minimal sections for an empty catalog", () => {
const instructions = render([])
expect(instructions).toContain("No tools are currently available.")
expect(instructions).toContain("## Language")
expect(instructions).not.toContain("## Available tools")
expect(instructions).not.toContain("## Workflow")
expect(instructions).not.toContain("## Rules")
expect(instructions).not.toContain("search(")
test("renders only the no-tools notice for an empty catalog", () => {
expect(render([])).toBe("No tools are currently available.")
})
})
@@ -193,14 +159,16 @@ describe("CodeModeInstructions.update", () => {
expect(text).toContain(
"The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.",
)
expect(text).toContain("## Available tools (PARTIAL")
expect(text).toContain("## Search")
expect(text).toContain("## Available tools")
})
test("falls back to full replacement when the delta is larger than the catalog", () => {
const previous = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
const text = update([...previous, echo], [echo])
expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
expect(text).toContain("## Available tools (COMPLETE list")
expect(text).toContain("## Available tools")
expect(text).not.toContain("## Search")
expect(text).not.toContain("must not be called")
})
@@ -38,7 +38,8 @@ describe("CodeModeInstructions", () => {
return Effect.gen(function* () {
const instructions = yield* CodeModeInstructions.Service
const initialized = yield* instructions.load({ id: agent.id, info: agent }).pipe(Effect.flatMap(readInitial))
expect(initialized.text).toContain("## Available tools (COMPLETE list")
expect(initialized.text).toContain("## Available tools")
expect(initialized.text).not.toContain("## Search")
expect(initialized.text).toContain(` - ${echo.signature} // Echo text`)
catalog = [echo, lookup]