refactor(codemode): simplify tool instructions
This commit is contained in:
@@ -1,35 +0,0 @@
|
||||
import { searchSignature } from "./packages/codemode/src"
|
||||
import { CodeModeCatalog } from "./packages/core/src/codemode/catalog"
|
||||
|
||||
const prompt = (catalog: CodeModeCatalog.Summary, hasMoreTools: boolean) => {
|
||||
return `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`.
|
||||
|
||||
Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`.
|
||||
|
||||
Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.<namespace>["tool-name"](input)\`.${hasMoreTools ? `
|
||||
|
||||
## Search
|
||||
|
||||
Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools:
|
||||
|
||||
- ${searchSignature}` : ""}
|
||||
|
||||
## Available tools`
|
||||
}
|
||||
|
||||
export function render(catalog: CodeModeCatalog.Summary) {
|
||||
const hasMoreTools = true
|
||||
const tools = catalog.namespaces.flatMap((namespace) => {
|
||||
const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools`
|
||||
const label = (() => {
|
||||
if (namespace.entries.length === namespace.count) return count
|
||||
if (namespace.entries.length === 0) return `${count}, none shown`
|
||||
return `${count}, ${namespace.entries.length} shown`
|
||||
})()
|
||||
return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)]
|
||||
})
|
||||
|
||||
return `${prompt(catalog, hasMoreTools)}
|
||||
|
||||
${tools.join("\n")}`
|
||||
}
|
||||
@@ -463,7 +463,7 @@ const resolve = <R>(root: ToolNode<R>, path: ReadonlyArray<string>): Tool<R> =>
|
||||
const node = lookup(root, segments)
|
||||
if (node === undefined) {
|
||||
throw new ToolRuntimeError("UnknownTool", `Unknown tool '${segments.join(".")}'.`, [
|
||||
"Use search({ query }) to find available described tools.",
|
||||
"The tool may have been removed or renamed. Use search to find available tools.",
|
||||
])
|
||||
}
|
||||
if (node.tool === undefined) {
|
||||
@@ -536,7 +536,11 @@ export const make = <R>(
|
||||
const input = yield* Effect.try({
|
||||
try: () => decodeToolInput(tool, externalArgs[0]),
|
||||
catch: (cause) =>
|
||||
new ToolRuntimeError("InvalidToolInput", `Invalid input for tool '${name}': ${String(cause)}`),
|
||||
new ToolRuntimeError(
|
||||
"InvalidToolInput",
|
||||
`Invalid input for tool '${name}': ${String(cause)}`,
|
||||
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
|
||||
),
|
||||
})
|
||||
const index = yield* recordAndObserve(name, input)
|
||||
return yield* observeEnd(
|
||||
|
||||
@@ -85,6 +85,9 @@ describe("callable namespaces", () => {
|
||||
const diagnostic = await failure(runtime, `return await tools.issues.missing({})`)
|
||||
expect(diagnostic.kind).toBe("UnknownTool")
|
||||
expect(diagnostic.message).toContain("Unknown tool 'issues.missing'")
|
||||
expect(diagnostic.suggestions).toEqual([
|
||||
"The tool may have been removed or renamed. Use search to find available tools.",
|
||||
])
|
||||
})
|
||||
|
||||
test("a namespace without its own tool stays non-callable", async () => {
|
||||
@@ -95,6 +98,31 @@ describe("callable namespaces", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("tool input diagnostics", () => {
|
||||
const runtime = CodeMode.make({
|
||||
tools: {
|
||||
"notes.echo": Tool.make({
|
||||
description: "Echo text",
|
||||
input: Schema.Struct({ text: Schema.String }),
|
||||
output: Schema.String,
|
||||
execute: ({ text }) => Effect.succeed(text),
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
test("a schema mismatch suggests searching for the current signature", async () => {
|
||||
const diagnostic = await failure(runtime, `return await tools.notes.echo({ message: "hello" })`)
|
||||
expect(diagnostic.kind).toBe("InvalidToolInput")
|
||||
expect(diagnostic.suggestions).toEqual(["The signature may have changed. Use search to get the current signature."])
|
||||
})
|
||||
|
||||
test("a wrong argument count keeps the existing error without a stale-signature hint", async () => {
|
||||
const diagnostic = await failure(runtime, `return await tools.notes.echo()`)
|
||||
expect(diagnostic.kind).toBe("InvalidToolInput")
|
||||
expect(diagnostic.suggestions).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("blocked member names on tool paths", () => {
|
||||
const runtime = CodeMode.make({
|
||||
tools: {
|
||||
|
||||
@@ -8,106 +8,44 @@ import { CodeMode } from "../codemode"
|
||||
import { Instructions } from "../instructions/index"
|
||||
import { CodeModeCatalog } from "./catalog"
|
||||
|
||||
const completeWorkflow = `## Workflow
|
||||
// prettier-ignore
|
||||
const prompt = (hasMoreTools: boolean) => `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`.
|
||||
|
||||
1. Pick a tool from the list under \`## Available tools\` - each line is the exact call signature; use it as-is rather than guessing segments.
|
||||
2. Call it using the exact signature shown: \`const result = await tools.<namespace>.<tool>(input)\`; bracket notation and quotes are part of the path.
|
||||
3. Return only the fields you need from structured results; narrow unknown results before reading fields, and avoid returning large raw payloads.`
|
||||
Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`.
|
||||
|
||||
const partialWorkflow = `## Workflow
|
||||
Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.<namespace>["tool-name"](input)\`.${hasMoreTools ? `
|
||||
|
||||
1. If needed, discover tools with the built-in search function: \`return search({ query: "<intent + key nouns>" })\`.
|
||||
2. In the next execution, copy a returned path exactly, call it, and return only the needed fields.`
|
||||
## Search
|
||||
|
||||
const language = `## Language
|
||||
Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools:
|
||||
|
||||
Use common JavaScript data operations, functions, control flow, selected standard-library methods, and awaited tool calls. Built-ins include Date, RegExp, Map, Set, URL, URLSearchParams, and URI encoding helpers.
|
||||
Modules/imports, classes, timers, fetch, eval, prototype access, and unlisted methods are unavailable. Use tools for external operations. Use await with try/catch.
|
||||
Prefer explicit \`return\`; otherwise only the final top-level expression becomes the result.
|
||||
Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to \`{}\`.`
|
||||
- ${searchSignature}` : ""}
|
||||
|
||||
## Available tools`
|
||||
|
||||
export function render(catalog: CodeModeCatalog.Summary) {
|
||||
const complete = catalog.shown === catalog.total
|
||||
const sections: Array<string> = []
|
||||
if (catalog.total === 0) return "No tools are currently available."
|
||||
|
||||
if (catalog.total === 0) {
|
||||
sections.push("This is a restricted JavaScript language for calling tools, not a general-purpose runtime.")
|
||||
} else {
|
||||
const availability = (() => {
|
||||
if (complete) return "listed below"
|
||||
return "listed or searchable below"
|
||||
})()
|
||||
sections.push(
|
||||
`This is a restricted JavaScript language for calling tools, not a general-purpose runtime. Inside the confined interpreter, \`tools\` contains the tools ${availability}; surrounding agent tools are not available.\nDo not infer or normalize tool names; use only exact signatures shown below or returned by search.`,
|
||||
)
|
||||
const tools = catalog.namespaces.flatMap((namespace) => {
|
||||
const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools`
|
||||
const label =
|
||||
namespace.entries.length === namespace.count
|
||||
? count
|
||||
: namespace.entries.length === 0
|
||||
? `${count}, none shown`
|
||||
: `${count}, ${namespace.entries.length} shown`
|
||||
return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)]
|
||||
})
|
||||
|
||||
if (complete) sections.push(completeWorkflow)
|
||||
if (!complete) sections.push(partialWorkflow)
|
||||
return `${prompt(catalog.shown < catalog.total)}
|
||||
|
||||
const availabilityRule = (() => {
|
||||
if (complete) return "- Only tools listed here are available; surrounding agent tools are not implicitly exposed."
|
||||
return "- Only tools listed here or returned by the built-in `search` function are available; surrounding agent tools are not implicitly exposed."
|
||||
})()
|
||||
const rules = [
|
||||
"## Rules",
|
||||
"",
|
||||
availabilityRule,
|
||||
"- Filter, aggregate, and transform collections in code - never return them raw or call a tool per item across messages.",
|
||||
"- A result typed `Promise<unknown>` may be structured data or text. Before reading fields, check that it is a non-null object and not an array; otherwise handle the returned text or primitive directly.",
|
||||
'- Run independent calls in parallel: `await Promise.all(items.map((item) => tools.<namespace>.<tool>(item)))`, or use `tools.<namespace>["tool-name"](item)` when the listed signature uses bracket notation.',
|
||||
"- Execution ends when the program returns; pending promises are interrupted, so await every call whose completion matters.",
|
||||
"- `Object.keys(tools)` lists namespaces; `Object.keys(tools.<namespace>)` lists its tools; `for...in` works on both.",
|
||||
]
|
||||
if (!complete) {
|
||||
rules.push(
|
||||
'- Browse one namespace: `search({ query: "", namespace: "<name>" })`.',
|
||||
"- If search returns `next`, repeat the same search with `offset: next.offset`.",
|
||||
)
|
||||
}
|
||||
sections.push(rules.join("\n"))
|
||||
}
|
||||
|
||||
sections.push(language)
|
||||
|
||||
if (catalog.total === 0) {
|
||||
sections.push("No tools are currently available.")
|
||||
return sections.join("\n\n")
|
||||
}
|
||||
|
||||
const tools: Array<string> = []
|
||||
if (complete) {
|
||||
tools.push("## Available tools (COMPLETE list - every tool is shown below with its full call signature)", "")
|
||||
} else {
|
||||
tools.push(
|
||||
`## Available tools (PARTIAL - ${catalog.shown} of ${catalog.total} shown; find the rest with search(...))`,
|
||||
"",
|
||||
)
|
||||
}
|
||||
|
||||
for (const namespace of catalog.namespaces) {
|
||||
const count = (() => {
|
||||
if (namespace.count === 1) return "1 tool"
|
||||
return `${namespace.count} tools`
|
||||
})()
|
||||
const label = (() => {
|
||||
if (namespace.entries.length === namespace.count) return count
|
||||
if (namespace.entries.length === 0) return `${count}, none shown`
|
||||
return `${count}, ${namespace.entries.length} shown`
|
||||
})()
|
||||
tools.push(`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line))
|
||||
}
|
||||
|
||||
if (!complete) tools.push("", "Search returns complete callable signatures:", `- ${searchSignature}`)
|
||||
sections.push(tools.join("\n"))
|
||||
|
||||
return sections.join("\n\n")
|
||||
${tools.join("\n")}`
|
||||
}
|
||||
|
||||
export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatalog.Summary) {
|
||||
const full = [
|
||||
"The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.",
|
||||
render(current),
|
||||
].join("\n\n")
|
||||
const full = `The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.
|
||||
|
||||
${render(current)}`
|
||||
const previousComplete = previous.shown === previous.total
|
||||
const currentComplete = current.shown === current.total
|
||||
if (previousComplete !== currentComplete) return full
|
||||
@@ -162,9 +100,10 @@ export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatal
|
||||
const parts = ["The Code Mode tool catalog has changed."]
|
||||
if (diff.added.length > 0) {
|
||||
parts.push(
|
||||
["New tools are available in addition to those previously listed:", ...diff.added.map((entry) => entry.line)].join(
|
||||
"\n",
|
||||
),
|
||||
[
|
||||
"New tools are available in addition to those previously listed:",
|
||||
...diff.added.map((entry) => entry.line),
|
||||
].join("\n"),
|
||||
)
|
||||
}
|
||||
if (diff.changed.length > 0) {
|
||||
@@ -200,9 +139,7 @@ const layer = Layer.effect(
|
||||
|
||||
return Service.of({
|
||||
load: Effect.fn("CodeModeInstructions.load")(function* (selection) {
|
||||
const entries = selection.info
|
||||
? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? [])
|
||||
: []
|
||||
const entries = selection.info ? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? []) : []
|
||||
const catalog = CodeModeCatalog.summarize(entries)
|
||||
return Instructions.make<CodeModeCatalog.Summary>({
|
||||
key: Instructions.Key.make("core/codemode"),
|
||||
|
||||
@@ -22,10 +22,7 @@ const update = (
|
||||
current: ReadonlyArray<CodeModeCatalog.Entry>,
|
||||
budget?: number,
|
||||
) =>
|
||||
CodeModeInstructions.update(
|
||||
CodeModeCatalog.summarize(previous, budget),
|
||||
CodeModeCatalog.summarize(current, budget),
|
||||
)
|
||||
CodeModeInstructions.update(CodeModeCatalog.summarize(previous, budget), CodeModeCatalog.summarize(current, budget))
|
||||
|
||||
describe("CodeModeCatalog.summarize", () => {
|
||||
test("retains namespace inventory without retaining tools outside the inline budget", () => {
|
||||
@@ -63,62 +60,37 @@ describe("CodeModeCatalog.summarize", () => {
|
||||
})
|
||||
|
||||
describe("CodeModeInstructions.render", () => {
|
||||
test("inlines complete catalogs with markdown sections and placeholder-only call forms", () => {
|
||||
test("inlines complete catalogs without search guidance", () => {
|
||||
const instructions = render([lookup])
|
||||
expect(instructions).toContain("## Available tools (COMPLETE list")
|
||||
expect(instructions).toContain("## Available tools")
|
||||
expect(instructions).toContain("- orders (1 tool)")
|
||||
expect(instructions).toContain(` - ${lookup.signature} // Look up an order by ID`)
|
||||
expect(instructions).not.toContain("search(")
|
||||
|
||||
expect(instructions).toContain("## Workflow")
|
||||
expect(instructions).toContain("## Rules")
|
||||
expect(instructions).toContain("## Language")
|
||||
expect(instructions.indexOf("## Workflow")).toBeLessThan(instructions.indexOf("## Rules"))
|
||||
expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Language"))
|
||||
expect(instructions.indexOf("## Language")).toBeLessThan(instructions.indexOf("\n## Available tools (COMPLETE"))
|
||||
expect(instructions).not.toContain("## Search")
|
||||
expect(instructions).toContain("Do not infer or normalize tool names")
|
||||
expect(instructions).toContain("bracket notation and quotes are part of the path")
|
||||
expect(instructions).toContain("surrounding agent tools are not available")
|
||||
expect(instructions).toContain("Only tools listed here are available")
|
||||
expect(instructions).toContain("`const result = await tools.<namespace>.<tool>(input)`")
|
||||
expect(instructions).toContain("check that it is a non-null object and not an array")
|
||||
expect(instructions).not.toContain("tools.orders.lookup({")
|
||||
expect(instructions).toContain("1. Pick a tool from the list under `## Available tools`")
|
||||
expect(instructions).not.toContain("Browse one namespace")
|
||||
expect(instructions).toContain('`tools.<namespace>["tool-name"](input)`')
|
||||
})
|
||||
|
||||
test("describes the restricted runtime without overclaiming", () => {
|
||||
test("describes the runtime and execution lifecycle concisely", () => {
|
||||
const instructions = render([lookup])
|
||||
expect(instructions).toContain("restricted JavaScript language for calling tools")
|
||||
expect(instructions).toContain("not a general-purpose runtime")
|
||||
for (const missing of ["Modules/imports", "classes", "fetch"]) {
|
||||
expect(instructions).toContain(missing)
|
||||
}
|
||||
// Generators are supported by the interpreter and must not be listed as unavailable.
|
||||
expect(instructions).not.toContain("generators")
|
||||
expect(instructions).toContain("URL, URLSearchParams, and URI encoding helpers")
|
||||
expect(instructions).toContain("Use tools for external operations")
|
||||
expect(instructions).toContain("Run JavaScript to orchestrate tool calls and compose their results.")
|
||||
expect(instructions).toContain("Imports, filesystem access, and timers are unavailable.")
|
||||
expect(instructions).toContain("Do not use `fetch`; all API calls go through `tools`.")
|
||||
expect(instructions).toContain(
|
||||
"Prefer explicit `return`; otherwise only the final top-level expression becomes the result.",
|
||||
)
|
||||
expect(instructions).toContain(
|
||||
"Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to `{}`.",
|
||||
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
|
||||
)
|
||||
expect(instructions).toContain("any calls still pending when execution ends are interrupted")
|
||||
expect(instructions).toContain("Run independent calls concurrently with `Promise.all`.")
|
||||
})
|
||||
|
||||
test("switches to search-first guidance when the catalog exceeds the budget", () => {
|
||||
test("adds search guidance when the catalog exceeds the budget", () => {
|
||||
const partial = render([lookup], 0)
|
||||
expect(partial).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))")
|
||||
expect(partial).toContain("## Available tools")
|
||||
expect(partial).toContain("- orders (1 tool, none shown)")
|
||||
expect(partial).toContain(
|
||||
'1. If needed, discover tools with the built-in search function: `return search({ query: "<intent + key nouns>" })`.',
|
||||
)
|
||||
expect(partial).toContain("In the next execution, copy a returned path exactly")
|
||||
expect(partial).toContain("Only tools listed here or returned by the built-in `search` function")
|
||||
expect(partial).toContain('- Browse one namespace: `search({ query: "", namespace: "<name>" })`.')
|
||||
expect(partial).toContain("repeat the same search with `offset: next.offset`")
|
||||
expect(partial).toContain("Search returns complete callable signatures:\n- search(input: {")
|
||||
expect(partial).toContain("## Search")
|
||||
expect(partial).toContain("Only some tool signatures are shown.")
|
||||
expect(partial).toContain("- search(input: {")
|
||||
expect(partial).toContain(" limit?: number,\n offset?: number,")
|
||||
expect(partial).toContain("or returned by `search`")
|
||||
expect(partial).not.toContain("tools.orders.lookup(input:")
|
||||
})
|
||||
|
||||
@@ -133,7 +105,7 @@ describe("CodeModeInstructions.render", () => {
|
||||
// Round 1 places alpha.cheap and beta.cheap; in round 2 alpha.expensive does not fit,
|
||||
// which marks only alpha done - it must NOT prevent other namespaces from inlining.
|
||||
const instructions = render([cheapAlpha, expensive, cheapBeta], 40)
|
||||
expect(instructions).toContain("## Available tools (PARTIAL - 2 of 3 shown; find the rest with search(...))")
|
||||
expect(instructions).toContain("## Search")
|
||||
expect(instructions).toContain("- alpha (2 tools, 1 shown)")
|
||||
expect(instructions).toContain(` - ${cheapAlpha.signature} // Cheap`)
|
||||
expect(instructions).not.toContain("tools.alpha.expensive(")
|
||||
@@ -148,18 +120,12 @@ describe("CodeModeInstructions.render", () => {
|
||||
`tools.records.lookup(input: {\n /** ${"A detailed identifier description. ".repeat(20).trim()} */\n id: string,\n}): Promise<string>`,
|
||||
)
|
||||
const instructions = render([documented], 40)
|
||||
expect(instructions).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))")
|
||||
expect(instructions).toContain("- records (1 tool, none shown)")
|
||||
expect(instructions).not.toContain("tools.records.lookup(input:")
|
||||
})
|
||||
|
||||
test("renders the no-tools notice with minimal sections for an empty catalog", () => {
|
||||
const instructions = render([])
|
||||
expect(instructions).toContain("No tools are currently available.")
|
||||
expect(instructions).toContain("## Language")
|
||||
expect(instructions).not.toContain("## Available tools")
|
||||
expect(instructions).not.toContain("## Workflow")
|
||||
expect(instructions).not.toContain("## Rules")
|
||||
expect(instructions).not.toContain("search(")
|
||||
test("renders only the no-tools notice for an empty catalog", () => {
|
||||
expect(render([])).toBe("No tools are currently available.")
|
||||
})
|
||||
})
|
||||
|
||||
@@ -193,14 +159,16 @@ describe("CodeModeInstructions.update", () => {
|
||||
expect(text).toContain(
|
||||
"The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.",
|
||||
)
|
||||
expect(text).toContain("## Available tools (PARTIAL")
|
||||
expect(text).toContain("## Search")
|
||||
expect(text).toContain("## Available tools")
|
||||
})
|
||||
|
||||
test("falls back to full replacement when the delta is larger than the catalog", () => {
|
||||
const previous = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
|
||||
const text = update([...previous, echo], [echo])
|
||||
expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
|
||||
expect(text).toContain("## Available tools (COMPLETE list")
|
||||
expect(text).toContain("## Available tools")
|
||||
expect(text).not.toContain("## Search")
|
||||
expect(text).not.toContain("must not be called")
|
||||
})
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ describe("CodeModeInstructions", () => {
|
||||
return Effect.gen(function* () {
|
||||
const instructions = yield* CodeModeInstructions.Service
|
||||
const initialized = yield* instructions.load({ id: agent.id, info: agent }).pipe(Effect.flatMap(readInitial))
|
||||
expect(initialized.text).toContain("## Available tools (COMPLETE list")
|
||||
expect(initialized.text).toContain("## Available tools")
|
||||
expect(initialized.text).not.toContain("## Search")
|
||||
expect(initialized.text).toContain(` - ${echo.signature} // Echo text`)
|
||||
|
||||
catalog = [echo, lookup]
|
||||
|
||||
Reference in New Issue
Block a user