diff --git a/instructions-rework.ts b/instructions-rework.ts deleted file mode 100644 index 10a3b52a1a..0000000000 --- a/instructions-rework.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { searchSignature } from "./packages/codemode/src" -import { CodeModeCatalog } from "./packages/core/src/codemode/catalog" - -const prompt = (catalog: CodeModeCatalog.Summary, hasMoreTools: boolean) => { - return `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`. - -Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`. - -Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.["tool-name"](input)\`.${hasMoreTools ? ` - -## Search - -Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools: - -- ${searchSignature}` : ""} - -## Available tools` -} - -export function render(catalog: CodeModeCatalog.Summary) { - const hasMoreTools = true - const tools = catalog.namespaces.flatMap((namespace) => { - const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools` - const label = (() => { - if (namespace.entries.length === namespace.count) return count - if (namespace.entries.length === 0) return `${count}, none shown` - return `${count}, ${namespace.entries.length} shown` - })() - return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)] - }) - - return `${prompt(catalog, hasMoreTools)} - -${tools.join("\n")}` -} diff --git a/packages/codemode/src/tool-runtime.ts b/packages/codemode/src/tool-runtime.ts index 598d133eb8..4af3fd5a29 100644 --- a/packages/codemode/src/tool-runtime.ts +++ b/packages/codemode/src/tool-runtime.ts @@ -463,7 +463,7 @@ const resolve = (root: ToolNode, path: ReadonlyArray): Tool => const node = lookup(root, segments) if (node === undefined) { throw new ToolRuntimeError("UnknownTool", `Unknown tool '${segments.join(".")}'.`, [ - "Use search({ query }) to find available described tools.", + "The tool may have been removed or renamed. Use search to find available tools.", ]) } if (node.tool === undefined) { @@ -536,7 +536,11 @@ export const make = ( const input = yield* Effect.try({ try: () => decodeToolInput(tool, externalArgs[0]), catch: (cause) => - new ToolRuntimeError("InvalidToolInput", `Invalid input for tool '${name}': ${String(cause)}`), + new ToolRuntimeError( + "InvalidToolInput", + `Invalid input for tool '${name}': ${String(cause)}`, + name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."], + ), }) const index = yield* recordAndObserve(name, input) return yield* observeEnd( diff --git a/packages/codemode/test/tool-paths.test.ts b/packages/codemode/test/tool-paths.test.ts index 38ef4c2850..6041ff997b 100644 --- a/packages/codemode/test/tool-paths.test.ts +++ b/packages/codemode/test/tool-paths.test.ts @@ -85,6 +85,9 @@ describe("callable namespaces", () => { const diagnostic = await failure(runtime, `return await tools.issues.missing({})`) expect(diagnostic.kind).toBe("UnknownTool") expect(diagnostic.message).toContain("Unknown tool 'issues.missing'") + expect(diagnostic.suggestions).toEqual([ + "The tool may have been removed or renamed. Use search to find available tools.", + ]) }) test("a namespace without its own tool stays non-callable", async () => { @@ -95,6 +98,31 @@ describe("callable namespaces", () => { }) }) +describe("tool input diagnostics", () => { + const runtime = CodeMode.make({ + tools: { + "notes.echo": Tool.make({ + description: "Echo text", + input: Schema.Struct({ text: Schema.String }), + output: Schema.String, + execute: ({ text }) => Effect.succeed(text), + }), + }, + }) + + test("a schema mismatch suggests searching for the current signature", async () => { + const diagnostic = await failure(runtime, `return await tools.notes.echo({ message: "hello" })`) + expect(diagnostic.kind).toBe("InvalidToolInput") + expect(diagnostic.suggestions).toEqual(["The signature may have changed. Use search to get the current signature."]) + }) + + test("a wrong argument count keeps the existing error without a stale-signature hint", async () => { + const diagnostic = await failure(runtime, `return await tools.notes.echo()`) + expect(diagnostic.kind).toBe("InvalidToolInput") + expect(diagnostic.suggestions).toBeUndefined() + }) +}) + describe("blocked member names on tool paths", () => { const runtime = CodeMode.make({ tools: { diff --git a/packages/core/src/codemode/instructions.ts b/packages/core/src/codemode/instructions.ts index 33bbe57c59..d12869daa1 100644 --- a/packages/core/src/codemode/instructions.ts +++ b/packages/core/src/codemode/instructions.ts @@ -8,106 +8,44 @@ import { CodeMode } from "../codemode" import { Instructions } from "../instructions/index" import { CodeModeCatalog } from "./catalog" -const completeWorkflow = `## Workflow +// prettier-ignore +const prompt = (hasMoreTools: boolean) => `Run JavaScript to orchestrate tool calls and compose their results. Imports, filesystem access, and timers are unavailable. Do not use \`fetch\`; all API calls go through \`tools\`. -1. Pick a tool from the list under \`## Available tools\` - each line is the exact call signature; use it as-is rather than guessing segments. -2. Call it using the exact signature shown: \`const result = await tools..(input)\`; bracket notation and quotes are part of the path. -3. Return only the fields you need from structured results; narrow unknown results before reading fields, and avoid returning large raw payloads.` +Prefer an explicit \`return\`; if omitted, the final top-level expression becomes the result. Await tool calls before returning; any calls still pending when execution ends are interrupted. Run independent calls concurrently with \`Promise.all\`. -const partialWorkflow = `## Workflow +Do not infer or normalize tool names; use only the exact signatures shown below${hasMoreTools ? " or returned by `search`" : ""}, preserving bracket notation such as \`tools.["tool-name"](input)\`.${hasMoreTools ? ` -1. If needed, discover tools with the built-in search function: \`return search({ query: "" })\`. -2. In the next execution, copy a returned path exactly, call it, and return only the needed fields.` +## Search -const language = `## Language +Only some tool signatures are shown. Use \`search\` to discover exact paths and signatures for additional tools: -Use common JavaScript data operations, functions, control flow, selected standard-library methods, and awaited tool calls. Built-ins include Date, RegExp, Map, Set, URL, URLSearchParams, and URI encoding helpers. -Modules/imports, classes, timers, fetch, eval, prototype access, and unlisted methods are unavailable. Use tools for external operations. Use await with try/catch. -Prefer explicit \`return\`; otherwise only the final top-level expression becomes the result. -Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to \`{}\`.` +- ${searchSignature}` : ""} + +## Available tools` export function render(catalog: CodeModeCatalog.Summary) { - const complete = catalog.shown === catalog.total - const sections: Array = [] + if (catalog.total === 0) return "No tools are currently available." - if (catalog.total === 0) { - sections.push("This is a restricted JavaScript language for calling tools, not a general-purpose runtime.") - } else { - const availability = (() => { - if (complete) return "listed below" - return "listed or searchable below" - })() - sections.push( - `This is a restricted JavaScript language for calling tools, not a general-purpose runtime. Inside the confined interpreter, \`tools\` contains the tools ${availability}; surrounding agent tools are not available.\nDo not infer or normalize tool names; use only exact signatures shown below or returned by search.`, - ) + const tools = catalog.namespaces.flatMap((namespace) => { + const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools` + const label = + namespace.entries.length === namespace.count + ? count + : namespace.entries.length === 0 + ? `${count}, none shown` + : `${count}, ${namespace.entries.length} shown` + return [`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)] + }) - if (complete) sections.push(completeWorkflow) - if (!complete) sections.push(partialWorkflow) + return `${prompt(catalog.shown < catalog.total)} - const availabilityRule = (() => { - if (complete) return "- Only tools listed here are available; surrounding agent tools are not implicitly exposed." - return "- Only tools listed here or returned by the built-in `search` function are available; surrounding agent tools are not implicitly exposed." - })() - const rules = [ - "## Rules", - "", - availabilityRule, - "- Filter, aggregate, and transform collections in code - never return them raw or call a tool per item across messages.", - "- A result typed `Promise` may be structured data or text. Before reading fields, check that it is a non-null object and not an array; otherwise handle the returned text or primitive directly.", - '- Run independent calls in parallel: `await Promise.all(items.map((item) => tools..(item)))`, or use `tools.["tool-name"](item)` when the listed signature uses bracket notation.', - "- Execution ends when the program returns; pending promises are interrupted, so await every call whose completion matters.", - "- `Object.keys(tools)` lists namespaces; `Object.keys(tools.)` lists its tools; `for...in` works on both.", - ] - if (!complete) { - rules.push( - '- Browse one namespace: `search({ query: "", namespace: "" })`.', - "- If search returns `next`, repeat the same search with `offset: next.offset`.", - ) - } - sections.push(rules.join("\n")) - } - - sections.push(language) - - if (catalog.total === 0) { - sections.push("No tools are currently available.") - return sections.join("\n\n") - } - - const tools: Array = [] - if (complete) { - tools.push("## Available tools (COMPLETE list - every tool is shown below with its full call signature)", "") - } else { - tools.push( - `## Available tools (PARTIAL - ${catalog.shown} of ${catalog.total} shown; find the rest with search(...))`, - "", - ) - } - - for (const namespace of catalog.namespaces) { - const count = (() => { - if (namespace.count === 1) return "1 tool" - return `${namespace.count} tools` - })() - const label = (() => { - if (namespace.entries.length === namespace.count) return count - if (namespace.entries.length === 0) return `${count}, none shown` - return `${count}, ${namespace.entries.length} shown` - })() - tools.push(`- ${namespace.name} (${label})`, ...namespace.entries.map((entry) => entry.line)) - } - - if (!complete) tools.push("", "Search returns complete callable signatures:", `- ${searchSignature}`) - sections.push(tools.join("\n")) - - return sections.join("\n\n") +${tools.join("\n")}` } export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatalog.Summary) { - const full = [ - "The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.", - render(current), - ].join("\n\n") + const full = `The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog. + +${render(current)}` const previousComplete = previous.shown === previous.total const currentComplete = current.shown === current.total if (previousComplete !== currentComplete) return full @@ -162,9 +100,10 @@ export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatal const parts = ["The Code Mode tool catalog has changed."] if (diff.added.length > 0) { parts.push( - ["New tools are available in addition to those previously listed:", ...diff.added.map((entry) => entry.line)].join( - "\n", - ), + [ + "New tools are available in addition to those previously listed:", + ...diff.added.map((entry) => entry.line), + ].join("\n"), ) } if (diff.changed.length > 0) { @@ -200,9 +139,7 @@ const layer = Layer.effect( return Service.of({ load: Effect.fn("CodeModeInstructions.load")(function* (selection) { - const entries = selection.info - ? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? []) - : [] + const entries = selection.info ? ((yield* codeMode.materialize(selection.info.permissions)).catalog ?? []) : [] const catalog = CodeModeCatalog.summarize(entries) return Instructions.make({ key: Instructions.Key.make("core/codemode"), diff --git a/packages/core/test/codemode/catalog.test.ts b/packages/core/test/codemode/catalog.test.ts index 9cef26e170..0cf0809d13 100644 --- a/packages/core/test/codemode/catalog.test.ts +++ b/packages/core/test/codemode/catalog.test.ts @@ -22,10 +22,7 @@ const update = ( current: ReadonlyArray, budget?: number, ) => - CodeModeInstructions.update( - CodeModeCatalog.summarize(previous, budget), - CodeModeCatalog.summarize(current, budget), - ) + CodeModeInstructions.update(CodeModeCatalog.summarize(previous, budget), CodeModeCatalog.summarize(current, budget)) describe("CodeModeCatalog.summarize", () => { test("retains namespace inventory without retaining tools outside the inline budget", () => { @@ -63,62 +60,37 @@ describe("CodeModeCatalog.summarize", () => { }) describe("CodeModeInstructions.render", () => { - test("inlines complete catalogs with markdown sections and placeholder-only call forms", () => { + test("inlines complete catalogs without search guidance", () => { const instructions = render([lookup]) - expect(instructions).toContain("## Available tools (COMPLETE list") + expect(instructions).toContain("## Available tools") expect(instructions).toContain("- orders (1 tool)") expect(instructions).toContain(` - ${lookup.signature} // Look up an order by ID`) - expect(instructions).not.toContain("search(") - - expect(instructions).toContain("## Workflow") - expect(instructions).toContain("## Rules") - expect(instructions).toContain("## Language") - expect(instructions.indexOf("## Workflow")).toBeLessThan(instructions.indexOf("## Rules")) - expect(instructions.indexOf("## Rules")).toBeLessThan(instructions.indexOf("## Language")) - expect(instructions.indexOf("## Language")).toBeLessThan(instructions.indexOf("\n## Available tools (COMPLETE")) + expect(instructions).not.toContain("## Search") expect(instructions).toContain("Do not infer or normalize tool names") - expect(instructions).toContain("bracket notation and quotes are part of the path") - expect(instructions).toContain("surrounding agent tools are not available") - expect(instructions).toContain("Only tools listed here are available") - expect(instructions).toContain("`const result = await tools..(input)`") - expect(instructions).toContain("check that it is a non-null object and not an array") - expect(instructions).not.toContain("tools.orders.lookup({") - expect(instructions).toContain("1. Pick a tool from the list under `## Available tools`") - expect(instructions).not.toContain("Browse one namespace") + expect(instructions).toContain('`tools.["tool-name"](input)`') }) - test("describes the restricted runtime without overclaiming", () => { + test("describes the runtime and execution lifecycle concisely", () => { const instructions = render([lookup]) - expect(instructions).toContain("restricted JavaScript language for calling tools") - expect(instructions).toContain("not a general-purpose runtime") - for (const missing of ["Modules/imports", "classes", "fetch"]) { - expect(instructions).toContain(missing) - } - // Generators are supported by the interpreter and must not be listed as unavailable. - expect(instructions).not.toContain("generators") - expect(instructions).toContain("URL, URLSearchParams, and URI encoding helpers") - expect(instructions).toContain("Use tools for external operations") + expect(instructions).toContain("Run JavaScript to orchestrate tool calls and compose their results.") + expect(instructions).toContain("Imports, filesystem access, and timers are unavailable.") + expect(instructions).toContain("Do not use `fetch`; all API calls go through `tools`.") expect(instructions).toContain( - "Prefer explicit `return`; otherwise only the final top-level expression becomes the result.", - ) - expect(instructions).toContain( - "Dates and URLs serialize to strings at data boundaries; Map/Set/RegExp/URLSearchParams serialize to `{}`.", + "Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.", ) + expect(instructions).toContain("any calls still pending when execution ends are interrupted") + expect(instructions).toContain("Run independent calls concurrently with `Promise.all`.") }) - test("switches to search-first guidance when the catalog exceeds the budget", () => { + test("adds search guidance when the catalog exceeds the budget", () => { const partial = render([lookup], 0) - expect(partial).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))") + expect(partial).toContain("## Available tools") expect(partial).toContain("- orders (1 tool, none shown)") - expect(partial).toContain( - '1. If needed, discover tools with the built-in search function: `return search({ query: "" })`.', - ) - expect(partial).toContain("In the next execution, copy a returned path exactly") - expect(partial).toContain("Only tools listed here or returned by the built-in `search` function") - expect(partial).toContain('- Browse one namespace: `search({ query: "", namespace: "" })`.') - expect(partial).toContain("repeat the same search with `offset: next.offset`") - expect(partial).toContain("Search returns complete callable signatures:\n- search(input: {") + expect(partial).toContain("## Search") + expect(partial).toContain("Only some tool signatures are shown.") + expect(partial).toContain("- search(input: {") expect(partial).toContain(" limit?: number,\n offset?: number,") + expect(partial).toContain("or returned by `search`") expect(partial).not.toContain("tools.orders.lookup(input:") }) @@ -133,7 +105,7 @@ describe("CodeModeInstructions.render", () => { // Round 1 places alpha.cheap and beta.cheap; in round 2 alpha.expensive does not fit, // which marks only alpha done - it must NOT prevent other namespaces from inlining. const instructions = render([cheapAlpha, expensive, cheapBeta], 40) - expect(instructions).toContain("## Available tools (PARTIAL - 2 of 3 shown; find the rest with search(...))") + expect(instructions).toContain("## Search") expect(instructions).toContain("- alpha (2 tools, 1 shown)") expect(instructions).toContain(` - ${cheapAlpha.signature} // Cheap`) expect(instructions).not.toContain("tools.alpha.expensive(") @@ -148,18 +120,12 @@ describe("CodeModeInstructions.render", () => { `tools.records.lookup(input: {\n /** ${"A detailed identifier description. ".repeat(20).trim()} */\n id: string,\n}): Promise`, ) const instructions = render([documented], 40) - expect(instructions).toContain("## Available tools (PARTIAL - 0 of 1 shown; find the rest with search(...))") + expect(instructions).toContain("- records (1 tool, none shown)") expect(instructions).not.toContain("tools.records.lookup(input:") }) - test("renders the no-tools notice with minimal sections for an empty catalog", () => { - const instructions = render([]) - expect(instructions).toContain("No tools are currently available.") - expect(instructions).toContain("## Language") - expect(instructions).not.toContain("## Available tools") - expect(instructions).not.toContain("## Workflow") - expect(instructions).not.toContain("## Rules") - expect(instructions).not.toContain("search(") + test("renders only the no-tools notice for an empty catalog", () => { + expect(render([])).toBe("No tools are currently available.") }) }) @@ -193,14 +159,16 @@ describe("CodeModeInstructions.update", () => { expect(text).toContain( "The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.", ) - expect(text).toContain("## Available tools (PARTIAL") + expect(text).toContain("## Search") + expect(text).toContain("## Available tools") }) test("falls back to full replacement when the delta is larger than the catalog", () => { const previous = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) const text = update([...previous, echo], [echo]) expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.") - expect(text).toContain("## Available tools (COMPLETE list") + expect(text).toContain("## Available tools") + expect(text).not.toContain("## Search") expect(text).not.toContain("must not be called") }) diff --git a/packages/core/test/codemode/instructions.test.ts b/packages/core/test/codemode/instructions.test.ts index 187b2ac156..9e7ee58553 100644 --- a/packages/core/test/codemode/instructions.test.ts +++ b/packages/core/test/codemode/instructions.test.ts @@ -38,7 +38,8 @@ describe("CodeModeInstructions", () => { return Effect.gen(function* () { const instructions = yield* CodeModeInstructions.Service const initialized = yield* instructions.load({ id: agent.id, info: agent }).pipe(Effect.flatMap(readInitial)) - expect(initialized.text).toContain("## Available tools (COMPLETE list") + expect(initialized.text).toContain("## Available tools") + expect(initialized.text).not.toContain("## Search") expect(initialized.text).toContain(` - ${echo.signature} // Echo text`) catalog = [echo, lookup]