From 7defaea8a013348cbe94248c3147f282a7fc47cd Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 31 Mar 2026 21:54:42 -0400 Subject: [PATCH] test(app): route prompt e2e through mock llm --- packages/app/e2e/prompt/prompt.spec.ts | 187 ++++++++++------- packages/opencode/test/lib/llm-server.ts | 243 +++++++++++++++++++++++ 2 files changed, 355 insertions(+), 75 deletions(-) diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.ts index 1acf17f5bf..80c05ce662 100644 --- a/packages/app/e2e/prompt/prompt.spec.ts +++ b/packages/app/e2e/prompt/prompt.spec.ts @@ -1,44 +1,66 @@ -import fs from "node:fs/promises" -import path from "node:path" import { test, expect } from "../fixtures" import { promptSelector } from "../selectors" import { sessionIDFromUrl } from "../actions" -import { createSdk } from "../utils" -async function config(dir: string, url: string) { - await fs.writeFile( - path.join(dir, "opencode.json"), - JSON.stringify({ - $schema: "https://opencode.ai/config.json", - enabled_providers: ["e2e-llm"], - provider: { - "e2e-llm": { - name: "E2E LLM", - npm: "@ai-sdk/openai-compatible", - env: [], - models: { - "test-model": { - name: "Test Model", - tool_call: true, - limit: { context: 128000, output: 32000 }, - }, - }, - options: { - apiKey: "test-key", - baseURL: url, - }, - }, - }, - agent: { - build: { - model: "e2e-llm/test-model", - }, - }, - }), - ) +const mdl = { providerID: "openai", modelID: "gpt-5.3-chat-latest" } + +async function pickModel(page: Parameters[0]["page"], value: { providerID: string; modelID: string }) { + await expect + .poll( + () => + page.evaluate(() => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + controls?: { + setModel?: (value: { providerID: string; modelID: string } | undefined) => void + } + } + } + } + return !!win.__opencode_e2e?.model?.controls?.setModel + }), + { timeout: 30_000 }, + ) + .toBe(true) + + await page.evaluate((value) => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + controls?: { + setModel?: (value: { providerID: string; modelID: string } | undefined) => void + } + } + } + } + const fn = win.__opencode_e2e?.model?.controls?.setModel + if (!fn) throw new Error("Model e2e model control is not enabled") + fn(value) + }, value) + + await expect + .poll( + () => + page.evaluate(() => { + const win = window as Window & { + __opencode_e2e?: { + model?: { + current?: { + model?: { providerID: string; modelID: string } + } + } + } + } + const model = win.__opencode_e2e?.model?.current?.model + return model ? `${model.providerID}/${model.modelID}` : null + }), + { timeout: 30_000 }, + ) + .toBe(`${value.providerID}/${value.modelID}`) } -test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => { +test("can send a prompt and receive a reply", async ({ page, llm, sdk, gotoSession }) => { test.setTimeout(120_000) const pageErrors: string[] = [] @@ -47,50 +69,65 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject }) } page.on("pageerror", onPageError) + const prev = await sdk.global.config.get().then((res) => res.data ?? {}) + try { - await withProject( - async (project) => { - const sdk = createSdk(project.directory) - const token = `E2E_OK_${Date.now()}` - - await llm.text(token) - await project.gotoSession() - - const prompt = page.locator(promptSelector) - await prompt.click() - await page.keyboard.type(`Reply with exactly: ${token}`) - await page.keyboard.press("Enter") - - await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) - - const sessionID = (() => { - const id = sessionIDFromUrl(page.url()) - if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) - return id - })() - project.trackSession(sessionID) - - await expect - .poll( - async () => { - const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) - return messages - .filter((m) => m.info.role === "assistant") - .flatMap((m) => m.parts) - .filter((p) => p.type === "text") - .map((p) => p.text) - .join("\n") + await sdk.global.config.update({ + config: { + ...prev, + model: `${mdl.providerID}/${mdl.modelID}`, + enabled_providers: ["openai"], + provider: { + ...prev.provider, + openai: { + ...prev.provider?.openai, + options: { + ...prev.provider?.openai?.options, + apiKey: "test-key", + baseURL: llm.url, }, - { timeout: 30_000 }, - ) - .toContain(token) + }, + }, }, - { - model: { providerID: "e2e-llm", modelID: "test-model" }, - setup: (dir) => config(dir, llm.url), - }, - ) + }) + + const token = `E2E_OK_${Date.now()}` + await llm.text("E2E Title") + await llm.text(token) + await gotoSession() + await pickModel(page, mdl) + + const prompt = page.locator(promptSelector) + await prompt.click() + await page.keyboard.type(`Reply with exactly: ${token}`) + await page.keyboard.press("Enter") + + await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 }) + + const sessionID = (() => { + const id = sessionIDFromUrl(page.url()) + if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`) + return id + })() + + await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(2) + + await expect + .poll( + async () => { + const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? []) + return messages + .filter((m) => m.info.role === "assistant") + .flatMap((m) => m.parts) + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n") + }, + { timeout: 30_000 }, + ) + .toContain(token) } finally { + await sdk.global.config.update({ config: prev }) page.off("pageerror", onPageError) } diff --git a/packages/opencode/test/lib/llm-server.ts b/packages/opencode/test/lib/llm-server.ts index 8e7365d97f..d651140b31 100644 --- a/packages/opencode/test/lib/llm-server.ts +++ b/packages/opencode/test/lib/llm-server.ts @@ -119,6 +119,225 @@ function bytes(input: Iterable) { return Stream.fromIterable([...input].map(line)).pipe(Stream.encodeText) } +function created(model: string) { + return { + type: "response.created", + sequence_number: 1, + response: { + id: "resp_test", + created_at: Math.floor(Date.now() / 1000), + model, + service_tier: null, + }, + } +} + +function completed(input: { seq: number; usage?: Usage }) { + return { + type: "response.completed", + sequence_number: input.seq, + response: { + incomplete_details: null, + service_tier: null, + usage: { + input_tokens: input.usage?.input ?? 0, + input_tokens_details: { cached_tokens: null }, + output_tokens: input.usage?.output ?? 0, + output_tokens_details: { reasoning_tokens: null }, + }, + }, + } +} + +function responses(item: Sse, model: string) { + let seq = 1 + let msg: string | undefined + let reason: string | undefined + let call: + | { + id: string + item: string + name: string + args: string + } + | undefined + let usage: Usage | undefined + const lines: unknown[] = [created(model)] + + const all = [...item.head, ...item.tail] + for (const part of all) { + if (!part || typeof part !== "object") continue + if (!("choices" in part) || !Array.isArray(part.choices)) continue + const choice = part.choices[0] + if (!choice || typeof choice !== "object") continue + const delta = "delta" in choice && choice.delta && typeof choice.delta === "object" ? choice.delta : undefined + + if (delta && "content" in delta && typeof delta.content === "string") { + msg ||= "msg_1" + if ( + !lines.some( + (item) => + typeof item === "object" && + item && + "type" in item && + item.type === "response.output_item.added" && + "item" in item && + item.item && + typeof item.item === "object" && + "id" in item.item && + item.item.id === msg, + ) + ) { + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "message", id: msg }, + }) + } + seq += 1 + lines.push({ + type: "response.output_text.delta", + sequence_number: seq, + item_id: msg, + delta: delta.content, + logprobs: null, + }) + } + + if (delta && "reasoning_content" in delta && typeof delta.reasoning_content === "string") { + reason ||= "rs_1" + if ( + !lines.some( + (item) => + typeof item === "object" && + item && + "type" in item && + item.type === "response.output_item.added" && + "item" in item && + item.item && + typeof item.item === "object" && + "id" in item.item && + item.item.id === reason, + ) + ) { + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id: reason, encrypted_content: null }, + }) + seq += 1 + lines.push({ + type: "response.reasoning_summary_part.added", + sequence_number: seq, + item_id: reason, + summary_index: 0, + }) + } + seq += 1 + lines.push({ + type: "response.reasoning_summary_text.delta", + sequence_number: seq, + item_id: reason, + summary_index: 0, + delta: delta.reasoning_content, + }) + } + + if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { + for (const tool of delta.tool_calls) { + if (!tool || typeof tool !== "object") continue + const fn = "function" in tool && tool.function && typeof tool.function === "object" ? tool.function : undefined + const id = "id" in tool && typeof tool.id === "string" ? tool.id : call?.id + const name = fn && "name" in fn && typeof fn.name === "string" ? fn.name : call?.name + const args = fn && "arguments" in fn && typeof fn.arguments === "string" ? fn.arguments : "" + if (!id || !name) continue + if (!call) { + call = { id, item: "fc_1", name, args: "" } + seq += 1 + lines.push({ + type: "response.output_item.added", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: call.item, + call_id: id, + name, + arguments: "", + status: "in_progress", + }, + }) + } + call.args += args + if (args) { + seq += 1 + lines.push({ + type: "response.function_call_arguments.delta", + sequence_number: seq, + output_index: 0, + item_id: call.item, + delta: args, + }) + } + } + } + + if ("usage" in part && part.usage && typeof part.usage === "object") { + const raw = part.usage as Record + if (typeof raw.prompt_tokens === "number" && typeof raw.completion_tokens === "number") { + usage = { input: raw.prompt_tokens, output: raw.completion_tokens } + } + } + } + + if (msg) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "message", id: msg }, + }) + } + if (reason) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { type: "reasoning", id: reason, encrypted_content: null }, + }) + } + if (call && !item.hang && !item.error) { + seq += 1 + lines.push({ + type: "response.output_item.done", + sequence_number: seq, + output_index: 0, + item: { + type: "function_call", + id: call.item, + call_id: call.id, + name: call.name, + arguments: call.args, + status: "completed", + }, + }) + } + if (!item.hang && !item.error) lines.push(completed({ seq: seq + 1, usage })) + return { ...item, head: lines, tail: [] } satisfies Sse +} + +function modelFrom(body: unknown) { + if (!body || typeof body !== "object") return "test-model" + if (!("model" in body) || typeof body.model !== "string") return "test-model" + return body.model +} + function send(item: Sse) { const head = bytes(item.head) const tail = bytes([...item.tail, ...(item.hang || item.error ? [] : [done])]) @@ -358,6 +577,9 @@ export class TestLLMServer extends ServiceMap.Service ({}))) + hits = [ + ...hits, + { + url: new URL(req.originalUrl, "http://localhost"), + body: body && typeof body === "object" ? (body as Record) : {}, + }, + ] + yield* notify() + if (next.type === "sse") return send(responses(next, modelFrom(body))) + return fail(next) + }), + ) + yield* server.serve(router.asHttpEffect()) return TestLLMServer.of({